From f8645fbca4d3b2e0116a7360714e7cefdf0bc91b Mon Sep 17 00:00:00 2001 From: Chen Zhiyang <1792266893@qq.com> Date: Sat, 23 Dec 2023 21:08:42 +0800 Subject: [PATCH 001/146] fix hsigmoid_loss_grad infermeta bug (#60264) --- paddle/phi/infermeta/backward.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 3f8eac6468381..71c4e5ecbca06 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -448,7 +448,9 @@ void GeneralTernaryGradInferMeta(const MetaTensor& x, dy->share_meta(y); } if (dz) { - dz->share_meta(z); + if (z) { + dz->share_meta(z); + } } } void GeneralQuaternaryGradInferMeta(const MetaTensor& x, From 300d22a3ce1146eda8470d1f55eb8acc8daa98d9 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Sun, 24 Dec 2023 10:44:27 +0800 Subject: [PATCH 002/146] Fix build ci (#60285) --- paddle/scripts/paddle_build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index ee6171510cdfe..0adff56ba7a1c 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -3487,9 +3487,15 @@ function build_pr_and_develop() { rm -f ${PADDLE_ROOT}/build/python/dist/*.whl && rm -f ${PADDLE_ROOT}/build/python/build/.timestamp git checkout $BRANCH - dev_commit=`git log -1|head -1|awk '{print $2}'` - dev_url="https://xly-devops.bj.bcebos.com/PR/build_whl/0/${dev_commit}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl" + dev_commit=`git log -2|grep -w 'commit'|awk '{print $2}'` + for commit_id in $dev_commit + do + dev_url="https://xly-devops.bj.bcebos.com/PR/build_whl/0/${commit_id}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl" url_return=`curl -s -m 5 -IL ${dev_url} |awk 'NR==1{print $2}'` + if [ "$url_return" == '200' ];then + break + fi + done if [ "$url_return" == '200' ];then mkdir ${PADDLE_ROOT}/build/dev_whl && wget -q -P ${PADDLE_ROOT}/build/dev_whl ${dev_url} cp ${PADDLE_ROOT}/build/dev_whl/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl ${PADDLE_ROOT}/build/python/dist From 4663496ced13010b6146a29c5a7aa9a7e9bb67d2 Mon Sep 17 00:00:00 2001 From: xiongkun Date: Sun, 24 Dec 2023 11:04:50 +0800 Subject: [PATCH 003/146] [Dy2static] fix paddle.grad copy node error. (#60267) --- .../eager/to_static/run_program_op_node.h | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index af30f605b9fd0..1fc63942a7669 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -864,7 +864,9 @@ inline void RunProgramGradAPI( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, 1); - VLOG(2) << "No interpretercore cahce, so create a new interpretercore"; + VLOG(2) << "No interpretercore cahce, so create a new interpretercore" + "for program: " + << program_id; details::ShareTensorsIntoScope(out_grad, global_inner_scope); bool in_pir_pt_mode = FLAGS_enable_pir_with_pt_in_dy2st; @@ -1154,12 +1156,14 @@ inline void PirRunProgramGradAPI( class GradNodeRunProgram : public egr::GradNodeBase { public: GradNodeRunProgram(size_t bwd_in_slot_num, size_t bwd_out_slot_num) - : egr::GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) {} + : egr::GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) { + VLOG(4) << "GradNodeRunProgram"; + } ~GradNodeRunProgram() override { - if (!executed_) { + if (!(*executed_)) { auto *out_scope_vec = &step_scope_; - VLOG(4) << "~GradNodeRunProgram"; + VLOG(4) << "~GradNodeRunProgram: " << this; // Normally out_scope_vec.size() == 1. for safty, we add for-loop here. for (size_t i = 0; i < out_scope_vec->size(); ++i) { paddle::framework::Scope *global_inner_scope = out_scope_vec->at(i); @@ -1226,9 +1230,9 @@ class GradNodeRunProgram : public egr::GradNodeBase { x_grad_ptr, params_grad_ptr, place_hash_keys_); - VLOG(3) << "End Eager Backward Node: GradNodeRunProgram"; + VLOG(3) << "End Eager Backward Node: GradNodeRunProgram: Ptr " << this; - executed_ = true; + *executed_ = true; egr::EagerUtils::FillZeroForEmptyOptionalGradOutput(&x_grad, this->OutputMeta()[0]); egr::EagerUtils::FillZeroForEmptyOptionalGradOutput(¶ms_grad, @@ -1332,7 +1336,9 @@ class GradNodeRunProgram : public egr::GradNodeBase { std::vector place_hash_keys_; - bool executed_{false}; + // why use shared_ptr. because paddle.grad will copy GradNode, if + // we use bool, the copied node have different executed states. + std::shared_ptr executed_ = std::make_shared(false); }; class PirGradNodeRunProgram : public egr::GradNodeBase { @@ -1341,7 +1347,7 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { : egr::GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) {} ~PirGradNodeRunProgram() override { - if (!executed_) { + if (!(*executed_)) { auto *out_scope_vec = &step_scope_; VLOG(4) << "~PirGradNodeRunProgram"; // Normally out_scope_vec.size() == 1. for safty, we add for-loop here. @@ -1414,7 +1420,7 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { place_hash_keys_); VLOG(3) << "End Eager Backward Node: PirGradNodeRunProgram"; - executed_ = true; + *executed_ = true; return {x_grad, params_grad}; } @@ -1519,5 +1525,5 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { std::vector place_hash_keys_; - bool executed_{false}; + std::shared_ptr executed_ = std::make_shared(false); }; From 78d0a2ec499197b0f5d2b15056417ff898ad46c6 Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Sun, 24 Dec 2023 15:41:02 +0800 Subject: [PATCH 004/146] [CodeStyle][ruff] clean some F401 step: 11 (#60290) --- pyproject.toml | 6 -- .../distributed/fleet/elastic/__init__.py | 16 +++-- .../distribute_transpiler/__init__.py | 70 ++++++++----------- .../distributed/models/moe/__init__.py | 4 +- .../distributed/models/moe/gate/__init__.py | 8 +-- .../incubate/distributed/utils/io/__init__.py | 4 +- 6 files changed, 46 insertions(+), 62 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb43be382de6f..64727e39f1d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,12 +126,6 @@ known-first-party = ["paddle"] # Ignore unnecessary lambda in dy2st unittest test_lambda "test/dygraph_to_static/test_lambda.py" = ["PLC3002"] -# temp ignore unused imports in all distributed files -"python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py" = ["F401", "I001"] -"python/paddle/incubate/distributed/models/moe/gate/__init__.py" = ["F401", "I001"] -"python/paddle/incubate/distributed/models/moe/__init__.py" = ["F401", "I001"] -"python/paddle/incubate/distributed/utils/io/__init__.py" = ["F401", "I001"] -"python/paddle/distributed/fleet/elastic/__init__.py" = ["F401", "I001"] # temp ignore isort "python/paddle/amp/__init__.py" = ["I001"] diff --git a/python/paddle/distributed/fleet/elastic/__init__.py b/python/paddle/distributed/fleet/elastic/__init__.py index 8dc190a10511a..345f2e61907c3 100644 --- a/python/paddle/distributed/fleet/elastic/__init__.py +++ b/python/paddle/distributed/fleet/elastic/__init__.py @@ -12,17 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import signal import os +import signal import sys -from .manager import ElasticManager -from .manager import ElasticStatus -from .manager import ELASTIC_EXIT_CODE -from .manager import ElasticLevel -from .collective import CollectiveLauncher +from paddle.distributed.fleet.launch_utils import DistributeMode # noqa: F401 -from paddle.distributed.fleet.launch_utils import DistributeMode +from .collective import CollectiveLauncher +from .manager import ( + ELASTIC_EXIT_CODE, + ElasticLevel, # noqa: F401 + ElasticManager, + ElasticStatus, +) def enable_elastic(args, distribute_mode): diff --git a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py index d590fe145ebc3..f810014e93b3b 100644 --- a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py +++ b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py @@ -17,63 +17,51 @@ import os import sys -import warnings import paddle -from paddle.framework import core -from paddle.static import ( - default_main_program, - default_startup_program, - Program, - Executor, -) from paddle.base.compiler import CompiledProgram - +from paddle.distributed.fleet.base.private_helper_function import ( + wait_server_ready, +) from paddle.distributed.transpiler.distribute_transpiler import ( DistributeTranspilerConfig, ) - -from paddle.incubate.distributed.fleet.base import Fleet -from paddle.incubate.distributed.fleet.base import Mode -from paddle.incubate.distributed.fleet.role_maker import MPISymetricRoleMaker - -from paddle.incubate.distributed.fleet.parameter_server import version -from paddle.incubate.distributed.fleet.parameter_server.pslib.optimizer_factory import ( - DistributedAdam, -) -from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( - get_sparse_tablenames, -) -from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( - _get_lr_ops, -) -from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( - _has_global_step, +from paddle.framework import core +from paddle.incubate.distributed.fleet.base import ( + DistributedOptimizer, + Fleet, + Mode, ) +from paddle.incubate.distributed.fleet.parameter_server import version from paddle.incubate.distributed.fleet.parameter_server.distribute_transpiler.distributed_strategy import ( - TrainerRuntimeConfig, - DistributedStrategy, - SyncStrategy, AsyncStrategy, - HalfAsyncStrategy, + DistributedStrategy, GeoStrategy, + HalfAsyncStrategy, StrategyFactory, + SyncStrategy, + TrainerRuntimeConfig, # noqa: F401 ) - -from paddle.distributed.fleet.base.private_helper_function import ( - wait_server_ready, -) -from paddle.incubate.distributed.fleet.base import DistributedOptimizer -from paddle.incubate.distributed.fleet.parameter_server.mode import PSMode - from paddle.incubate.distributed.fleet.parameter_server.ir import ( + pserver_pass as server, + public, trainer_pass as worker, ) -from paddle.incubate.distributed.fleet.parameter_server.ir import ( - pserver_pass as server, +from paddle.incubate.distributed.fleet.parameter_server.ir.public import ( + _get_lr_ops, + _has_global_step, + get_sparse_tablenames, ) -from paddle.incubate.distributed.fleet.parameter_server.ir import ( - public, +from paddle.incubate.distributed.fleet.parameter_server.mode import PSMode +from paddle.incubate.distributed.fleet.parameter_server.pslib.optimizer_factory import ( + DistributedAdam, # noqa: F401 +) +from paddle.incubate.distributed.fleet.role_maker import MPISymetricRoleMaker +from paddle.static import ( + Executor, + Program, + default_main_program, + default_startup_program, ) diff --git a/python/paddle/incubate/distributed/models/moe/__init__.py b/python/paddle/incubate/distributed/models/moe/__init__.py index 795c939e81fbb..716571dac5382 100644 --- a/python/paddle/incubate/distributed/models/moe/__init__.py +++ b/python/paddle/incubate/distributed/models/moe/__init__.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .gate import GShardGate, BaseGate, SwitchGate, NaiveGate -from .moe_layer import MoELayer +from .gate import BaseGate, GShardGate, NaiveGate, SwitchGate # noqa: F401 from .grad_clip import ClipGradForMOEByGlobalNorm +from .moe_layer import MoELayer # noqa: F401 ClipGradByGlobalNorm = ClipGradForMOEByGlobalNorm diff --git a/python/paddle/incubate/distributed/models/moe/gate/__init__.py b/python/paddle/incubate/distributed/models/moe/gate/__init__.py index 2bfa5cd62cd49..ce52a117ee715 100644 --- a/python/paddle/incubate/distributed/models/moe/gate/__init__.py +++ b/python/paddle/incubate/distributed/models/moe/gate/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .gshard_gate import GShardGate -from .switch_gate import SwitchGate -from .naive_gate import NaiveGate -from .base_gate import BaseGate +from .base_gate import BaseGate # noqa: F401 +from .gshard_gate import GShardGate # noqa: F401 +from .naive_gate import NaiveGate # noqa: F401 +from .switch_gate import SwitchGate # noqa: F401 diff --git a/python/paddle/incubate/distributed/utils/io/__init__.py b/python/paddle/incubate/distributed/utils/io/__init__.py index de970a1339038..cd2b618beb927 100644 --- a/python/paddle/incubate/distributed/utils/io/__init__.py +++ b/python/paddle/incubate/distributed/utils/io/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .dist_save import save, save_for_auto_inference -from .dist_load import load +from .dist_load import load # noqa: F401 +from .dist_save import save, save_for_auto_inference # noqa: F401 From 5faf595c036d58f9108f5118861201be78070113 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Mon, 25 Dec 2023 10:03:29 +0800 Subject: [PATCH 005/146] [Dy2St] Move transformers to a separate dir (#60246) --- python/paddle/jit/dy2static/__init__.py | 3 +- python/paddle/jit/dy2static/ast_utils.py | 5 ++++ .../jit/dy2static/program_translator.py | 2 +- .../jit/dy2static/transformers/__init__.py | 15 ++++++++++ .../{ => transformers}/assert_transformer.py | 2 +- .../base.py} | 30 ------------------- .../basic_api_transformer.py | 4 +-- .../break_continue_transformer.py | 2 +- .../{ => transformers}/call_transformer.py | 4 +-- .../{ => transformers}/cast_transformer.py | 2 +- .../create_variable_transformer.py | 6 ++-- .../decorator_transformer.py | 4 +-- .../early_return_transformer.py | 2 +- .../{ => transformers}/ifelse_transformer.py | 4 +-- .../{ => transformers}/logical_transformer.py | 4 +-- .../{ => transformers}/loop_transformer.py | 16 +++++----- .../{ => transformers}/return_transformer.py | 6 ++-- .../tensor_shape_transformer.py | 4 +-- .../tensorhook_transformer.py | 2 +- .../transform.py} | 6 ++-- .../typehint_transformer.py | 2 +- python/paddle/static/nn/control_flow.py | 4 ++- python/setup.py.in | 1 + setup.py | 1 + test/dygraph_to_static/test_logical.py | 4 ++- test/dygraph_to_static/test_loop.py | 2 +- 26 files changed, 66 insertions(+), 71 deletions(-) create mode 100644 python/paddle/jit/dy2static/transformers/__init__.py rename python/paddle/jit/dy2static/{ => transformers}/assert_transformer.py (96%) rename python/paddle/jit/dy2static/{base_transformer.py => transformers/base.py} (95%) rename python/paddle/jit/dy2static/{ => transformers}/basic_api_transformer.py (99%) rename python/paddle/jit/dy2static/{ => transformers}/break_continue_transformer.py (99%) rename python/paddle/jit/dy2static/{ => transformers}/call_transformer.py (96%) rename python/paddle/jit/dy2static/{ => transformers}/cast_transformer.py (96%) rename python/paddle/jit/dy2static/{ => transformers}/create_variable_transformer.py (89%) rename python/paddle/jit/dy2static/{ => transformers}/decorator_transformer.py (97%) rename python/paddle/jit/dy2static/{ => transformers}/early_return_transformer.py (98%) rename python/paddle/jit/dy2static/{ => transformers}/ifelse_transformer.py (99%) rename python/paddle/jit/dy2static/{ => transformers}/logical_transformer.py (97%) rename python/paddle/jit/dy2static/{ => transformers}/loop_transformer.py (99%) rename python/paddle/jit/dy2static/{ => transformers}/return_transformer.py (99%) rename python/paddle/jit/dy2static/{ => transformers}/tensor_shape_transformer.py (94%) rename python/paddle/jit/dy2static/{ => transformers}/tensorhook_transformer.py (98%) rename python/paddle/jit/dy2static/{ast_transformer.py => transformers/transform.py} (97%) rename python/paddle/jit/dy2static/{ => transformers}/typehint_transformer.py (96%) diff --git a/python/paddle/jit/dy2static/__init__.py b/python/paddle/jit/dy2static/__init__.py index 68e348142616b..83535ac17aee6 100644 --- a/python/paddle/jit/dy2static/__init__.py +++ b/python/paddle/jit/dy2static/__init__.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .assert_transformer import AssertTransformer # noqa: F401 -from .ast_transformer import DygraphToStaticAst # noqa: F401 from .convert_call_func import convert_call as Call # noqa: F401 from .convert_operators import ( # noqa: F401 convert_assert as Assert, @@ -33,6 +31,7 @@ ) from .program_translator import convert_to_static # noqa: F401 from .static_analysis import NodeVarType, StaticAnalysisVisitor # noqa: F401 +from .transformers import DygraphToStaticAst # noqa: F401 from .utils import UndefinedVar, ast_to_source_code, saw # noqa: F401 from .variable_trans_func import ( # noqa: F401 create_bool_as_type, diff --git a/python/paddle/jit/dy2static/ast_utils.py b/python/paddle/jit/dy2static/ast_utils.py index 7724cc1b4a13f..fc703dd6f6e49 100644 --- a/python/paddle/jit/dy2static/ast_utils.py +++ b/python/paddle/jit/dy2static/ast_utils.py @@ -14,6 +14,7 @@ import ast +import sys import astor @@ -32,6 +33,10 @@ def ast_to_source_code(ast_node): if isinstance(ast_node, gast.AST): ast_node = gast.gast_to_ast(ast_node) + if sys.version_info >= (3, 9): + ast.fix_missing_locations(ast_node) + return ast.unparse(ast_node) + # Do not wrap lines even if they are too long def pretty_source(source): return ''.join(source) diff --git a/python/paddle/jit/dy2static/program_translator.py b/python/paddle/jit/dy2static/program_translator.py index 7a929c78fdab5..6c79cf0090d70 100644 --- a/python/paddle/jit/dy2static/program_translator.py +++ b/python/paddle/jit/dy2static/program_translator.py @@ -37,7 +37,6 @@ from paddle.utils import flatten, gast from . import error, logging_utils -from .ast_transformer import DygraphToStaticAst from .function_spec import ( FunctionSpec, _hash_spec_names, @@ -53,6 +52,7 @@ from .pir_partial_program import ( PartialProgramLayerHook as PirPartialProgramLayerHook, ) +from .transformers import DygraphToStaticAst from .utils import ( ALREADY_D2S, NO_SHAPE_VAR_TYPE, diff --git a/python/paddle/jit/dy2static/transformers/__init__.py b/python/paddle/jit/dy2static/transformers/__init__.py new file mode 100644 index 0000000000000..56cf74247476a --- /dev/null +++ b/python/paddle/jit/dy2static/transformers/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .transform import DygraphToStaticAst # noqa: F401 diff --git a/python/paddle/jit/dy2static/assert_transformer.py b/python/paddle/jit/dy2static/transformers/assert_transformer.py similarity index 96% rename from python/paddle/jit/dy2static/assert_transformer.py rename to python/paddle/jit/dy2static/transformers/assert_transformer.py index acf17b5861813..2ec42be7e6614 100644 --- a/python/paddle/jit/dy2static/assert_transformer.py +++ b/python/paddle/jit/dy2static/transformers/assert_transformer.py @@ -15,7 +15,7 @@ from paddle.jit.dy2static.utils import ast_to_source_code from paddle.utils import gast -from .base_transformer import BaseTransformer +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/base_transformer.py b/python/paddle/jit/dy2static/transformers/base.py similarity index 95% rename from python/paddle/jit/dy2static/base_transformer.py rename to python/paddle/jit/dy2static/transformers/base.py index 7d61b3a7417b7..81d5f22902787 100644 --- a/python/paddle/jit/dy2static/base_transformer.py +++ b/python/paddle/jit/dy2static/transformers/base.py @@ -23,7 +23,6 @@ ORIGI_INFO, ast_to_source_code, create_assign_node, - get_attribute_full_name, ) from paddle.utils import gast @@ -50,35 +49,6 @@ def visit(self, node): return result -class RenameTransformer(BaseTransformer): - def __init__(self, node): - assert isinstance( - node, gast.AST - ), "RenameTransformer only accepts gast.AST as input" - self.root = node - self.old_name = "" - self.new_name = "" - - def rename(self, old_name, new_name): - self.old_name = old_name - self.new_name = new_name - self.visit(self.root) - - def visit_Name(self, node): - self.generic_visit(node) - if node.id == self.old_name: - node.id = self.new_name - return node - - def visit_Attribute(self, node): - self.generic_visit(node) - attr_full_name = get_attribute_full_name(node) - if attr_full_name == self.old_name: - new_name_node = gast.parse(self.new_name).body[0].value - return new_name_node - return node - - class NameNodeReplaceTransformer(BaseTransformer): """ This class replaces specified gast.Name node by replace_node. diff --git a/python/paddle/jit/dy2static/basic_api_transformer.py b/python/paddle/jit/dy2static/transformers/basic_api_transformer.py similarity index 99% rename from python/paddle/jit/dy2static/basic_api_transformer.py rename to python/paddle/jit/dy2static/transformers/basic_api_transformer.py index fcb2e21071217..1d9c865bf75b2 100644 --- a/python/paddle/jit/dy2static/basic_api_transformer.py +++ b/python/paddle/jit/dy2static/transformers/basic_api_transformer.py @@ -17,8 +17,8 @@ from paddle.utils import gast -from . import utils -from .base_transformer import BaseTransformer +from .. import utils +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/break_continue_transformer.py b/python/paddle/jit/dy2static/transformers/break_continue_transformer.py similarity index 99% rename from python/paddle/jit/dy2static/break_continue_transformer.py rename to python/paddle/jit/dy2static/transformers/break_continue_transformer.py index 4c6cad0e788e3..39ea02f00db6a 100644 --- a/python/paddle/jit/dy2static/break_continue_transformer.py +++ b/python/paddle/jit/dy2static/transformers/break_continue_transformer.py @@ -17,7 +17,7 @@ from paddle.jit.dy2static.variable_trans_func import create_bool_node from paddle.utils import gast -from .base_transformer import BaseTransformer, ForNodeVisitor +from .base import BaseTransformer, ForNodeVisitor __all__ = [] diff --git a/python/paddle/jit/dy2static/call_transformer.py b/python/paddle/jit/dy2static/transformers/call_transformer.py similarity index 96% rename from python/paddle/jit/dy2static/call_transformer.py rename to python/paddle/jit/dy2static/transformers/call_transformer.py index 7ed0a4681bcc1..8ba0bde462029 100644 --- a/python/paddle/jit/dy2static/call_transformer.py +++ b/python/paddle/jit/dy2static/transformers/call_transformer.py @@ -15,8 +15,8 @@ from paddle.jit.dy2static.utils import ast_to_source_code, is_paddle_api from paddle.utils import gast -from .base_transformer import BaseTransformer -from .utils import is_builtin # noqa: F401 +from ..utils import is_builtin # noqa: F401 +from .base import BaseTransformer PDB_SET = "pdb.set_trace" diff --git a/python/paddle/jit/dy2static/cast_transformer.py b/python/paddle/jit/dy2static/transformers/cast_transformer.py similarity index 96% rename from python/paddle/jit/dy2static/cast_transformer.py rename to python/paddle/jit/dy2static/transformers/cast_transformer.py index c556f374104e8..1622159f37aad 100644 --- a/python/paddle/jit/dy2static/cast_transformer.py +++ b/python/paddle/jit/dy2static/transformers/cast_transformer.py @@ -15,7 +15,7 @@ from paddle.jit.dy2static.utils import ast_to_source_code from paddle.utils import gast -from .base_transformer import BaseTransformer +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/create_variable_transformer.py b/python/paddle/jit/dy2static/transformers/create_variable_transformer.py similarity index 89% rename from python/paddle/jit/dy2static/create_variable_transformer.py rename to python/paddle/jit/dy2static/transformers/create_variable_transformer.py index f0d5583834b2d..1be824349656a 100644 --- a/python/paddle/jit/dy2static/create_variable_transformer.py +++ b/python/paddle/jit/dy2static/transformers/create_variable_transformer.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .base_transformer import BaseTransformer -from .utils import FunctionNameLivenessAnalysis -from .variable_trans_func import create_undefined_var +from ..utils import FunctionNameLivenessAnalysis +from ..variable_trans_func import create_undefined_var +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/decorator_transformer.py b/python/paddle/jit/dy2static/transformers/decorator_transformer.py similarity index 97% rename from python/paddle/jit/dy2static/decorator_transformer.py rename to python/paddle/jit/dy2static/transformers/decorator_transformer.py index 2597c5199c1bc..4641592626656 100644 --- a/python/paddle/jit/dy2static/decorator_transformer.py +++ b/python/paddle/jit/dy2static/transformers/decorator_transformer.py @@ -18,8 +18,8 @@ from paddle.utils import gast -from .base_transformer import BaseTransformer -from .utils import RE_PYMODULE, RE_PYNAME, ast_to_source_code +from ..utils import RE_PYMODULE, RE_PYNAME, ast_to_source_code +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/early_return_transformer.py b/python/paddle/jit/dy2static/transformers/early_return_transformer.py similarity index 98% rename from python/paddle/jit/dy2static/early_return_transformer.py rename to python/paddle/jit/dy2static/transformers/early_return_transformer.py index 4613f2b6ecbc4..4dab1e5ab1638 100644 --- a/python/paddle/jit/dy2static/early_return_transformer.py +++ b/python/paddle/jit/dy2static/transformers/early_return_transformer.py @@ -14,7 +14,7 @@ from paddle.utils import gast -from .base_transformer import BaseTransformer +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/ifelse_transformer.py b/python/paddle/jit/dy2static/transformers/ifelse_transformer.py similarity index 99% rename from python/paddle/jit/dy2static/ifelse_transformer.py rename to python/paddle/jit/dy2static/transformers/ifelse_transformer.py index 02129b02bf103..bf9a753ccf2c8 100644 --- a/python/paddle/jit/dy2static/ifelse_transformer.py +++ b/python/paddle/jit/dy2static/transformers/ifelse_transformer.py @@ -41,8 +41,8 @@ # See details in https://github.com/serge-sans-paille/gast/ from paddle.utils import gast -from .base_transformer import BaseTransformer -from .utils import FALSE_FUNC_PREFIX, TRUE_FUNC_PREFIX +from ..utils import FALSE_FUNC_PREFIX, TRUE_FUNC_PREFIX +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/logical_transformer.py b/python/paddle/jit/dy2static/transformers/logical_transformer.py similarity index 97% rename from python/paddle/jit/dy2static/logical_transformer.py rename to python/paddle/jit/dy2static/transformers/logical_transformer.py index c2719d2c177f1..b6837c61869c4 100644 --- a/python/paddle/jit/dy2static/logical_transformer.py +++ b/python/paddle/jit/dy2static/transformers/logical_transformer.py @@ -14,8 +14,8 @@ from paddle.utils import gast -from .base_transformer import BaseTransformer -from .utils import ast_to_source_code +from ..utils import ast_to_source_code +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/loop_transformer.py b/python/paddle/jit/dy2static/transformers/loop_transformer.py similarity index 99% rename from python/paddle/jit/dy2static/loop_transformer.py rename to python/paddle/jit/dy2static/transformers/loop_transformer.py index e96cfa943abca..42c2a40a5ca98 100644 --- a/python/paddle/jit/dy2static/loop_transformer.py +++ b/python/paddle/jit/dy2static/transformers/loop_transformer.py @@ -18,14 +18,8 @@ from paddle.base import unique_name from paddle.utils import gast -from .base_transformer import ( - BaseTransformer, - ForLoopTuplePreTransformer, - ForNodeVisitor, -) -from .ifelse_transformer import ARGS_NAME -from .static_analysis import NodeVarType, StaticAnalysisVisitor -from .utils import ( +from ..static_analysis import NodeVarType, StaticAnalysisVisitor +from ..utils import ( FOR_BODY_PREFIX, FOR_CONDITION_PREFIX, WHILE_BODY_PREFIX, @@ -39,6 +33,12 @@ create_set_args_node, get_attribute_full_name, ) +from .base import ( + BaseTransformer, + ForLoopTuplePreTransformer, + ForNodeVisitor, +) +from .ifelse_transformer import ARGS_NAME __all__ = [] diff --git a/python/paddle/jit/dy2static/return_transformer.py b/python/paddle/jit/dy2static/transformers/return_transformer.py similarity index 99% rename from python/paddle/jit/dy2static/return_transformer.py rename to python/paddle/jit/dy2static/transformers/return_transformer.py index 3fc259e0a303a..6aafe1a991215 100644 --- a/python/paddle/jit/dy2static/return_transformer.py +++ b/python/paddle/jit/dy2static/transformers/return_transformer.py @@ -15,14 +15,14 @@ from paddle.base import unique_name from paddle.utils import gast -from .base_transformer import BaseTransformer -from .break_continue_transformer import ForToWhileTransformer -from .utils import ( +from ..utils import ( ORIGI_INFO, Dygraph2StaticException, ast_to_source_code, index_in_list, ) +from .base import BaseTransformer +from .break_continue_transformer import ForToWhileTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/tensor_shape_transformer.py b/python/paddle/jit/dy2static/transformers/tensor_shape_transformer.py similarity index 94% rename from python/paddle/jit/dy2static/tensor_shape_transformer.py rename to python/paddle/jit/dy2static/transformers/tensor_shape_transformer.py index 13b81608f7920..0e71a21fde1c6 100644 --- a/python/paddle/jit/dy2static/tensor_shape_transformer.py +++ b/python/paddle/jit/dy2static/transformers/tensor_shape_transformer.py @@ -14,8 +14,8 @@ from paddle.utils import gast -from .base_transformer import BaseTransformer -from .utils import ast_to_source_code +from ..utils import ast_to_source_code +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/jit/dy2static/tensorhook_transformer.py b/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py similarity index 98% rename from python/paddle/jit/dy2static/tensorhook_transformer.py rename to python/paddle/jit/dy2static/transformers/tensorhook_transformer.py index 5b17ae2efd021..73e377d189526 100644 --- a/python/paddle/jit/dy2static/tensorhook_transformer.py +++ b/python/paddle/jit/dy2static/transformers/tensorhook_transformer.py @@ -16,7 +16,7 @@ from paddle.utils import gast -from .base_transformer import BaseTransformer +from .base import BaseTransformer class RegisterHookTransformer(BaseTransformer): diff --git a/python/paddle/jit/dy2static/ast_transformer.py b/python/paddle/jit/dy2static/transformers/transform.py similarity index 97% rename from python/paddle/jit/dy2static/ast_transformer.py rename to python/paddle/jit/dy2static/transformers/transform.py index bdb3e5b59cca7..2e9282259ecf0 100644 --- a/python/paddle/jit/dy2static/ast_transformer.py +++ b/python/paddle/jit/dy2static/transformers/transform.py @@ -19,9 +19,10 @@ import os -from . import logging_utils +from .. import logging_utils +from ..utils import ast_to_source_code from .assert_transformer import AssertTransformer -from .base_transformer import BaseTransformer +from .base import BaseTransformer from .basic_api_transformer import BasicApiTransformer, NameloadJstTransformer from .break_continue_transformer import ( BreakContinueTransformer, @@ -39,7 +40,6 @@ from .tensor_shape_transformer import TensorShapeTransformer from .tensorhook_transformer import RegisterHookTransformer from .typehint_transformer import TypeHintTransformer -from .utils import ast_to_source_code __all__ = [] diff --git a/python/paddle/jit/dy2static/typehint_transformer.py b/python/paddle/jit/dy2static/transformers/typehint_transformer.py similarity index 96% rename from python/paddle/jit/dy2static/typehint_transformer.py rename to python/paddle/jit/dy2static/transformers/typehint_transformer.py index da2f625ce5a90..ab6e3c3c6e807 100644 --- a/python/paddle/jit/dy2static/typehint_transformer.py +++ b/python/paddle/jit/dy2static/transformers/typehint_transformer.py @@ -13,7 +13,7 @@ # limitations under the License. -from .base_transformer import BaseTransformer +from .base import BaseTransformer __all__ = [] diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py index 06ae72db65dcb..2361cdd2c0088 100644 --- a/python/paddle/static/nn/control_flow.py +++ b/python/paddle/static/nn/control_flow.py @@ -1611,7 +1611,9 @@ def expand_undefined_var(nest1, nest2, names): nest2: Var2, ([1,2,3,4], UndefinedVar) In this case, we should not expand recursively. """ - from paddle.jit.dy2static.return_transformer import RETURN_VALUE_PREFIX + from paddle.jit.dy2static.transformers.return_transformer import ( + RETURN_VALUE_PREFIX, + ) from paddle.jit.dy2static.utils import UndefinedVar def pack_undefined_var_as(seq): diff --git a/python/setup.py.in b/python/setup.py.in index c0c8e81ebe399..f8581129642c2 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -462,6 +462,7 @@ packages=['paddle', 'paddle.framework', 'paddle.jit', 'paddle.jit.dy2static', + 'paddle.jit.dy2static.transformers', 'paddle.jit.pir_dy2static', 'paddle.jit.sot', 'paddle.jit.sot.opcode_translator', diff --git a/setup.py b/setup.py index 6bbb613bacd8f..1b688f15e9885 100644 --- a/setup.py +++ b/setup.py @@ -1465,6 +1465,7 @@ def get_setup_parameters(): 'paddle.framework', 'paddle.jit', 'paddle.jit.dy2static', + 'paddle.jit.dy2static.transformers', 'paddle.jit.pir_dy2static', 'paddle.jit.sot', 'paddle.jit.sot.opcode_translator', diff --git a/test/dygraph_to_static/test_logical.py b/test/dygraph_to_static/test_logical.py index edbb6fc594781..84916395a8e31 100644 --- a/test/dygraph_to_static/test_logical.py +++ b/test/dygraph_to_static/test_logical.py @@ -21,7 +21,9 @@ from dygraph_to_static_utils import Dy2StTestBase, enable_to_static_guard import paddle -from paddle.jit.dy2static.logical_transformer import cmpop_node_to_str +from paddle.jit.dy2static.transformers.logical_transformer import ( + cmpop_node_to_str, +) from paddle.utils import gast SEED = 2020 diff --git a/test/dygraph_to_static/test_loop.py b/test/dygraph_to_static/test_loop.py index 8ce72a7688502..517e32fcdbaf5 100644 --- a/test/dygraph_to_static/test_loop.py +++ b/test/dygraph_to_static/test_loop.py @@ -26,7 +26,7 @@ import paddle import paddle.nn.functional as F from paddle import base -from paddle.jit.dy2static.loop_transformer import NameVisitor +from paddle.jit.dy2static.transformers.loop_transformer import NameVisitor from paddle.utils import gast SEED = 2020 From 4055a7ab51441fadafdc833703020120e48172ac Mon Sep 17 00:00:00 2001 From: zhaoyingli <86812880+zhaoyinglia@users.noreply.github.com> Date: Mon, 25 Dec 2023 10:40:49 +0800 Subject: [PATCH 006/146] [AutoParallel] complete vpp with struct name (#60187) * [AutoParallel] complete vpp from struct name * update ut * update ut * update func name * add comment * update chunk cond * add comment * add comment * update comment add fix bug * update cond for complete chunk * tiny fix --- python/paddle/base/framework.py | 72 +++++ .../distributed/auto_parallel/constants.py | 2 + .../auto_parallel/static/completion.py | 301 +++++++++++------- .../auto_parallel/static/parallelizer_v2.py | 10 +- .../distributed/auto_parallel/static/utils.py | 25 ++ .../paddle/distributed/passes/pass_utils.py | 6 +- python/paddle/nn/layer/layers.py | 4 +- .../pipeline_scheduler_vpp_unittest.py | 169 ++++------ 8 files changed, 369 insertions(+), 220 deletions(-) diff --git a/python/paddle/base/framework.py b/python/paddle/base/framework.py index f0751b426ee13..e44e8d157623f 100644 --- a/python/paddle/base/framework.py +++ b/python/paddle/base/framework.py @@ -1133,6 +1133,66 @@ def name_scope(prefix=None): _name_scope = _name_scope.parent() +class NameStruct: + def __init__(self, name="", parent=None): + self._children = {} + self._name = name + self._parent = parent + + def child(self, prefix): + if prefix not in self._children: + new_child = NameStruct(prefix, self) + self._children[prefix] = [new_child] + else: + new_child = NameStruct( + prefix + "_%d" % len(self._children[prefix]), self + ) + self._children[prefix].append(new_child) + return new_child + + def parent(self): + return self._parent + + def name(self): + return self._name + + +_name_struct = NameStruct() + + +@signature_safe_contextmanager +def name_struct(prefix=None): + """ + Note: This should only used in Paddle/python/paddle/nn/layer/layers.py + to record the call path for the operators in Static Graph of AutoParallel. + + Args: + prefix(str, optional): prefix. Default is none. + """ + # TODO(panyx0718): Only [0-9a-z]. + # in dygraph we don't need namescope since it will cause mem leak + if in_dygraph_mode(): + yield + else: + assert prefix, "namescope prefix can not be empty." + global _name_struct + _name_struct = _name_struct.child(prefix) + try: + yield + finally: + _name_struct = _name_struct.parent() + + +def _full_name_struct(): + global _name_struct + struct = _name_struct + name = "" + while struct: + name = struct.name() + "/" + name + struct = struct.parent() + return name + + def _full_name_scope(): global _name_scope scope = _name_scope @@ -2982,6 +3042,9 @@ def __init__( self._amp_options: AmpOptions = DEFAULT_AMP_OPTIONS + # record the call path of op, only used in AutoParallel + self._struct_name = _full_name_struct() + op_maker = core.op_proto_and_checker_maker if op_maker.kOpRoleAttrName() not in op_attrs: @@ -3758,6 +3821,14 @@ def amp_options(self): """ return self._amp_options + @property + def struct_name(self): + return self._struct_name + + @struct_name.setter + def struct_name(self, struct_name): + self._struct_name = struct_name + @signature_safe_contextmanager def _stride_in_no_check_dy2st_diff(): @@ -7077,6 +7148,7 @@ def _copy_operator_info_from(self, other: Program): for dst_block, src_block in zip(self.blocks, other.blocks): for dst_op, src_op in zip(dst_block.ops, src_block.ops): dst_op.set_amp_options(src_op.amp_options) + dst_op.struct_name = src_op.struct_name def list_vars(self): """ diff --git a/python/paddle/distributed/auto_parallel/constants.py b/python/paddle/distributed/auto_parallel/constants.py index 57a8cf7a36ad8..2d2073f293ed7 100644 --- a/python/paddle/distributed/auto_parallel/constants.py +++ b/python/paddle/distributed/auto_parallel/constants.py @@ -110,6 +110,8 @@ def set_field_default_config(category, field, default_value): PIPELINE = "pipeline" set_field_default_config(PIPELINE, "enable", False) set_field_default_config(PIPELINE, "schedule_mode", "1F1B") +set_field_default_config(PIPELINE, "vpp_degree", 1) +set_field_default_config(PIPELINE, "vpp_seg_method", "") set_field_default_config(PIPELINE, "micro_batch_size", 1) set_field_default_config(PIPELINE, "accumulate_steps", 1) set_field_default_config(PIPELINE, "generation_batch_size", 1) diff --git a/python/paddle/distributed/auto_parallel/static/completion.py b/python/paddle/distributed/auto_parallel/static/completion.py index befcf18c0c454..76c6a9d181766 100644 --- a/python/paddle/distributed/auto_parallel/static/completion.py +++ b/python/paddle/distributed/auto_parallel/static/completion.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import copy import logging import os +import re import paddle from paddle.base.core import ( # noqa: F401 @@ -39,10 +41,13 @@ from .utils import ( __no_shape_var_type__, _g_gradient_clip_ops, + get_pp_degree, is_gradient_clip_op, is_loss_grad_op, is_loss_op, is_naive_data_parallel, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, + set_var_dist_attr, ) _logger = get_logger( @@ -1041,11 +1046,138 @@ def complete_forward_annotation(self, serial_main_program=None): # NOTE:[HighOrderGrad] update vars and ops distributed attribute in high order gradient self._complete_high_order_grad_annotation(serial_main_program) + self._complete_chunk_id(serial_main_program) # Do the validation check and amend some completion self._dist_context.amend_dist_attr_for_program() self._dist_context.validate_dist_attr_for_program() return serial_main_program + def _complete_chunk_id(self, serial_main_program): + def set_chunk_id(block, op, chunk_id, var_to_chunk_id): + dist_op = self._dist_context.get_dist_op_for_program(op) + dist_op.dist_attr.chunk_id = chunk_id + for name in op.input_arg_names + op.output_arg_names: + var = block._find_var_recursive(name) + if "lod_tensor_blocking_queue" in name: + continue + if name not in var_to_chunk_id: + op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program(op) + ) + tensor_dist_attr = ( + self._dist_context.get_tensor_dist_attr_for_program(var) + ) + if ( + op_dist_attr.process_mesh + == tensor_dist_attr.process_mesh + ): + tensor_dist_attr.chunk_id = op_dist_attr.chunk_id + var_to_chunk_id[var.name] = op_dist_attr.chunk_id + + if ( + not self._dist_context.strategy + or not self._dist_context.strategy.pipeline.enable + ): + return + + pp_degree = get_pp_degree(self._dist_context) + vpp_degree = self._dist_context.strategy.pipeline.vpp_degree + seg_method = self._dist_context.strategy.pipeline.vpp_seg_method + schedule_mode = self._dist_context.strategy.pipeline.schedule_mode + + if pp_degree < 2 and vpp_degree > 1: + raise ValueError( + "VPP schedule mode only can be set in pipeline mode." + ) + if vpp_degree > 1 and (not seg_method or schedule_mode != "VPP"): + raise ValueError( + "Please set right schedule_mode and vpp_seg_method for VPP." + ) + if vpp_degree < 2: + return + + block = serial_main_program.global_block() + ops = block.ops + + # 1. search seg_method in op's struct_name, and get all ops of segments + seg_op_deps = collections.OrderedDict() + regex = re.compile(seg_method, re.IGNORECASE) + for i, op in enumerate(ops): + struct_name = op.struct_name + m = regex.search(struct_name) + if not m: + continue + + struct_name = struct_name[m.start(0) :].split("/")[0] + if struct_name not in seg_op_deps: + seg_op_deps[struct_name] = [i] + else: + assert ( + seg_op_deps[struct_name][-1] + 1 == i + ), "The segment's ops should be continuous." + pre_op = ops[seg_op_deps[struct_name][-1]] + pre_dist_op = self._dist_context.get_dist_op_for_program(pre_op) + dist_op = self._dist_context.get_dist_op_for_program(op) + assert ( + pre_dist_op.dist_attr.process_mesh + == dist_op.dist_attr.process_mesh + ), "The segment's ops should have same process_mesh." + seg_op_deps[struct_name].extend([i]) + + # the num of chunk is equal to vpp_degree + num_parts = pp_degree * vpp_degree + assert ( + len(seg_op_deps.keys()) % num_parts == 0 + ), "number of layers[{}] ({}) should be devided by part number ({}).".format( + seg_method, len(seg_op_deps.keys()), num_parts + ) + + part_size = len(seg_op_deps.keys()) // vpp_degree + + # 2. get boundary index of each chunk + results = [0] * (vpp_degree + 1) + memory_counter = 0 + result_idx = 1 + for struct_name, idxs in seg_op_deps.items(): + memory_counter += 1 + if memory_counter == part_size: + results[result_idx] = idxs[-1] + 1 + result_idx += 1 + memory_counter = 0 + results[vpp_degree] = len(ops) + + # 3. set right chunk_id for each op + var_to_chunk_id = {} + for chunk_id in range(len(results) - 1): + start_idx = results[chunk_id] + end_idx = results[chunk_id + 1] + _logger.info( + "[chunk_{}] start op: [{}]: [{}] [{}]".format( + chunk_id, + ops[start_idx].type, + ops[start_idx].input_arg_names, + ops[start_idx].output_arg_names, + ) + ) + _logger.info( + "[chunk_{}] end op: [{}]: [{}] [{}]".format( + chunk_id, + ops[end_idx - 1].type, + ops[end_idx - 1].input_arg_names, + ops[end_idx - 1].output_arg_names, + ) + ) + + for idx in range(start_idx, end_idx): + op = ops[idx] + if op.has_attr("sub_block"): + block_id = op.attr('sub_block').id + sub_block = serial_main_program.blocks[block_id] + for op in sub_block.ops: + set_chunk_id(sub_block, op, chunk_id, var_to_chunk_id) + else: + set_chunk_id(block, op, chunk_id, var_to_chunk_id) + def _update_dist_attr_for_dp(self): # TODO: we must ensure the world process group contains all ranks ranks = get_world_process_group().ranks @@ -1231,6 +1363,9 @@ def _get_op_by_id(ops, id): dist_op_context = self._dist_context.dist_op_context grad_var_to_var = dist_op_context.grad_var_to_var + if len(grad_var_to_var) < 2: + return + appended_grad_times = 0 for idx in range(0, len(ops)): op = ops[idx] @@ -1446,6 +1581,7 @@ def _complete_grad_op_with_forward_op(forward_op, grad_op, vars): ) grad_op_dist_attr = OperatorDistAttr() ref_process_mesh = fwd_op_dist_attr.process_mesh + ref_chunk_id = fwd_op_dist_attr.chunk_id if grad_op.type == "concat" and forward_op.type == "split": split_input_var_name = forward_op.input("X")[0] @@ -1454,11 +1590,12 @@ def _complete_grad_op_with_forward_op(forward_op, grad_op, vars): ) # var output_var = vars[grad_op.desc.output('Out')[0]] - output_var_dist_attr = TensorDistAttr() - output_var_dist_attr.dims_mapping = ref_dims_mapping - output_var_dist_attr.process_mesh = ref_process_mesh - self._dist_context.set_tensor_dist_attr_for_program( - output_var, output_var_dist_attr + set_var_dist_attr( + self._dist_context, + output_var, + ref_dims_mapping, + ref_process_mesh, + chunk_id=ref_chunk_id, ) # op for input_name in grad_op.input_arg_names: @@ -1511,14 +1648,15 @@ def _complete_grad_op_with_forward_op(forward_op, grad_op, vars): for output_name in grad_op.output_arg_names: if output_name == "@EMPTY@": output_var = vars[output_name] - tensor_dist_attr = TensorDistAttr() ref_dims_mapping = [ -1 for _ in range(len(output_var.shape)) ] - tensor_dist_attr.dims_mapping = ref_dims_mapping - tensor_dist_attr.process_mesh = ref_process_mesh - self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr + set_var_dist_attr( + self._dist_context, + output_var, + ref_dims_mapping, + ref_process_mesh, + chunk_id=ref_chunk_id, ) grad_op_dist_attr.set_output_dims_mapping( output_name, ref_dims_mapping @@ -1532,11 +1670,12 @@ def _complete_grad_op_with_forward_op(forward_op, grad_op, vars): ) # var output_var = vars[output_name] - tensor_dist_attr = TensorDistAttr() - tensor_dist_attr.dims_mapping = ref_dims_mapping - tensor_dist_attr.process_mesh = ref_process_mesh - self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr + set_var_dist_attr( + self._dist_context, + output_var, + ref_dims_mapping, + ref_process_mesh, + chunk_id=ref_chunk_id, ) # op grad_op_dist_attr.set_output_dims_mapping( @@ -1544,6 +1683,7 @@ def _complete_grad_op_with_forward_op(forward_op, grad_op, vars): ) grad_op_dist_attr.process_mesh = ref_process_mesh + grad_op_dist_attr.chunk_id = ref_chunk_id grad_op_dist_attr.impl_type = fwd_op_dist_attr.impl_type grad_op_dist_attr.impl_idx = fwd_op_dist_attr.impl_idx grad_op_dist_attr.chunk_id = fwd_op_dist_attr.chunk_id @@ -1669,37 +1809,31 @@ def infer_backward_op_partial_status( len(grad_op.output_arg_names) ) - loss_grad_var = vars[grad_op.output_arg_names[0]] loss_var = vars[loss_op.output_arg_names[0]] + loss_grad_var = vars[grad_op.output_arg_names[0]] assert loss_var.name + "@GRAD" == loss_grad_var.name - loss_var_distr_attr = ( - self._dist_context.get_tensor_dist_attr_for_program( - loss_var - ) + dist_loss_var = self._dist_context.get_dist_tensor_for_program( + loss_var ) - - # TODO complete other attribute for grad var - tensor_dist_attr = TensorDistAttr() - tensor_dist_attr.dims_mapping = loss_var_distr_attr.dims_mapping - tensor_dist_attr.process_mesh = loss_var_distr_attr.process_mesh - self._dist_context.set_tensor_dist_attr_for_program( - loss_grad_var, tensor_dist_attr + dist_loss_op = self._dist_context.get_dist_op_for_program( + loss_op ) - loss_op_dist_attr = ( - self._dist_context.get_op_dist_attr_for_program(loss_op) - ) - grad_op_dist_attr = OperatorDistAttr() - grad_op_dist_attr.process_mesh = loss_op_dist_attr.process_mesh - grad_op_dist_attr.chunk_id = loss_op_dist_attr.chunk_id - ref_dims_mapping = loss_op_dist_attr.get_output_dims_mapping( - loss_var.name - ) - grad_op_dist_attr.set_output_dims_mapping( - loss_grad_var.name, ref_dims_mapping + set_var_dist_attr( + self._dist_context, + loss_grad_var, + dist_loss_var.dist_attr.dims_mapping, + dist_loss_var.dist_attr.process_mesh, + chunk_id=dist_loss_var.dist_attr.chunk_id, ) - self._dist_context.set_op_dist_attr_for_program( - grad_op, grad_op_dist_attr + naive_set_dist_op_attr_for_program_by_mesh_and_mapping( + grad_op, + dist_loss_op.dist_attr.process_mesh, + dist_loss_op.dist_attr.get_output_dims_mapping( + loss_var.name + ), + self._dist_context, + chunk_id=dist_loss_op.dist_attr.chunk_id, ) continue @@ -1760,14 +1894,16 @@ def infer_backward_op_partial_status( ) ref_fwd_dims_mapping = ref_fwd_dist_attr.dims_mapping ref_fwd_process_mesh = ref_fwd_dist_attr.process_mesh + ref_fwd_chunk_id = ref_fwd_dist_attr.chunk_id # output - tensor_dist_attr = TensorDistAttr() - tensor_dist_attr.dims_mapping = ref_fwd_dims_mapping - tensor_dist_attr.process_mesh = ref_fwd_process_mesh output_var = vars[output_name] - self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr + set_var_dist_attr( + self._dist_context, + output_var, + ref_fwd_dims_mapping, + ref_fwd_process_mesh, + chunk_id=ref_fwd_chunk_id, ) # op @@ -1780,34 +1916,12 @@ def infer_backward_op_partial_status( output_name, ref_fwd_dims_mapping ) grad_op_dist_attr.process_mesh = ref_fwd_process_mesh - # NOTE(zhaoyingli): - # The sum op is used to accmulate the grads' value of the same forward var, - # sum op's chunk_id is same with the last op which generate the grad. - chunk_id = None - for pre_idx in range( - idx - 1, first_backward_op_idx + 1, -1 - ): - pre_grad_op = ops[pre_idx] - inter_arg_name = list( - set(pre_grad_op.output_arg_names) - & set(grad_op.input_arg_names) - ) - if len(inter_arg_name) > 0: - pre_op_dist_attr = ( - self._dist_context.get_op_dist_attr_for_program( - pre_grad_op - ) - ) - chunk_id = pre_op_dist_attr.chunk_id - break - assert chunk_id is not None - grad_op_dist_attr.chunk_id = chunk_id + grad_op_dist_attr.chunk_id = ref_fwd_chunk_id self._dist_context.set_op_dist_attr_for_program( grad_op, grad_op_dist_attr ) elif grad_op.type == 'fill_any_like': - # TODO: support complete chunk_id ref_var_name = grad_op.input_arg_names[0] ref_var = vars[ref_var_name] ref_dist_attr = ( @@ -1817,18 +1931,21 @@ def infer_backward_op_partial_status( ) ref_dims_mapping = ref_dist_attr.dims_mapping ref_process_mesh = ref_dist_attr.process_mesh + ref_chunk_id = ref_dist_attr.chunk_id # var output_var_name = grad_op.output_arg_names[0] output_var = vars[output_var_name] - tensor_dist_attr = TensorDistAttr() - tensor_dist_attr.dims_mapping = ref_dims_mapping - tensor_dist_attr.process_mesh = ref_process_mesh - self._dist_context.set_tensor_dist_attr_for_program( - output_var, tensor_dist_attr + set_var_dist_attr( + self._dist_context, + output_var, + ref_dims_mapping, + ref_process_mesh, + chunk_id=ref_chunk_id, ) # op grad_op_dist_attr = OperatorDistAttr() grad_op_dist_attr.process_mesh = ref_process_mesh + grad_op_dist_attr.chunk_id = ref_chunk_id grad_op_dist_attr.set_input_dims_mapping( ref_var_name, ref_dims_mapping ) @@ -2072,44 +2189,6 @@ def complete_update_annotation(self, serial_main_program): ) continue - def _complete_var_chunk_id(self, serial_main_program=None): - """ - NOTE(zhaoyingli): Temporary methods. - This func is for completing the chunk_id attr for every var - """ - - if serial_main_program is None: - serial_main_program = self._dist_context.serial_main_program - else: - self._dist_context._serial_main_program = serial_main_program - - var_to_chunk_id = {} - for block in serial_main_program.blocks: - for op in block.ops: - for name in op.input_arg_names + op.output_arg_names: - var = block._find_var_recursive(name) - if "lod_tensor_blocking_queue" in name: - continue - if name not in var_to_chunk_id: - op_dist_attr = ( - self._dist_context.get_op_dist_attr_for_program(op) - ) - tensor_dist_attr = ( - self._dist_context.get_tensor_dist_attr_for_program( - var - ) - ) - if ( - op_dist_attr.process_mesh - == tensor_dist_attr.process_mesh - ): - tensor_dist_attr.chunk_id = op_dist_attr.chunk_id - var_to_chunk_id[var.name] = op_dist_attr.chunk_id - - self._dist_context._num_model_chunks = len( - set(var_to_chunk_id.values()) - ) - def complete_prim_annotation(self, serial_main_program=None): """ fill default data parallel annotation for program with primitive operators. diff --git a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py index 4ce9ffef79960..115f260873d62 100644 --- a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py +++ b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py @@ -26,7 +26,11 @@ from .partitioner import Partitioner from .process_group import get_world_process_group from .reshard import Resharder -from .utils import get_pp_stage, is_sequential_run, use_new_executor +from .utils import ( + get_pp_stage, + is_sequential_run, + use_new_executor, +) NEW_IR_PASS = [ 'fused_gemm_epilogue_pass', @@ -238,8 +242,6 @@ def _generate_backward( ) self._completer.complete_backward_annotation(main_program) self._dist_context.block_state.parse_backward_blocks(main_program) - # NOTE(zhaoyingli): temporary method: complete all vars' chunk_id attr of main_program - self._completer._complete_var_chunk_id(main_program) return params_grads def _generate_optimizer( @@ -512,6 +514,6 @@ def _apply_post_optimization( "num_micro_batches": self._strategy.pipeline.accumulate_steps, "pp_degree": len(self._dist_context.process_meshes), "pp_stage": get_pp_stage(self._dist_context, rank), - "vpp_degree": self._dist_context._num_model_chunks, + "vpp_degree": self._strategy.pipeline.vpp_degree, "dist_context": self._dist_context, } diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 4e93f4f9ab132..296196230d086 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -2313,6 +2313,31 @@ def is_sequential_run(): ) +def get_pp_degree(dist_context): + if len(dist_context.process_meshes) < 2: + return 0 + + process_ids = set() + process_meshes = copy.deepcopy(dist_context.process_meshes) + + for pm in process_meshes: + process_ids |= set(pm.process_ids) + + global_pm_idx = [] + has_sub_pm = False + for idx, pm in enumerate(process_meshes): + if len(set(pm.process_ids)) == len(process_ids): + global_pm_idx.append(idx) + elif set(pm.process_ids) < process_ids: + has_sub_pm = True + + if has_sub_pm: + for idx in reversed(global_pm_idx): + process_meshes.pop(idx) + + return len(process_meshes) + + def get_pp_stage(dist_context, rank): pp_idx = None for idx, process_mesh in enumerate(dist_context.process_meshes): diff --git a/python/paddle/distributed/passes/pass_utils.py b/python/paddle/distributed/passes/pass_utils.py index 7a9fb89bbeee2..cb9bc37abec54 100644 --- a/python/paddle/distributed/passes/pass_utils.py +++ b/python/paddle/distributed/passes/pass_utils.py @@ -25,8 +25,6 @@ get_logger, is_backward_op, is_forward_op, - is_loss_grad_op, - is_loss_op, is_optimize_op, use_new_executor, ) @@ -676,9 +674,9 @@ def _split_ops(block): type_to_ops["fetch"] = [] for ip, op in enumerate(block.ops): - if is_forward_op(op) or is_loss_op(op): + if is_forward_op(op): type = oprole_type[0] - elif is_backward_op(op) or is_loss_grad_op(op): + elif is_backward_op(op): type = oprole_type[1] elif is_optimize_op(op): type = oprole_type[2] diff --git a/python/paddle/nn/layer/layers.py b/python/paddle/nn/layer/layers.py index fdcfd388fc260..91a993ddc4563 100644 --- a/python/paddle/nn/layer/layers.py +++ b/python/paddle/nn/layer/layers.py @@ -43,6 +43,7 @@ default_main_program, in_dygraph_mode, in_pir_mode, + name_struct, ) from paddle.base.layer_helper_base import LayerHelperBase from paddle.base.param_attr import ParamAttr @@ -1405,7 +1406,8 @@ def _dygraph_call_func(self, *inputs, **kwargs): ): outputs = self.forward(*inputs, **kwargs) else: - outputs = self.forward(*inputs, **kwargs) + with name_struct(self.__class__.__name__): + outputs = self.forward(*inputs, **kwargs) for forward_post_hook in self._forward_post_hooks.values(): hook_result = forward_post_hook(self, inputs, outputs) diff --git a/test/auto_parallel/pipeline_scheduler_vpp_unittest.py b/test/auto_parallel/pipeline_scheduler_vpp_unittest.py index 8b6a986468efc..431e782cb073e 100644 --- a/test/auto_parallel/pipeline_scheduler_vpp_unittest.py +++ b/test/auto_parallel/pipeline_scheduler_vpp_unittest.py @@ -21,6 +21,11 @@ import paddle.nn.functional as F from paddle import nn from paddle.distributed import ParallelEnv +from paddle.distributed.auto_parallel.static.utils import ( + is_backward_op, + is_forward_op, + is_optimize_op, +) from paddle.distributed.fleet import auto paddle.enable_static() @@ -29,81 +34,68 @@ PP_MESH_1 = auto.ProcessMesh([1]) -class MLPLayer(nn.Layer): +class MyLinear(nn.Layer): def __init__( self, hidden_size=1024, intermediate_size=4 * 1024, dropout_ratio=0.1, - initializer_range=0.02, + weight_attr=None, ): super().__init__() - weight_attr = paddle.ParamAttr( - initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) - ) - self.linear0 = nn.Linear( hidden_size, intermediate_size, weight_attr, bias_attr=None ) self.linear1 = nn.Linear( intermediate_size, hidden_size, weight_attr, bias_attr=None ) - self.linear2 = nn.Linear( - hidden_size, intermediate_size, weight_attr, bias_attr=None - ) - self.linear3 = nn.Linear( - intermediate_size, hidden_size, weight_attr, bias_attr=None - ) - self.linear4 = nn.Linear( - hidden_size, intermediate_size, weight_attr, bias_attr=None - ) - self.linear5 = nn.Linear( - intermediate_size, hidden_size, weight_attr, bias_attr=None - ) - self.linear6 = nn.Linear( - hidden_size, intermediate_size, weight_attr, bias_attr=None - ) - self.linear7 = nn.Linear( - intermediate_size, hidden_size, weight_attr, bias_attr=None - ) - - self.linear8 = nn.Linear(hidden_size, 1, weight_attr, bias_attr=None) - self.norm = nn.LayerNorm(hidden_size, epsilon=1e-5) self.dropout = nn.Dropout(dropout_ratio, mode="upscale_in_train") def forward(self, input): - out = auto.shard_op(self.norm, PP_MESH_0)(input) + out = self.linear0(input) + out = F.gelu(out, approximate=True) + out = self.linear1(out) + out = self.dropout(out) - out = auto.shard_op(self.linear0, PP_MESH_0, chunk_id=0)(out) - out = auto.shard_op(F.gelu, PP_MESH_0, chunk_id=0)( - out, approximate=True - ) - out = auto.shard_op(self.linear1, PP_MESH_0, chunk_id=0)(out) - out = auto.shard_op(self.dropout, PP_MESH_0, chunk_id=0)(out) + return out - out = auto.shard_op(self.linear2, PP_MESH_1, chunk_id=0)(out) - out = auto.shard_op(F.gelu, PP_MESH_1, chunk_id=0)( - out, approximate=True - ) - out = auto.shard_op(self.linear3, PP_MESH_1, chunk_id=0)(out) - out = auto.shard_op(self.dropout, PP_MESH_1, chunk_id=0)(out) - out = auto.shard_op(self.linear4, PP_MESH_0, chunk_id=1)(out) - out = auto.shard_op(F.gelu, PP_MESH_0, chunk_id=1)( - out, approximate=True +class MLPLayer(nn.Layer): + def __init__( + self, + hidden_size=1024, + intermediate_size=4 * 1024, + dropout_ratio=0.1, + initializer_range=0.02, + ): + super().__init__() + + weight_attr = paddle.ParamAttr( + initializer=nn.initializer.Normal(mean=0.0, std=initializer_range) ) - out = auto.shard_op(self.linear5, PP_MESH_0, chunk_id=1)(out) - out = auto.shard_op(self.dropout, PP_MESH_0, chunk_id=1)(out) - out = auto.shard_op(self.linear6, PP_MESH_1, chunk_id=1)(out) - out = auto.shard_op(F.gelu, PP_MESH_1, chunk_id=1)( - out, approximate=True + self.layers = nn.LayerList( + [ + MyLinear( + hidden_size, intermediate_size, dropout_ratio, weight_attr + ) + for _ in range(4) + ] ) - out = auto.shard_op(self.linear7, PP_MESH_1, chunk_id=1)(out) - out = auto.shard_op(self.dropout, PP_MESH_1, chunk_id=1)(out) - out = auto.shard_op(self.linear8, PP_MESH_1, chunk_id=1)(out) + self.linear = nn.Linear(hidden_size, 1, weight_attr, bias_attr=None) + self.norm = nn.LayerNorm(hidden_size, epsilon=1e-5) + self.layer_to_mesh = [PP_MESH_0, PP_MESH_1, PP_MESH_0, PP_MESH_1] + + def forward(self, input): + out = self.norm(input) + + for i, layer in enumerate(self.layers): + auto.shard_tensor(out, self.layer_to_mesh[i], [None, None]) + out = layer(out) + + out = self.linear(out) return out @@ -116,6 +108,8 @@ def apply_pass(schedule_mode, acc_step): pipeline.enable = True pipeline.schedule_mode = schedule_mode pipeline.accumulate_steps = acc_step + pipeline.vpp_degree = 2 + pipeline.vpp_seg_method = "MyLinear" return strategy @@ -123,6 +117,7 @@ def apply_pass(schedule_mode, acc_step): def reset_prog(): paddle.base.framework.switch_main_program(paddle.static.Program()) paddle.base.framework.switch_startup_program(paddle.static.Program()) + paddle.utils.unique_name.switch() class MyDataset(paddle.io.Dataset): @@ -163,62 +158,36 @@ def get_engine(self, schedule_mode, acc_step): clip = paddle.nn.ClipGradByGlobalNorm(self.clip_norm) opt = paddle.optimizer.AdamW(learning_rate=0.00001, grad_clip=clip) model = MLPLayer() - loss = auto.shard_op( - paddle.nn.CrossEntropyLoss(), PP_MESH_1, chunk_id=1 - ) + loss = paddle.nn.CrossEntropyLoss() engine = auto.Engine(model, loss, opt, strategy=strategy) self.init(engine) return engine - def check_results(self, ref_losses, check_losses): - np.testing.assert_allclose( - ref_losses, - check_losses, - rtol=self.rtol, - atol=self.atol, - err_msg='pass {} has wrong results!, \nu={}\nv={}\ndiff={}'.format( - __class__, ref_losses, check_losses, ref_losses - check_losses - ), - ) - def test_pp_pass(self): - # pp2-fthenb - engine_fthenb = self.get_engine(schedule_mode="FThenB", acc_step=2) - history_fthenb = engine_fthenb.fit( - self.dataset, batch_size=self.batch_size, log_freq=1 - ) - assert engine_fthenb._strategy.pipeline.schedule_mode == "FThenB" - # pp2-vpp - engine_vpp_acc2 = self.get_engine(schedule_mode="VPP", acc_step=2) - history_vpp_acc2 = engine_vpp_acc2.fit( - self.dataset, batch_size=self.batch_size, log_freq=1 - ) - assert engine_vpp_acc2._strategy.pipeline.schedule_mode == "VPP" - - # pp2-1f1b - engine_1f1b = self.get_engine(schedule_mode="1F1B", acc_step=4) - history_1f1b = engine_1f1b.fit( - self.dataset, batch_size=self.batch_size, log_freq=1 - ) - assert engine_1f1b._strategy.pipeline.schedule_mode == "1F1B" - - # pp2-vpp - engine_vpp_acc4 = self.get_engine(schedule_mode="VPP", acc_step=4) - history_vpp_acc4 = engine_vpp_acc4.fit( - self.dataset, batch_size=self.batch_size, log_freq=1 - ) - assert engine_vpp_acc4._strategy.pipeline.schedule_mode == "VPP" - - if paddle.distributed.get_rank() == 1: - losses_fthenb = np.array(history_fthenb.history["loss"]) - losses_vpp_acc2 = np.array(history_vpp_acc2.history["loss"]) - self.check_results(losses_fthenb, losses_vpp_acc2) - - losses_1f1b = np.array(history_1f1b.history["loss"]) - losses_vpp_acc4 = np.array(history_vpp_acc4.history["loss"]) - self.check_results(losses_1f1b, losses_vpp_acc4) + engine = self.get_engine(schedule_mode="VPP", acc_step=4) + engine.fit(self.dataset, batch_size=self.batch_size, log_freq=1) + assert engine._strategy.pipeline.schedule_mode == "VPP" + + fw_chunk_ids = [] + bw_chunk_ids = [] + for op in engine.main_program.global_block().ops: + if is_optimize_op(op): + break + + dist_op = engine.dist_context.get_dist_op_for_program(op) + if is_forward_op(op): + fw_chunk_ids.append(dist_op.dist_attr.chunk_id) + if is_backward_op(op): + bw_chunk_ids.append(dist_op.dist_attr.chunk_id) + + if paddle.distributed.get_rank() == 0: + assert sum(fw_chunk_ids) == 8 + assert sum(bw_chunk_ids) == 13 + else: + assert sum(fw_chunk_ids) == 12 + assert sum(bw_chunk_ids) == 18 if __name__ == "__main__": From b8113f2cc862712b2badd703682594aa8d890fe2 Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Mon, 25 Dec 2023 11:09:51 +0800 Subject: [PATCH 007/146] [XPU][Phi Kernel] nonzero kernel support simulator XPUSIM_SKIP_RUN mode (#60224) * [XPU][Phi Kernel] nonzero kernel support simulator XPUSIM_SKIP_RUN mode * [XPU][Phi Kernel] add unittest for nonzero simulator skip_run mode --- paddle/phi/kernels/xpu/nonzero_kernel.cc | 10 +++++++ test/xpu/test_where_index_xpu.py | 34 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/paddle/phi/kernels/xpu/nonzero_kernel.cc b/paddle/phi/kernels/xpu/nonzero_kernel.cc index f3d665afaa664..e2a1339504bae 100644 --- a/paddle/phi/kernels/xpu/nonzero_kernel.cc +++ b/paddle/phi/kernels/xpu/nonzero_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/nonzero_kernel.h" +#include "glog/logging.h" + #include "paddle/phi/backends/xpu/enforce_xpu.h" #include "paddle/phi/common/memory_utils.h" #include "paddle/phi/core/kernel_registry.h" @@ -40,6 +42,14 @@ void NonZeroKernel(const Context& dev_ctx, dev_ctx.GetPlace(), static_cast(true_num), sizeof(int32_t)); + if (std::getenv("XPUSIM_SKIP_RUN") && + std::strcmp(std::getenv("XPUSIM_SKIP_RUN"), "1") == 0) { + VLOG(3) << "WARNING: In the simulator mode, the variable true_num_cpu " + "stores an uninitialized value. To avoid allocating a memory of " + "random size, we limit the value of true_num_cpu to the range 0 " + "<= true_num_cpu < numel"; + true_num_cpu = std::min(std::max(true_num_cpu, 0), static_cast(numel)); + } out->Resize(common::make_ddim({static_cast(true_num_cpu), rank})); auto* out_data = dev_ctx.template Alloc(out); diff --git a/test/xpu/test_where_index_xpu.py b/test/xpu/test_where_index_xpu.py index 6d0cf79032ef6..be5692ea290bb 100644 --- a/test/xpu/test_where_index_xpu.py +++ b/test/xpu/test_where_index_xpu.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest import numpy as np @@ -114,5 +115,38 @@ def test_type(): self.assertRaises(AttributeError, test_type) +class TestWhereSimulatorMode(unittest.TestCase): + def test_skip_run_on(self): + os.environ['XPUSIM_SKIP_RUN'] = '1' + cond = paddle.static.data(name='cond', shape=[-1, 4], dtype='bool') + result = paddle.nonzero(cond) + + exe = base.Executor(paddle.XPUPlace(0)) + exe.run(base.default_startup_program()) + cond_i = np.array([True, False, False, False]).astype("bool") + out = exe.run(base.default_main_program(), feed={'cond': cond_i}) + del os.environ['XPUSIM_SKIP_RUN'] + + def test_skip_run_off1(self): + cond = paddle.static.data(name='cond', shape=[-1, 4], dtype='bool') + result = paddle.nonzero(cond) + + exe = base.Executor(paddle.XPUPlace(0)) + exe.run(base.default_startup_program()) + cond_i = np.array([True, False, False, False]).astype("bool") + out = exe.run(base.default_main_program(), feed={'cond': cond_i}) + + def test_skip_run_off2(self): + os.environ['XPUSIM_SKIP_RUN'] = '0' + cond = paddle.static.data(name='cond', shape=[-1, 4], dtype='bool') + result = paddle.nonzero(cond) + + exe = base.Executor(paddle.XPUPlace(0)) + exe.run(base.default_startup_program()) + cond_i = np.array([True, False, False, False]).astype("bool") + out = exe.run(base.default_main_program(), feed={'cond': cond_i}) + del os.environ['XPUSIM_SKIP_RUN'] + + if __name__ == "__main__": unittest.main() From 6eb0a84b84764d9de162e68e4e9e2aa9619fdbee Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Mon, 25 Dec 2023 11:41:54 +0800 Subject: [PATCH 008/146] [SOT][3.11] Combine PRECALL and CALL as a super instruction in simulation (#60280) --- .../executor/function_graph.py | 17 +++++---- .../executor/opcode_executor.py | 37 +++++++++++++++---- test/sot/test_break_graph.py | 21 +++++++++++ 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py index a2fb85734c7be..a188f56154a85 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py +++ b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py @@ -255,15 +255,16 @@ def load(self, var): self._pycode_gen.gen_load(self._store_var_info[var.id]) origin_instrs = get_instructions(self.pycode_gen._origin_code) + is_precall = origin_instrs[instr_idx].opname == "PRECALL" + current_idx = instr_idx + # skip CALL if current instr is PRECALL + next_idx = instr_idx + 1 + int(is_precall) - restore_instrs = origin_instrs[:instr_idx] + restore_instrs = origin_instrs[:current_idx] restore_instr_names = [ - instr.opname for instr in restore_instrs[:instr_idx] + instr.opname for instr in restore_instrs[:current_idx] ] - # NOTE(SigureMo): Trailing KW_NAMES or PRECALL is no need to restore in Python 3.11+ - if restore_instr_names[-1:] == ["PRECALL"]: - restore_instrs = restore_instrs[:-1] - restore_instr_names = restore_instr_names[:-1] + # NOTE(SigureMo): Trailing KW_NAMES is no need to restore in Python 3.11+ if restore_instr_names[-1:] == ["KW_NAMES"]: restore_instrs = restore_instrs[:-1] restore_instr_names = restore_instr_names[:-1] @@ -272,12 +273,12 @@ def load(self, var): nop = self.pycode_gen._add_instr("NOP") for instr in origin_instrs: - if instr.jump_to == origin_instrs[instr_idx]: + if instr.jump_to == origin_instrs[current_idx]: instr.jump_to = nop self.pycode_gen.hooks.append( lambda: self.pycode_gen.extend_instrs( - iter(origin_instrs[instr_idx + 1 :]) + iter(origin_instrs[next_idx:]) ) ) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index eabdeb9889e7c..c7664d354d92e 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -558,8 +558,10 @@ def step(self, instr: Instruction): print(log_message) breakpoint() # noqa: T100 - with EventGuard(f"{instr.opname}", event_level=1): - return getattr(self, instr.opname)(instr) # run single step. + opname = instr.opname if instr.opname != "PRECALL" else "PRECALL__CALL" + assert opname != "CALL", "CALL should fused with PRECALL" + with EventGuard(f"{opname}", event_level=1): + return getattr(self, opname)(instr) # run single step. def indexof(self, instr: Instruction): """ @@ -1027,6 +1029,19 @@ def BUILD_MAP_UNPACK_WITH_CALL(self, instr: Instruction): ) ) + @call_break_graph_decorator(push_n=1) + def PRECALL__CALL(self, instr: Instruction): + """ + presudo super-instruction for PRECALL + CALL + """ + assert isinstance(instr.arg, int) + assert instr.opname == "PRECALL" + self.PRECALL(instr) + next_instr = self._instructions[self._lasti] + self._lasti += 1 + assert next_instr.opname == "CALL" + self.CALL(next_instr) + def PRECALL(self, instr: Instruction): assert isinstance(instr.arg, int) is_method_layout = not isinstance( @@ -1045,7 +1060,6 @@ def KW_NAMES(self, instr: Instruction): assert isinstance(instr.arg, int) self._call_shape = self._co_consts[instr.arg].get_py_value() - @call_break_graph_decorator(push_n=1) def CALL(self, instr: Instruction): assert isinstance(instr.arg, int) assert instr.arg + 2 <= len(self.stack) @@ -1660,19 +1674,24 @@ def _break_graph_in_call( """ push_n = push_n(instr.arg) if callable(push_n) else push_n + is_precall = instr.opname == "PRECALL" index = self.indexof(instr) + # Use CALL instead of PRECALL to calculate the real stack effect + call_instr = self._instructions[index + int(is_precall)] + # skip CALL if current instr is PRECALL + next_index = index + 1 + int(is_precall) self.stack = origin_stack # gen call static fn opcode - resume_input_name = analysis_inputs(self._instructions, index + 1) + resume_input_name = analysis_inputs(self._instructions, next_index) var_loader = self.gen_compute_in_break_with_name_store( - resume_input_name, self.indexof(instr) + resume_input_name, index ) # gen graph break call fn opcode - stack_effect = calc_stack_effect(instr) + stack_effect = calc_stack_effect(call_instr) pop_n = push_n - stack_effect for i, stack_arg in enumerate(self.stack): @@ -1681,11 +1700,13 @@ def _break_graph_in_call( # gen call resume fn opcode # NOTE(SigureMo): In Python 3.11，we need generate KW_NAMES if the call shape is not None. self._graph.pycode_gen.gen_kw_names(self._call_shape) - self._graph.pycode_gen.extend_instrs([instr]) + self._graph.pycode_gen.extend_instrs( + self._instructions[index:next_index] + ) self.stack.pop_n(pop_n) stack_size = len(self.stack) + push_n - resume_fn, _ = self._create_resume_fn(index + 1, stack_size) + resume_fn, _ = self._create_resume_fn(next_index, stack_size) if resume_fn: self._graph.pycode_gen.gen_load_object( diff --git a/test/sot/test_break_graph.py b/test/sot/test_break_graph.py index cc1aca51caec3..b6908f4d229b5 100644 --- a/test/sot/test_break_graph.py +++ b/test/sot/test_break_graph.py @@ -164,5 +164,26 @@ def test_break_graph_resume_pass_null(self): self.assert_results(break_graph_resume_pass_null, x, y) +class MyLayer(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.head = paddle.nn.Linear(3, 10) + + def forward_features(self, x): + paddle.jit.sot.psdb.breakgraph() + return x + + def forward(self, x): + x = self.forward_features(x) + return self.head(x) + + +class TestBreakGraphInLayer(TestCaseBase): + def test_break_graph_in_layer(self): + x = paddle.rand([2, 3], dtype=paddle.float32) + net = MyLayer() + self.assert_results(net.forward, x) + + if __name__ == "__main__": unittest.main() From b68cfb86ba2cb53ba80ab8c1639329377a1f5737 Mon Sep 17 00:00:00 2001 From: houj04 <35131887+houj04@users.noreply.github.com> Date: Mon, 25 Dec 2023 11:43:30 +0800 Subject: [PATCH 009/146] [XPU] remove generate_proposals and sequence_conv from xpu3_op_list (#60288) --- paddle/phi/backends/xpu/xpu3_op_list.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/phi/backends/xpu/xpu3_op_list.cc b/paddle/phi/backends/xpu/xpu3_op_list.cc index 0265a8b6e9fa7..24a35b7029aae 100644 --- a/paddle/phi/backends/xpu/xpu3_op_list.cc +++ b/paddle/phi/backends/xpu/xpu3_op_list.cc @@ -484,7 +484,6 @@ XPUOpMap& get_kl3_ops() { {"gelu_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"gelu", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, - {"generate_proposals_v2", XPUKernelSet({phi::DataType::FLOAT32})}, {"generate_sequence_xpu", XPUKernelSet({ phi::DataType::FLOAT32, @@ -1152,8 +1151,6 @@ XPUOpMap& get_kl3_ops() { XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, // AddMore - {"sequence_conv", XPUKernelSet({phi::DataType::FLOAT32})}, - {"sequence_conv_grad", XPUKernelSet({phi::DataType::FLOAT32})}, {"sequence_unpad", XPUKernelSet({phi::DataType::FLOAT32})}, // Fused op {"resnet_basic_block_grad", XPUKernelSet({phi::DataType::FLOAT32})}, From f191a9737ba7fac8374a5fcb2f4313d226ba6a62 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Mon, 25 Dec 2023 12:46:03 +0800 Subject: [PATCH 010/146] [PIR] Support some while_loop op_test (#60271) --- python/paddle/static/nn/control_flow.py | 27 ++++++++---- test/legacy_test/test_while_loop_op.py | 55 +++++++++++++++---------- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py index 2361cdd2c0088..5ba3a14469d8e 100644 --- a/python/paddle/static/nn/control_flow.py +++ b/python/paddle/static/nn/control_flow.py @@ -675,25 +675,34 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): pre_cond = cond(*loop_vars) + check_variable_and_dtype( + pre_cond, 'var of cond returned', ['bool'], 'static.nn.while_loop' + ) + if reduce(lambda a, b: a * b, pre_cond.shape, 1) != 1: + raise TypeError( + "the shape of the variable returned by cond should be [1]," + f"but given shape as {list(pre_cond.shape)}." + ) + if in_pir_mode(): while_op = build_while_op(pre_cond, flatten(loop_vars)) with while_op.body() as cur_block: args = cur_block.args() next_var = body(*args) + try: + assert_same_structure( + flatten(next_var), flatten(loop_vars), check_types=False + ) + except ValueError as e: + raise ValueError( + "body in while_loop should return the same arity " + f"(length and structure) as loop_vars: {e}" + ) next_cond = cond(*next_var) next_cond.stop_gradient = True cf_yield([next_cond, *next_var]) return while_op.as_operation().results() - check_variable_and_dtype( - pre_cond, 'var of cond returned', ['bool'], 'static.nn.while_loop' - ) - if reduce(lambda a, b: a * b, pre_cond.shape, 1) != 1: - raise TypeError( - "the shape of the variable returned by cond should be [1]," - f"but given shape as {list(pre_cond.shape)}." - ) - if in_dygraph_mode(): now_cond = pre_cond.item() while now_cond: diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index 4feddf5a7c2df..c75670d95c6a8 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -523,7 +523,8 @@ def fn_add_one(): class TestApiWhileLoop_Error(unittest.TestCase): @compare_legacy_with_pt - def test_error(self): + @test_with_pir_api + def test_error1(self): def cond_returns_constant(i): return 1 @@ -549,27 +550,9 @@ def body_returns_error_length(i): def body_returns_error_type(i, ten): return paddle.increment(i) - def cond_returns_with_mutable_dict(i, test_dict): - return i > 0 - - def body_returns_with_mutable_dict(i, test_dict): - test_dict['new_key'] = paddle.tensor.fill_constant( - shape=[1], dtype='int64', value=1 - ) - return paddle.increment(i), test_dict - - def cond_returns_with_mutable_list(i, test_list): - return i > 0 - - def body_returns_with_mutable_list(i, test_list): - test_list.append( - paddle.tensor.fill_constant(shape=[1], dtype='int64', value=1) - ) - return paddle.increment(i), test_list - main_program = paddle.static.Program() startup_program = paddle.static.Program() - with program_guard(main_program, startup_program): + with paddle.static.program_guard(main_program, startup_program): data = paddle.tensor.fill_constant( shape=[1], dtype='int64', value=1 ) @@ -656,7 +639,35 @@ def value_error_body_returns_error_type(): self.assertRaises(ValueError, value_error_body_returns_error_type) + @compare_legacy_with_pt + def test_error2(self): + def cond_returns_with_mutable_dict(i, test_dict): + return i > 0 + + def body_returns_with_mutable_dict(i, test_dict): + test_dict['new_key'] = paddle.tensor.fill_constant( + shape=[1], dtype='int64', value=1 + ) + return paddle.increment(i), test_dict + + def cond_returns_with_mutable_list(i, test_list): + return i > 0 + + def body_returns_with_mutable_list(i, test_list): + test_list.append( + paddle.tensor.fill_constant(shape=[1], dtype='int64', value=1) + ) + return paddle.increment(i), test_list + + main_program = paddle.static.Program() + startup_program = paddle.static.Program() + with paddle.static.program_guard(main_program, startup_program): + data = paddle.tensor.fill_constant( + shape=[1], dtype='int64', value=1 + ) + # The length of `output_vars` with mutable value should keep same with `loop_vars` + # TODO(zhangbo): slice error need to fix, loop_vars support list/dict def value_error_body_returns_with_mutable_dict(): test_dict = { "int_constant": paddle.tensor.fill_constant( @@ -673,6 +684,7 @@ def value_error_body_returns_with_mutable_dict(): ValueError, value_error_body_returns_with_mutable_dict ) + # TODO(zhangbo): loop_vars support list/dict def value_error_body_returns_with_mutable_list(): test_list = [ paddle.tensor.fill_constant( @@ -691,7 +703,8 @@ def value_error_body_returns_with_mutable_list(): class TestApiWhileLoopSliceInBody(unittest.TestCase): - # @compare_legacy_with_pt + @compare_legacy_with_pt + # @test_with_pir_api (need to fix slice bug in pir) def test_var_slice(self): def cond(z, i): return i + 1 <= x_shape[0] From 1468a0a29de82b9da195eb5fa17b56b6af07b90c Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:12:45 +0800 Subject: [PATCH 011/146] Fix optional output error when input is none (#60269) --- .../fluid/pir/dialect/op_generator/api_gen.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/paddle/fluid/pir/dialect/op_generator/api_gen.py b/paddle/fluid/pir/dialect/op_generator/api_gen.py index 1223cd0404d47..7fbfa092cc011 100644 --- a/paddle/fluid/pir/dialect/op_generator/api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/api_gen.py @@ -85,6 +85,7 @@ {in_combine} {compute_op} {handle_optional_outputs} + {set_null_type} {out_split} {return_result}""" @@ -148,6 +149,12 @@ optional_{name} = paddle::make_optional>(optional_{name}_slice_op.outputs()); }}""" +SET_NULL_TYPE_TEMPLATE = """ + if (!{input}) {{ + {op_name}_op.result({index}).set_type(pir::Type()); + }}""" + + COMBINE_OP_TEMPLATE = """ auto {op_name} = ApiBuilder::Instance().GetBuilder()->Build({in_name});""" @@ -435,6 +442,21 @@ def _gen_handle_optional_outputs(self, op_info, op_name): ) return ret + def _gen_set_null_type(self, op_info, op_name): + name_list = op_info.output_name_list + inplace_map = op_info.inplace_map + if inplace_map is None: + return "" + + ret = "" + for i, out_name in enumerate(name_list): + if self._is_optional_output(op_info, out_name): + in_name = inplace_map[out_name] + ret += SET_NULL_TYPE_TEMPLATE.format( + input=in_name, op_name=op_name, index=i + ) + return ret + def _gen_in_combine(self, op_info, is_mutable_attr, is_vector_mutable_attr): name_list = op_info.input_name_list type_list = op_info.input_type_list @@ -727,6 +749,7 @@ def _gen_one_impl( handle_optional_outputs=self._gen_handle_optional_outputs( op_info, kernel_name ), + set_null_type=self._gen_set_null_type(op_info, kernel_name), out_split=out_split, return_result=self._gen_return_result(ret_list), ) @@ -782,6 +805,7 @@ def _gen_one_impl( handle_optional_outputs=self._gen_handle_optional_outputs( op_info, op_name ), + set_null_type=self._gen_set_null_type(op_info, op_name), out_split=out_split, return_result=self._gen_return_result(ret_list), ) From 5dc746766b1acd9a47ac3925934087c181ae3a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:17:20 +0800 Subject: [PATCH 012/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.109?= =?UTF-8?q?=E3=80=91identity=5Floss=20(#58880)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/incubate/nn/loss.py | 4 ++-- test/legacy_test/test_identity_loss_op.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python/paddle/incubate/nn/loss.py b/python/paddle/incubate/nn/loss.py index c6eb7df467a79..d8aea8b33b589 100644 --- a/python/paddle/incubate/nn/loss.py +++ b/python/paddle/incubate/nn/loss.py @@ -15,7 +15,7 @@ from paddle import _C_ops from paddle.base.data_feeder import check_variable_and_dtype from paddle.base.layer_helper import LayerHelper -from paddle.framework import in_dynamic_mode +from paddle.framework import in_dynamic_or_pir_mode def identity_loss(x, reduction="none"): @@ -59,7 +59,7 @@ def identity_loss(x, reduction="none"): if reduction is None: raise Exception("Unsupported reduction type.") - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.identity_loss(x, reduction) check_variable_and_dtype(x, 'x', ['float32', 'float64'], "identity_loss") diff --git a/test/legacy_test/test_identity_loss_op.py b/test/legacy_test/test_identity_loss_op.py index f751aa8959309..f0264b17950e0 100644 --- a/test/legacy_test/test_identity_loss_op.py +++ b/test/legacy_test/test_identity_loss_op.py @@ -20,6 +20,7 @@ import paddle from paddle import base from paddle.base import Program, program_guard +from paddle.pir_utils import test_with_pir_api class TestIdentityLossOp(OpTest): @@ -48,12 +49,12 @@ def setUp(self): def test_check_output(self): paddle.enable_static() - self.check_output() + self.check_output(check_pir=True) paddle.disable_static() def test_check_grad_normal(self): paddle.enable_static() - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', check_pir=True) paddle.disable_static() def initTestCase(self): @@ -124,6 +125,7 @@ def identity_loss_ref(self, input, reduction): else: return input + @test_with_pir_api def test_api_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): From 3c4c50c0b5edb84d9fe8b34dcd514240c7557707 Mon Sep 17 00:00:00 2001 From: Lu Qi <61354321+MarioLulab@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:22:25 +0800 Subject: [PATCH 013/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.311?= =?UTF-8?q?=E3=80=81312=E3=80=81301=E3=80=81290=E3=80=81259=E3=80=91=20Mig?= =?UTF-8?q?rate=20some=20loss=20api=20into=20pir=20(#60291)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/legacy_test/test_hinge_embedding_loss.py | 30 +++++++++++++------ test/legacy_test/test_poisson_nll_loss.py | 11 +++++-- test/legacy_test/test_soft_margin_loss.py | 2 ++ test/legacy_test/test_triplet_margin_loss.py | 4 +++ .../test_triplet_margin_with_distance_loss.py | 4 +++ 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/test/legacy_test/test_hinge_embedding_loss.py b/test/legacy_test/test_hinge_embedding_loss.py index 5bfeb0f0f143a..1e45c80685c4a 100644 --- a/test/legacy_test/test_hinge_embedding_loss.py +++ b/test/legacy_test/test_hinge_embedding_loss.py @@ -17,7 +17,7 @@ import numpy as np import paddle -from paddle.static import Program, program_guard +from paddle.pir_utils import test_with_pir_api np.random.seed(42) @@ -45,7 +45,7 @@ def setUp(self): def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) input = paddle.to_tensor(self.input_np) - label = paddle.to_tensor(self.label_np, dtype=paddle.float64) + label = paddle.to_tensor(self.label_np, dtype="float64") dy_result = paddle.nn.functional.hinge_embedding_loss(input, label) expected = calc_hinge_embedding_loss(self.input_np, self.label_np) @@ -70,18 +70,21 @@ def run_dynamic_check(self, place=paddle.CPUPlace()): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertEqual(dy_result.shape, list(self.shape)) + @test_with_pir_api def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: expected = calc_hinge_embedding_loss( self.input_np, self.label_np, reduction=reduction ) - with program_guard(Program(), Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input = paddle.static.data( - name="input", shape=self.shape, dtype=paddle.float64 + name="input", shape=self.shape, dtype="float64" ) label = paddle.static.data( - name="label", shape=self.shape, dtype=paddle.float64 + name="label", shape=self.shape, dtype="float64" ) st_result = paddle.nn.functional.hinge_embedding_loss( input, label, reduction=reduction @@ -93,10 +96,12 @@ def run_static_check(self, place=paddle.CPUPlace): ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) + @test_with_pir_api def test_cpu(self): self.run_dynamic_check(place=paddle.CPUPlace()) self.run_static_check(place=paddle.CPUPlace()) + @test_with_pir_api def test_gpu(self): if not paddle.is_compiled_with_cuda(): return @@ -104,6 +109,7 @@ def test_gpu(self): self.run_static_check(place=paddle.CUDAPlace(0)) # test case the raise message + @test_with_pir_api def test_reduce_errors(self): def test_value_error(): loss = paddle.nn.functional.hinge_embedding_loss( @@ -124,7 +130,7 @@ def setUp(self): def run_dynamic_check(self, place=paddle.CPUPlace()): paddle.disable_static(place=place) input = paddle.to_tensor(self.input_np) - label = paddle.to_tensor(self.label_np, dtype=paddle.float64) + label = paddle.to_tensor(self.label_np, dtype="float64") hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss() dy_result = hinge_embedding_loss(input, label) expected = calc_hinge_embedding_loss(self.input_np, self.label_np) @@ -151,18 +157,21 @@ def run_dynamic_check(self, place=paddle.CPUPlace()): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertTrue(dy_result.shape, list(self.shape)) + @test_with_pir_api def run_static_check(self, place=paddle.CPUPlace): paddle.enable_static() for reduction in ['none', 'mean', 'sum']: expected = calc_hinge_embedding_loss( self.input_np, self.label_np, reduction=reduction ) - with program_guard(Program(), Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): input = paddle.static.data( - name="input", shape=self.shape, dtype=paddle.float64 + name="input", shape=self.shape, dtype="float64" ) label = paddle.static.data( - name="label", shape=self.shape, dtype=paddle.float64 + name="label", shape=self.shape, dtype="float64" ) hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( reduction=reduction @@ -175,10 +184,12 @@ def run_static_check(self, place=paddle.CPUPlace): ) np.testing.assert_allclose(result_numpy, expected, rtol=1e-05) + @test_with_pir_api def test_cpu(self): self.run_dynamic_check(place=paddle.CPUPlace()) self.run_static_check(place=paddle.CPUPlace()) + @test_with_pir_api def test_gpu(self): if not paddle.is_compiled_with_cuda(): return @@ -186,6 +197,7 @@ def test_gpu(self): self.run_static_check(place=paddle.CUDAPlace(0)) # test case the raise message + @test_with_pir_api def test_reduce_errors(self): def test_value_error(): hinge_embedding_loss = paddle.nn.loss.HingeEmbeddingLoss( diff --git a/test/legacy_test/test_poisson_nll_loss.py b/test/legacy_test/test_poisson_nll_loss.py index 4cfa517856780..f58caac91e642 100644 --- a/test/legacy_test/test_poisson_nll_loss.py +++ b/test/legacy_test/test_poisson_nll_loss.py @@ -19,6 +19,7 @@ import paddle import paddle.nn.functional as F from paddle.base import core +from paddle.pir_utils import test_with_pir_api np.random.seed(100) @@ -75,6 +76,7 @@ def setUp(self, dtype="float32"): else paddle.CPUPlace() ) + @test_with_pir_api def test_static_case( self, dtype="float32", @@ -90,8 +92,6 @@ def test_static_case( with paddle.static.program_guard(prog, startup_prog): input = paddle.static.data('input', self.shape, dtype) label = paddle.static.data('label', self.shape, dtype) - input.desc.set_need_check_feed(False) - label.desc.set_need_check_feed(False) out1 = F.poisson_nll_loss( input, label, @@ -203,6 +203,7 @@ def test_api(self): class TestPoissonNLLLossFloat16Case(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): if core.is_compiled_with_cuda(): self.test_static_case(dtype="float16") @@ -210,6 +211,7 @@ def test_api(self): class TestPoissonNLLLossBfloat16Case(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): if core.is_compiled_with_cuda(): self.test_static_case(dtype="uint16") @@ -217,30 +219,35 @@ def test_api(self): class TestPoissonNLLLossFloat32Case(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): self.test_static_case(dtype="float32") self.test_dynamic_case(dtype="float32") class TestPoissonNLLLossFloat64Case(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): self.test_static_case(dtype="float64") self.test_dynamic_case(dtype="float64") class TestPoissonNLLLossNoLoginputCase(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): self.test_static_case(log_input=False) self.test_dynamic_case(log_input=False) class TestPoissonNLLLossFulllossCase(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): self.test_static_case(full=True) self.test_dynamic_case(full=True) class TestPoissonNLLLossSumReductionCase(TestPoissonNLLLossBasicCase): + @test_with_pir_api def test_api(self): self.test_static_case(reduction="sum") self.test_dynamic_case(reduction="sum") diff --git a/test/legacy_test/test_soft_margin_loss.py b/test/legacy_test/test_soft_margin_loss.py index 9396d07e8680e..afeab4eebd5e1 100644 --- a/test/legacy_test/test_soft_margin_loss.py +++ b/test/legacy_test/test_soft_margin_loss.py @@ -17,6 +17,7 @@ import numpy as np import paddle +from paddle.pir_utils import test_with_pir_api def test_static_layer( @@ -122,6 +123,7 @@ def calc_softmarginloss( class TestSoftMarginLoss(unittest.TestCase): + @test_with_pir_api def test_SoftMarginLoss(self): input_np = np.random.uniform(0.1, 0.8, size=(5, 5)).astype(np.float64) types = [np.int32, np.int64, np.float32, np.float64] diff --git a/test/legacy_test/test_triplet_margin_loss.py b/test/legacy_test/test_triplet_margin_loss.py index 66b150df214df..aa43fc67d07ef 100644 --- a/test/legacy_test/test_triplet_margin_loss.py +++ b/test/legacy_test/test_triplet_margin_loss.py @@ -17,6 +17,7 @@ import numpy as np import paddle +from paddle.pir_utils import test_with_pir_api def call_TripletMarginLoss_layer( @@ -193,6 +194,7 @@ def calc_triplet_margin_loss( class TestTripletMarginLoss(unittest.TestCase): + @test_with_pir_api def test_TripletMarginLoss(self): shape = (2, 2) input = np.random.uniform(0.1, 0.8, size=shape).astype(np.float64) @@ -305,6 +307,7 @@ def test_TripletMarginLoss_dimension(self): ) paddle.enable_static() + @test_with_pir_api def test_TripletMarginLoss_swap(self): reduction = 'mean' place = paddle.CPUPlace() @@ -389,6 +392,7 @@ def test_TripletMarginLoss_margin(self): ) paddle.enable_static() + @test_with_pir_api def test_TripletMarginLoss_p(self): p = 3 shape = (2, 2) diff --git a/test/legacy_test/test_triplet_margin_with_distance_loss.py b/test/legacy_test/test_triplet_margin_with_distance_loss.py index 3c102fdc655c2..9bdd18e92d9bf 100644 --- a/test/legacy_test/test_triplet_margin_with_distance_loss.py +++ b/test/legacy_test/test_triplet_margin_with_distance_loss.py @@ -17,6 +17,7 @@ import numpy as np import paddle +from paddle.pir_utils import test_with_pir_api def call_TripletMarginDistanceLoss_layer( @@ -192,6 +193,7 @@ def calc_triplet_margin_distance_loss( class TestTripletMarginWithDistanceLossnew(unittest.TestCase): + @test_with_pir_api def test_TripletMarginDistanceLoss(self): shape = (5, 5) np.random.seed(1234) @@ -286,6 +288,7 @@ def test_TripletMarginDistanceLoss_error(self): class TestTripletMarginWithDistanceLossDF(unittest.TestCase): + @test_with_pir_api def test_TripletMarginDistanceLoss_distance_function(self): def distance_function_1(x1, x2): return 1.0 - paddle.nn.functional.cosine_similarity(x1, x2) @@ -399,6 +402,7 @@ def test_TripletMarginDistanceLoss_dimension(self): class TestTripletMarginWithDistanceLossSwap(unittest.TestCase): + @test_with_pir_api def test_TripletMarginWithDistanceLoss_swap(self): reduction = 'mean' place = paddle.CPUPlace() From 39693325e01d3a75462d42a5be720570bdffc190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=98=A5=E4=B9=94?= <83450930+Liyulingyue@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:28:02 +0800 Subject: [PATCH 014/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.124?= =?UTF-8?q?=E3=80=91=20optimizer.Lamb=20(#58881)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/optimizer/lamb.py | 2 +- python/paddle/tensor/linalg.py | 2 +- test/legacy_test/test_imperative_base.py | 10 ++++---- .../test_imperative_optimizer_v2.py | 2 ++ test/legacy_test/test_lambv2_op.py | 24 +++++++++++-------- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/python/paddle/optimizer/lamb.py b/python/paddle/optimizer/lamb.py index b409e88e338e9..105bddf07b839 100644 --- a/python/paddle/optimizer/lamb.py +++ b/python/paddle/optimizer/lamb.py @@ -243,7 +243,7 @@ def _append_optimize_op(self, block, param_and_grad): else: master_weight = None - if framework.in_dygraph_mode(): + if framework.in_dynamic_or_pir_mode(): _C_ops.lamb_( param_and_grad[0], param_and_grad[1], diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 920be5283adcf..fe80aaa10d635 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -428,7 +428,7 @@ def vector_norm( name (str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): if axis is None: axis = -1 return _C_ops.p_norm(input, porder, axis, 1e-12, keepdim, asvector) diff --git a/test/legacy_test/test_imperative_base.py b/test/legacy_test/test_imperative_base.py index f3a2d0dc503ff..175db17f07be7 100644 --- a/test/legacy_test/test_imperative_base.py +++ b/test/legacy_test/test_imperative_base.py @@ -14,15 +14,15 @@ import contextlib -from paddle import base +from paddle import base, static @contextlib.contextmanager def new_program_scope(main=None, startup=None, scope=None): - prog = main if main else base.Program() - startup_prog = startup if startup else base.Program() + prog = main if main else static.Program() + startup_prog = startup if startup else static.Program() scope = scope if scope else base.core.Scope() - with base.scope_guard(scope): - with base.program_guard(prog, startup_prog): + with static.scope_guard(scope): + with static.program_guard(prog, startup_prog): with base.unique_name.guard(): yield diff --git a/test/legacy_test/test_imperative_optimizer_v2.py b/test/legacy_test/test_imperative_optimizer_v2.py index 0000bd49ccb08..413d83d9bd34b 100644 --- a/test/legacy_test/test_imperative_optimizer_v2.py +++ b/test/legacy_test/test_imperative_optimizer_v2.py @@ -22,6 +22,7 @@ from paddle import base from paddle.base import core from paddle.distributed.fleet.meta_optimizers import DGCMomentumOptimizer +from paddle.pir_utils import test_with_pir_api # Note(wangzhongpu) # In dygraph, don't support ModelAverage, DGCMomentumOptimizer, ExponentialMovingAverage, PipelineOptimizer, LookaheadOptimizer, RecomputeOptimizer. @@ -790,6 +791,7 @@ def get_optimizer(self): return optimizer # should fix: may fail in CI-windows + @test_with_pir_api def _test_lamb(self): self._check_mlp() diff --git a/test/legacy_test/test_lambv2_op.py b/test/legacy_test/test_lambv2_op.py index 42dab23cc221f..54bb9cc955acf 100644 --- a/test/legacy_test/test_lambv2_op.py +++ b/test/legacy_test/test_lambv2_op.py @@ -20,6 +20,7 @@ from paddle import base from paddle.base import core from paddle.base.dygraph.base import switch_to_static_graph +from paddle.pir_utils import test_with_pir_api class LAMBOptimizer(paddle.optimizer.Lamb): @@ -113,6 +114,7 @@ def test_lamb_op(self): class TestLambOpWithCombinedOp(unittest.TestCase): + @test_with_pir_api def test_lamb_op_with_multi_steps(self): paddle.enable_static() @@ -124,7 +126,10 @@ def _build_static_model(main, startup, seed=100): name='X', shape=[-1, 13], dtype='float32' ) y = paddle.static.data(name='Y', shape=[-1, 1], dtype='float32') - prediction = paddle.static.nn.fc(x, size=1, activation=None) + linear = paddle.nn.Linear( + in_features=x.shape[-1], out_features=1 + ) + prediction = linear(x) loss = paddle.nn.functional.square_error_cost( input=prediction, label=y ) @@ -138,8 +143,8 @@ def _build_static_model(main, startup, seed=100): feed_x = np.random.random(size=(10, 13)).astype('float32') feed_y = np.random.random(size=(10, 1)).astype('float32') - main_program = base.Program() - startup_program = base.Program() + main_program = paddle.static.Program() + startup_program = paddle.static.Program() with base.program_guard(main_program, startup_program): avg_loss = _build_static_model(main_program, startup_program) lamb_kernel = paddle.optimizer.Lamb(learning_rate=0.2) @@ -150,11 +155,11 @@ def _build_static_model(main, startup, seed=100): output = executor.run( program=main_program, feed={'X': feed_x, 'Y': feed_y}, - fetch_list=[avg_loss.name], + fetch_list=[avg_loss], ) - main = base.Program() - startup = base.Program() + main = paddle.static.Program() + startup = paddle.static.Program() with base.program_guard(main, startup): loss = _build_static_model(main, startup) lamb = LAMBOptimizer(learning_rate=0.2) @@ -165,7 +170,7 @@ def _build_static_model(main, startup, seed=100): out = exe.run( program=main, feed={'X': feed_x, 'Y': feed_y}, - fetch_list=[loss.name], + fetch_list=[loss], ) np.testing.assert_allclose(out, output, rtol=1e-05) @@ -226,8 +231,7 @@ def check_main(self, x_np, place, multi_precision=False, seed=10, n=10): weight, bias = linear.weight, linear.bias exe = paddle.static.Executor(place) scope = paddle.static.Scope() - x = main_prog.global_block().var(x.name) - if x.dtype == core.VarDesc.VarType.FP16: + if x.dtype in (core.VarDesc.VarType.FP16, core.DataType.FLOAT16): x_np = x_np.astype(np.float16) def get_parameter(var): @@ -256,7 +260,7 @@ def get_parameter(var): weight_np, bias_np = None, None for i in range(n): - feed_dict = {x.name: x_np} + feed_dict = {'x': x_np} weight_np, bias_np = exe.run( main_prog, feed=feed_dict, fetch_list=[weight, bias] ) From 42c2b70092553903c665332803bf78a227afdb84 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Mon, 25 Dec 2023 14:37:35 +0800 Subject: [PATCH 015/146] del cuda10.2 dockerfile (#60299) --- tools/dockerfile/Dockerfile.ubuntu | 214 ----------------------------- tools/dockerfile/ci_dockerfile.sh | 37 ----- 2 files changed, 251 deletions(-) delete mode 100644 tools/dockerfile/Dockerfile.ubuntu diff --git a/tools/dockerfile/Dockerfile.ubuntu b/tools/dockerfile/Dockerfile.ubuntu deleted file mode 100644 index 1f9edcbf4ca96..0000000000000 --- a/tools/dockerfile/Dockerfile.ubuntu +++ /dev/null @@ -1,214 +0,0 @@ -# A image for building paddle binaries -# Use cuda devel base image for both cpu and gpu environment -# When you modify it, please be aware of cudnn-runtime version -FROM nvidia/cuda: -MAINTAINER PaddlePaddle Authors - -# ENV variables -ARG WITH_GPU -ARG WITH_AVX - -ENV WITH_GPU=${WITH_GPU:-ON} -ENV WITH_AVX=${WITH_AVX:-ON} - -ENV HOME /root -# Add bash enhancements -COPY paddle/scripts/docker/root/ /root/ - -RUN chmod 777 /tmp -# Prepare packages for Python -RUN apt-get update --allow-unauthenticated && \ - apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev - -RUN apt-get update && \ - apt-get install -y --allow-downgrades --allow-change-held-packages \ - patchelf git python-pip python-dev python-opencv openssh-server bison \ - wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ - curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-matplotlib \ - automake locales swig \ - liblapack-dev liblapacke-dev \ - net-tools libtool module-init-tools && \ - apt-get clean -y - -RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \ - tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \ - rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1 - -# Downgrade gcc&&g++ - - -# install cmake -WORKDIR /home -RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz -ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH - -# Install Python3.7 -RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \ - tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.7.0 --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null && ldconfig - -# Install Python3.8 -RUN wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz && \ - tar -xzf Python-3.8.0.tgz && cd Python-3.8.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null && ldconfig - -# Install Python3.9 -RUN wget -q https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz && \ - tar -xzf Python-3.9.0.tgz && cd Python-3.9.0 && \ - CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \ - make -j8 > /dev/null && make altinstall > /dev/null && ldconfig - -ENV PATH=/usr/local/python3.7.0/include:${PATH} -ENV PATH=/usr/local/python3.7.0/bin:${PATH} -ENV LD_LIBRARY_PATH=/usr/local/python3.7.0/lib:${LD_LIBRARY_PATH} -ENV CPLUS_INCLUDE_PATH=/usr/local/python3.7.0/include/python3.7:$CPLUS_INCLUDE_PATH -RUN ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/local/bin/python3 && \ - ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/bin/python3 && \ - ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/local/bin/python && \ - ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/bin/python - -RUN rm -rf /root/python_build - -# Replace pip and setuptools with updated version -WORKDIR /home -RUN python3.9 -m pip uninstall -y pip setuptools && \ - python3.8 -m pip uninstall -y pip setuptools && \ - python3.7 -m pip uninstall -y pip setuptools - -RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip -WORKDIR /home/setuptools-50.3.2 -RUN python3.9 setup.py build && python3.9 setup.py install && \ - python3.8 setup.py build && python3.8 setup.py install && \ - python3.7 setup.py build && python3.7 setup.py install - -WORKDIR /home -RUN wget https://files.pythonhosted.org/packages/a3/50/c4d2727b99052780aad92c7297465af5fe6eec2dbae490aa9763273ffdc1/pip-22.3.1.tar.gz && tar -zxvf pip-22.3.1.tar.gz -WORKDIR pip-22.3.1 -RUN python setup.py install && \ - python3.9 setup.py install && \ - python3.8 setup.py install && \ - python3.7 setup.py install && \ - rm /usr/local/bin/pip && \ - ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip && \ - rm /usr/local/bin/pip3 && \ - ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3 - -WORKDIR /home -RUN rm setuptools-50.3.2.zip pip-22.3.1.tar.gz && \ - rm -r setuptools-50.3.2 pip-22.3.1 - -# Install Go and glide -WORKDIR /home -RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \ - tar -xz -C /usr/local && \ - mkdir /root/gopath && \ - mkdir /root/gopath/bin && \ - mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# Install TensorRT -# following TensorRT.tar.gz is not the default official one, we do two miny changes: -# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now, -# and its size is only one-third of the official one. -# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle. -# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details. - -# Downgrade TensorRT - -# Older versions of patchelf limited the size of the files being processed and were fixed in this pr. -# # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa -# # So install a newer version here. -COPY tools/dockerfile/build_scripts /build_scripts -RUN bash /build_scripts/install_trt.sh && \ - bash /build_scripts/install_nccl2.sh && \ - bash /build_scripts/install_patchelf.sh -RUN rm -rf /build_scripts - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -# specify sphinx version as 1.5.6 and remove -U option for [pip install -U -# sphinx-rtd-theme] since -U option will cause sphinx being updated to newest -# version(1.7.1 for now), which causes building documentation failed. -RUN pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.7 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.7 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.8 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.8 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.8 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark && \ - pip3.9 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \ - pip3.9 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \ - pip3.9 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark - -RUN pip3.7 --no-cache-dir install 'ipython==5.3.0' && \ - pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.8 --no-cache-dir install 'ipython==5.3.0' && \ - pip3.8 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip3.9 --no-cache-dir install 'ipython==5.3.0' && \ - pip3.9 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0' - -# For PaddleTest CE -RUN pip3.7 --no-cache-dir install pytest && \ - pip3.8 --no-cache-dir install pytest && \ - pip3.9 --no-cache-dir install pytest - -# For pre-commit -RUN pip3.7 --no-cache-dir install pre-commit==2.17.0 && \ - pip3.8 --no-cache-dir install pre-commit==2.17.0 && \ - pip3.9 --no-cache-dir install pre-commit==2.17.0 && \ - pip3.7 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ - pip3.8 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \ - pip3.9 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 - -RUN pip3.7 --no-cache-dir install coverage && \ - pip3.8 --no-cache-dir install coverage && \ - pip3.9 --no-cache-dir install coverage - -COPY ./python/requirements.txt /root/ -RUN pip3.7 --no-cache-dir install -r /root/requirements.txt && \ - pip3.8 --no-cache-dir install -r /root/requirements.txt && \ - pip3.9 --no-cache-dir install -r /root/requirements.txt - -# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use -# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 -RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y && \ - pip3.7 --no-cache-dir install certifi urllib3[secure] && \ - pip3.8 --no-cache-dir install certifi urllib3[secure] && \ - pip3.9 --no-cache-dir install certifi urllib3[secure] - -# ar mishandles 4GB files -# https://sourceware.org/bugzilla/show_bug.cgi?id=14625 -# remove them when apt-get support 2.27 and higher version -RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \ - tar -xzf binutils_2.27.orig.tar.gz && \ - cd binutils-2.27 && \ - ./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz - -RUN apt-get install libprotobuf-dev -y - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc - -# ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ - tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ - ./configure -prefix=/usr/local/ccache-3.7.9 && \ - make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache && \ - cd ../ && rm -rf ccache-3.7.9 ccache-3.7.9.tar.gz - - -EXPOSE 22 diff --git a/tools/dockerfile/ci_dockerfile.sh b/tools/dockerfile/ci_dockerfile.sh index 14cc1270c73a7..cb4a12891efb8 100644 --- a/tools/dockerfile/ci_dockerfile.sh +++ b/tools/dockerfile/ci_dockerfile.sh @@ -15,41 +15,6 @@ # limitations under the License. -function make_ubuntu_trt7_dockerfile(){ - dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16" - sed "s//10.2-cudnn8-devel-ubuntu16.04/g" ./Dockerfile.ubuntu >${dockerfile_name} - sed -i "s#liblzma-dev#liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev#g" ${dockerfile_name} - dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}') - sed -i "${dockerfile_line}i RUN apt remove -y libcudnn* --allow-change-held-packages \&\& \ - apt-get install -y --allow-unauthenticated libsndfile1 libcudnn8=8.1.0.77-1+cuda10.2 libcudnn8-dev=8.1.0.77-1+cuda10.2 --allow-change-held-packages" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q \ - https://developer.download.nvidia.com/compute/cuda/10.2/Prod/patches/2/cuda_10.2.2_linux.run \&\& \ - bash cuda_10.2.2_linux.run --silent --toolkit \&\& ldconfig" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q \ - https://developer.download.nvidia.com/compute/cuda/10.2/Prod/patches/1/cuda_10.2.1_linux.run \&\& \ - bash cuda_10.2.1_linux.run --silent --toolkit \&\& ldconfig" ${dockerfile_name} - sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh ubuntu1604-7234#g' ${dockerfile_name} - sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \ - tar -xzf hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name} - sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt install -y libcurl4-openssl-dev gettext zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \ - tar -xvf git-2.17.1.tar.gz \&\& \ - cd git-2.17.1 \&\& \ - ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \ - make -j8 \&\& make install " ${dockerfile_name} - sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel \&\& pip3.8 install PyGithub distro" ${dockerfile_name} - sed -i "s##WORKDIR /usr/bin \\ - COPY tools/dockerfile/build_scripts /build_scripts \\ - RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \\ - RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \\ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \\ - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \\ - RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \\ - RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \\ - ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name} - sed -i "s#bash /build_scripts/install_nccl2.sh#wget -q --no-proxy https://nccl2-deb.gz.bcebos.com/nccl-local-repo-ubuntu1604-2.8.4-cuda10.2_1.0-1_amd64.deb \\ - RUN apt remove -y libnccl* --allow-change-held-packages \&\& apt-get install -y --allow-unauthenticated libsndfile1 libnccl2=2.8.4-1+cuda10.2 libnccl-dev=2.8.4-1+cuda10.2 zstd pigz --allow-change-held-packages #g" ${dockerfile_name} -} - function make_ubuntu_trt7_dockerfile_temp_ues(){ dockerfile_name="Dockerfile.cuda102_cudnn8_gcc82_ubuntu16" sed "s##nvidia/cuda:12.0.1-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name} @@ -146,12 +111,10 @@ function make_ubuntu20_cu112_dockerfile(){ } function main() { - make_ubuntu_trt7_dockerfile_temp_ues make_cpu_dockerfile make_ce_framework_dockcerfile make_unbuntu20_cu12_dockerfile make_ubuntu20_cu112_dockerfile - cp Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage Dockerfile.cuda102_cudnn8_gcc82_ubuntu16 } main "$@" From 7bcba0e2dbc2a30ae21de3e23cbdcbfe652859c0 Mon Sep 17 00:00:00 2001 From: liuzhenhai93 Date: Mon, 25 Dec 2023 14:49:52 +0800 Subject: [PATCH 016/146] cross entropy grad infer spmd (#60258) * cross entropy with softmax * poish * cross entropy with softmax * cross entropy with softmax --- paddle/phi/api/yaml/backward.yaml | 1 + paddle/phi/api/yaml/ops.yaml | 1 + .../spmd_rules/cross_entropy_with_softmax.cc | 105 +++++++++++++++--- .../spmd_rules/cross_entropy_with_softmax.h | 9 ++ .../spmd_rules/spmd_rule_macro_define.h | 60 ++++++++++ test/cpp/auto_parallel/CMakeLists.txt | 9 ++ .../cross_entropy_softmax_spmd_rule_test.cc | 101 +++++++++++++++++ 7 files changed, 273 insertions(+), 13 deletions(-) create mode 100644 paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h create mode 100644 test/cpp/auto_parallel/cross_entropy_softmax_spmd_rule_test.cc diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index 469a18888e515..21ec2126c8f94 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -479,6 +479,7 @@ output : Tensor(input_grad) infer_meta : func : CrossEntropyWithSoftmaxGradInferMeta + spmd_rule : CrossEntropyWithSoftmaxGradInferSpmd kernel : func : cross_entropy_with_softmax_grad data_type : loss_grad diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index 7dd1f0fda4174..c15fb2fdb1998 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -606,6 +606,7 @@ inplace : (input -> softmax) infer_meta : func : CrossEntropyWithSoftmaxInferMeta + spmd_rule: CrossEntropyWithSoftmaxInferSpmd kernel : func : cross_entropy_with_softmax data_type : input diff --git a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc index af3a738a8aee6..325672ca62f3c 100644 --- a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc +++ b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.cc @@ -19,6 +19,7 @@ limitations under the License. */ #include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" #include "paddle/phi/core/distributed/auto_parallel/inferspmd_utils.h" #include "paddle/phi/core/distributed/auto_parallel/utils.h" +#include "paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h" #include "paddle/phi/infermeta/spmd_rules/utils.h" namespace phi { @@ -61,14 +62,9 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmd(const DistMetaTensor& x, int ignore_index, int axis) { // Step0: Verify input args based on cross_entropy_with_softmax logic - auto x_shape = phi::vectorize(x.dims()); - int x_ndim = x_shape.size(); - TensorDistAttr x_dist_attr_src = x.dist_attr(); - std::vector x_dims_mapping_src = x_dist_attr_src.dims_mapping(); - auto label_shape = phi::vectorize(label.dims()); - TensorDistAttr label_dist_attr_src = label.dist_attr(); - std::vector label_dims_mapping_src = - label_dist_attr_src.dims_mapping(); + + EXTRACT_SHAPE_AND_DIST_ATTR(x); + EXTRACT_SHAPE_AND_DIST_ATTR(label); VLOG(6) << "CrossEntropyWithSoftmaxSPMDRule InferForward Inputs: " << "X shape: [" << str_join(x_shape) << "], x_dims_mapping_src: [" @@ -189,11 +185,9 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdReverse( int ignore_index, int axis) { // Step0: Verify input args based on cross_entropy_with_softmax logic - auto loss_shape = phi::vectorize(loss.dims()); - int loss_ndim = loss_shape.size(); - TensorDistAttr loss_dist_attr_src = loss.dist_attr(); - std::vector loss_dims_mapping_src = - loss_dist_attr_src.dims_mapping(); + + EXTRACT_SHAPE_AND_DIST_ATTR_WITH_DIM_CK(loss); + auto s_out_shape = phi::vectorize(softmax_out.dims()); int s_out_ndim = s_out_shape.size(); TensorDistAttr s_out_dist_attr_src = softmax_out.dist_attr(); @@ -315,5 +309,90 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdReverse( {s_out_dist_attr_dst, loss_dist_attr_dst}}; } +void GetCrossEntropyGradNotations(int loss_ndim, + int axis, + bool soft_label, + bool use_softmax, + std::string* label_axes, + std::string* softmax_axes, + std::string* loss_grad_axes) { + std::string alphabet = + "abcdefghijlmnopqrstuvwxyz"; // k for softmax_normalize axis + auto x_axes = alphabet.substr(0, loss_ndim); + x_axes[axis] = 'k'; + *label_axes = x_axes; + if (!soft_label) { + (*label_axes)[axis] = '1'; + } + + *loss_grad_axes = x_axes; + (*loss_grad_axes)[axis] = '1'; + // optional output + if (use_softmax) { + *softmax_axes = x_axes; + } else { + *softmax_axes = ""; + } +} + +SpmdInfo CrossEntropyWithSoftmaxGradInferSpmd(const DistMetaTensor& label, + const DistMetaTensor& softmax, + const DistMetaTensor& loss_grad, + bool soft_label, + bool use_softmax, + bool numeric_stable_mode, + int ignore_index, + int axis) { + EXTRACT_SHAPE_AND_DIST_ATTR_WITH_DIM_CK(label); + EXTRACT_SHAPE_AND_DIST_ATTR(softmax); + EXTRACT_SHAPE_AND_DIST_ATTR_WITH_DIM_CK(loss_grad); + + if (axis < 0) { + axis = loss_grad_ndim + axis; + } + + std::string label_axes, softmax_axes, loss_grad_axes; + GetCrossEntropyGradNotations(loss_grad_ndim, + axis, + soft_label, + use_softmax, + &label_axes, + &softmax_axes, + &loss_grad_axes); + + std::unordered_map axis_to_dim_map = + ShardingMergeForTensors({{label_axes, label_dims_mapping_src}, + {softmax_axes, softmax_dims_mapping_src}, + {loss_grad_axes, loss_grad_dims_mapping_src}}); + + auto label_dist_attr_dst = CopyTensorDistAttrForOutput(label_dist_attr_src); + auto label_dims_mapping_dst = + GetDimsMappingForAxes(label_axes, axis_to_dim_map, true); + label_dist_attr_dst.set_dims_mapping(label_dims_mapping_dst); + + auto softmax_dist_attr_dst = + CopyTensorDistAttrForOutput(softmax_dist_attr_src); + auto softmax_dims_mapping_dst = + GetDimsMappingForAxes(softmax_axes, axis_to_dim_map, true); + softmax_dist_attr_dst.set_dims_mapping(softmax_dims_mapping_dst); + + auto loss_grad_dist_attr_dst = + CopyTensorDistAttrForOutput(loss_grad_dist_attr_src); + auto loss_grad_dims_mapping_dst = + GetDimsMappingForAxes(loss_grad_axes, axis_to_dim_map, true); + loss_grad_dist_attr_dst.set_dims_mapping(loss_grad_dims_mapping_dst); + + auto x_grad = CopyTensorDistAttrForOutput(label_dist_attr_dst); + x_grad.set_dims_mapping(label_dims_mapping_dst); + + LOG_SPMD_INPUT(label); + LOG_SPMD_INPUT(softmax); + LOG_SPMD_INPUT(loss_grad); + LOG_SPMD_OUTPUT(x_grad); + + return {{label_dist_attr_dst, softmax_dist_attr_dst, loss_grad_dist_attr_dst}, + {x_grad}}; +} + } // namespace distributed } // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.h b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.h index 52ff5d3e85176..fed37f5ead264 100644 --- a/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.h +++ b/paddle/phi/infermeta/spmd_rules/cross_entropy_with_softmax.h @@ -39,5 +39,14 @@ SpmdInfo CrossEntropyWithSoftmaxInferSpmdReverse( int ignore_index, int axis); +SpmdInfo CrossEntropyWithSoftmaxGradInferSpmd(const DistMetaTensor& label, + const DistMetaTensor& softmax, + const DistMetaTensor& loss_grad, + bool soft_label, + bool use_softmax, + bool numeric_stable_mode, + int ignore_index, + int axis); + } // namespace distributed } // namespace phi diff --git a/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h b/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h new file mode 100644 index 0000000000000..281f438e8f624 --- /dev/null +++ b/paddle/phi/infermeta/spmd_rules/spmd_rule_macro_define.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +using phi::distributed::auto_parallel::str_join; + +#define EXTRACT_SHAPE_AND_DIST_ATTR(x) \ + auto x##_shape = phi::vectorize(x.dims()); \ + int x##_ndim = x##_shape.size(); \ + auto x##_dist_attr_src = x.dist_attr(); \ + const auto& x##_dims_mapping_src = x##_dist_attr_src.dims_mapping(); \ + PADDLE_ENFORCE_EQ(x##_ndim, \ + x##_dims_mapping_src.size(), \ + phi::errors::InvalidArgument( \ + "[%d] [%d] The Tensor [%d]'s rank [%d] and Loss's " \ + "dims_mapping size [%d] are not matched.", \ + __FILE__, \ + __LINE__, \ + #x, \ + x##_ndim, \ + x##_dims_mapping_src.size())) + +#define EXTRACT_SHAPE_AND_DIST_ATTR_WITH_DIM_CK(x) \ + EXTRACT_SHAPE_AND_DIST_ATTR(x); \ + PADDLE_ENFORCE_EQ(x##_ndim, \ + x##_dims_mapping_src.size(), \ + phi::errors::InvalidArgument( \ + "[%d] [%d] The Tensor [%d]'s rank [%d] and Loss's " \ + "dims_mapping size [%d] are not matched.", \ + __FILE__, \ + __LINE__, \ + #x, \ + x##_ndim, \ + x##_dims_mapping_src.size())) + +#define LOG_SPMD_INPUT(name) \ + do { \ + VLOG(4) << #name; \ + VLOG(4) << "shape: [" << str_join(name##_shape) << "] " \ + << "src_dist_attr: [" << name##_dist_attr_src.to_string() << "] " \ + << "src_dist_attr: [" << name##_dist_attr_dst.to_string() << "]"; \ + } while (0) + +#define LOG_SPMD_OUTPUT(name) \ + do { \ + VLOG(4) << #name; \ + VLOG(4) << "src_dist_attr: [" << name.to_string() << "]"; \ + } while (0) diff --git a/test/cpp/auto_parallel/CMakeLists.txt b/test/cpp/auto_parallel/CMakeLists.txt index d5bf3f1cf5d3a..5911712dffdf2 100644 --- a/test/cpp/auto_parallel/CMakeLists.txt +++ b/test/cpp/auto_parallel/CMakeLists.txt @@ -49,6 +49,15 @@ if(WITH_DISTRIBUTE) spmd_rule_test_util spmd_rules phi) + + paddle_test( + cross_entropy_softmax_spmd_rule_test + SRCS + cross_entropy_softmax_spmd_rule_test.cc + DEPS + spmd_rule_test_util + spmd_rules + phi) endif() cc_test( diff --git a/test/cpp/auto_parallel/cross_entropy_softmax_spmd_rule_test.cc b/test/cpp/auto_parallel/cross_entropy_softmax_spmd_rule_test.cc new file mode 100644 index 0000000000000..f4f4e455081e1 --- /dev/null +++ b/test/cpp/auto_parallel/cross_entropy_softmax_spmd_rule_test.cc @@ -0,0 +1,101 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "test/cpp/auto_parallel/spmd_rule_test_util.h" + +namespace paddle { +namespace distributed { +namespace auto_parallel { + +TEST(CrossEntropyInferSpmd, Ctor) { + std::vector x_shape = {32, 48}; + + std::vector mesh_shape = {2, 3}; + std::vector process_ids = {0, 1, 2, 3, 4, 5}; + std::vector dim_names = {"x", "y"}; + ProcessMesh process_mesh(mesh_shape, process_ids, dim_names); + + TensorDistAttr x_dist_attr = TensorDistAttr(); + x_dist_attr.set_process_mesh(process_mesh); + x_dist_attr.set_dims_mapping(std::vector({0, -1})); + x_dist_attr.set_dynamic_dims(std::vector({false, false})); + + TensorDistAttr label_dist_attr = TensorDistAttr(); + label_dist_attr.set_process_mesh(process_mesh); + label_dist_attr.set_dims_mapping(std::vector({0, -1})); + label_dist_attr.set_dynamic_dims(std::vector({false, false})); + + // forward + { + phi::distributed::DistMetaTensor x(phi::make_ddim(x_shape), x_dist_attr); + phi::distributed::DistMetaTensor label(phi::make_ddim(x_shape), + label_dist_attr); + int axis = 1; + + auto spmdinfo = + CrossEntropyWithSoftmaxInferSpmd(x, label, false, true, true, 1, axis); + + EXPECT_EQ(spmdinfo.first.size(), 2UL); + EXPECT_EQ(spmdinfo.second.size(), 2UL); + check_dim_mapping(spmdinfo.first[0], {0, -1}); + check_dim_mapping(spmdinfo.first[1], {0, -1}); + check_dim_mapping(spmdinfo.second[0], {0, -1}); + check_dim_mapping(spmdinfo.second[1], {0, -1}); + check_partial_dims(spmdinfo.second[0], {}); + + VLOG(4) << "Test CrossEntropyWithSoftmaxInferSpmd sharding on other axes." + << std::endl + << std::endl + << std::endl; + } + + // backward + { + std::vector loss_shape = {32, 1}; + // Sharding along softmax axis. + x_dist_attr.set_dims_mapping(std::vector{0, 1}); + label_dist_attr.set_dims_mapping(std::vector({0, 1})); + auto label = phi::distributed::DistMetaTensor(phi::make_ddim(x_shape), + label_dist_attr); + auto softmax = + phi::distributed::DistMetaTensor(phi::make_ddim(x_shape), x_dist_attr); + + auto loss_dist_attr = x_dist_attr; + loss_dist_attr.set_dims_mapping(std::vector({0, -1})); + auto loss_grad = phi::distributed::DistMetaTensor( + phi::make_ddim(loss_shape), x_dist_attr); + + int axis = 1; + auto spmdinfo = CrossEntropyWithSoftmaxGradInferSpmd( + label, softmax, loss_grad, true, true, true, 1, axis); + + EXPECT_EQ(spmdinfo.first.size(), 3UL); + EXPECT_EQ(spmdinfo.second.size(), 1UL); + check_dim_mapping(spmdinfo.first[0], {0, 1}); + check_dim_mapping(spmdinfo.first[1], {0, 1}); + check_dim_mapping(spmdinfo.first[2], {0, -1}); + check_dim_mapping(spmdinfo.second[0], {0, 1}); + check_partial_dims(spmdinfo.second[0], {}); + + VLOG(4) + << "Test CrossEntropyWithSoftmaxGradInferSpmd sharding on softmax axis." + << std::endl + << std::endl + << std::endl; + } +} + +} // namespace auto_parallel +} // namespace distributed +} // namespace paddle From 27a818467be3daf47f1ec393551df2a6897ec172 Mon Sep 17 00:00:00 2001 From: lzydev Date: Mon, 25 Dec 2023 14:51:10 +0800 Subject: [PATCH 017/146] support bf16 of sync_op (#60278) --- paddle/fluid/operators/collective/c_sync_calc_stream_op.cu.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cu.cc b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cu.cc index ab5634bdbb4e7..526726ae3c772 100644 --- a/paddle/fluid/operators/collective/c_sync_calc_stream_op.cu.cc +++ b/paddle/fluid/operators/collective/c_sync_calc_stream_op.cu.cc @@ -24,4 +24,5 @@ PD_REGISTER_STRUCT_KERNEL(c_sync_calc_stream, double, int, int64_t, - plat::float16) {} + plat::float16, + plat::bfloat16) {} From 269deddb564e7cb47dbc2621863a8e3a03db0d9c Mon Sep 17 00:00:00 2001 From: lzydev Date: Mon, 25 Dec 2023 14:51:31 +0800 Subject: [PATCH 018/146] fix bug in amp-bf16 (#60268) --- python/paddle/distributed/auto_parallel/static/engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/distributed/auto_parallel/static/engine.py b/python/paddle/distributed/auto_parallel/static/engine.py index de645882c219f..7e95bfdd74906 100644 --- a/python/paddle/distributed/auto_parallel/static/engine.py +++ b/python/paddle/distributed/auto_parallel/static/engine.py @@ -848,7 +848,9 @@ def _initialize(self, mode, init_parameters=True): # for amp if dest_type == core.VarDesc.VarType.BF16: buffer_tensor.set( - _convert_float_to_bfloat16(buffer.numpy()), + _convert_float_to_bfloat16( + self._place, buffer.numpy() + ), self._place, ) elif dest_type == core.VarDesc.VarType.FP16: From de7b2880bb8473d67c8861ed668c17072033e653 Mon Sep 17 00:00:00 2001 From: xingmingyyj <135400902+xingmingyyj@users.noreply.github.com> Date: Mon, 25 Dec 2023 14:52:37 +0800 Subject: [PATCH 019/146] =?UTF-8?q?=E3=80=90PIR=20OpTest=20Fix=20No.38?= =?UTF-8?q?=E3=80=91=20fix=20test=5Fsemi=5Fauto=5Fparallel=5Fc=5Fcross=5Fe?= =?UTF-8?q?ntropy=20(#59893)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * register c_softmax * register c_softmax * Update ops_backward.yaml * Update utils.cc * add test_semi_auto_parallel_c_cross_entropy to whitelist * Revert "add test_semi_auto_parallel_c_cross_entropy to whitelist" This reverts commit 75b3605d0e7fb4523b5bd31f4decaa91eae6a9b5. * add pit test * Update ops.yaml --- .../pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 10 +++++ .../pir/dialect/operator/ir/ops_backward.yaml | 10 +++++ .../fluid/pir/dialect/operator/utils/utils.cc | 2 + paddle/phi/api/yaml/op_compat.yaml | 7 +++ paddle/phi/infermeta/backward.cc | 12 +++++- paddle/phi/infermeta/backward.h | 10 +++++ paddle/phi/infermeta/binary.cc | 43 +++++++++++++++++++ paddle/phi/infermeta/binary.h | 10 +++++ ...test_semi_auto_parallel_c_cross_entropy.py | 5 +++ 10 files changed, 109 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 86abd12c82dfd..0225bd45f2700 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -115,6 +115,7 @@ 'c_identity', 'c_reduce_sum', 'c_reducescatter', + 'c_softmax_with_cross_entropy', 'decayed_adagrad', 'dpsgd', 'embedding_grad_sparse', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index cf0b64565978d..cdb45eb034c06 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1391,6 +1391,16 @@ output : Tensor(out) invoke : full_like(x, 0, dtype, place) +- op: c_softmax_with_cross_entropy + args: (Tensor logits, Tensor label, int64_t ignore_index=-100, int ring_id=0, int rank=0, int nranks=0) + output: Tensor(softmax), Tensor(loss) + infer_meta: + func : CSoftmaxWithCrossEntropyInferMeta + kernel: + func: c_softmax_with_cross_entropy + data_type : logits + backward: c_softmax_with_cross_entropy_grad + - op: dpsgd args: (Tensor param, Tensor grad, Tensor learning_rate, float clip = 10.0f, float batch_size = 16.0f, float sigma = 1.0f, int seed = 0) output: Tensor(param_out) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml index 3fc33f72b565c..bf0b939267e1b 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml @@ -119,6 +119,16 @@ func : c_embedding_grad no_need_buffer : weight +- backward_op : c_softmax_with_cross_entropy_grad + forward: c_softmax_with_cross_entropy (Tensor logits, Tensor label, int64_t ignore_index=-100, int ring_id=0, int rank=0, int nranks=0) -> Tensor(softmax), Tensor(loss) + args: (Tensor softmax, Tensor label, Tensor loss_grad,int64_t ignore_index=-100, int ring_id=0, int rank=0, int nranks=0) + output: Tensor(logits_grad) + infer_meta : + func: CSoftmaxWithCrossEntropyGradInferMeta + kernel: + func: c_softmax_with_cross_entropy_grad + data_type: loss_grad + - backward_op : cast_grad forward : cast (Tensor x, DataType dtype) -> Tensor(out) args : (Tensor x, Tensor out_grad) diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 696d4ee34dcde..783ecbd567554 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -49,6 +49,8 @@ const std::unordered_set LegacyOpList = { CReduceSum_Op::name(), CAllreduceMax_Op::name(), CAllgatherOp::name(), + CSoftmaxWithCrossEntropyOp::name(), + CSoftmaxWithCrossEntropyGradOp::name(), SeedOp::name(), ShareDataOp::name(), SparseMomentumOp::name(), diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index e64e837b24d49..8150e6cdd55cd 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -452,6 +452,13 @@ outputs : out : Out +- op : c_softmax_with_cross_entropy + backward : c_softmax_with_cross_entropy_grad + inputs : + {logits : Logits, label : Label} + outputs : + {softmax : Softmax, loss : Loss} + - op : cast inputs : x : X diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 71c4e5ecbca06..6d6eab8097337 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -201,7 +201,17 @@ void CropGradInferMeta(const MetaTensor& out_grad, x_grad->set_dtype(x.dtype()); } } - +void CSoftmaxWithCrossEntropyGradInferMeta(const MetaTensor& softmax, + const MetaTensor& label, + const MetaTensor& loss_grad, + int64_t ignore_index, + int ring_id, + int rank, + int nranks, + MetaTensor* logits_grad, + MetaConfig config) { + logits_grad->set_dims(softmax.dims()); +} void FlashAttnGradInferMeta(const MetaTensor& q, const MetaTensor& k, const MetaTensor& v, diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h index 125a0ec5ffcd1..86878c5feb082 100644 --- a/paddle/phi/infermeta/backward.h +++ b/paddle/phi/infermeta/backward.h @@ -123,6 +123,16 @@ void CrossEntropyWithSoftmaxGradInferMeta(const MetaTensor& label, MetaTensor* logits_grad, MetaConfig config = MetaConfig()); +void CSoftmaxWithCrossEntropyGradInferMeta(const MetaTensor& softmax, + const MetaTensor& label, + const MetaTensor& loss_grad, + int64_t ignore_index, + int ring_id, + int rank, + int nranks, + MetaTensor* logits_grad, + MetaConfig config = MetaConfig()); + void DeformableConvGradInferMeta(const MetaTensor& x, const MetaTensor& offset, const MetaTensor& filter, diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc index c15455e07182c..8b85a3efc4dd8 100644 --- a/paddle/phi/infermeta/binary.cc +++ b/paddle/phi/infermeta/binary.cc @@ -1009,6 +1009,49 @@ void CrossEntropyWithSoftmaxInferMeta(const MetaTensor& logits, loss->share_lod(logits); } +void CSoftmaxWithCrossEntropyInferMeta(const MetaTensor& logits, + const MetaTensor& label, + int64_t ignore_index, + int ring_id, + int rank, + int nranks, + MetaTensor* softmax, + MetaTensor* loss, + MetaConfig config) { + auto logits_dims = logits.dims(); + auto labels_dims = label.dims(); + + auto logits_rank = logits_dims.size(); + auto axis = logits_rank - 1; + for (int i = 0; i < logits_rank; i++) { + if (i != axis) { + if (config.is_runtime || (logits_dims[i] > 0 && labels_dims[i] > 0)) { + PADDLE_ENFORCE_EQ(logits_dims[i], + labels_dims[i], + phi::errors::InvalidArgument( + "Input(Logits) and Input(Label) should in " + "same shape in dimensions except axis.")); + } + } + } + + PADDLE_ENFORCE_EQ( + labels_dims[logits_rank - 1], + 1UL, + phi::errors::InvalidArgument( + "the last dimension of Input(Label) should be 1." + "But received: the last dimension of Input(Label) is [%d]," + "the last dimension is [%d]", + labels_dims[logits_rank - 1], + logits_rank - 1)); + + softmax->set_dims(logits_dims); + logits_dims[axis] = 1; + loss->set_dims(logits_dims); + softmax->share_lod(logits); + loss->share_lod(logits); +} + void DepthwiseConvInferMeta(const MetaTensor& input, const MetaTensor& filter, const std::vector& strides, diff --git a/paddle/phi/infermeta/binary.h b/paddle/phi/infermeta/binary.h index 92443d66d42ce..d082caea28636 100644 --- a/paddle/phi/infermeta/binary.h +++ b/paddle/phi/infermeta/binary.h @@ -159,6 +159,16 @@ void CrossEntropyWithSoftmaxInferMeta(const MetaTensor& logits, MetaTensor* loss, MetaConfig config = MetaConfig()); +void CSoftmaxWithCrossEntropyInferMeta(const MetaTensor& logits, + const MetaTensor& label, + int64_t ignore_index, + int ring_id, + int rank, + int nranks, + MetaTensor* softmax, + MetaTensor* loss, + MetaConfig config = MetaConfig()); + void DepthwiseConvInferMeta(const MetaTensor& input, const MetaTensor& filter, const std::vector& strides, diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_c_cross_entropy.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_c_cross_entropy.py index bc37a8716d066..96a6227ef469a 100644 --- a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_c_cross_entropy.py +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_c_cross_entropy.py @@ -35,6 +35,11 @@ def test_mp(self): "semi_auto_parallel_c_cross_entropy_mp.py", ) + def test_mp_pir(self): + os.environ["FLAGS_enable_pir_in_executor"] = "True" + self.test_mp() + os.environ["FLAGS_enable_pir_in_executor"] = "False" + class TestParallelCrossEntropyHybrid(test_base.CommunicationTestDistBase): def setUp(self): From f048329a9c091003b38c87195c9369573fba271f Mon Sep 17 00:00:00 2001 From: JYChen Date: Mon, 25 Dec 2023 15:10:56 +0800 Subject: [PATCH 020/146] Optimize performance of advanced getitem (#60254) * replace broadcast_tensors to expand * static mode still use broadcast-tensors, since dynamic shape -1 cannot be compared * optimize bool index getting --- paddle/fluid/pybind/slice_utils.h | 39 +++++++++++++++---- .../kernels/cpu/masked_select_grad_kernel.cc | 10 ++++- .../phi/kernels/cpu/masked_select_kernel.cc | 10 ++++- .../kernels/gpu/masked_select_grad_kernel.cu | 8 +++- .../phi/kernels/gpu/masked_select_kernel.cu | 8 +++- python/paddle/base/variable_index.py | 3 +- 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/paddle/fluid/pybind/slice_utils.h b/paddle/fluid/pybind/slice_utils.h index 918d2eeae4272..f4eef3af16bcf 100644 --- a/paddle/fluid/pybind/slice_utils.h +++ b/paddle/fluid/pybind/slice_utils.h @@ -25,6 +25,7 @@ #include "paddle/fluid/eager/api/generated/eager_generated/forwards/dygraph_functions.h" #include "paddle/fluid/framework/convert_utils.h" #include "paddle/fluid/framework/scope_guard.h" +#include "paddle/fluid/operators/common_infer_shape_functions.h" #include "paddle/fluid/operators/utils.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/core/compat/convert_utils.h" @@ -483,13 +484,15 @@ static paddle::Tensor getValueForBoolTensor(const paddle::Tensor& tensor, i++; } - auto bool_2_idx = nonzero_ad_func(bool_index); - const phi::distributed::ProcessMesh* mesh = nullptr; - if (InputsContainDistTensor(&mesh, tensor, bool_2_idx)) { - ConvertAllInputsToDistTensor(mesh, tensor, bool_2_idx); + if (InputsContainDistTensor(&mesh, tensor, bool_index)) { + ConvertAllInputsToDistTensor(mesh, tensor, bool_index); } + if (bool_index.shape().size() == tensor_shape.size()) { + return masked_select_ad_func(tensor, bool_index); + } + auto bool_2_idx = nonzero_ad_func(bool_index); return gather_nd_ad_func(tensor, bool_2_idx); } @@ -504,10 +507,30 @@ static void ParseBoolAndBroadcastIndices( } } if (advanced_index->size() > 1) { - // Here advanced_index has been checked ContainDistTensor - // and transed in dealWithAdvancedIndex - auto broadcasted_index = broadcast_tensors_ad_func(*advanced_index); - advanced_index->assign(broadcasted_index.begin(), broadcasted_index.end()); + bool need_broadcast = false; + common::DDim common_shape = common::make_ddim((*advanced_index)[0].shape()); + for (size_t i = 1; i < advanced_index->size(); ++i) { + common::DDim current_shape = + common::make_ddim((*advanced_index)[i].shape()); + if (current_shape != common_shape) { + need_broadcast = true; + common_shape = operators::details::BroadcastTwoDims( + current_shape, common_shape, -1); + } + } + + if (need_broadcast) { + // Here advanced_index has been checked ContainDistTensor + // and transed in dealWithAdvancedIndex + auto common_shape_vec = common::vectorize(common_shape); + for (size_t i = 0; i < advanced_index->size(); ++i) { + auto current_shape = (*advanced_index)[i].shape(); + if (current_shape != common_shape_vec) { + (*advanced_index)[i] = + expand_ad_func((*advanced_index)[i], common_shape_vec); + } + } + } } } diff --git a/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc b/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc index fa120de4b7952..49b1de9446c3e 100644 --- a/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/masked_select_grad_kernel.cc @@ -100,7 +100,15 @@ PD_REGISTER_KERNEL(masked_select_grad, CPU, ALL_LAYOUT, phi::MaskedSelectGradKernel, + bool, float, double, int, - int64_t) {} + int8_t, + int64_t, + int16_t, + uint8_t, + phi::dtype::float16, + phi::dtype::bfloat16, + phi::dtype::complex, + phi::dtype::complex) {} diff --git a/paddle/phi/kernels/cpu/masked_select_kernel.cc b/paddle/phi/kernels/cpu/masked_select_kernel.cc index 8e9e3bbebecd4..7c7c134248bd4 100644 --- a/paddle/phi/kernels/cpu/masked_select_kernel.cc +++ b/paddle/phi/kernels/cpu/masked_select_kernel.cc @@ -87,9 +87,17 @@ PD_REGISTER_KERNEL(masked_select, CPU, ALL_LAYOUT, phi::MaskedSelectKernel, + bool, float, double, int, - int64_t) { + int8_t, + int64_t, + int16_t, + uint8_t, + phi::dtype::float16, + phi::dtype::bfloat16, + phi::dtype::complex, + phi::dtype::complex) { kernel->InputAt(1).SetDataType(phi::DataType::BOOL); } diff --git a/paddle/phi/kernels/gpu/masked_select_grad_kernel.cu b/paddle/phi/kernels/gpu/masked_select_grad_kernel.cu index 4bf5949f084fe..0e717ecc13ff8 100644 --- a/paddle/phi/kernels/gpu/masked_select_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/masked_select_grad_kernel.cu @@ -108,9 +108,15 @@ PD_REGISTER_KERNEL(masked_select_grad, GPU, ALL_LAYOUT, phi::MaskedSelectGradKernel, + bool, float, double, int, + int8_t, int64_t, + int16_t, + uint8_t, phi::dtype::float16, - phi::dtype::bfloat16) {} + phi::dtype::bfloat16, + phi::dtype::complex, + phi::dtype::complex) {} diff --git a/paddle/phi/kernels/gpu/masked_select_kernel.cu b/paddle/phi/kernels/gpu/masked_select_kernel.cu index 9739f9799a4ec..0bf8a8789d0a1 100644 --- a/paddle/phi/kernels/gpu/masked_select_kernel.cu +++ b/paddle/phi/kernels/gpu/masked_select_kernel.cu @@ -94,11 +94,17 @@ PD_REGISTER_KERNEL(masked_select, GPU, ALL_LAYOUT, phi::MaskedSelectKernel, + bool, float, double, int, + int8_t, int64_t, + int16_t, + uint8_t, phi::dtype::float16, - phi::dtype::bfloat16) { + phi::dtype::bfloat16, + phi::dtype::complex, + phi::dtype::complex) { kernel->InputAt(1).SetDataType(phi::DataType::BOOL); } diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index f47afddde84f0..0df9ebc5513da 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -134,7 +134,8 @@ def get_value_for_bool_tensor(var, item): ) ) i += 1 - + if len(item.shape) == len(var.shape): + return paddle.masked_select(var, item) bool_2_idx = paddle.nonzero(item) return paddle.gather_nd(var, bool_2_idx) From 85e3693c62440c16c31252fcaecd4badff356906 Mon Sep 17 00:00:00 2001 From: JYChen Date: Mon, 25 Dec 2023 15:11:10 +0800 Subject: [PATCH 021/146] Fix set value grad (#59034) * first fix the UT * fix set value grad * polish code * add static mode backward test * always has input valuetensor * add dygraph test --- paddle/fluid/operators/set_value_op.cc | 44 +++++----- paddle/phi/api/yaml/legacy_backward.yaml | 6 +- .../phi/kernels/cpu/set_value_grad_kernel.cc | 17 ++++ .../phi/kernels/gpu/set_value_grad_kernel.cu | 17 ++++ .../kernels/impl/set_value_grad_kernel_impl.h | 22 +++++ paddle/phi/kernels/set_value_grad_kernel.h | 10 +++ .../phi/kernels/xpu/set_value_grad_kernel.cc | 31 +++++++ test/legacy_test/test_set_value_op.py | 82 +++++++++++++++++++ 8 files changed, 201 insertions(+), 28 deletions(-) diff --git a/paddle/fluid/operators/set_value_op.cc b/paddle/fluid/operators/set_value_op.cc index 2be7f24ce7157..5eeb356817a2a 100644 --- a/paddle/fluid/operators/set_value_op.cc +++ b/paddle/fluid/operators/set_value_op.cc @@ -152,32 +152,26 @@ class SetValueGradMaker : public framework::SingleGradOpMaker { protected: void Apply(GradOpPtr op) const override { - if (this->HasInput("ValueTensor")) { - op->SetType("set_value_grad"); - - op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); - op->SetInput("ValueTensor", this->Input("ValueTensor")); - if (this->HasInput("StartsTensorList")) { - op->SetInput("StartsTensorList", this->Input("StartsTensorList")); - } - if (this->HasInput("EndsTensorList")) { - op->SetInput("EndsTensorList", this->Input("EndsTensorList")); - } - if (this->HasInput("StepsTensorList")) { - op->SetInput("StepsTensorList", this->Input("StepsTensorList")); - } - - op->SetAttrMap(this->Attrs()); - - op->SetOutput(framework::GradVarName("ValueTensor"), - this->InputGrad("ValueTensor")); - op->SetOutput(framework::GradVarName("Input"), this->InputGrad("Input")); - - } else { - op->SetType("assign"); - op->SetInput("X", this->OutputGrad("Out")); - op->SetOutput("Out", this->InputGrad("Input")); + op->SetType("set_value_grad"); + op->SetInput("ValueTensor", this->Input("ValueTensor")); + op->SetOutput(framework::GradVarName("ValueTensor"), + this->InputGrad("ValueTensor")); + + op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); + + if (this->HasInput("StartsTensorList")) { + op->SetInput("StartsTensorList", this->Input("StartsTensorList")); + } + if (this->HasInput("EndsTensorList")) { + op->SetInput("EndsTensorList", this->Input("EndsTensorList")); } + if (this->HasInput("StepsTensorList")) { + op->SetInput("StepsTensorList", this->Input("StepsTensorList")); + } + + op->SetAttrMap(this->Attrs()); + + op->SetOutput(framework::GradVarName("Input"), this->InputGrad("Input")); } }; diff --git a/paddle/phi/api/yaml/legacy_backward.yaml b/paddle/phi/api/yaml/legacy_backward.yaml index 3d47f2cbedbc6..7bda4331420a5 100755 --- a/paddle/phi/api/yaml/legacy_backward.yaml +++ b/paddle/phi/api/yaml/legacy_backward.yaml @@ -614,14 +614,14 @@ - backward_op : set_value_grad forward : set_value (Tensor x, IntArray starts, IntArray ends, IntArray steps, int64_t[] axes, int64_t[] decrease_axes, int64_t[] none_axes, int64_t[] shape, Scalar[] values) -> Tensor(out) - args : (Tensor out_grad) + args : (Tensor out_grad, IntArray starts, IntArray ends, IntArray steps, int64_t[] axes, int64_t[] decrease_axes, int64_t[] none_axes) output : Tensor(x_grad) infer_meta: func: UnchangedInferMeta param: [out_grad] kernel: - func: assign - param: [out_grad] + func: set_value_with_scalar_grad + param: [out_grad, starts, ends, steps, axes, decrease_axes, none_axes] - backward_op : set_value_with_tensor_grad forward: set_value_with_tensor (Tensor x, Tensor values, IntArray starts, IntArray ends, IntArray steps, int64_t[] axes, int64_t[] decrease_axes, int64_t[] none_axes) -> Tensor(out) diff --git a/paddle/phi/kernels/cpu/set_value_grad_kernel.cc b/paddle/phi/kernels/cpu/set_value_grad_kernel.cc index ed35513d98550..237a892dbb356 100644 --- a/paddle/phi/kernels/cpu/set_value_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/set_value_grad_kernel.cc @@ -35,3 +35,20 @@ PD_REGISTER_KERNEL(set_value_grad, phi::dtype::float16, phi::dtype::complex, phi::dtype::complex) {} + +PD_REGISTER_KERNEL(set_value_with_scalar_grad, + CPU, + ALL_LAYOUT, + phi::SetValueWithScalarGradKernel, + float, + double, + int, + int64_t, + bool, + int16_t, + uint8_t, + int8_t, + phi::dtype::bfloat16, + phi::dtype::float16, + phi::dtype::complex, + phi::dtype::complex) {} diff --git a/paddle/phi/kernels/gpu/set_value_grad_kernel.cu b/paddle/phi/kernels/gpu/set_value_grad_kernel.cu index 66688b417ae30..42ff5b912eccd 100644 --- a/paddle/phi/kernels/gpu/set_value_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/set_value_grad_kernel.cu @@ -35,3 +35,20 @@ PD_REGISTER_KERNEL(set_value_grad, phi::dtype::bfloat16, phi::dtype::complex, phi::dtype::complex) {} + +PD_REGISTER_KERNEL(set_value_with_scalar_grad, + GPU, + ALL_LAYOUT, + phi::SetValueWithScalarGradKernel, + float, + double, + int, + int64_t, + bool, + int16_t, + uint8_t, + int8_t, + phi::dtype::float16, + phi::dtype::bfloat16, + phi::dtype::complex, + phi::dtype::complex) {} diff --git a/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h b/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h index 3f78361b92b8b..99f05f80c17ff 100644 --- a/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h +++ b/paddle/phi/kernels/impl/set_value_grad_kernel_impl.h @@ -341,4 +341,26 @@ void SetValueGradKernel(const Context& dev_ctx, } } +template +void SetValueWithScalarGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + const IntArray& starts, + const IntArray& ends, + const IntArray& steps, + const std::vector& axes, + const std::vector& decrease_axes, + const std::vector& none_axes, + DenseTensor* x_grad) { + SetValueGradKernel(dev_ctx, + out_grad, + starts, + ends, + steps, + axes, + decrease_axes, + none_axes, + x_grad, + nullptr); +} + } // namespace phi diff --git a/paddle/phi/kernels/set_value_grad_kernel.h b/paddle/phi/kernels/set_value_grad_kernel.h index e4dad683e40a9..04592cd2002d1 100644 --- a/paddle/phi/kernels/set_value_grad_kernel.h +++ b/paddle/phi/kernels/set_value_grad_kernel.h @@ -32,4 +32,14 @@ void SetValueGradKernel(const Context& dev_ctx, DenseTensor* x_grad, DenseTensor* value_grad); +template +void SetValueWithScalarGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + const IntArray& starts, + const IntArray& ends, + const IntArray& steps, + const std::vector& axes, + const std::vector& decrease_axes, + const std::vector& none_axes, + DenseTensor* x_grad); } // namespace phi diff --git a/paddle/phi/kernels/xpu/set_value_grad_kernel.cc b/paddle/phi/kernels/xpu/set_value_grad_kernel.cc index d1ad332cd626c..c5d33ae4ac8d0 100644 --- a/paddle/phi/kernels/xpu/set_value_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/set_value_grad_kernel.cc @@ -397,6 +397,28 @@ void SetValueGradKernel(const Context& dev_ctx, } } +template +void SetValueWithScalarGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + const IntArray& starts, + const IntArray& ends, + const IntArray& steps, + const std::vector& axes, + const std::vector& decrease_axes, + const std::vector& none_axes, + DenseTensor* x_grad) { + SetValueGradKernel(dev_ctx, + out_grad, + starts, + ends, + steps, + axes, + decrease_axes, + none_axes, + x_grad, + nullptr); +} + } // namespace phi PD_REGISTER_KERNEL(set_value_grad, @@ -407,3 +429,12 @@ PD_REGISTER_KERNEL(set_value_grad, phi::dtype::float16, int, int64_t) {} + +PD_REGISTER_KERNEL(set_value_with_scalar_grad, + XPU, + ALL_LAYOUT, + phi::SetValueWithScalarGradKernel, + float, + phi::dtype::float16, + int, + int64_t) {} diff --git a/test/legacy_test/test_set_value_op.py b/test/legacy_test/test_set_value_op.py index 65c9f69765d11..c42026fb9caee 100644 --- a/test/legacy_test/test_set_value_op.py +++ b/test/legacy_test/test_set_value_op.py @@ -1978,5 +1978,87 @@ def test_check_grad(self): self.check_grad_with_place(place, ['Input'], 'Out', check_dygraph=False) +class TestSetValueWithScalarInStatic(unittest.TestCase): + def setUp(self): + paddle.enable_static() + self.shape = (10, 2) + self.exe = paddle.static.Executor() + self.train_program = paddle.static.Program() + self.startup_program = paddle.static.Program() + + def test_value_input_is_scalar(self): + with paddle.static.program_guard( + self.train_program, self.startup_program + ): + x = paddle.ones(self.shape) + x.stop_gradient = False + y = x * 1 + + # mock test case x[0, 0] = 10 with no ValueTensor input + inputs = { + 'Input': y, + } + attrs = { + 'axes': [0, 1], + 'starts': [0, 0], + 'ends': [1, 1], + 'steps': [1, 1], + 'values': [10], + 'shape': [1], + } + + helper = LayerHelper("set_value") + out = helper.create_variable_for_type_inference(dtype=y.dtype) + + helper.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': out}, + attrs=attrs, + ) + + np_data = np.ones(self.shape).astype('float32') + + paddle.static.append_backward(out.sum()) + res = self.exe.run( + self.train_program, fetch_list=[out, x.grad_name] + ) + + np_data[0, 0] = 10 + expected_x_grad = np.ones(self.shape) + expected_x_grad[0, 0] = 0 + + np.testing.assert_array_equal(res[0], np_data) + np.testing.assert_array_equal(res[1], expected_x_grad) + + +class TestSetValueWithScalarInDygraph(unittest.TestCase): + def setUp(self): + paddle.disable_static() + self.shape = (10, 2) + + def test_value_input_is_scalar(self): + x = paddle.ones(self.shape) + x.stop_gradient = False + y = x * 1 + + # mock test case x[0, 0] = 10 with no ValueTensor input + out = paddle._C_ops.set_value( + y, [0, 0], [1, 1], [1, 1], [0, 1], [], [], [1], [10.0] + ) + + loss = out.sum() + loss.backward() + + np_data = np.ones(self.shape).astype('float32') + np_data[0, 0] = 10 + + expected_x_grad = np.ones(self.shape) + expected_x_grad[0, 0] = 0 + + np.testing.assert_array_equal(out, np_data) + np.testing.assert_array_equal(x.grad, expected_x_grad) + + if __name__ == '__main__': unittest.main() From df420a417a20966fe5e7f9dc246a6e2b5a0e9a36 Mon Sep 17 00:00:00 2001 From: kangguangli Date: Mon, 25 Dec 2023 15:12:18 +0800 Subject: [PATCH 022/146] [PIR] mark_mask_of_dropout_as_intermediate (#60221) * mark_mask_of_dropout_as_intermediate * fix * fix test_dropout_op --- .../pir_adaptor/pir_adaptor_util.cc | 2 +- .../ir_adaptor/translator/op_translator.cc | 2 +- paddle/fluid/ir_adaptor/translator/utils.cc | 31 ------ paddle/fluid/ir_adaptor/translator/utils.h | 2 - paddle/fluid/pir/dialect/operator/ir/ops.yaml | 3 +- paddle/phi/api/yaml/legacy_ops.yaml | 1 + python/paddle/nn/functional/common.py | 2 +- test/legacy_test/test_dropout_op.py | 96 +++++++++---------- 8 files changed, 52 insertions(+), 87 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 3573e2b44d638..8717c7d4fd2e1 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -425,7 +425,7 @@ void HandleForSpecialOp(pir::Operation* op, if (place.GetType() == phi::AllocationType::UNDEFINED) { place = phi::CPUPlace(); } - if (phi::product(dim) >= 0) { + if (!common::contain_unknown_dim(dim)) { phi::DenseTensorMeta meta(dtype.data(), dim); t->set_meta(meta); auto* dev_ctx = platform::DeviceContextPool::Instance().Get(place); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index ad703039b37a1..fa569b0df5ac6 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1995,7 +1995,7 @@ struct SelectInputOpTranscriber : public OpTranscriber { undefine_value.defining_op()->set_attribute( "dtype", dialect::DataTypeAttribute::get( - ctx, PirTypeToPhiDType(undefined_var_type.dtype()))); + ctx, dialect::TransToPhiDataType(undefined_var_type.dtype()))); auto& attribute_translator = AttributeTranslator::instance(); undefine_value.defining_op()->set_attribute( "shape", diff --git a/paddle/fluid/ir_adaptor/translator/utils.cc b/paddle/fluid/ir_adaptor/translator/utils.cc index 808ae739f0889..ebba4428220f7 100644 --- a/paddle/fluid/ir_adaptor/translator/utils.cc +++ b/paddle/fluid/ir_adaptor/translator/utils.cc @@ -105,36 +105,5 @@ std::vector CheckUnregisteredOperation( return unregistered_ops; } -phi::DataType PirTypeToPhiDType(pir::Type type) { - if (type.isa()) { - return phi::DataType::UINT8; - } else if (type.isa()) { - return phi::DataType::INT8; - } else if (type.isa()) { - return phi::DataType::INT16; - } else if (type.isa()) { - return phi::DataType::INT32; - } else if (type.isa()) { - return phi::DataType::INT64; - } else if (type.isa()) { - return phi::DataType::FLOAT32; - } else if (type.isa()) { - return phi::DataType::FLOAT64; - } else if (type.isa()) { - return phi::DataType::BOOL; - } else if (type.isa()) { - return phi::DataType::FLOAT16; - } else if (type.isa()) { - return phi::DataType::BFLOAT16; - } else if (type.isa()) { - return phi::DataType::COMPLEX64; - } else if (type.isa()) { - return phi::DataType::COMPLEX128; - } else { - PADDLE_THROW(phi::errors::Unimplemented( - "Unsupported pirType `%s` when casting it into phi::DataType.", type)); - } -} - } // namespace translator } // namespace paddle diff --git a/paddle/fluid/ir_adaptor/translator/utils.h b/paddle/fluid/ir_adaptor/translator/utils.h index 053e83145cb31..a4765940d0a78 100644 --- a/paddle/fluid/ir_adaptor/translator/utils.h +++ b/paddle/fluid/ir_adaptor/translator/utils.h @@ -100,7 +100,5 @@ inline DataType VarTypeToDataType( } } -phi::DataType PirTypeToPhiDType(pir::Type type); - } // namespace translator } // namespace paddle diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index cdb45eb034c06..4872f701bd795 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -371,7 +371,8 @@ kernel : func : dropout data_type : x - optional : seed_tensor, mask + optional : seed_tensor + intermediate : mask backward : dropout_grad - op : einsum diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index a7c5cdedc8852..7cd2b4b6e3f32 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -334,6 +334,7 @@ func : dropout data_type : x optional : seed_tensor + intermediate : mask backward : dropout_grad - op : einsum diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index dcadcb2409ad5..8988e89111c09 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -1137,7 +1137,7 @@ def dropout( if default_main_program().random_seed != 0: seed = default_main_program().random_seed - out, mask = _C_ops.dropout( + out = _C_ops.dropout( x, None, p, diff --git a/test/legacy_test/test_dropout_op.py b/test/legacy_test/test_dropout_op.py index 433b9eeff7056..7fb54f570522b 100644 --- a/test/legacy_test/test_dropout_op.py +++ b/test/legacy_test/test_dropout_op.py @@ -20,7 +20,7 @@ from utils import static_guard import paddle -from paddle import _C_ops, base, static +from paddle import base, static from paddle.autograd.ir_backward import grad from paddle.base import Program, Scope, core, program_guard from paddle.base.executor import scope_guard @@ -79,6 +79,9 @@ def setUp(self): 'Out': self.inputs['X'], 'Mask': np.ones((32, 64)).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. # Because prim op compare res with dygraph # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] @@ -108,6 +111,9 @@ def setUp(self): # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] self.enable_check_static_comp = False + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. class TestDropoutOpInput1d(OpTest): @@ -122,6 +128,9 @@ def setUp(self): 'Out': self.inputs['X'], 'Mask': np.ones(2000).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. # Because prim op compare res with dygraph # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] @@ -147,6 +156,9 @@ def setUp(self): 'Out': np.zeros((32, 64)).astype('float32'), 'Mask': np.zeros((32, 64)).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. class TestDropoutOp2_ZeroDim(TestDropoutOp2): @@ -161,6 +173,9 @@ def setUp(self): 'Out': np.zeros(()).astype('float32'), 'Mask': np.zeros(()).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. class TestDropoutOp3(TestDropoutOp): @@ -179,6 +194,9 @@ def setUp(self): # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] self.enable_check_static_comp = False + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. @skip_check_grad_ci(reason="For inference, check_grad is not required.") @@ -193,6 +211,9 @@ def setUp(self): self.outputs = { 'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob']) } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def test_check_output(self): self.check_output(check_prim=True, check_prim_pir=True, check_pir=True) @@ -210,6 +231,9 @@ def setUp(self): self.outputs = { 'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob']) } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def test_check_output(self): self.check_output(check_prim=True, check_prim_pir=True, check_pir=True) @@ -232,6 +256,9 @@ def setUp(self): 'Out': np.zeros((32, 64)).astype('float32'), 'Mask': np.zeros((32, 64)).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. class TestDropoutOp7(TestDropoutOp): @@ -255,6 +282,9 @@ def setUp(self): # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] self.enable_check_static_comp = False + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. @skip_check_grad_ci(reason="For inference, check_grad is not required.") @@ -272,6 +302,9 @@ def setUp(self): 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def test_check_output(self): self.check_output(check_prim=True, check_prim_pir=True, check_pir=True) @@ -291,6 +324,9 @@ def setUp(self): 'dropout_implementation': 'upscale_in_train', } self.outputs = {'Out': self.inputs['X']} + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def test_check_output(self): self.check_output(check_prim=True, check_prim_pir=True, check_pir=True) @@ -313,6 +349,9 @@ def setUp(self): 'Out': self.inputs['X'], 'Mask': np.ones((32, 64)).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. # Because prim op compare res with dygraph # when p = 0 dropout api return x,in dygraph mode x_grad = out_grad, # but in static mode x_grad = [] @@ -355,6 +394,9 @@ def setUp(self): 'is_test': True, } self.outputs = {'Out': out} + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def init_test_case(self): self.input_size = [32, 64] @@ -404,6 +446,9 @@ def setUp(self): ), 'Mask': np.zeros((32, 64)).astype('uint8'), } + self.python_out_sig = [ + "Out" + ] # python out sig is customized output signature. def test_check_output(self): self.check_output(check_prim=True, check_prim_pir=True, check_pir=True) @@ -1328,55 +1373,6 @@ def cal_grad_upscale_train(self, mask, prob): def cal_grad_downscale_in_infer(self, mask): return mask.astype("float32") - def test_backward_downscale_in_infer(self): - for place in self.places: - with base.dygraph.guard(place): - input = paddle.uniform([40, 40], dtype="float32") - input.stop_gradient = False - out, mask = _C_ops.dropout( - input, None, 0.5, False, "downgrade_in_infer", 0, False - ) - out.backward() - - np.testing.assert_array_equal( - input.gradient(), - self.cal_grad_downscale_in_infer(mask.numpy()), - ) - - def test_backward_upscale_train(self): - for place in self.places: - with base.dygraph.guard(place): - prob = 0.5 - input = paddle.uniform([40, 40], dtype="float32") - input.stop_gradient = False - out, mask = _C_ops.dropout( - input, None, 0.5, False, "upscale_in_train", 0, False - ) - out.backward() - - np.testing.assert_allclose( - input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob), - rtol=1e-05, - ) - - def test_backward_upscale_train_2(self): - for place in self.places: - with base.dygraph.guard(place): - prob = 0.3 - input = paddle.uniform([40, 40], dtype="float32") - input.stop_gradient = False - out, mask = _C_ops.dropout( - input, None, 0.3, False, "upscale_in_train", 0, False - ) - out.backward() - - np.testing.assert_allclose( - input.gradient(), - self.cal_grad_upscale_train(mask.numpy(), prob), - rtol=1e-05, - ) - class TestDropOutWithProbTensor(unittest.TestCase): def setUp(self): From 4df90b4d79a4f5efb5fffa42cdef6b104e855412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= Date: Mon, 25 Dec 2023 15:34:53 +0800 Subject: [PATCH 023/146] add dsl test cases for dynamic shape schedule (#60266) --- paddle/cinn/pybind/schedule.cc | 19 +++-- .../ir/test_llir_schedule_cache_read_write.py | 70 +++++++++++++------ test/cinn/ir/test_llir_schedule_fuse_split.py | 2 +- 3 files changed, 63 insertions(+), 28 deletions(-) diff --git a/paddle/cinn/pybind/schedule.cc b/paddle/cinn/pybind/schedule.cc index b1485fc7ef6ef..d9a8e418cabdb 100644 --- a/paddle/cinn/pybind/schedule.cc +++ b/paddle/cinn/pybind/schedule.cc @@ -35,13 +35,18 @@ void BindSchedule(py::module *m) { py::arg("debug_flag") = false, py::arg("err_msg_level") = utils::ErrorMessageLevel::kGeneral, py::arg("is_dynamic_shape") = false) - .def_static( - "make", - [](ir::LoweredFunc &ir_func) { - ir::ModuleExpr *module_expr = new ir::ModuleExpr({ir_func->body}); - auto scheduler = std::make_unique(*module_expr); - return scheduler; - }) + .def_static("make", + [](ir::LoweredFunc &ir_func) { + ir::ModuleExpr *module_expr = + new ir::ModuleExpr({ir_func->body}); + auto scheduler = std::make_unique( + *module_expr, + -1, + false, + utils::ErrorMessageLevel::kGeneral, + true); + return scheduler; + }) .def("fuse", py::overload_cast &>(&ir::IRSchedule::Fuse)) .def("split", diff --git a/test/cinn/ir/test_llir_schedule_cache_read_write.py b/test/cinn/ir/test_llir_schedule_cache_read_write.py index 85badc819f8f5..41f1fc8d342ab 100644 --- a/test/cinn/ir/test_llir_schedule_cache_read_write.py +++ b/test/cinn/ir/test_llir_schedule_cache_read_write.py @@ -12,35 +12,65 @@ # See the License for the specific language governing permissions and # limitations under the License. -from test.cinn.utils.testing import assert_llir_equal from cinn import ir, to_cinn_llir from cinn.runtime.data_array import DataArray from cinn.schedule import IRSchedule as sch +# (Note:LiuYang): Here the temp tensor is created in cache_read or cache_write +# so that the two ir is not equal and we just judge them by string of them def test_cache_read_elementwise(): - @to_cinn_llir - def elementwise_add_cache_read( - X: DataArray((128, 128)), - Y: DataArray((128, 128)), - A: DataArray((128, 128)), - ): - for i in range(128): - for j in range(128): - with ir.ScheduleBlockContext("A") as A_block: - i1, j1 = ir.AxisMap("SS", [i, j]) - A[i1, j1] = X[i1, j1] * 2.0 - for i3 in range(128): - for j3 in range(128): - with ir.ScheduleBlockContext("B") as B_block: - i1, j1 = ir.AxisMap("SS", [i3, j3]) - Y[i1, j1] = -A[i1, j1] + 3.0 + class origin: + @to_cinn_llir + def elementwise_add_cache_read( + X: DataArray((128, 128)), + Y: DataArray((128, 128)), + A: DataArray((128, 128)), + A_local_temp_buffer: DataArray((128, 128)), + ): + for i in range(128): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for i3 in range(128): + for j3 in range(128): + with ir.ScheduleBlockContext("B") as B_block: + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A[i1, j1] + 3.0 + + cached_b = sch.cache_read(B_block.block, 0, "local") - cached_a = sch.cache_read(A_block.block, 0, "global") - cached_b = sch.cache_read(B_block.block, 0, "local") + class expected: + @to_cinn_llir + def elementwise_add_cache_read( + X: DataArray((128, 128)), + Y: DataArray((128, 128)), + A: DataArray((128, 128)), + A_local_temp_buffer: DataArray((128, 128)), + ): + for i in range(128): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for cache_ax0 in range(128): + for cache_ax1 in range(128): + with ir.ScheduleBlockContext( + "A_local_temp_buffer" + ) as A_local_temp_buffer_block: + v0, v1 = ir.AxisMap("SS", [cache_ax0, cache_ax1]) + A_local_temp_buffer[v0, v1] = A[v0, v1] + for i3 in range(128): + for j3 in range(128): + with ir.ScheduleBlockContext("B") as B_block: + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A_local_temp_buffer[i1, j1] + 3.0 - assert_llir_equal(elementwise_add_cache_read, elementwise_add_cache_read) + assert str(origin.elementwise_add_cache_read) == str( + expected.elementwise_add_cache_read + ) def test_cache_write_elementwise(): diff --git a/test/cinn/ir/test_llir_schedule_fuse_split.py b/test/cinn/ir/test_llir_schedule_fuse_split.py index f22b1a1f8d3a9..07712590b9ac1 100644 --- a/test/cinn/ir/test_llir_schedule_fuse_split.py +++ b/test/cinn/ir/test_llir_schedule_fuse_split.py @@ -37,7 +37,7 @@ def elementwise_fuse_assign_loop( def elementwise_fuse_assign_loop_gt( X: DataArray((128, 128, 128)), Y: DataArray((128, 128, 128)) ): - for i in range(2097152): + for i in range(((1 * 128) * 128) * 128): with ir.ScheduleBlockContext("Y") as block_y: i1_1, j1_1, k1_1 = ir.AxisMap( "SSS", [(i / 128) / 128, (i / 128) % 128, i % 128] From 644cbcc920da200fbc734523cac3baa85e6e546c Mon Sep 17 00:00:00 2001 From: xingmingyyj <135400902+xingmingyyj@users.noreply.github.com> Date: Mon, 25 Dec 2023 15:37:47 +0800 Subject: [PATCH 024/146] =?UTF-8?q?=E3=80=90PIR=20OpTest=20Fix=20No.5?= =?UTF-8?q?=E3=80=91=20fix=20test=5Ftril=5Ftriu=5Fop=20(#59734)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix * fix * Update pir_op_test_white_list --- .../ir_adaptor/translator/op_translator.cc | 23 +++++++++++++++++++ test/white_list/pir_op_test_white_list | 1 + 2 files changed, 24 insertions(+) diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index fa569b0df5ac6..76a787cda64bf 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -1343,6 +1343,28 @@ struct TrilAndTriuOpTranscriber : public OpTranscriber { } }; +struct TrilAndTriuGradOpTranscriber : public OpTranscriber { + pir::OpInfo LoopkUpOpInfo(pir::IrContext* ctx, + const OpDesc& op_desc) override { + bool lower = PADDLE_GET_CONST(bool, op_desc.GetAttr("lower")); + std::string target_op_name = ""; + if (lower) { + target_op_name = "pd_op.tril_grad"; + } else { + target_op_name = "pd_op.triu_grad"; + } + const auto& op_info = ctx->GetRegisteredOpInfo(target_op_name); + if (!op_info) { + IR_THROW( + "Op tril_triu_grad should have corresponding OpInfo pd_op.tril_grad " + "or " + "pd_op.triu_grad."); + } + + return op_info; + } +}; + using ValueInfo = std::tuple, dialect::DenseTensorType, pir::OpResult>; @@ -2988,6 +3010,7 @@ OpTranslator::OpTranslator() { special_handlers["split"] = SplitOpTranscriber(); special_handlers["sum"] = AddNOpTranscriber(); special_handlers["tril_triu"] = TrilAndTriuOpTranscriber(); + special_handlers["tril_triu_grad"] = TrilAndTriuGradOpTranscriber(); special_handlers["matrix_rank"] = MatrixRankOpTranscriber(); special_handlers["mul"] = MulOpTranscriber(); special_handlers["mul_grad"] = MulGradOpTranscriber(); diff --git a/test/white_list/pir_op_test_white_list b/test/white_list/pir_op_test_white_list index fe57bbe32693f..9e4de5ccffcfc 100644 --- a/test/white_list/pir_op_test_white_list +++ b/test/white_list/pir_op_test_white_list @@ -301,6 +301,7 @@ test_transpose_int8_mkldnn_op test_transpose_op test_triangular_solve_op test_tril_indices_op +test_tril_triu_op test_trilinear_interp_v2_op test_triu_indices_op test_trunc_op From 23af8cfda07af9363d96f57cd041867066de6236 Mon Sep 17 00:00:00 2001 From: BiynXu <62832681+BiynXu@users.noreply.github.com> Date: Mon, 25 Dec 2023 15:51:56 +0800 Subject: [PATCH 025/146] Add arrange storage tactic (#60155) * [CINN] Add arrange storage tactic * [CINN] Apply tactics in dy group scheduler * Polish codes --- paddle/cinn/common/integer_set.cc | 57 ++- paddle/cinn/common/integer_set.h | 20 +- paddle/cinn/common/integer_set_test.cc | 13 + paddle/cinn/ir/group_schedule/CMakeLists.txt | 2 + .../ir/group_schedule/base_group_scheduler.cc | 8 + .../ir/group_schedule/base_group_scheduler.h | 2 + .../dy_shape_group_scheduler.cc | 20 + .../group_schedule/dy_shape_group_scheduler.h | 11 +- .../st_shape_group_scheduler.cc | 9 - .../group_schedule/st_shape_group_scheduler.h | 3 - .../ir/group_schedule/tactic/CMakeLists.txt | 3 + .../tactic/arrange_storage_tactic.cc | 407 ++++++++++++++++++ .../tactic/arrange_storage_tactic.h | 36 ++ .../group_schedule/tactic/schedule_tactic.h | 29 ++ paddle/cinn/ir/ir.h | 9 + paddle/cinn/ir/ir_analyzer/ir_analyzer.cc | 161 +++++++ paddle/cinn/ir/ir_analyzer/ir_analyzer.h | 22 + 17 files changed, 781 insertions(+), 31 deletions(-) create mode 100644 paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt create mode 100644 paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc create mode 100644 paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h create mode 100644 paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h diff --git a/paddle/cinn/common/integer_set.cc b/paddle/cinn/common/integer_set.cc index 9b623bf7bb467..762c273caef7c 100644 --- a/paddle/cinn/common/integer_set.cc +++ b/paddle/cinn/common/integer_set.cc @@ -58,6 +58,14 @@ std::optional SymbolicExprAnalyzer::ProveEQ(const ir::Expr& lhs, if (diff.is_constant()) { return diff.get_constant() == 0; } + ir::Expr diff_lower_bound = LowerBound(diff); + VLOG(6) << "lower bound of " << diff << " = " << diff_lower_bound; + ir::Expr diff_upper_bound = UpperBound(diff); + VLOG(6) << "upper bound of " << diff << " = " << diff_upper_bound; + if (diff_lower_bound.is_constant() && diff_upper_bound.is_constant() && + diff_lower_bound.get_constant() == diff_upper_bound.get_constant()) { + return diff_lower_bound.get_constant() == 0; + } std::optional prove_gt = ProveGT(lhs, rhs); if (prove_gt.has_value() && prove_gt.value()) { return false; @@ -71,22 +79,11 @@ std::optional SymbolicExprAnalyzer::ProveEQ(const ir::Expr& lhs, std::optional SymbolicExprAnalyzer::ProveNE(const ir::Expr& lhs, const ir::Expr& rhs) const { - if (lhs == rhs) { - return false; - } - ir::Expr diff = AutoSimplify(ir::Sub::Make(lhs, rhs), var_intervals_); - if (diff.is_constant()) { - return diff.get_constant() != 0; - } - std::optional prove_gt = ProveGT(lhs, rhs); - if (prove_gt.has_value() && prove_gt.value()) { - return true; - } - std::optional prove_lt = ProveLT(lhs, rhs); - if (prove_lt.has_value() && prove_lt.value()) { - return true; + std::optional prove_eq = ProveEQ(lhs, rhs); + if (!prove_eq.has_value()) { + return std::nullopt; } - return std::nullopt; + return !prove_eq.value(); } std::optional SymbolicExprAnalyzer::ProveGE(const ir::Expr& lhs, @@ -456,5 +453,35 @@ std::optional SingleIntervalIntSet::ProveSuperSet( return std::nullopt; } +ir::Expr EnhancedSimplifyModExpr( + ir::Expr e, + const absl::flat_hash_map& var_intervals) { + struct Mutator : public ir::IRMutator { + explicit Mutator( + const absl::flat_hash_map& var_intervals) + : var_intervals_(var_intervals), analyzer_(var_intervals_) {} + + void operator()(ir::Expr* expr) { Visit(expr); } + void Visit(ir::Expr* expr) { ir::IRMutator<>::Visit(expr, expr); } + + private: + void Visit(const ir::Mod* op, ir::Expr* expr) override { + std::optional prove_lt = analyzer_.ProveLT(op->a(), op->b()); + if (prove_lt.has_value() && prove_lt.value()) { + *expr = op->a(); + } + } + + private: + const absl::flat_hash_map& var_intervals_; + SymbolicExprAnalyzer analyzer_; + }; + + Mutator mutator(var_intervals); + ir::Expr copied = ir::ir_utils::IRCopy(e); + mutator(&copied); + return copied; +} + } // namespace common } // namespace cinn diff --git a/paddle/cinn/common/integer_set.h b/paddle/cinn/common/integer_set.h index 520b88c582b75..e0f23da2e744f 100644 --- a/paddle/cinn/common/integer_set.h +++ b/paddle/cinn/common/integer_set.h @@ -58,9 +58,10 @@ struct SymbolicExprLimit { // The set consisting of all integers in the interval from min to max class SingleIntervalIntSet { public: - explicit SingleIntervalIntSet(const ir::Expr& min, - const ir::Expr& max, - cas_intervals_t var_intervals = {}); + explicit SingleIntervalIntSet( + const ir::Expr& min = SymbolicExprLimit::positive_inf, + const ir::Expr& max = SymbolicExprLimit::negative_inf, + cas_intervals_t var_intervals = {}); SingleIntervalIntSet(const SingleIntervalIntSet& set) = default; SingleIntervalIntSet(SingleIntervalIntSet&& set) = default; SingleIntervalIntSet& operator=(const SingleIntervalIntSet& set) = default; @@ -92,5 +93,18 @@ class SingleIntervalIntSet { cas_intervals_t var_intervals_; }; +std::optional ProveEQ(const SingleIntervalIntSet& lhs, + const SingleIntervalIntSet& rhs); +std::optional ProvedUnion(const SingleIntervalIntSet& a, + const SingleIntervalIntSet& b); +std::optional ProvedIntersect( + const SingleIntervalIntSet& a, const SingleIntervalIntSet& b); +cas_intervals_t MergeVarIntervals(const SingleIntervalIntSet& a, + const SingleIntervalIntSet& b); + +ir::Expr EnhancedSimplifyModExpr( + ir::Expr e, + const absl::flat_hash_map& var_intervals); + } // namespace common } // namespace cinn diff --git a/paddle/cinn/common/integer_set_test.cc b/paddle/cinn/common/integer_set_test.cc index c8154e65f66f5..23406ec2f770e 100644 --- a/paddle/cinn/common/integer_set_test.cc +++ b/paddle/cinn/common/integer_set_test.cc @@ -278,5 +278,18 @@ TEST(SingleIntervalIntSet, case_1) { ProvedIntersect(set_0, single_point).value().ProveEmpty().value()); } +TEST(SingleIntervalIntSet, case_2) { + ir::Var S = ir::Var(ir::Expr(0), ir::Expr(0), "S"); + + SingleIntervalIntSet set_0{S, S + Expr(1)}; + SingleIntervalIntSet set_1{Expr(0), Expr(1)}; + SingleIntervalIntSet set_2{Expr(0), Expr(2)}; + + EXPECT_TRUE(ProveEQ(set_0, set_1).value()); + EXPECT_FALSE(ProveEQ(set_0, set_2).value()); + EXPECT_TRUE(set_0.ProveSubSet(set_2).value()); + EXPECT_TRUE(set_2.ProveSuperSet(set_0).value()); +} + } // namespace common } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/CMakeLists.txt b/paddle/cinn/ir/group_schedule/CMakeLists.txt index 61b774245597f..d53ce85347b61 100644 --- a/paddle/cinn/ir/group_schedule/CMakeLists.txt +++ b/paddle/cinn/ir/group_schedule/CMakeLists.txt @@ -3,3 +3,5 @@ core_gather_headers() gather_srcs(cinnapi_src SRCS base_group_scheduler.cc) gather_srcs(cinnapi_src SRCS st_shape_group_scheduler.cc) gather_srcs(cinnapi_src SRCS dy_shape_group_scheduler.cc) + +add_subdirectory(tactic) diff --git a/paddle/cinn/ir/group_schedule/base_group_scheduler.cc b/paddle/cinn/ir/group_schedule/base_group_scheduler.cc index 687122741aa2e..a740ad268cb09 100644 --- a/paddle/cinn/ir/group_schedule/base_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/base_group_scheduler.cc @@ -33,5 +33,13 @@ std::unique_ptr GroupScheduler::Make( } } +std::unordered_set GroupScheduler::OutputTensorNames() const { + std::unordered_set output_tensor_names{output_tensor_names_}; + for (ir::ScheduleBlockNode* node : schedule_block_graph_->EndPoints()) { + output_tensor_names.insert(node->id()); + } + return output_tensor_names; +} + } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/base_group_scheduler.h b/paddle/cinn/ir/group_schedule/base_group_scheduler.h index 6a277f01d43bf..33cce051f1845 100644 --- a/paddle/cinn/ir/group_schedule/base_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/base_group_scheduler.h @@ -48,6 +48,8 @@ class GroupScheduler { virtual std::vector> GetIRs() = 0; + std::unordered_set OutputTensorNames() const; + protected: ir::IRSchedule* ir_sch_; const std::unordered_set& output_tensor_names_; diff --git a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc index abaeb76d5ceea..f0804e16aee36 100644 --- a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc @@ -13,10 +13,16 @@ // limitations under the License. #include "paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h" +#include "paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h" namespace cinn { namespace ir { +void DynamicShapeGroupScheduler::Init() { + std::unordered_set output_names = OutputTensorNames(); + tactics_.emplace_back(new ArrangeStorageTactic(output_names)); +} + void DynamicShapeGroupScheduler::Schedule() { // Fake schedule for test std::vector all_blocks = ir_sch_->GetAllBlocks(); @@ -36,12 +42,26 @@ void DynamicShapeGroupScheduler::Schedule() { auto splited_loops1 = ir_sch_->Split(block0_loops[0], {1024, -1}); ir_sch_->Bind(splited_loops1[0], "threadIdx.x"); + + ApplyTactics(); + ir::Expr predicate1 = ir::LE::Make(Expr(1023), Expr(1024)); std::unique_ptr new_ir_sch1 = std::make_unique(*ir_sch_); ir_schs_.emplace_back(predicate1, std::move(new_ir_sch1)); } +void DynamicShapeGroupScheduler::ApplyTactics() { + schedule_block_graph_->Update(*ir_sch_); + for (const auto& tactic : tactics_) { + auto ApplyTacticFunc = [&](ir::ScheduleBlockNode* node) { + tactic->Apply(ir_sch_, node->id()); + }; + schedule_block_graph_->DFSTopoWalk(ApplyTacticFunc); + schedule_block_graph_->Update(*ir_sch_); + } +} + std::vector> DynamicShapeGroupScheduler::GetIRs() { std::vector> irs; diff --git a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h index 6b89a0eff0003..7d2f9115776dc 100644 --- a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h @@ -14,6 +14,7 @@ #pragma once #include "paddle/cinn/ir/group_schedule/base_group_scheduler.h" +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" namespace cinn { namespace ir { @@ -28,15 +29,23 @@ class DynamicShapeGroupScheduler : public GroupScheduler { ir::IRSchedule* ir_sch, const std::unordered_set& output_tensor_names, const cinn::common::Target& target) - : GroupScheduler(ir_sch, output_tensor_names, target) {} + : GroupScheduler(ir_sch, output_tensor_names, target) { + Init(); + } void Schedule() override; std::vector> GetIRs() override; + private: + void Init(); + + void ApplyTactics(); + private: std::vector>> ir_schs_; + std::vector> tactics_; }; } // namespace ir diff --git a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc index 36e3279810778..a5cb17dc5a2a7 100644 --- a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc @@ -205,15 +205,6 @@ ir::ScheduleBlockNode* StaticShapeGroupScheduler::FindGlobalMasterNode() const { return master; } -std::unordered_set StaticShapeGroupScheduler::OutputTensorNames() - const { - std::unordered_set output_tensor_names{output_tensor_names_}; - for (ir::ScheduleBlockNode* node : schedule_block_graph_->EndPoints()) { - output_tensor_names.insert(node->id()); - } - return output_tensor_names; -} - void StaticShapeGroupScheduler::DoLoopAlignment() { VLOG(5) << "[Start LoopAlignment] func body: " << ir_sch_->GetModule().GetExprs().front(); diff --git a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h index 294fc3d1b4181..337817995eb0f 100644 --- a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h +++ b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.h @@ -94,9 +94,6 @@ class StaticShapeGroupScheduler : public GroupScheduler { // throughout the entire IR. void UpdateBlockOrder(); - // Get output tensor names of group. - std::unordered_set OutputTensorNames() const; - /** * @brief Determine whether the graph level dependency is still maintained * after the schedule_block is placed in the insert position of target_loop. diff --git a/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt new file mode 100644 index 0000000000000..50e8500ae38bc --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt @@ -0,0 +1,3 @@ +core_gather_headers() + +gather_srcs(cinnapi_src SRCS arrange_storage_tactic.cc) diff --git a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc new file mode 100644 index 0000000000000..fad7097d09787 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.cc @@ -0,0 +1,407 @@ +// Copyright (c) 2023 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h" +#include "paddle/cinn/common/cas.h" +#include "paddle/cinn/common/integer_set.h" +#include "paddle/cinn/ir/ir.h" +#include "paddle/cinn/ir/ir_analyzer/ir_analyzer.h" +#include "paddle/cinn/ir/op/ir_operators.h" +#include "paddle/cinn/ir/utils/ir_copy.h" +#include "paddle/cinn/optim/replace_var_with_expr.h" + +namespace cinn { +namespace ir { + +// [block_name, [var_name, for_node]] +using VarToForMap = + std::unordered_map>; +using IntSet = common::SingleIntervalIntSet; + +enum class CudaAxisType : int { + kCudaBlock = 0, + kCudaThread = 1, +}; + +struct CudaAxisSpace { + IntSet x{Expr(0), Expr(0)}; + IntSet y{Expr(0), Expr(0)}; + IntSet z{Expr(0), Expr(0)}; + CudaAxisType type; +}; + +struct CudaIterVarName { + static constexpr char* kCudaBlockX = "cuda_block_x"; + static constexpr char* kCudaBlockY = "cuda_block_y"; + static constexpr char* kCudaBlockZ = "cuda_block_z"; + static constexpr char* kCudaThreadX = "cuda_thread_x"; + static constexpr char* kCudaThreadY = "cuda_thread_y"; + static constexpr char* kCudaThreadZ = "cuda_thread_z"; +}; + +std::optional IsSubCudaAxisSpace(const CudaAxisSpace& lhs, + const CudaAxisSpace& rhs) { + CHECK(lhs.type == rhs.type); + std::optional prove_sub_x = lhs.x.ProveSubSet(rhs.x); + std::optional prove_sub_y = lhs.y.ProveSubSet(rhs.y); + std::optional prove_sub_z = lhs.z.ProveSubSet(rhs.z); + if (!prove_sub_x.has_value() || !prove_sub_y.has_value() || + !prove_sub_z.has_value()) { + return std::nullopt; + } + return prove_sub_x.value() && prove_sub_y.value() && prove_sub_z.value(); +} + +std::tuple GetCudaAxisSpace( + const VarToForMap& var2for_map, const std::string block_name) { + CudaAxisSpace cuda_block_space{IntSet{Expr(0), Expr(0)}, + IntSet{Expr(0), Expr(0)}, + IntSet{Expr(0), Expr(0)}, + CudaAxisType::kCudaBlock}; + CudaAxisSpace cuda_thread_space{IntSet{Expr(0), Expr(0)}, + IntSet{Expr(0), Expr(0)}, + IntSet{Expr(0), Expr(0)}, + CudaAxisType::kCudaThread}; + CHECK_GT(var2for_map.count(block_name), 0); + for (const auto& var2for : var2for_map.at(block_name)) { + const Expr& for_expr = var2for.second; + const ir::For* for_node = for_expr.As(); + CHECK_NOTNULL(for_node); + IntSet interval{ + for_node->min, + common::AutoSimplify(for_node->min + for_node->extent - Expr(1))}; + if (for_node->is_gpu_thread_binded()) { + if (for_node->bind_info().offset == 0) { + cuda_thread_space.x = interval; + } else if (for_node->bind_info().offset == 1) { + cuda_thread_space.y = interval; + } else if (for_node->bind_info().offset == 2) { + cuda_thread_space.z = interval; + } + } else if (for_node->is_gpu_block_binded()) { + if (for_node->bind_info().offset == 0) { + cuda_block_space.x = interval; + } else if (for_node->bind_info().offset == 1) { + cuda_block_space.y = interval; + } else if (for_node->bind_info().offset == 2) { + cuda_block_space.z = interval; + } + } + } + VLOG(6) << "GetCudaAxisSpace of block: " << block_name + << "\n cuda_block_space: [" + << "x = [" << cuda_block_space.x.Min() << " : " + << cuda_block_space.x.Max() << "] " + << "y = [" << cuda_block_space.y.Min() << " : " + << cuda_block_space.y.Max() << "] " + << "z = [" << cuda_block_space.z.Min() << " : " + << cuda_block_space.z.Max() << "]]" + << "\n cuda_thread_space: [" + << "x = [" << cuda_thread_space.x.Min() << " : " + << cuda_thread_space.x.Max() << "] " + << "y = [" << cuda_thread_space.y.Min() << " : " + << cuda_thread_space.y.Max() << "] " + << "z = [" << cuda_thread_space.z.Min() << " : " + << cuda_thread_space.z.Max() << "]]"; + return {cuda_block_space, cuda_thread_space}; +} + +IntSet Evaluate(Expr expr, + const std::unordered_map& fixed, + const std::unordered_map& var_domain) { + Expr copy_for_upper_bound = ir::ir_utils::IRCopy(expr); + Expr copy_for_lower_bound = ir::ir_utils::IRCopy(expr); + common::cas_intervals_t var_intervals; + std::set var_set = ir::ir_utils::CollectIRNodesWithoutTensor( + expr, [](const ir::Expr* x) { return x->as_var(); }); + for (Expr var_expr : var_set) { + ir::Var var = var_expr.as_var_ref(); + if (fixed.count(var) != 0) { + const ir::Var& fixed_var = fixed.at(var); + var_intervals.emplace( + fixed_var->name, + common::CasInterval(fixed_var->lower_bound, fixed_var->upper_bound)); + optim::ReplaceVarWithExpr(©_for_lower_bound, var, Expr(fixed_var)); + optim::ReplaceVarWithExpr(©_for_upper_bound, var, Expr(fixed_var)); + } else if (var_domain.count(var) != 0) { + Expr var_min = var_domain.at(var).Min(); + Expr var_max = var_domain.at(var).Max(); + optim::ReplaceVarWithExpr(©_for_lower_bound, var, var_min); + optim::ReplaceVarWithExpr(©_for_upper_bound, var, var_max); + } else { + CHECK(var->lower_bound.defined()); + CHECK(var->upper_bound.defined()); + optim::ReplaceVarWithExpr(©_for_lower_bound, var, var->lower_bound); + optim::ReplaceVarWithExpr(©_for_upper_bound, var, var->upper_bound); + } + } + ir::Expr lower_bound = + common::AutoSimplify(copy_for_lower_bound, var_intervals); + ir::Expr upper_bound = + common::AutoSimplify(copy_for_upper_bound, var_intervals); + lower_bound = common::EnhancedSimplifyModExpr(lower_bound, var_intervals); + upper_bound = common::EnhancedSimplifyModExpr(upper_bound, var_intervals); + return IntSet(lower_bound, upper_bound, var_intervals); +} + +std::unordered_map GetFixedVar( + const VarToForMap& var2for_map, + const std::string& block_name, + const CudaAxisSpace& cuda_space) { + if (var2for_map.count(block_name) == 0) return {}; + std::unordered_map fix_var_map; + const CudaAxisType& type = cuda_space.type; + for (const std::pair& var2for : + var2for_map.at(block_name)) { + const ir::For* for_node = var2for.second.As(); + if (type == CudaAxisType::kCudaBlock && for_node->is_gpu_block_binded()) { + if (for_node->bind_info().offset == 0) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.x.Min(), + cuda_space.x.Max(), + CudaIterVarName::kCudaBlockX, + var2for.first->is_reduce_axis)}); + } else if (for_node->bind_info().offset == 1) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.y.Min(), + cuda_space.y.Max(), + CudaIterVarName::kCudaBlockY, + var2for.first->is_reduce_axis)}); + } else if (for_node->bind_info().offset == 2) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.z.Min(), + cuda_space.z.Max(), + CudaIterVarName::kCudaBlockZ, + var2for.first->is_reduce_axis)}); + } + } else if (type == CudaAxisType::kCudaThread && + for_node->is_gpu_thread_binded()) { + if (for_node->bind_info().offset == 0) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.x.Min(), + cuda_space.x.Max(), + CudaIterVarName::kCudaThreadX, + var2for.first->is_reduce_axis)}); + } else if (for_node->bind_info().offset == 1) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.y.Min(), + cuda_space.y.Max(), + CudaIterVarName::kCudaThreadY, + var2for.first->is_reduce_axis)}); + } else if (for_node->bind_info().offset == 2) { + fix_var_map.insert({var2for.first, + ir::_Var_::Make(cuda_space.z.Min(), + cuda_space.z.Max(), + CudaIterVarName::kCudaThreadZ, + var2for.first->is_reduce_axis)}); + } + } + } + return fix_var_map; +} + +std::unordered_map GetVarDomainOfSBlock( + const VarToForMap& var2for_map, const std::string& block_name) { + if (var2for_map.count(block_name) == 0) return {}; + std::unordered_map var_domains; + for (const std::pair& var2for : + var2for_map.at(block_name)) { + const ir::For* for_node = var2for.second.As(); + var_domains.emplace( + var2for.first, + IntSet(for_node->min, + common::AutoSimplify(for_node->min + for_node->extent - + ir::Expr(1)))); + } + return var_domains; +} + +std::optional AnalyzeCrossType(const VarToForMap& var2for_map, + Expr store, + Expr load, + Expr store_block, + Expr load_block) { + CHECK(store_block.As()); + CHECK(load_block.As()); + std::string store_block_name = store_block.As() + ->schedule_block.As() + ->name; + std::string load_block_name = load_block.As() + ->schedule_block.As() + ->name; + VLOG(6) << "Analyzing cross type of Store: [" << store << "] and Load: [" + << load << "]"; + + // 1. Determine domain range + CudaAxisSpace cuda_block_space_of_store; + CudaAxisSpace cuda_thread_space_of_store; + std::tie(cuda_block_space_of_store, cuda_thread_space_of_store) = + GetCudaAxisSpace(var2for_map, store_block_name); + CudaAxisSpace cuda_block_space_of_load; + CudaAxisSpace cuda_thread_space_of_load; + std::tie(cuda_block_space_of_load, cuda_thread_space_of_load) = + GetCudaAxisSpace(var2for_map, load_block_name); + std::optional is_block_sub_space = + IsSubCudaAxisSpace(cuda_block_space_of_load, cuda_block_space_of_store); + if (!is_block_sub_space.has_value() || !is_block_sub_space.value()) { + VLOG(6) << "load cuda block space is not sub space of store"; + return CudaAxisType::kCudaBlock; + } + VLOG(6) << "load cuda block space is sub space of store"; + std::optional is_thread_sub_space = + IsSubCudaAxisSpace(cuda_thread_space_of_load, cuda_thread_space_of_store); + if (!is_thread_sub_space.has_value() || !is_thread_sub_space.value()) { + VLOG(6) << "load cuda thread space is not sub space of store"; + return CudaAxisType::kCudaThread; + } + VLOG(6) << "load cuda thread space is sub space of store"; + + // 2. Determine value range + std::unordered_map cuda_block_fixed_var_of_store = + GetFixedVar(var2for_map, store_block_name, cuda_block_space_of_load); + std::unordered_map cuda_block_fixed_var_of_load = + GetFixedVar(var2for_map, load_block_name, cuda_block_space_of_load); + std::unordered_map cuda_thread_fixed_var_of_store = + GetFixedVar(var2for_map, store_block_name, cuda_thread_space_of_load); + std::unordered_map cuda_thread_fixed_var_of_load = + GetFixedVar(var2for_map, load_block_name, cuda_thread_space_of_load); + std::unordered_map cuda_block_thread_fixed_var_of_store = + cuda_block_fixed_var_of_store; + cuda_block_thread_fixed_var_of_store.insert( + cuda_thread_fixed_var_of_store.begin(), + cuda_thread_fixed_var_of_store.end()); + std::unordered_map cuda_block_thread_fixed_var_of_load = + cuda_block_fixed_var_of_load; + cuda_block_thread_fixed_var_of_store.insert( + cuda_thread_fixed_var_of_load.begin(), + cuda_thread_fixed_var_of_load.end()); + std::unordered_map store_var_domain = + GetVarDomainOfSBlock(var2for_map, store_block_name); + std::unordered_map load_var_domain = + GetVarDomainOfSBlock(var2for_map, load_block_name); + std::vector iter_values_of_store = + analyzer::GetIterValuesOfAccess(store, store_block); + std::vector iter_values_of_load = + analyzer::GetIterValuesOfAccess(load, load_block); + CHECK_EQ(iter_values_of_load.size(), iter_values_of_store.size()); + + for (int i = 0; i < iter_values_of_load.size(); ++i) { + IntSet block_store_range = Evaluate(iter_values_of_store[i], + cuda_block_fixed_var_of_store, + store_var_domain); + IntSet block_load_range = Evaluate( + iter_values_of_load[i], cuda_block_fixed_var_of_load, load_var_domain); + VLOG(6) << "block_store_range of [" << iter_values_of_store[i] << "] = [" + << block_store_range.Min() << " : " << block_store_range.Max() + << "]"; + VLOG(6) << "block_load_range of [" << iter_values_of_load[i] << "] = [" + << block_load_range.Min() << " : " << block_load_range.Max() << "]"; + std::optional is_block_sub_set = + block_load_range.ProveSubSet(block_store_range); + if (!is_block_sub_set.has_value() || !is_block_sub_set.value()) { + VLOG(6) << "load range of a cuda block is not sub set of store"; + return CudaAxisType::kCudaBlock; + } + + IntSet thread_store_range = Evaluate(iter_values_of_store[i], + cuda_block_thread_fixed_var_of_store, + store_var_domain); + IntSet thread_load_range = Evaluate(iter_values_of_load[i], + cuda_block_thread_fixed_var_of_load, + load_var_domain); + VLOG(6) << "thread_store_range of [" << iter_values_of_store[i] << "] = [" + << thread_store_range.Min() << " : " << thread_store_range.Max() + << "]"; + VLOG(6) << "thread_load_range of [" << iter_values_of_load[i] << "] = [" + << thread_load_range.Min() << " : " << thread_load_range.Max() + << "]"; + std::optional is_thread_sub_set = + thread_load_range.ProveSubSet(thread_store_range); + if (!is_thread_sub_set.has_value() || !is_thread_sub_set.value()) { + VLOG(6) << "load range of a cuda thread is not sub set of store"; + return CudaAxisType::kCudaThread; + } + } + + return std::nullopt; +} + +ArrangeStorageTactic::ArrangeStorageTactic( + const std::unordered_set& output_names) + : output_names_(output_names) {} + +void ArrangeStorageTactic::Apply(ir::IRSchedule* sch, + const std::string& block_id) { + ir::Expr store_block = sch->GetBlock(block_id); + ir::Expr root_block = sch->GetRootBlock(store_block); + ir::Expr store = *ir::ir_utils::CollectIRNodesWithoutTensor( + store_block, + [&](const ir::Expr* x) { return x->As(); }, + true) + .begin(); + + VarToForMap var2for_map = + analyzer::CollectVarToForMap({root_block}, sch->GetAllBlocks()); + + // Traverse load nodes to check if there are loads that cross cuda blocks or + // threads + std::vector> loads_and_blocks = + analyzer::GetConsumerLoadsAndSBlocks(store_block, root_block); + + ir::MemoryType memory_type = ir::MemoryType::GPULocal; + for (const auto& load_and_block : loads_and_blocks) { + ir::Expr load = load_and_block.first; + ir::Expr load_block = load_and_block.second; + std::optional cross_type = + AnalyzeCrossType(var2for_map, store, load, store_block, load_block); + if (!cross_type.has_value()) { + memory_type = ir::MemoryType::GPULocal; + } else if (cross_type.value() == CudaAxisType::kCudaThread) { + memory_type = ir::MemoryType::GPUShared; + } else if (cross_type.value() == CudaAxisType::kCudaBlock) { + LOG(FATAL) << "Fusion requires synchronization across blocks, but " + "currently we do not support it."; + break; + } + } + + // Set output tensor to global + if (output_names_.count(block_id) > 0) { + memory_type = ir::MemoryType::Auto; + } + // Set the reduce_init tensor and the real tensor to the same memory + if (ir::IsReduceInitTensorName(block_id)) { + ir::Expr block = sch->GetBlock(ir::GetOriginalReduceTensorName(block_id)); + memory_type = analyzer::GetStoreTensorOfSBlock(block)->buffer->memory_type; + } + // Do schedule + std::unordered_set sync_mark; + if (memory_type == ir::MemoryType::Auto) { + VLOG(6) << "Set store tensor of block " << block_id << " to global"; + } else if (memory_type == ir::MemoryType::GPUShared) { + VLOG(6) << "Set store tensor of block " << block_id << " to shared"; + sch->SetBuffer(store_block, "shared"); + std::vector loops = sch->GetLoops(store_block); + if (sync_mark.count(ir::GetOriginalReduceTensorName(block_id)) == 0) { + sch->SyncThreads(loops.back(), true); + sync_mark.insert(ir::GetOriginalReduceTensorName(block_id)); + } + } else if (memory_type == ir::MemoryType::GPULocal) { + VLOG(6) << "Set store tensor of block " << block_id << " to register"; + sch->SetBuffer(store_block, "local"); + } +} + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h new file mode 100644 index 0000000000000..05c9e67225a8f --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h @@ -0,0 +1,36 @@ +// Copyright (c) 2023 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" + +namespace cinn { +namespace ir { + +class ArrangeStorageTactic : public ScheduleTactic { + public: + explicit ArrangeStorageTactic( + const std::unordered_set& output_names); + + void Apply(ir::IRSchedule* sch, const std::string& block_id) final; + + private: + std::unordered_set output_names_; +}; + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h new file mode 100644 index 0000000000000..49c4d8b623f45 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h @@ -0,0 +1,29 @@ +// Copyright (c) 2023 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/cinn/ir/schedule/ir_schedule.h" + +namespace cinn { +namespace ir { + +class ScheduleTactic { + public: + virtual void Apply(ir::IRSchedule* sch, const std::string& block_id) = 0; +}; + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/ir.h b/paddle/cinn/ir/ir.h index 85e4cbeac983e..7859a7181c527 100644 --- a/paddle/cinn/ir/ir.h +++ b/paddle/cinn/ir/ir.h @@ -1076,3 +1076,12 @@ using ir::Var; // @} } // namespace cinn + +namespace std { +template <> +struct hash { + std::size_t operator()(const cinn::ir::Var& var) const { + return std::hash()(var->name); + } +}; +} // namespace std diff --git a/paddle/cinn/ir/ir_analyzer/ir_analyzer.cc b/paddle/cinn/ir/ir_analyzer/ir_analyzer.cc index 724cca3e6279c..bdb37d4189ce4 100644 --- a/paddle/cinn/ir/ir_analyzer/ir_analyzer.cc +++ b/paddle/cinn/ir/ir_analyzer/ir_analyzer.cc @@ -212,6 +212,167 @@ Expr AddUnitLoop(const std::vector& exprs, const Expr& block) { return Expr{nullptr}; } +Expr GetStoreOfSBlock(const Expr& block) { + CHECK(block.As()); + std::set find_store = ir_utils::CollectIRNodesWithoutTensor( + block, [&](const Expr* x) { return x->As(); }, true); + CHECK_EQ(find_store.size(), 1U) + << "One block should only have one Store node!(except for root block)"; + return *find_store.begin(); +} + +Tensor GetStoreTensorOfSBlock(const Expr& block) { + CHECK(block.As()); + Expr find_store = GetStoreOfSBlock(block); + CHECK(find_store.As()->tensor.as_tensor()); + return find_store.As()->tensor.as_tensor_ref(); +} + +std::vector GetConsumerSBlocks(const Expr& block, const Expr& root) { + CHECK(block.As()); + CHECK(root.As()); + std::vector consumers; + std::string store_tensor_name = GetStoreTensorOfSBlock(block)->name; + if (IsReduceInitTensorName(store_tensor_name)) { + std::string consumer_name = GetOriginalReduceTensorName(store_tensor_name); + auto consumer = + ir_utils::CollectIRNodesWithoutTensor(root, [&](const Expr* x) { + return x->As() && + x->As() + ->schedule_block.As() + ->name == consumer_name; + }); + CHECK_EQ(consumer.size(), 1); + return {*consumer.begin()}; + } + + auto find_blocks = + ir_utils::CollectIRNodesWithoutTensor(root, [&](const Expr* x) { + return x->As() && *x != block && *x != root; + }); + for (auto& find_block : find_blocks) { + CHECK(find_block.As() + ->schedule_block.As()); + auto block_body = find_block.As() + ->schedule_block.As() + ->body; + auto find_load_or_call = + ir_utils::CollectIRNodesWithoutTensor(block_body, [&](const Expr* x) { + if (x->As()) { + const std::vector& read_args = x->As()->read_args; + for (const Expr& arg : read_args) { + if (arg.as_tensor() && + arg.as_tensor_ref()->name == store_tensor_name) { + return true; + } + } + } + return x->As() && x->As()->tensor.as_tensor_ref()->name == + store_tensor_name; + }); + if (!find_load_or_call.empty()) consumers.emplace_back(find_block); + } + return consumers; +} + +std::vector> GetConsumerLoadsAndSBlocks( + const Expr& block, const Expr& root) { + CHECK(block.As()); + CHECK(root.As()); + + Expr store = GetStoreOfSBlock(block); + std::vector consumer_blocks = GetConsumerSBlocks(block, root); + std::vector> loads_and_blocks; + for (const Expr& consumer_block : consumer_blocks) { + ir_utils::CollectIRNodesWithoutTensor(consumer_block, [&](const Expr* x) { + if (x->As() && + (x->As()->name() == store.As()->name())) { + loads_and_blocks.emplace_back(*x, consumer_block); + } + return false; + }); + } + return loads_and_blocks; +} + +std::unordered_map> +CollectVarToForMap(const std::vector& exprs, + const std::vector& blocks) { + std::unordered_map> + for_map; + for (const ir::Expr& block : blocks) { + std::string block_name = block.As() + ->schedule_block.As() + ->name; + std::vector for_exprs = GetLoops(exprs, block); + for (ir::Expr for_expr : for_exprs) { + for_map[block_name][for_expr.As()->loop_var] = for_expr; + VLOG(6) << "for_map.insert: <" << block_name << ", " + << for_expr.As()->loop_var->name << ">"; + } + } + return for_map; +} + +std::unordered_map GetIterVarToValueOfSBlock( + ir::Expr block) { + ir::ScheduleBlockRealize* s_block_realize = + block.As(); + CHECK_NOTNULL(s_block_realize); + ir::ScheduleBlock* s_block = + s_block_realize->schedule_block.As(); + CHECK_NOTNULL(s_block); + CHECK_EQ(s_block_realize->iter_values.size(), s_block->iter_vars.size()); + std::unordered_map iter_var2iter_values; + for (size_t i = 0; i < s_block_realize->iter_values.size(); ++i) { + iter_var2iter_values.emplace(s_block->iter_vars[i], + s_block_realize->iter_values[i]); + } + return iter_var2iter_values; +} + +ir::Expr ReplaceVarWithExpr(const ir::Expr& source, + const std::vector& candidates, + const std::vector& targets) { + CHECK_EQ(candidates.size(), targets.size()) + << "In ReplaceExpr, the size of Vars to be replaces must be equal to the " + "size of targets Exprs! Please check."; + ir::Expr copied = ir::ir_utils::IRCopy(source); + if (candidates.empty()) return copied; + std::map replacing_map; + for (int i = 0; i < candidates.size(); ++i) { + // If the Var to be candidates is equal to the candidate, we skip it. + if (targets[i].is_var() && targets[i].as_var_ref() == candidates[i]) + continue; + replacing_map[candidates[i]] = targets[i]; + } + MappingVarToExprMutator mapper(replacing_map); + mapper(&copied); + return copied; +} + +std::vector GetIterValuesOfAccess(ir::Expr load_or_store, + ir::Expr block) { + CHECK(load_or_store.As() || load_or_store.As()); + std::vector indices = load_or_store.As() + ? load_or_store.As()->indices + : load_or_store.As()->indices; + ir::ScheduleBlockRealize* s_block_realize = + block.As(); + CHECK_NOTNULL(s_block_realize); + ir::ScheduleBlock* s_block = + s_block_realize->schedule_block.As(); + CHECK_NOTNULL(s_block); + + std::vector iter_values; + for (ir::Expr index : indices) { + ir::Expr index_value = ReplaceVarWithExpr( + index, s_block->iter_vars, s_block_realize->iter_values); + iter_values.push_back(common::AutoSimplify(index_value)); + } + return iter_values; +} + } // namespace analyzer } // namespace ir } // namespace cinn diff --git a/paddle/cinn/ir/ir_analyzer/ir_analyzer.h b/paddle/cinn/ir/ir_analyzer/ir_analyzer.h index 7a6f86a39d4a9..a24e9726b8ce7 100644 --- a/paddle/cinn/ir/ir_analyzer/ir_analyzer.h +++ b/paddle/cinn/ir/ir_analyzer/ir_analyzer.h @@ -45,6 +45,28 @@ DeviceAPI GetDeviceAPI(const std::vector& exprs); Expr AddUnitLoop(const std::vector& exprs, const Expr& block); +Expr GetStoreOfSBlock(const Expr& block); + +Tensor GetStoreTensorOfSBlock(const Expr& block); + +std::vector GetConsumerSBlocks(const Expr& block, const Expr& root); + +std::vector> GetConsumerLoadsAndSBlocks(const Expr& block, + const Expr& root); + +std::unordered_map> +CollectVarToForMap(const std::vector& exprs, + const std::vector& blocks); + +std::unordered_map GetIterVarToValueOfSBlock(ir::Expr block); + +ir::Expr ReplaceVarWithExpr(const ir::Expr& source, + const std::vector& candidates, + const std::vector& targets); + +std::vector GetIterValuesOfAccess(ir::Expr load_or_store, + ir::Expr block); + } // namespace analyzer } // namespace ir } // namespace cinn From 021bfd0ffd2bcfb68e4cb72d49710d2fe8d96df0 Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Mon, 25 Dec 2023 16:03:00 +0800 Subject: [PATCH 026/146] [XPU] update XPU XCCL version to latest 1.1.8.1 (#60305) - the new XCCL version supports bf16 collective ops --- cmake/external/xpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index f43868068d66e..d45d0ad2a7245 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -31,7 +31,7 @@ endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) set(XPU_XHPC_BASE_DATE "20231215") endif() -set(XPU_XCCL_BASE_VERSION "1.1.7.1") +set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) set(XPU_XFT_BASE_VERSION "20230602") endif() From c475cd665119aa9b890bc073d311c8286f8c18c2 Mon Sep 17 00:00:00 2001 From: JYChen Date: Mon, 25 Dec 2023 17:36:57 +0800 Subject: [PATCH 027/146] fix error in PIR slice (#60259) * fix error in PIR slice, still has probrolem * fix ut --- python/paddle/utils/layers_utils.py | 10 +++++----- test/legacy_test/test_while_loop_op.py | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/paddle/utils/layers_utils.py b/python/paddle/utils/layers_utils.py index 3e4077b3f3c89..8242a64d3ba86 100644 --- a/python/paddle/utils/layers_utils.py +++ b/python/paddle/utils/layers_utils.py @@ -380,10 +380,10 @@ def _is_symmetric_padding(padding, data_dim): def _contain_var(list_or_tuple): """ - Check whether list or tuple contains variable / OpResult. + Check whether list or tuple contains variable / OpResult / Value. """ for item in list_or_tuple: - if isinstance(item, (Variable, paddle.pir.OpResult)): + if isinstance(item, (Variable, paddle.pir.OpResult, paddle.pir.Value)): return True return False @@ -394,7 +394,7 @@ def get_int_tensor_list(ele_list, place=None, default_dtype='int64'): int_tensor_list = [] for ele in ele_list: - if isinstance(ele, paddle.pir.OpResult): + if isinstance(ele, (paddle.pir.OpResult, paddle.pir.Value)): ele.stop_gradient = True if convert_dtype(ele.dtype) != default_dtype: ele = paddle.cast(x=ele, dtype=default_dtype) @@ -466,13 +466,13 @@ def _get_shape_tensor(list_shape): def _convert_to_tensor_list(old_list, dtype="int32"): """ - Converts all elements of a list to Variable / OpResult. + Converts all elements of a list to Variable / OpResult / Value. """ from paddle.tensor import fill_constant new_list_tensor = [] for ele in old_list: - if isinstance(ele, (Variable, paddle.pir.OpResult)): + if isinstance(ele, (Variable, paddle.pir.OpResult, paddle.pir.Value)): ele.stop_gradient = True new_list_tensor.append(ele) else: diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index c75670d95c6a8..0926048d37e50 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -704,7 +704,7 @@ def value_error_body_returns_with_mutable_list(): class TestApiWhileLoopSliceInBody(unittest.TestCase): @compare_legacy_with_pt - # @test_with_pir_api (need to fix slice bug in pir) + @test_with_pir_api def test_var_slice(self): def cond(z, i): return i + 1 <= x_shape[0] @@ -716,7 +716,8 @@ def body(z, i): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with program_guard(main_program, startup_program): + + with paddle.static.program_guard(main_program, startup_program): x = paddle.static.data(name='x', shape=[-1, 5], dtype='int32') z = paddle.tensor.fill_constant([], 'int32', 0) x_shape = paddle.shape(x) From d424d15936c487f4b672d79ace19a78b25e340bd Mon Sep 17 00:00:00 2001 From: Tian <121000916+SylarTiaNII@users.noreply.github.com> Date: Mon, 25 Dec 2023 18:55:19 +0800 Subject: [PATCH 028/146] [AutoTuner] get enhanced report in dp estimation mode (#60294) --- .../paddle/distributed/auto_tuner/recorder.py | 38 ++++++++++++++++--- python/paddle/distributed/launch/main.py | 22 ++++++++++- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/python/paddle/distributed/auto_tuner/recorder.py b/python/paddle/distributed/auto_tuner/recorder.py index 143c74332b7ae..9ee891e350479 100644 --- a/python/paddle/distributed/auto_tuner/recorder.py +++ b/python/paddle/distributed/auto_tuner/recorder.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import csv import os from typing import Tuple @@ -21,9 +22,12 @@ class HistoryRecorder: # NOTE increase extenable ablitity - def __init__(self) -> None: + def __init__(self, tuner_cfg) -> None: + self.tuner_cfg = tuner_cfg + self.search_algo = self.tuner_cfg['search_algo']['name'] self.history = [] self.store_path = None + self.additional_metric_key = None def add_cfg(self, **kwargs): cur_configs = {} @@ -76,11 +80,10 @@ def get_best(self, metric, direction, mode=None) -> Tuple[dict, bool]: return (self.history[0], True) return (self.history[0], False) - def store_history(self, path="./history.csv"): + def _store_history_impl(self, data, path="./history.csv"): """Store history to csv file.""" - self.store_path = path # convert to pd dataframe - df = pd.DataFrame(self.history) + df = pd.DataFrame(data) # move 'job_id' to the first column cols = df.columns.tolist() cols.insert(0, cols.pop(cols.index('job_id'))) @@ -91,7 +94,32 @@ def store_history(self, path="./history.csv"): if 'has_error' in df.columns: df = df.drop(columns=['has_error']) # write to csv - df.to_csv(self.store_path, index=False) + df.to_csv(path, index=False) + + def store_history(self, path="./history.csv"): + # get enhanced report in dp-estimation mode + if self.search_algo == "dp_estimation": + metric_name = self.tuner_cfg['metric_cfg']['name'] + _history = [] + for cfg in self.history: + if ( + "sharding_overlap" not in cfg.keys() + or cfg["sharding_overlap"] is None + ) and cfg["error_info"] is None: + _history.append(copy.deepcopy(cfg)) + _history.sort( + key=lambda x: x[self.additional_metric_key] + if x[self.additional_metric_key] is not None + else float('-inf'), + reverse=True, + ) + self._store_history_impl( + data=_history, path=path.split('.csv')[0] + '_enhanced.csv' + ) + + """Store history to csv file.""" + self.store_path = path + self._store_history_impl(data=self.history, path=path) def load_history(self, path="./history.csv") -> Tuple[list, bool]: """Load history from csv file.""" diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index 6d821860c0fb5..457d095d5806e 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -436,7 +436,7 @@ def launch(): is_first_task = True # build history recorder - recorder = HistoryRecorder() + recorder = HistoryRecorder(tuner_cfg) job_id = 0 error_task_nums = 0 @@ -898,6 +898,15 @@ def launch(): ) ) amp = tuner_cfg["search_algo"]["conversion"].get("amp", False) + num_gpus = int(cur_cfg["num_gpus"]) + seq_length = int( + tuner_cfg["model_cfg"].get("max_seq_length", 2048) + ) + cur_cfg[f"unified_{tuner_cfg['metric_cfg']['name']}"] = ( + round(single_dp_performance / num_gpus * seq_length, 2) + if single_dp_performance + else None + ) for bw in comm_bw: if amp: comm_time = model_size_b * (4 + 2) / bw @@ -916,6 +925,17 @@ def launch(): cur_cfg[ f"bw_{bw}_{tuner_cfg['metric_cfg']['name']}" ] = multi_dp_performace + cur_cfg[ + f"unified_bw_{bw}_{tuner_cfg['metric_cfg']['name']}" + ] = ( + round(multi_dp_performace / num_gpus * seq_length, 2) + if multi_dp_performace + else None + ) + if recorder.additional_metric_key is None: + recorder.additional_metric_key = ( + f"unified_bw_{bw}_{tuner_cfg['metric_cfg']['name']}" + ) error_info = None cur_cfg["has_error"] = has_error From 0c5a31c68a7b05e40c0f40488ade4d292f544e77 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Mon, 25 Dec 2023 20:01:21 +0800 Subject: [PATCH 029/146] fix layer_norm decomp (#60303) --- paddle/fluid/primitive/base/decomp_trans.cc | 44 +++++++++++--------- paddle/fluid/primitive/composite/composite.h | 24 ++++++----- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/paddle/fluid/primitive/base/decomp_trans.cc b/paddle/fluid/primitive/base/decomp_trans.cc index 20d2011bb0182..6dde6c8b94002 100644 --- a/paddle/fluid/primitive/base/decomp_trans.cc +++ b/paddle/fluid/primitive/base/decomp_trans.cc @@ -130,26 +130,27 @@ void DecompProgram::check_decomp_outputs( PADDLE_ENFORCE( !paddle::dialect::IsEmptyValue(orig_outs[i]), paddle::platform::errors::PreconditionNotMet( - "[Prim] For op %s, its origin output index %d is invalid", + "[Prim] For op %s, its origin %d-index output is invalid", op_name, i)); PADDLE_ENFORCE( !paddle::dialect::IsEmptyValue(decomp_outs[i]), paddle::platform::errors::PreconditionNotMet( - "[Prim] For op %s, its decomp output index %d is invalid", + "[Prim] For op %s, its decomp %d-index output is invalid", op_name, i)); auto orig_dtype = GetValueDtype(orig_outs[i]); auto decomp_dtype = GetValueDtype(decomp_outs[i]); - PADDLE_ENFORCE( - orig_dtype == decomp_dtype, - paddle::platform::errors::PreconditionNotMet( - "[Prim] For op %s, its origin output dtype %s is not equal to " - "decomp output dtype %s ", - op_name, - orig_dtype, - decomp_dtype)); + PADDLE_ENFORCE(orig_dtype == decomp_dtype, + paddle::platform::errors::PreconditionNotMet( + "[Prim] For op %s, its origin %d-index output dtype " + "%s is not equal to " + "decomp output dtype %s ", + op_name, + i, + orig_dtype, + decomp_dtype)); auto orig_dim = GetValueDims(orig_outs[i]); auto decomp_dim = GetValueDims(decomp_outs[i]); @@ -158,23 +159,26 @@ void DecompProgram::check_decomp_outputs( LOG(WARNING) << "[Prim] Decomp op does not support dynamic shape -1, but got " "shape [" - << orig_dim << "] in output of origin op " << op_name; + << orig_dim << "] in " << i << "-index output of origin op " + << op_name; } if (find_value(common::vectorize(decomp_dim), -1)) { LOG(WARNING) << "[Prim] Decomp op does not support dynamic shape -1, but got " "shape [" - << decomp_dim << "] in output of decomp op " << op_name; + << decomp_dim << "] in " << i << "-index output of decomp op " + << op_name; } - PADDLE_ENFORCE( - orig_dim == decomp_dim, - paddle::platform::errors::PreconditionNotMet( - "[Prim] For op %s, its origin output shape [%s] is not equal to " - "decomp output shape [%s] ", - op_name, - orig_dim, - decomp_dim)); + PADDLE_ENFORCE(orig_dim == decomp_dim, + paddle::platform::errors::PreconditionNotMet( + "[Prim] For op %s, its origin %d-index output shape " + "[%s] is not equal to " + "decomp output shape [%s] ", + op_name, + i, + orig_dim, + decomp_dim)); } } return; diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index dc31b32c6dd0b..1ab1f33f4f5f6 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -365,15 +365,19 @@ std::tuple layer_norm_decomp( auto scale_ptr = scale.get_ptr(); auto bias_ptr = bias.get_ptr(); - std::vector slice_shape; - for (int64_t i = begin_norm_axis; i < static_cast(x_dim.size()); - i++) { - slice_shape.push_back(x_dim[i]); + std::vector slice_shape_l; + std::vector slice_shape_r; + for (int64_t i = 0; i < static_cast(x_dim.size()); i++) { + if (i < begin_norm_axis) { + slice_shape_l.push_back(x_dim[i]); + } else { + slice_shape_r.push_back(x_dim[i]); + } } Tensor scale_cast; if (scale_ptr) { - if (slice_shape != scale_ptr->shape()) { - scale_cast = reshape(*scale_ptr, slice_shape); + if (slice_shape_r != scale_ptr->shape()) { + scale_cast = reshape(*scale_ptr, slice_shape_r); } else { scale_cast = *scale_ptr; } @@ -384,8 +388,8 @@ std::tuple layer_norm_decomp( } Tensor bias_cast; if (bias_ptr) { - if (slice_shape != bias_ptr->shape()) { - bias_cast = reshape(*bias_ptr, slice_shape); + if (slice_shape_r != bias_ptr->shape()) { + bias_cast = reshape(*bias_ptr, slice_shape_r); } else { bias_cast = *bias_ptr; } @@ -394,8 +398,8 @@ std::tuple layer_norm_decomp( } out = out + bias_cast; } - mean_ = reshape(mean_, std::vector({-1})); - variance = reshape(variance, std::vector({-1})); + mean_ = reshape(mean_, slice_shape_l); + variance = reshape(variance, slice_shape_l); // same as LayerNormInferMeta // x: float32 --> out: float32, mean: float32, variance: float32 From d70b7ad141142776fb6af2cf216a56a6cb920e6f Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:19:21 +0800 Subject: [PATCH 030/146] =?UTF-8?q?=E3=80=90pir=E3=80=91support=20test=5Fw?= =?UTF-8?q?hile=5Fapi=20case=20with=20same=20loop=5Fvars=20and=20extra=20i?= =?UTF-8?q?nput=20(#60262)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimize backward * [PIR] add vjp interface for while op * [PIR] fix ci error. * modify while stopgradient * merge * modify while grad bug * modify while grad op * modify * increment vp * [PIR] add get_used_external_value interface for block. * while case * delete print * delete print * Update python/paddle/autograd/ir_backward.py * [PIR] add unit_test for get_used_external_value * modify while_loop * code_style * modofy ci bug * modify while api * modify ci * Update python/paddle/autograd/ir_backward.py --------- Co-authored-by: winter-wang <1030748926@qq.com> --- python/paddle/autograd/ir_backward.py | 164 +++++++++++++++----------- test/ir/pir/test_while_api.py | 39 +++++- 2 files changed, 130 insertions(+), 73 deletions(-) diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index f804b3cdc9171..a8ac124e6e2b1 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -80,8 +80,12 @@ def append_full_like(float_value, copy_value, value, state, backward_ops): def get_real_op_inputs(op): - if op.name() in ["pd_op.if", "pd_op.while"]: + if op.name() == "pd_op.if": return get_used_external_value(op) + elif op.name() == "pd_op.while": + return op.operands_source() + get_used_external_value( + op.as_while_op().body() + ) else: return op.operands_source() @@ -373,7 +377,7 @@ def append_backward_ops( no_grad_set, backward_ops, state, - bwd_block_argument_to_value_map, + bwd_value_to_block_argument_map=ValueDict(), ): ''' add grad_op in order of topological inverse sort @@ -415,12 +419,10 @@ def append_backward_ops( else continue to next op. ''' - def return_value_to_copyvalue_map( - value, control_flow_value_to_copyvalue_map - ): + def return_map_value(value, map): output = value - while output in control_flow_value_to_copyvalue_map: - output = control_flow_value_to_copyvalue_map[output] + while output in map: + output = map[output] return output def append_add_n(value): @@ -446,9 +448,7 @@ def make_output_with_output_grad(op): output_grads = [] for i, value in enumerate(op.results()): new_value = [ - return_value_to_copyvalue_map( - value, control_flow_value_to_copyvalue_map - ) + return_map_value(value, control_flow_value_to_copyvalue_map) ] while value in state.inside_value_to_outside_value_map: value = state.inside_value_to_outside_value_map[value] @@ -496,33 +496,11 @@ def make_output_with_output_grad(op): outputs.append(new_value) grad_value = state.value_to_valuegrad[value][0] output_grads.append( - bwd_block_argument_to_value_map[grad_value[0]] - if grad_value[0] in bwd_block_argument_to_value_map + [bwd_value_to_block_argument_map[grad_value[0]]] + if grad_value[0] in bwd_value_to_block_argument_map else grad_value ) - if op.name() == "pd_op.while": - for i, input in enumerate(get_real_op_inputs(op)): - if i <= len(op.results()): - continue - if ( - input in state.value_to_valuegrad - and len(state.value_to_valuegrad[input]) > 1 - ): - append_add_n(input) - - if ( - input not in state.value_to_valuegrad - or state.value_to_valuegrad[input] == [] - ): - append_full_like(0.0, input, input, state, backward_ops) - - grad_value = state.value_to_valuegrad[input][0] - output_grads.append( - bwd_block_argument_to_value_map[grad_value[0]] - if grad_value[0] in bwd_block_argument_to_value_map - else grad_value - ) return zero_flag, outputs, output_grads def get_grad_semantic_info(op): @@ -555,7 +533,7 @@ def make_input_with_input_stopgradient(op): tmp_input = [] for tmp in input.get_defining_op().operands_source(): tmp_input.append( - return_value_to_copyvalue_map( + return_map_value( tmp, control_flow_value_to_copyvalue_map ) ) @@ -563,7 +541,7 @@ def make_input_with_input_stopgradient(op): inputs.append(tmp_input) else: tmp_input = [ - return_value_to_copyvalue_map( + return_map_value( input, control_flow_value_to_copyvalue_map ) ] @@ -584,9 +562,7 @@ def make_input_with_input_stopgradient(op): ) else: tmp_input = [ - return_value_to_copyvalue_map( - input, control_flow_value_to_copyvalue_map - ) + return_map_value(input, control_flow_value_to_copyvalue_map) ] inputs.append(tmp_input) @@ -597,13 +573,13 @@ def make_input_with_input_stopgradient(op): return inputs, input_grad_stopgradients - def update_input_grad_map(op, input_grads, origin_inputs): + def update_input_grad_map(op, input_grads, all_inputs): + _, fwd_value_to_block_argument_map = argument_to_value(op) i = 0 - for input, grad_semantic in zip( - origin_inputs, get_grad_semantic_info(op) - ): + for input, grad_semantic in zip(all_inputs, get_grad_semantic_info(op)): if not grad_semantic: continue + if ( input.get_defining_op() is not None and input.get_defining_op().name() == "builtin.combine" @@ -615,9 +591,6 @@ def update_input_grad_map(op, input_grads, origin_inputs): ) else: input_grad = input_grads[i] - if input in fwd_block_argument_to_value_map: - input = fwd_block_argument_to_value_map[input] - if isinstance(input_grad, list): state.value_to_valuegrad[input].append(input_grad) else: @@ -625,27 +598,29 @@ def update_input_grad_map(op, input_grads, origin_inputs): i += 1 def append_yield( - block, base_op, base_grad_op, base_inputs, base_inputs_grad + block, + base_op, + base_grad_op, + base_inputs, + base_inputs_grad, ): + ( + fwd_block_argument_to_value_map, + fwd_value_to_block_argument_map, + ) = argument_to_value(base_op) with block: inputs_grad = [] if base_op.name() == "pd_op.while": new_cond = paddle.base.libpaddle.pir.cf_has_elements(base_op) inputs_grad.append(new_cond) - output_grads = base_grad_op.operands_source() - # output_grad = [new_cond, loop_vars(fwd_output_grad)] - # base_inputs = [cond, loop_vars(fwd_input)] - assert len(output_grads) <= len( - base_inputs - ), "while op's inputs size should less than while_grad op's inputs size" - - else: - output_grads = [None] * len(base_inputs) + for idx in range(len(base_inputs[: base_op.num_operands()])): + operands = base_inputs[idx] + if operands in fwd_value_to_block_argument_map: + operands = fwd_value_to_block_argument_map[operands] + base_inputs[idx] = operands - for value, value_grad, output_grad in zip( - base_inputs, base_inputs_grad, output_grads - ): + for value, value_grad in zip(base_inputs, base_inputs_grad): if value_grad is None: continue @@ -659,12 +634,6 @@ def append_yield( value_grad = append_full_like( 0.0, value, value, state, backward_ops ) - - # if base_op.name() == "pd_op.while": - # input_grad = paddle.add( - # output_grad, state.value_to_valuegrad[value][0][0] - # ) - # else: input_grad = state.value_to_valuegrad[value][0][0] inputs_grad.append(input_grad) @@ -672,6 +641,9 @@ def append_yield( paddle.base.libpaddle.pir.cf_yield(inputs_grad) def argument_to_value(while_op): + if while_op.name() != "pd_op.while": + return ValueDict(), ValueDict() + assert len(while_op.as_while_op().block_arguments()) + 1 == len( while_op.operands_source() ), "while op's block_arguments size + 1 should same to whiel op's operands_source" @@ -682,7 +654,7 @@ def argument_to_value(while_op): while_op.operands_source()[1:], ): arg_to_value_map[arg] = value - value_to_arg_map[value] = [arg] + value_to_arg_map[value] = arg return arg_to_value_map, value_to_arg_map # there are four patterns: @@ -695,9 +667,6 @@ def argument_to_value(while_op): # tuple_push value to pop value control_flow_value_to_copyvalue_map = ValueDict() control_flow_copyvalue_to_value_map = ValueDict() - # fwd_whileop's blockargument to fwd_whileop's input value - fwd_block_argument_to_value_map = ValueDict() - # bwd_whileop's input value to bwd_whileop's blockargument if ( len(effective_forward_ops) > 1 @@ -708,7 +677,6 @@ def argument_to_value(while_op): # while op yield [cond, loop_vars], # but outputs only has loop_vars. inside_outputs = yield_op.operands_source()[1:] - fwd_block_argument_to_value_map, _ = argument_to_value(base_op) else: inside_outputs = yield_op.operands_source() @@ -776,8 +744,8 @@ def argument_to_value(while_op): if len(output_grads) == 0 or all(zero_flag): continue - if op.name() in ["pd_op.if", "pd_op.while"]: - origin_inputs = get_used_external_value(op) + if op.name() == "pd_op.if": + origin_inputs = get_real_op_inputs(op) for sub_block in op.blocks(): build_pipe_for_block(sub_block) with dynamic_shape_prim_vjp_guard(op, inputs): @@ -820,6 +788,58 @@ def argument_to_value(while_op): ) # update input_grad map update_input_grad_map(op, input_grads, origin_inputs) + elif op.name() == "pd_op.while": + origin_inputs = get_real_op_inputs(op) + # prepare while[cond, loop_vars, other_input] other_input's grad + while_block = op.as_while_op().body() + sub_state = state.copy(while_block) + for i, input in enumerate( + get_used_external_value(while_block) + ): + append_full_like( + 0.0, input, input, sub_state, backward_ops + ) + grad_value = sub_state.value_to_valuegrad[input][0] + output_grads.append( + [bwd_value_to_block_argument_map[grad_value[0]]] + if grad_value[0] + in bwd_value_to_block_argument_map + else grad_value + ) + + build_pipe_for_block(while_block) + with dynamic_shape_prim_vjp_guard(op, inputs): + input_grads = paddle.framework.core.call_vjp( + op, + inputs, + outputs, + output_grads, + input_grad_stopgradients, + ) + grad_op = bwd_block.ops[-1] + bwd_ops = [grad_op] + + # update grad_op structure + ( + _, + sub_bwd_value_to_block_argument_map, + ) = argument_to_value(grad_op) + while_grad_block = grad_op.as_while_op().body() + sub_backward_ops = [] + append_backward_ops( + op, + [input[0] for input in inputs], + [input_grad[0] for input_grad in input_grads], + while_block, + while_grad_block, + while_block.ops, + no_grad_set, + sub_backward_ops, + sub_state, + sub_bwd_value_to_block_argument_map, + ) + # update input_grad map + update_input_grad_map(op, input_grads, origin_inputs) else: # create grad_op before_ops_num = len(bwd_block.ops) diff --git a/test/ir/pir/test_while_api.py b/test/ir/pir/test_while_api.py index 9165cec5ac077..45b68b9fcf125 100644 --- a/test/ir/pir/test_while_api.py +++ b/test/ir/pir/test_while_api.py @@ -152,7 +152,7 @@ def body2(i, j, ten): class TestBuildModuleWithWhile2Op(unittest.TestCase): - def test_add_n_program(self): + def test_backward(self): main_program = paddle.static.Program() with paddle.pir.core.program_guard(main_program): i = paddle.full( @@ -189,6 +189,43 @@ def test_add_n_program(self): "cf.has_elements", ) + def test_backward_with_loop_var_same_to_extral_var(self): + main_program = paddle.static.Program() + with paddle.pir.core.program_guard(main_program): + i = paddle.full(shape=[1], fill_value=0) + x = paddle.full(shape=[1], fill_value=5) + y = paddle.full(shape=[1], fill_value=10) + i.stop_gradient = False + x.stop_gradient = False + y.stop_gradient = False + new_i, new_x = paddle.static.nn.while_loop( + lambda p, q: p < q, lambda p, q: [p + y, q + x], [i, x] + ) + + out = new_i - new_x + grad_outs = grad(out, [i, x, y]) + + self.assertEqual( + grad_outs[0].get_defining_op().name(), "pd_op.while" + ) + self.assertEqual( + grad_outs[1].get_defining_op().name(), "pd_op.add_n" + ) + self.assertEqual( + grad_outs[2].get_defining_op().name(), "pd_op.while" + ) + self.assertEqual( + main_program.global_block() + .ops[-3] + .as_while_op() + .body() + .ops[-1] + .operand_source(1) + .get_defining_op() + .name(), + "pd_op.add_grad", + ) + if __name__ == "__main__": unittest.main() From 92dd1825609f67c8852acde7d2c4634a588571d7 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:48:40 +0800 Subject: [PATCH 031/146] [PIr] Fix has elementwise instruction exe bug (#60312) * fix * fix --- .../instruction/has_elements_instruction.cc | 2 +- .../instruction/instruction_base.cc | 2 ++ .../instruction/tuple_pop_instruction.cc | 1 + .../instruction/tuple_push_instruction.cc | 3 ++- .../instruction/while_instruction.cc | 2 ++ test/legacy_test/test_while_loop_op.py | 27 +++++++------------ 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc b/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc index 819a6cea2efe6..958daf2239eaf 100644 --- a/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc @@ -56,7 +56,7 @@ HasElementsInstruction::HasElementsInstruction( void HasElementsInstruction::Run() { VLOG(6) << "run has_elements instruction"; - *has_elements_ = stack_element_var_array_->empty(); + *has_elements_ = !stack_element_var_array_->empty(); } } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/instruction_base.cc b/paddle/fluid/framework/new_executor/instruction/instruction_base.cc index 74cc8acf8e476..78796b3e4192f 100644 --- a/paddle/fluid/framework/new_executor/instruction/instruction_base.cc +++ b/paddle/fluid/framework/new_executor/instruction/instruction_base.cc @@ -174,6 +174,8 @@ static double GetDenseTensorEleSum(const Scope& scope, const phi::dtype::float16* data = cpu_tensor.data(); sum += static_cast(data[0]); + } else if (cpu_tensor.dtype() == phi::DataType::BOOL) { + sum += static_cast(cpu_tensor.data()[i]); } else { return std::numeric_limits::quiet_NaN(); } diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc b/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc index 74cf9d9ce6456..d86ee66a9d1e9 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc @@ -69,6 +69,7 @@ static std::stack PopElements(VariableRefArray* var_array, for (uint64_t i = 0; i < num; i++) { rtn.push(var_array->back()); var_array->pop_back(); + VLOG(6) << "tuple pop " << rtn.top() << " from : " << var_array; } return rtn; } diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc b/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc index edb8374da8f8a..78a174ba1c977 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc @@ -86,7 +86,8 @@ void TuplePushInstruction::Run() { DeepCopyVariable(var, copy_var, value_exe_info_, stack_size); VLOG(10) << "done DeepCopyVariable " << new_name; stack_element_var_array_->emplace_back(copy_var); - VLOG(6) << "push back var: " << new_name << "[" << copy_var << "]"; + VLOG(6) << "push back var: " << new_name << "[" << copy_var << "]" + << "to: " << stack_element_var_array_; } } } diff --git a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc b/paddle/fluid/framework/new_executor/instruction/while_instruction.cc index 2f3787118d2e4..f2a6e92e2f4b2 100644 --- a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/while_instruction.cc @@ -205,6 +205,8 @@ void WhileInstruction::ShareDatasToOutputs() { if (out_var->IsType()) { outputs_[i]->GetMutable()->ShareDataWith( out_var->Get()); + VLOG(6) << "share data from " << out_var_name << "[" << out_var << "]" + << " -> " << i << " output[" << outputs_[i] << "]"; } else if (out_var->IsType()) { const auto& inner_array = out_var->Get(); auto* output_array = outputs_[i]->GetMutable(); diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index 0926048d37e50..4675c2b30e73e 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -315,13 +315,9 @@ def body(i, x): np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) - # TODO(zhangbo): Support while grad exe for pir - # @test_with_pir_api + @test_with_pir_api def test_while_loop_backward2(self): - def cond1(i, x): - return i < 2 - - def cond2(i, x): + def cond(i, x): return i < 3 def body(i, x): @@ -339,7 +335,7 @@ def body(i, x): x.stop_gradient = False x.persistable = True - out = paddle.static.nn.while_loop(cond1, body, [i, x]) + out = paddle.static.nn.while_loop(cond, body, [i, x]) mean = paddle.mean(out[1]) grad_list = append_backward(mean) @@ -353,21 +349,17 @@ def body(i, x): feed_i = np.ones(1).astype('float32') feed_x = np.ones(1).astype('float32') data = np.asarray([2]).astype('float32') - ans = np.asarray([1]).astype('float32') - x1_grad = np.asarray([1]).astype('float32') i_grad = np.asarray([3]).astype('float32') x_grad = np.asarray([2]).astype('float32') if paddle.framework.in_pir_mode(): + fetch_list = [out[1]] for p, g in grad_list: - if p == i: - di = g - if p == x: - dx = g + fetch_list.append(g) res = exe.run( main_program, feed={'i': feed_i, 'x': feed_x}, - fetch_list=[out[1], di, dx], + fetch_list=fetch_list, ) else: res = exe.run( @@ -375,10 +367,9 @@ def body(i, x): feed={'i': feed_i, 'x': feed_x}, fetch_list=[out[1].name, i.grad_name, x.grad_name], ) - - np.testing.assert_allclose(np.asarray(res[0]), ans, rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[1]), ans, rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[2]), ans, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[2]), x_grad, rtol=1e-05) class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): From 0852820026412984e11856bc3f561e0bc055cd6c Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Tue, 26 Dec 2023 10:12:20 +0800 Subject: [PATCH 032/146] [Shape]Remove LOG(Waring) in ShapeOptimPass (#60249) * [Shape]Remove LOG(Waring) in ShapeOptimPass * fix VLOG --- paddle/fluid/pir/transforms/shape_optimization_pass.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index cf43113e1db14..1ab54ad2133a2 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -111,8 +111,8 @@ class InferSymbolicShapePass : public pir::Pass { if (it != infer_sym_shape_map.end()) { it->second(op, shape_analysis_); } else { - LOG(WARNING) << "[" << op.name() - << "] is not supported for infer_symbolic_shape pass."; + VLOG(3) << "[" << op.name() + << "] is not supported for infer_symbolic_shape pass."; } } @@ -206,7 +206,7 @@ struct ExpandShapeOfOpPattern : public OpRewritePattern { bool MatchAndRewrite(shape::ShapeOfOp op, PatternRewriter& rewriter) const override { - VLOG(3) << "Apply ExpandShapeOfOpPattern..."; + VLOG(5) << "Apply ExpandShapeOfOpPattern..."; auto type = op.out().type().dyn_cast(); @@ -762,7 +762,7 @@ class ShapeOptimizationPass : public pir::Pass { ShapeOptimizationPass() : pir::Pass("shape_optimization_pass", 0) {} void Run(pir::Operation* op) override { - VLOG(0) << "===================== ShapeOptimizationPass Run start... " + VLOG(5) << "===================== ShapeOptimizationPass Run start... " "============================="; auto module_op = op->dyn_cast(); IR_ENFORCE(module_op, "ShapeOptimizationPass should run on module op."); @@ -777,7 +777,7 @@ class ShapeOptimizationPass : public pir::Pass { // if (!OptimizeShapeComputation(module_op, runner)) { // return; // } - VLOG(0) << "===================== ShapeOptimizationPass Run End. " + VLOG(5) << "===================== ShapeOptimizationPass Run End. " "============================="; } From 2ca590bed3c69470f88211c8963b82889ee7bca6 Mon Sep 17 00:00:00 2001 From: "Zhang,Lirong" <56445728+zhanglirong1999@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:27:31 +0800 Subject: [PATCH 033/146] [oneDNN] fix gru error for make foramt any (#60298) --- paddle/phi/kernels/fusion/onednn/fusion_gru_kernel.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/phi/kernels/fusion/onednn/fusion_gru_kernel.cc b/paddle/phi/kernels/fusion/onednn/fusion_gru_kernel.cc index 8e7fe89ec1f7f..c11ce83458f18 100644 --- a/paddle/phi/kernels/fusion/onednn/fusion_gru_kernel.cc +++ b/paddle/phi/kernels/fusion/onednn/fusion_gru_kernel.cc @@ -108,7 +108,7 @@ class GRUOneDNNHandler // Create memory descriptors auto input_md = OneDNNMemDesc( - {Ti, N, IC}, OneDNNGetDataType(), OneDNNMemoryFormat::ntc); + {Ti, N, IC}, OneDNNGetDataType(), OneDNNMemoryFormat::any); auto weight_x_md = OneDNNMemDesc({L, D, IC, G, OC}, weights_dt, OneDNNMemoryFormat::any); auto weight_h_md = @@ -116,7 +116,7 @@ class GRUOneDNNHandler auto bias_md = OneDNNMemDesc( {L, D, G, OC}, OneDNNGetDataType(), OneDNNMemoryFormat::ldgo); auto hidden_md = OneDNNMemDesc( - {Ti, N, OC}, OneDNNGetDataType(), OneDNNMemoryFormat::ntc); + {Ti, N, OC}, OneDNNGetDataType(), OneDNNMemoryFormat::any); auto h0_md = OneDNNMemDesc( {L, D, N, OC}, OneDNNGetDataType(), OneDNNMemoryFormat::ldnc); From 9b40c67f672f130d22336de40851187443fb926f Mon Sep 17 00:00:00 2001 From: Haohongxiang <86215757+haohongxiang@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:32:43 +0800 Subject: [PATCH 034/146] [develop] Support release_grads in Pipeline Parallel and Sharding stage1 v1/v2 (#59739) --- .../framework/distributed_strategy.proto | 1 + .../dygraph_sharding_optimizer.py | 16 +- .../fleet/meta_parallel/pipeline_parallel.py | 20 +- .../fleet/utils/tensor_fusion_helper.py | 177 +++++++++++++++--- 4 files changed, 183 insertions(+), 31 deletions(-) diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto index 062a816c058c2..2042a313c41e6 100755 --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -81,6 +81,7 @@ message PpConfig { optional bool enable_timer = 3 [ default = false ]; optional bool sharding_comm_overlap = 4 [ default = false ]; optional bool profiling = 5 [ default = false ]; + optional bool release_gradients = 6 [ default = false ]; } message DygraphShardingConfig { diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py index 04ff96087a808..b6b4c3c01842f 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py @@ -535,6 +535,7 @@ def __init__(self, optimizer, hcg): # Setting pipeline parallelism overlap self.pp_overlap = pp_config.sharding_comm_overlap + self.pp_release_grads = pp_config.release_gradients # TODO(liuzhenhai):support it latter assert not self.comm_overlap, "not supported yet" @@ -604,6 +605,7 @@ def _build_comm_buffers(self, acc_steps, group_size=256 * 1024 * 1024): comm_group, acc_steps, act=HOOK_ACTION.REDUCE_SCATTER, + release_grads=self.pp_release_grads, ) self._comm_buffer_list.append(buffer) @@ -611,7 +613,8 @@ def clear_grad(self, set_to_zero=True): """ should clear grad for all parameters in model """ - assert set_to_zero, "should not erase grad buffer" + if not self.pp_release_grads: + assert set_to_zero, "should not erase grad buffer" def clear_grad_func(p): if hasattr(p, "main_grad") and p.main_grad is not None: @@ -634,6 +637,10 @@ def clear_grad_func(p): for p in self._parameter_list: clear_grad_func(p) + if self.pp_release_grads and not self.pp_overlap: + for comm_buffer in self._comm_buffer_list: + comm_buffer._clear_grad_storage() + def filter_parameters(self, parameter_list, hcg): parameter_list = [ self._slice_params[param.name] for param in parameter_list @@ -648,6 +655,10 @@ def reduce_gradients(self, parameter_list, hcg): logger.debug("sharding start gradients sync") with framework.no_grad(): for comm_buffer in self._comm_buffer_list: + if self.pp_release_grads and comm_buffer.grad_storage is None: + for param in comm_buffer.params: + comm_buffer._copy_grad_to_buffer(param) + if not self.comm_overlap: comm_buffer._comm_grads() @@ -713,6 +724,9 @@ def _assign_slice_grad(self): for param in comm_buffer.params: assert param.name in self._slice_params slice_param = self._slice_params[param.name] + if self.pp_release_grads and hasattr(slice_param, "main_grad"): + assert not slice_param.main_grad._is_initialized() + del slice_param.main_grad comm_buffer.assign_slice_grad(param, slice_param) assert param_num == len(self._parameter_list) diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index a152c919c882e..c7605b18c30ae 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -208,6 +208,9 @@ def __init__(self, layers, hcg, strategy): self._enable_timer = self._strategy.hybrid_configs[ "pp_configs" ].enable_timer + self._release_gradients = self._strategy.hybrid_configs[ + "pp_configs" + ].release_gradients self._sharding_split_param = self._strategy.hybrid_configs[ "sharding_configs" @@ -372,7 +375,13 @@ def fused_gradient( for group_idx, parameters in var_groups.items(): buffer = FusedCommBuffer( - group_idx, parameters, comm_group, acc_steps, act, dst + group_idx, + parameters, + comm_group, + acc_steps, + act, + dst, + release_grads=self._release_gradients, ) self._chunk_2_comm_buffers[chunk_idx].append(buffer) @@ -862,7 +871,14 @@ def _optimizer_step(self): else: self.optimizer.step() - self.optimizer.clear_grad() + if self._release_gradients: + self.optimizer.clear_grad(set_to_zero=False) + for _, buffers in self._chunk_2_comm_buffers.items(): + for buffer in buffers: + buffer._clear_grad_storage() + else: + self.optimizer.clear_grad() + if self.lr_scheduler: self.lr_scheduler.step() diff --git a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py index 6f24c0e07865b..ba2f4fb2cc016 100644 --- a/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py +++ b/python/paddle/distributed/fleet/utils/tensor_fusion_helper.py @@ -57,7 +57,11 @@ def assign_group_by_size(parameters, group_size=128 * 1024 * 1024): def flatten_dense_tensors( - parameters, use_main_grad=False, fuse_param=True, warp_buffer=False + parameters, + use_main_grad=False, + fuse_param=True, + warp_buffer=False, + release_grad=False, ): from paddle.distributed.fleet.meta_parallel.sharding.group_sharded_storage import ( GradStorage, @@ -66,6 +70,7 @@ def flatten_dense_tensors( _buffer_size = 0 _param2align = {} + _param2offset = {} dtype = parameters[0].dtype for param in parameters: @@ -74,9 +79,13 @@ def flatten_dense_tensors( remaining = size % alignment["gpu"] ali = 0 if remaining == 0 else alignment["gpu"] - remaining align_ = ali // align[dtype] + _param2offset[param.name] = _buffer_size _buffer_size += np.prod(param.shape) + align_ _param2align[param.name] = align_ + if release_grad: + return None, _buffer_size, _param2offset + if fuse_param: param_storage = ParamStorage( size=_buffer_size, dtype=dtype, device="gpu" @@ -101,6 +110,7 @@ def flatten_dense_tensors( param_storage.warp_buffer() grad_storage.warp_buffer() + outputs = (grad_storage,) if fuse_param: if not use_main_grad: # param_storage --> grad_storage @@ -108,9 +118,15 @@ def flatten_dense_tensors( else: param_storage.buffer.main_grad = grad_storage.buffer param_storage.buffer.stop_gradient = False - return param_storage, grad_storage - else: - return grad_storage + outputs = (param_storage,) + outputs + + if release_grad: + outputs = outputs + ( + _buffer_size, + _param2offset, + ) + + return outputs def bw_hook_func(buffer, param): @@ -132,6 +148,7 @@ def __init__( sharding_degree, rank, use_main_grad=False, + release_grad=False, ): self._param = param self._param_buffer = param_buffer @@ -140,6 +157,8 @@ def __init__( self._padded_size = padded_size self._sharding_degree = sharding_degree self._rank = rank + self._use_main_grad = use_main_grad + self._release_grad = release_grad shard_size = param_buffer._numel() // sharding_degree rank_begin = rank * shard_size rank_end = rank_begin + shard_size @@ -150,20 +169,32 @@ def __init__( self._param_end = param_end self._slice_grad = None - if param_begin < param_end: - self._slice_grad = grad_buffer._slice(param_begin, param_end) - # share grad buffer - tmp_grad = grad_buffer._slice(self._index, self._index + param._numel()) - tmp_grad.get_tensor()._set_dims(param.shape) - if not use_main_grad: - self._param._copy_gradient_from(tmp_grad) - else: - self._param.main_grad = tmp_grad + if not self._release_grad: + self._link_grad_to_buffer() # share param buffer self._share_param_buffer() + def _slice_grad_from_buffer(self): + assert self._grad_buffer is not None + if self._param_begin < self._param_end: + self._slice_grad = self._grad_buffer._slice( + self._param_begin, self._param_end + ) + tmp_grad = self._grad_buffer._slice( + self._index, self._index + self._param._numel() + ) + return tmp_grad + + def _link_grad_to_buffer(self): + tmp_grad = self._slice_grad_from_buffer() + tmp_grad.get_tensor()._set_dims(self._param.shape) + if not self._use_main_grad: + self._param._copy_gradient_from(tmp_grad) + else: + self._param.main_grad = tmp_grad + def _share_param_buffer(self): param_shape = self._param.shape stop_gradient = self._param.stop_gradient @@ -208,9 +239,18 @@ def assign_slice_grad(self, slice_param): else: assert slice_param.grad._is_shared_buffer_with(slice_grad) + def _reset_grad_buffer(self): + if self._slice_grad is not None: + self._slice_grad._clear_dataptr() + self._slice_grad = None + + if self._grad_buffer is not None: + self._grad_buffer._clear_dataptr() + self._grad_buffer = None + def build_reduce_scatter_buffer( - parameters, sharding_degree, rank, use_main_grad=False + parameters, sharding_degree, rank, use_main_grad=False, release_grad=False ): total_buffer_size = 0 param2index = {} @@ -231,7 +271,11 @@ def get_padded_size(param): grad_dtype = paddle.float32 if use_main_grad else dtype param_buffer = paddle.zeros(shape=[total_buffer_size], dtype=dtype) - grad_buffer = paddle.zeros(shape=[total_buffer_size], dtype=grad_dtype) + grad_buffer = ( + paddle.zeros(shape=[total_buffer_size], dtype=grad_dtype) + if not release_grad + else None + ) sharding_grad_view = {} for param in parameters: @@ -245,10 +289,11 @@ def get_padded_size(param): sharding_degree, rank, use_main_grad, + release_grad, ) # hack main_grad sharding_grad_view[param.name] = grad_view - return sharding_grad_view, param_buffer, grad_buffer + return sharding_grad_view, total_buffer_size, param_buffer, grad_buffer def get_grad_address(param, use_main_grad): @@ -274,6 +319,7 @@ def __init__( use_main_grad=None, fuse_param=False, scale_after_comm=True, + release_grads=False, ): self._id = id self._params = params @@ -281,6 +327,11 @@ def __init__( self._comm_group = comm_group self._scale_after_comm = scale_after_comm self._fuse_param = fuse_param + self._release_grads = release_grads + + assert not ( + self._fuse_param and self._release_grads + ), "It's not supported when using fuse_param and release_grad at the same time." self.use_main_grad = ( use_main_grad @@ -289,6 +340,9 @@ def __init__( ) self._task = None + self._dtype = ( + paddle.float32 if self.use_main_grad else self._params[0].dtype + ) self._params_step_dict = {} self._params_checked_in = 0 self._grads_to_addr = {} @@ -317,6 +371,22 @@ def __init__( ) self.param_storage = self.param_storage.buffer self.grad_storage = self.grad_storage.buffer + elif self._release_grads: + self.param_storage = None + ( + grad_storage, + self.buffer_size, + self.param2offset, + ) = flatten_dense_tensors( + self._params, + use_main_grad=self.use_main_grad, + fuse_param=False, + warp_buffer=False, + release_grad=True, + ) + self.grad_storage = ( + None if grad_storage is None else grad_storage.buffer + ) else: self.param_storage = None self.grad_storage = flatten_dense_tensors( @@ -324,11 +394,12 @@ def __init__( use_main_grad=self.use_main_grad, fuse_param=False, warp_buffer=False, - ).buffer + )[0].buffer else: assert not self._fuse_param, "not supported" ( self._sharding_param_grad_view, + self.buffer_size, self.param_storage, self.grad_storage, ) = build_reduce_scatter_buffer( @@ -336,10 +407,12 @@ def __init__( self._comm_group.nranks, self._comm_group.rank, use_main_grad=self.use_main_grad, + release_grad=self._release_grads, ) # hack, for parameter sync in dygraph sharding optimizer after step self._params[0].comm_buffer_ref = weakref.ref(self) - self._record_addr() + if not self._release_grads: + self._record_addr() def _record_addr(self): for param in self._params: @@ -347,10 +420,56 @@ def _record_addr(self): param, self.use_main_grad ) + def _clear_grad_storage(self): + self.grad_storage._clear_dataptr() + self.grad_storage = None + if self._act == HOOK_ACTION.REDUCE_SCATTER: + for param in self._params: + self._sharding_param_grad_view[param.name]._reset_grad_buffer() + def _init_step_dict(self): for p in self._params: self._params_step_dict[p.name] = 0 + def _copy_grad_to_buffer(self, param): + if self._params_step_dict[param.name] > 0: + return + + if self.grad_storage is None: + assert self._params_step_dict[param.name] == 0 + + self.grad_storage = paddle.zeros( + [self.buffer_size], dtype=self._dtype + ) + + if self._act == HOOK_ACTION.REDUCE_SCATTER: + self._sharding_param_grad_view[ + param.name + ]._grad_buffer = self.grad_storage + tmp_var = self._sharding_param_grad_view[ + param.name + ]._slice_grad_from_buffer() + else: + grad_end = self.param2offset[param.name] + np.prod(param.shape) + assert grad_end <= self.buffer_size + tmp_var = self.grad_storage._slice( + self.param2offset[param.name], grad_end + ) + + grad_var = param.main_grad if self.use_main_grad else param.grad + grad_var.stop_gradient = True + grad_var.flatten_() + + tmp_var.add_(grad_var) + tmp_var.get_tensor()._set_dims(param.shape) + + if self.use_main_grad: + param.main_grad._clear() + param.main_grad = tmp_var + param.main_grad.name = "main_grad@" + param.name + else: + param._copy_gradient_from(tmp_var) + def _reset_params_checked_in(self): self._task = None self._init_step_dict() @@ -366,16 +485,18 @@ def _all_params_checked_in(self): def add_grad(self, param, use_comm=True): assert param.name in self._params_step_dict - current_ptr = get_grad_address(param, self.use_main_grad) - - if self._grads_to_addr[param.name] != current_ptr: - raise ValueError( - "The address of the grad/main_grad of the param has been changed during training, " - "which is not allowed for dp/sharding overlap with pp. " - "This may be caused by some non-inplace operations on the grad/main_grad. Here are some examples: " - "1. The grad/main_grad of the param is changed by other operations, such as: clear_grad, " - "2. Using non-inplace operations on the grad/main_grad, such as: add, sub, mul, div, etc. " - ) + if not self._release_grads: + current_ptr = get_grad_address(param, self.use_main_grad) + if self._grads_to_addr[param.name] != current_ptr: + raise ValueError( + "The address of the grad/main_grad of the param has been changed during training, " + "which is not allowed for dp/sharding overlap with pp. " + "This may be caused by some non-inplace operations on the grad/main_grad. Here are some examples: " + "1. The grad/main_grad of the param is changed by other operations, such as: clear_grad, " + "2. Using non-inplace operations on the grad/main_grad, such as: add, sub, mul, div, etc. " + ) + else: + self._copy_grad_to_buffer(param) self._params_step_dict[param.name] += 1 From 868f2a85889120977b1d837fc21b1d03aa056287 Mon Sep 17 00:00:00 2001 From: cyberslack_lee Date: Tue, 26 Dec 2023 11:04:07 +0800 Subject: [PATCH 035/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.51?= =?UTF-8?q?=E3=80=81175=E3=80=91=20Migrate=20paddle.vision.ops.decode=5Fjp?= =?UTF-8?q?eg=EF=BC=8Cpaddle.vision.ops.read=5Ffile=20into=20pir=20(#58955?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paddle/phi/infermeta/nullary.cc | 1 + python/paddle/vision/ops.py | 16 ++++++--- test/legacy_test/test_read_file.py | 56 +++++++++++++++++++----------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/paddle/phi/infermeta/nullary.cc b/paddle/phi/infermeta/nullary.cc index 9dd2181ceeb87..d1bd204a682d9 100644 --- a/paddle/phi/infermeta/nullary.cc +++ b/paddle/phi/infermeta/nullary.cc @@ -316,6 +316,7 @@ void TriuIndicesInferMeta( void ReadFileInferMeta(const std::string& filename, MetaTensor* out) { auto out_dims = std::vector(1, -1); out->set_dims(phi::make_ddim(out_dims)); + out->set_dtype(phi::DataType::UINT8); } } // namespace phi diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 4fdcaf32991b7..6395151376b37 100755 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -15,13 +15,18 @@ import numpy as np import paddle -from paddle import _C_ops, _legacy_C_ops +from paddle import _C_ops from paddle.tensor.math import _add_with_axis from paddle.utils import convert_to_list from ..base import core from ..base.data_feeder import check_type, check_variable_and_dtype -from ..base.framework import Variable, in_dygraph_mode, in_dynamic_or_pir_mode +from ..base.framework import ( + Variable, + convert_np_dtype_to_dtype_, + in_dygraph_mode, + in_dynamic_or_pir_mode, +) from ..base.layer_helper import LayerHelper from ..framework import _current_expected_place from ..nn import BatchNorm2D, Conv2D, Layer, ReLU, Sequential @@ -1319,8 +1324,9 @@ def read_file(filename, name=None): [142773] """ - if in_dygraph_mode(): - return _legacy_C_ops.read_file('filename', filename) + attr_dtype = convert_np_dtype_to_dtype_('uint8') + if in_dynamic_or_pir_mode(): + return _C_ops.read_file(filename, attr_dtype, paddle.CPUPlace()) else: inputs = {} attrs = {'filename': filename} @@ -1368,7 +1374,7 @@ def decode_jpeg(x, mode='unchanged', name=None): >>> print(img.shape) [3, 400, 300] """ - if in_dygraph_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.decode_jpeg(x, mode, _current_expected_place()) else: inputs = {'X': x} diff --git a/test/legacy_test/test_read_file.py b/test/legacy_test/test_read_file.py index 354553dca7d33..c6fc9befd3aba 100644 --- a/test/legacy_test/test_read_file.py +++ b/test/legacy_test/test_read_file.py @@ -18,12 +18,14 @@ import cv2 import numpy as np +from op_test import paddle_static_guard import paddle +from paddle.pir_utils import test_with_pir_api from paddle.vision.ops import decode_jpeg, read_file -class TestReadFile(unittest.TestCase): +class TestReadFileWithDynamic(unittest.TestCase): def setUp(self): fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8') self.temp_dir = tempfile.TemporaryDirectory() @@ -33,38 +35,50 @@ def setUp(self): def tearDown(self): self.temp_dir.cleanup() - def read_file_decode_jpeg(self): + def test_read_file_decode_jpeg_dynamic(self): if not paddle.is_compiled_with_cuda(): return - img_bytes = read_file(self.img_path) - img = decode_jpeg(img_bytes, mode='gray') img = decode_jpeg(img_bytes, mode='rgb') - img = decode_jpeg(img_bytes) - img_cv2 = cv2.imread(self.img_path) - if paddle.in_dynamic_mode(): - np.testing.assert_equal(img.shape, img_cv2.transpose(2, 0, 1).shape) - else: - place = paddle.CUDAPlace(0) - exe = paddle.static.Executor(place) - exe.run(paddle.static.default_startup_program()) - out = exe.run( - paddle.static.default_main_program(), fetch_list=[img] - ) + np.testing.assert_equal(img.shape, img_cv2.transpose(2, 0, 1).shape) - np.testing.assert_equal( - out[0].shape, img_cv2.transpose(2, 0, 1).shape - ) - def test_read_file_decode_jpeg_dynamic(self): - self.read_file_decode_jpeg() +class TestReadFileWithStatic(unittest.TestCase): + def setUp(self): + fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8') + self.temp_dir = tempfile.TemporaryDirectory() + self.img_path = os.path.join(self.temp_dir.name, 'fake.jpg') + cv2.imwrite(self.img_path, fake_img) + + def tearDown(self): + self.temp_dir.cleanup() + @test_with_pir_api def test_read_file_decode_jpeg_static(self): paddle.enable_static() - self.read_file_decode_jpeg() + if not paddle.is_compiled_with_cuda(): + return + place = paddle.CUDAPlace(0) + with paddle_static_guard(): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + img_bytes = read_file(self.img_path) + img = decode_jpeg(img_bytes, mode='gray') + img = decode_jpeg(img_bytes, mode='rgb') + img = decode_jpeg(img_bytes) + img_cv2 = cv2.imread(self.img_path) + exe = paddle.static.Executor(place) + out = exe.run( + paddle.static.default_main_program(), fetch_list=[img] + ) + + np.testing.assert_equal( + out[0].shape, img_cv2.transpose(2, 0, 1).shape + ) paddle.disable_static() From 52e38507803a614e50b04b1c057df198844e4d00 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:20:17 +0800 Subject: [PATCH 036/146] [PIR][DynamicShape] fix some log level (#60326) change the use of 'LOG(Waring)' & 'VLOG(0)' to log level 3-5 --- paddle/fluid/pir/transforms/shape_optimization_pass.cc | 6 +++--- paddle/fluid/pybind/pir.cc | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index 1ab54ad2133a2..a7d32c6577906 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -513,7 +513,7 @@ void print_program(pir::ModuleOp m, std::string mgs) { print_stream << "\n\n"; m.program()->Print(print_stream); print_stream << "\n\n"; - VLOG(0) << "===================== " << mgs << "\n" << print_stream.str(); + VLOG(5) << "===================== " << mgs << "\n" << print_stream.str(); } bool IsShapeSpecialOp(const pir::Operation& op) { @@ -675,7 +675,7 @@ void InferSymbolicShapeReshape(pir::Operation* op, void debug_print_op_info( pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis = nullptr) { - VLOG(0) << op->name() << ", num_operands: " << op->num_operands(); + VLOG(5) << op->name() << ", num_operands: " << op->num_operands(); for (auto& rst : op->results()) { auto type = rst.type(); auto value_id = pir::GetValueId(&rst); @@ -694,7 +694,7 @@ void debug_print_op_info( } print_stream << "]\n"; } - VLOG(0) << print_stream.str(); + VLOG(5) << print_stream.str(); } } diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 74d2ae5992341..2af4d5eb55c02 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1572,7 +1572,6 @@ void ApplyPirPass(Program &forward_program) { // NOLINT : nullptr; pir::PassManager pass_manager(ctx); - VLOG(0) << "========= AddPass ==========="; pass_manager.AddPass(pir::CreateShapeOptimizationPass()); cinn::dialect::ir::PdOp2CinnOpConverter(&forward_program); From c9d7f8b257fb83121672c68b29d0c8016deded9b Mon Sep 17 00:00:00 2001 From: pangengzheng <117730991+pangengzheng@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:32:47 +0800 Subject: [PATCH 037/146] construct dtensor with dtensor_from_local api (#60206) --- python/paddle/distributed/auto_parallel/api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index b1ee251fd3522..bef09fa95fc8d 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -1309,15 +1309,13 @@ def build_distributed_tensor(local_tensor, dist_attr): ) else: raise ValueError(f"dim {dim} is not supported.") - # TODO(pangengzheng): construct dist_tensor with _dtensor_from_local api when it is ready. - global_tensor = paddle.zeros(global_shape, dtype=local_tensor.dtype) mesh = dist.ProcessMesh( np.array(dist_attr["process_group"]).reshape( dist_attr["process_shape"] ) ) placements = to_placements(dist_attr["dims_mapping"], mesh) - dist_tensor = dist.shard_tensor(global_tensor, mesh, placements) + dist_tensor = dtensor_from_local(local_tensor, mesh, placements) assert ( dist_tensor._local_value().shape == local_tensor.shape ), f"local tensor shape {dist_tensor._local_value().shape} not equal to local_tensor.shape:{local_tensor.shape}" From 7b5c0abc3b5c4088abbb5efba2c2fe0bb8f1a124 Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:43:59 +0800 Subject: [PATCH 038/146] [CodeStyle][ruff] fix `E226`, `NPY201` (#60245) * fix ruff preview * fix cast --- .../paddle/distributed/launch/controllers/collective.py | 4 +++- python/paddle/distributed/launch/main.py | 8 ++++---- python/paddle/hapi/callbacks.py | 4 ++-- .../jit/sot/opcode_translator/executor/opcode_executor.py | 2 +- .../paddle/static/quantization/quant2_int8_mkldnn_pass.py | 6 +++--- python/paddle/tensor/einsum.py | 6 +++--- test/custom_runtime/test_custom_cpu_to_static.py | 6 +++--- test/legacy_test/test_fill_constant_op.py | 4 ++-- test/legacy_test/test_seed_op.py | 5 ++++- test/xpu/test_increment_op_xpu.py | 2 +- tools/gen_ut_cmakelists.py | 4 ++-- 11 files changed, 28 insertions(+), 23 deletions(-) diff --git a/python/paddle/distributed/launch/controllers/collective.py b/python/paddle/distributed/launch/controllers/collective.py index 13d8ef403504a..ce69ec0a4c781 100644 --- a/python/paddle/distributed/launch/controllers/collective.py +++ b/python/paddle/distributed/launch/controllers/collective.py @@ -92,7 +92,9 @@ def _build_pod_with_args(self): ips = self.ctx.args.ips.split(',') job_endpoints = [ - f"{h}:{p+start_port}" for h in ips for p in range(self.pod.replicas) + f"{h}:{p + start_port}" + for h in ips + for p in range(self.pod.replicas) ] self.ctx.logger.debug(f"job endpoints: {job_endpoints}") diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index 457d095d5806e..0869ac7bbfcd9 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -576,10 +576,10 @@ def launch(): end_time = time.time() ctx.logger.info( - f"AtuoTuner for GBS search ends in {end_time-start_time}s." + f"AtuoTuner for GBS search ends in {end_time - start_time}s." ) logger.info( - f"AtuoTuner for GBS search ends in {end_time-start_time}s." + f"AtuoTuner for GBS search ends in {end_time - start_time}s." ) # build AutoTuner to get new config @@ -1118,8 +1118,8 @@ def launch(): assert best_cfg and best_cfg["time"] != -1 end_time = time.time() - ctx.logger.info(f"AutoTuner ended in {end_time-start_time}s.") - logger.info(f"AutoTuner ended in {end_time-start_time}s.") + ctx.logger.info(f"AutoTuner ended in {end_time - start_time}s.") + logger.info(f"AutoTuner ended in {end_time - start_time}s.") # launch best cfg # estimation search need not run best cfg if not tuner_cfg.get("run_best", True) or tuner_cfg["search_algo"].get( diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py index d2ed7238d52c4..dbb3f4e3840b4 100644 --- a/python/paddle/hapi/callbacks.py +++ b/python/paddle/hapi/callbacks.py @@ -1279,10 +1279,10 @@ def _reset(self): self.mode == 'auto' and 'acc' not in self.monitor ): self.monitor_op = lambda a, b: np.less(a, b - self.min_delta) - self.best = np.Inf + self.best = np.inf else: self.monitor_op = lambda a, b: np.greater(a, b + self.min_delta) - self.best = -np.Inf + self.best = -np.inf self.cooldown_counter = 0 self.wait = 0 diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index c7664d354d92e..17e74c9bfb0be 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -497,7 +497,7 @@ def error_message_summary(original_error: Exception) -> str: ) if current_line != -1: message_lines.append( - f"{indent} {lines[current_line-start].rstrip()}" + f"{indent} {lines[current_line - start].rstrip()}" ) error_message = traceback.format_exception_only( type(original_error), original_error diff --git a/python/paddle/static/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/static/quantization/quant2_int8_mkldnn_pass.py index cbb46200a6029..8e370dbf72918 100644 --- a/python/paddle/static/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/static/quantization/quant2_int8_mkldnn_pass.py @@ -203,7 +203,7 @@ def _gather_input_scales_from_fake(self, graph): scale = np.array( 1.0 / self._load_param(self._scope, scale_name)[0] ).astype(np.float64) - scale[scale == np.Inf] = 0.0 + scale[scale == np.inf] = 0.0 lod_tensor = self._convert_scale2tensor(scale) use_unsigned_int = False self._add_scale_for_vars( @@ -238,7 +238,7 @@ def _gather_output_scales_from_attr(self, graph): if attr_scale == 0.0: continue scale = np.array(1.0 / attr_scale).astype(np.float64) - scale[scale == np.Inf] = 0.0 + scale[scale == np.inf] = 0.0 scale_lod_tensor = self._convert_scale2tensor(scale) use_unsigned_int = False for output_name in op.op().outputs(): @@ -561,7 +561,7 @@ def _compute_var_scales(ops, w_name, axis): ), axis=axis, ) - scales[scales == np.Inf] = 0.0 + scales[scales == np.inf] = 0.0 lod_tensor = self._convert_scale2tensor(scales) use_unsigned_int = False diff --git a/python/paddle/tensor/einsum.py b/python/paddle/tensor/einsum.py index 9f26de2f216da..7b7af555cd04e 100644 --- a/python/paddle/tensor/einsum.py +++ b/python/paddle/tensor/einsum.py @@ -672,7 +672,7 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): if any(ax != dim for ax, dim in enumerate(view[:nout])): perm = [dim for dim in view if dim >= 0] if sorted(perm) != perm: - varname = f'op{nop-1}' + varname = f'op{nop - 1}' step = transpose, [varname], varname, perm plan.add_step(step) dim = 0 @@ -684,14 +684,14 @@ def plan_einsum(operands, g_view, g_shape, g_supports, g_count, n_bcast): if d == -1: unsqueeze_dims.append(ax) if unsqueeze_dims: - varname = f'op{nop-1}' + varname = f'op{nop - 1}' step = unsqueeze, [varname], varname, unsqueeze_dims plan.add_step(step) squeeze_dims = [dim for dim in view[nout:] if dim != -1] if squeeze_dims: # plan_reduce(plan, nop-1, reduce_dims, keepdim=False) - varname = f'op{nop-1}' + varname = f'op{nop - 1}' step = squeeze, [varname], varname, squeeze_dims plan.add_step(step) diff --git a/test/custom_runtime/test_custom_cpu_to_static.py b/test/custom_runtime/test_custom_cpu_to_static.py index 60ba27004afbd..b365f8ab39811 100644 --- a/test/custom_runtime/test_custom_cpu_to_static.py +++ b/test/custom_runtime/test_custom_cpu_to_static.py @@ -42,7 +42,7 @@ def train_func_base(epoch_id, train_loader, model, cost, optimizer): ) epoch_end = time.time() print( - f"Epoch ID: {epoch_id+1}, FP32 train epoch time: {(epoch_end - epoch_start) * 1000} ms" + f"Epoch ID: {epoch_id + 1}, FP32 train epoch time: {(epoch_end - epoch_start) * 1000} ms" ) @@ -75,7 +75,7 @@ def train_func_ampo1(epoch_id, train_loader, model, cost, optimizer, scaler): ) epoch_end = time.time() print( - f"Epoch ID: {epoch_id+1}, AMPO1 train epoch time: {(epoch_end - epoch_start) * 1000} ms" + f"Epoch ID: {epoch_id + 1}, AMPO1 train epoch time: {(epoch_end - epoch_start) * 1000} ms" ) @@ -96,7 +96,7 @@ def test_func(epoch_id, test_loader, model, cost): avg_acc[1].append(acc_top5.numpy()) model.train() print( - f"Epoch ID: {epoch_id+1}, Top1 accurary: {np.array(avg_acc[0]).mean()}, Top5 accurary: {np.array(avg_acc[1]).mean()}" + f"Epoch ID: {epoch_id + 1}, Top1 accurary: {np.array(avg_acc[0]).mean()}, Top5 accurary: {np.array(avg_acc[1]).mean()}" ) diff --git a/test/legacy_test/test_fill_constant_op.py b/test/legacy_test/test_fill_constant_op.py index d898567291a99..a660921d32ffe 100644 --- a/test/legacy_test/test_fill_constant_op.py +++ b/test/legacy_test/test_fill_constant_op.py @@ -413,9 +413,9 @@ def test_inf(self): def test_ninf(self): with base.dygraph.guard(): - res = paddle.tensor.fill_constant([1], 'float32', np.NINF) + res = paddle.tensor.fill_constant([1], 'float32', -np.inf) self.assertTrue(np.isinf(res.numpy().item(0))) - self.assertEqual(np.NINF, res.numpy().item(0)) + self.assertEqual(-np.inf, res.numpy().item(0)) class TestFillConstantOpError(unittest.TestCase): diff --git a/test/legacy_test/test_seed_op.py b/test/legacy_test/test_seed_op.py index a15b8099a5cf3..6dace0a0c6103 100644 --- a/test/legacy_test/test_seed_op.py +++ b/test/legacy_test/test_seed_op.py @@ -69,7 +69,10 @@ def check_static_result(self, place): (out1,) = exe.run( static.default_main_program(), fetch_list=res_list ) - self.assertEqual(out1, np.cast['int32'](self.rng1.random())) + self.assertEqual( + out1, + np.asarray(self.rng1.random()).astype(np.int32), + ) def test_static(self): for place in self.places: diff --git a/test/xpu/test_increment_op_xpu.py b/test/xpu/test_increment_op_xpu.py index 5ef28f30b44a9..6bf870e8980db 100644 --- a/test/xpu/test_increment_op_xpu.py +++ b/test/xpu/test_increment_op_xpu.py @@ -41,7 +41,7 @@ def setUp(self): self.initTestCase() x = np.random.uniform(-100, 100, [1]).astype(self.dtype) - output = x + np.cast[self.dtype](self.step) + output = x + np.asarray(self.step).astype(self.dtype) output = output.astype(self.dtype) self.inputs = {'X': x} diff --git a/tools/gen_ut_cmakelists.py b/tools/gen_ut_cmakelists.py index c1796ca4bb52e..9749fe320b414 100644 --- a/tools/gen_ut_cmakelists.py +++ b/tools/gen_ut_cmakelists.py @@ -275,7 +275,7 @@ def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name): assert re.compile("^test_[0-9a-zA-Z_]+").search( name ), f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong - at line {k-1}, in file {cmake_file_name} + at line {k - 1}, in file {cmake_file_name} ''' self.gset_port(name, port) @@ -559,7 +559,7 @@ def _gen_cmakelists(self, current_work_dir, depth=0): print("===============PARSE LINE ERRORS OCCUR==========") print(e) print(f"[ERROR FILE]: {current_work_dir}/testslist.csv") - print(f"[ERROR LINE {i+1}]: {line.strip()}") + print(f"[ERROR LINE {i + 1}]: {line.strip()}") sys.exit(1) for sub in sub_dirs: From a134bf901b33f3d5037346ecfb4385808f056b33 Mon Sep 17 00:00:00 2001 From: LoneRanger <836253168@qq.com> Date: Tue, 26 Dec 2023 11:46:22 +0800 Subject: [PATCH 039/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.51,?= =?UTF-8?q?=2060=E3=80=91Migrate=20some=20ops=20into=20pir=20=20(#58684)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/vision/ops.py | 2 +- test/legacy_test/test_detection.py | 24 +++++++++++++++---- .../test_distribute_fpn_proposals_op.py | 4 +++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 6395151376b37..ee34bf5e0ee69 100755 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -1231,7 +1231,7 @@ def distribute_fpn_proposals( num_lvl < 100 ), "Only support max to 100 levels, (max_level - min_level + 1 < 100)" - if in_dygraph_mode(): + if in_dynamic_or_pir_mode(): assert ( rois_num is not None ), "rois_num should not be None in dygraph mode." diff --git a/test/legacy_test/test_detection.py b/test/legacy_test/test_detection.py index 99e0836c4c1f5..49ee53ea6fb87 100644 --- a/test/legacy_test/test_detection.py +++ b/test/legacy_test/test_detection.py @@ -22,6 +22,7 @@ from paddle.base import core from paddle.base.dygraph import base as imperative_base from paddle.base.framework import Program, program_guard +from paddle.pir_utils import test_with_pir_api paddle.enable_static() @@ -66,9 +67,9 @@ def get_static_graph_result( self, feed, fetch_list, with_lod=False, force_to_use_cpu=False ): exe = base.Executor(self._get_place(force_to_use_cpu)) - exe.run(base.default_startup_program()) + exe.run(paddle.static.default_startup_program()) return exe.run( - base.default_main_program(), + paddle.static.default_main_program(), feed=feed, fetch_list=fetch_list, return_numpy=(not with_lod), @@ -183,9 +184,7 @@ def test_multiclass_nms2(self): class TestDistributeFpnProposals(LayerTest): - def test_distribute_fpn_proposals(self): - rois_np = np.random.rand(10, 4).astype('float32') - rois_num_np = np.array([4, 6]).astype('int32') + def static_distribute_fpn_proposals(self, rois_np, rois_num_np): with self.static_graph(): rois = paddle.static.data( name='rois', shape=[10, 4], dtype='float32' @@ -216,7 +215,9 @@ def test_distribute_fpn_proposals(self): output_np = np.array(output) if len(output_np) > 0: output_stat_np.append(output_np) + return output_stat_np + def dynamic_distribute_fpn_proposals(self, rois_np, rois_num_np): with self.dynamic_graph(): rois_dy = imperative_base.to_variable(rois_np) rois_num_dy = imperative_base.to_variable(rois_num_np) @@ -239,6 +240,19 @@ def test_distribute_fpn_proposals(self): output_np = output.numpy() if len(output_np) > 0: output_dy_np.append(output_np) + return output_dy_np + + @test_with_pir_api + def test_distribute_fpn_proposals(self): + rois_np = np.random.rand(10, 4).astype('float32') + rois_num_np = np.array([4, 6]).astype('int32') + + output_stat_np = self.static_distribute_fpn_proposals( + rois_np, rois_num_np + ) + output_dy_np = self.dynamic_distribute_fpn_proposals( + rois_np, rois_num_np + ) for res_stat, res_dy in zip(output_stat_np, output_dy_np): np.testing.assert_array_equal(res_stat, res_dy) diff --git a/test/legacy_test/test_distribute_fpn_proposals_op.py b/test/legacy_test/test_distribute_fpn_proposals_op.py index 956c435298781..3947a657306f4 100644 --- a/test/legacy_test/test_distribute_fpn_proposals_op.py +++ b/test/legacy_test/test_distribute_fpn_proposals_op.py @@ -18,6 +18,7 @@ from op_test import OpTest import paddle +from paddle.pir_utils import test_with_pir_api def distribute_fpn_proposals_wrapper( @@ -142,7 +143,7 @@ def setUp(self): self.set_data() def test_check_output(self): - self.check_output(check_dygraph=False) + self.check_output(check_dygraph=False, check_pir=False) class TestDistributeFPNProposalsOpWithRoisNum(TestDistributeFPNProposalsOp): @@ -200,6 +201,7 @@ def setUp(self): self.rois_np = np.random.rand(10, 4).astype('float32') self.rois_num_np = np.array([4, 6]).astype('int32') + @test_with_pir_api def test_dygraph_with_static(self): paddle.enable_static() rois = paddle.static.data(name='rois', shape=[10, 4], dtype='float32') From 23808ae56d39c201486a7aab9c29e2d96e476202 Mon Sep 17 00:00:00 2001 From: zhangyuqin1998 <75946871+zhangyuqin1998@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:48:41 +0800 Subject: [PATCH 040/146] [pir]Adding Set and Get attr method for pir passes (#60253) * [pir]Adding Set and Get method for pir passes * fix codestyle * Update constant_folding_pass.cc --- .../fluid/inference/api/analysis_predictor.cc | 15 +++- .../pir/transforms/constant_folding_pass.cc | 30 +++++--- .../pir/transforms/constant_folding_pass.h | 3 +- .../params_sync_among_devices_pass.cc | 29 +++++-- .../params_sync_among_devices_pass.h | 3 +- paddle/pir/pass/pass.h | 77 +++++++++++++++++++ .../drr_attention_fuse_test.cc | 9 ++- .../pattern_rewrite/pattern_rewrite_test.cc | 34 ++++++-- 8 files changed, 167 insertions(+), 33 deletions(-) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 78a38ef175ef1..c70ef74e94baa 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -806,9 +806,18 @@ bool AnalysisPredictor::PrepareExecutor() { //----------------------------------------------------------------------------------------------// // Basic pass required by the framework - gpu_pm.AddPass( - ::pir::CreateParamsSyncAmongDevicesPass(place_, sub_scope_)); - gpu_pm.AddPass(::pir::CreateConstantFoldingPass(place_, sub_scope_)); + auto params_sync_among_devices_pass = + ::pir::CreateParamsSyncAmongDevicesPass(); + params_sync_among_devices_pass->SetNotOwned(pir::kPlaceAttr, &place_); + params_sync_among_devices_pass->SetNotOwned(pir::kParamScopeAttr, + sub_scope_); + + auto constant_folding_pass = ::pir::CreateConstantFoldingPass(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place_); + constant_folding_pass->SetNotOwned(pir::kParamScopeAttr, sub_scope_); + + gpu_pm.AddPass(std::move(params_sync_among_devices_pass)); + gpu_pm.AddPass(std::move(constant_folding_pass)); gpu_pm.AddPass(::pir::CreateDeadCodeEliminationPass()); gpu_pm.AddPass(::pir::CreateReplaceFetchWithShadowOutputPass()); //----------------------------------------------------------------------------------------------// diff --git a/paddle/fluid/pir/transforms/constant_folding_pass.cc b/paddle/fluid/pir/transforms/constant_folding_pass.cc index 8f25f5c0f25fb..553cf3967dd68 100644 --- a/paddle/fluid/pir/transforms/constant_folding_pass.cc +++ b/paddle/fluid/pir/transforms/constant_folding_pass.cc @@ -309,15 +309,28 @@ class ConstantFoldingPattern : public pir::RewritePattern { class ConstantFoldingPass : public pir::Pass { public: - explicit ConstantFoldingPass(const phi::Place& place, - paddle::framework::Scope* scope) - : pir::Pass("constant_folding_pass", 1), place_(place), scope_(scope) { - PADDLE_ENFORCE_NOT_NULL( - scope_, phi::errors::InvalidArgument("scope can not be nullptr")); - } + ConstantFoldingPass() + : pir::Pass("constant_folding_pass", 1), + place_(phi::CPUPlace{}), + scope_(nullptr) {} private: bool Initialize(pir::IrContext* context) override { + IR_ENFORCE(Has(pir::kPlaceAttr), + "Pass initialize failed." + "When using ConstantFoldingPass, place attribute is required!" + "Use Set method to set the place attribute."); + IR_ENFORCE(Has(pir::kParamScopeAttr), + "Pass initialize failed." + "When using ConstantFoldingPass, scope attribute is required!" + "Use Set method to set the scope attribute."); + + place_ = Get(pir::kPlaceAttr); + scope_ = &Get(pir::kParamScopeAttr); + + PADDLE_ENFORCE_NOT_NULL( + scope_, phi::errors::InvalidArgument("scope can not be nullptr")); + pir::RewritePatternSet ps(context); ps.Add( context, &counter_, place_, scope_, &exe_config_, &deleted_vars_); @@ -354,9 +367,8 @@ class ConstantFoldingPass : public pir::Pass { namespace pir { -std::unique_ptr CreateConstantFoldingPass( - const phi::Place& place, paddle::framework::Scope* scope) { - return std::make_unique(place, scope); +std::unique_ptr CreateConstantFoldingPass() { + return std::make_unique(); } } // namespace pir diff --git a/paddle/fluid/pir/transforms/constant_folding_pass.h b/paddle/fluid/pir/transforms/constant_folding_pass.h index 0939ee589d448..eff49e88898e4 100644 --- a/paddle/fluid/pir/transforms/constant_folding_pass.h +++ b/paddle/fluid/pir/transforms/constant_folding_pass.h @@ -28,7 +28,6 @@ namespace pir { class Pass; -IR_API std::unique_ptr CreateConstantFoldingPass( - const phi::Place& place, paddle::framework::Scope* scope); +IR_API std::unique_ptr CreateConstantFoldingPass(); } // namespace pir diff --git a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc index 4a673022fc0a5..794b5bfe29484 100644 --- a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc +++ b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc @@ -33,16 +33,30 @@ namespace { class ParamsSyncAmongDevicesPass : public pir::Pass { public: - ParamsSyncAmongDevicesPass(const phi::Place& place, - paddle::framework::Scope* scope) - : pir::Pass("params_sync_among_devices_pass", 0), - place_(place), - scope_(scope) { + ParamsSyncAmongDevicesPass() + : pir::Pass("params_sync_among_devices_pass", 0) {} + + bool Initialize(pir::IrContext* context) override { + IR_ENFORCE(Has(pir::kPlaceAttr), + "Pass initialize failed." + "When using ConstantFoldingPass, place attribute is required!" + "Use Set method to set the place attribute."); + IR_ENFORCE(Has(pir::kParamScopeAttr), + "Pass initialize failed." + "When using ConstantFoldingPass, scope attribute is required!" + "Use Set method to set the scope attribute."); + + place_ = Get(pir::kPlaceAttr); + scope_ = &Get(pir::kParamScopeAttr); + + PADDLE_ENFORCE_NOT_NULL( + scope_, phi::errors::InvalidArgument("scope can not be nullptr")); PADDLE_ENFORCE( paddle::platform::is_gpu_place(place_) || paddle::platform::is_cpu_place(place_), phi::errors::PreconditionNotMet( "params_sync_among_devices_pass should run on cpu or gpu.")); + return true; } void Run(pir::Operation* op) override { @@ -94,9 +108,8 @@ class ParamsSyncAmongDevicesPass : public pir::Pass { namespace pir { -std::unique_ptr CreateParamsSyncAmongDevicesPass( - const phi::Place& place, paddle::framework::Scope* scope) { - return std::make_unique(place, scope); +std::unique_ptr CreateParamsSyncAmongDevicesPass() { + return std::make_unique(); } } // namespace pir diff --git a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.h b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.h index c046176bc1995..06a0830c1e56f 100644 --- a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.h +++ b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.h @@ -28,7 +28,6 @@ namespace pir { class Pass; -IR_API std::unique_ptr CreateParamsSyncAmongDevicesPass( - const phi::Place& place, paddle::framework::Scope* scope); +IR_API std::unique_ptr CreateParamsSyncAmongDevicesPass(); } // namespace pir diff --git a/paddle/pir/pass/pass.h b/paddle/pir/pass/pass.h index 14ead7e0c499f..a8a1d15345ae3 100644 --- a/paddle/pir/pass/pass.h +++ b/paddle/pir/pass/pass.h @@ -14,10 +14,13 @@ #pragma once +#include #include #include +#include #include +#include "paddle/common/enforce.h" #include "paddle/pir/core/builtin_op.h" #include "paddle/pir/pass/analysis_manager.h" #include "paddle/pir/pattern_rewrite/pattern_rewrite_driver.h" @@ -68,6 +71,9 @@ struct PassInfo { } // namespace detail +static const char kParamScopeAttr[] = "__param_scope__"; +static const char kPlaceAttr[] = "__place__"; + /// We can access pass only from PassManager. class IR_API Pass { public: @@ -82,6 +88,74 @@ class IR_API Pass { const detail::PassInfo& pass_info() const { return pass_info_; } + // Get a reference to the attributed previously set. + template + AttrType& Get(const std::string& attr_name) const { + IR_ENFORCE(attrs_.find(attr_name) != attrs_.end(), + "Attribute %s not registered for pass.", + attr_name); + try { + return *std::any_cast(attrs_.at(attr_name)); + } catch (std::bad_any_cast&) { + auto TypeToString = [](const std::type_info& info) -> std::string { + if (std::type_index(info) == std::type_index(typeid(bool*))) { + return "bool"; + } else if (std::type_index(info) == std::type_index(typeid(int*))) { + return "int"; + } else if (std::type_index(info) == + std::type_index(typeid(const int*))) { + return "const int"; + } else if (std::type_index(info) == + std::type_index(typeid(std::string*))) { + return "std::string"; + } + return info.name(); + }; + + IR_THROW("Invalid type for attritube %s, expected: %s, actual: %s.", + attr_name, + TypeToString(typeid(AttrType*)), + TypeToString(attrs_.at(attr_name).type())); + } + } + + bool Has(const std::string& attr_name) const { + return attrs_.count(attr_name) > 0; + } + + void Erase(const std::string& attr_name) { + if (!Has(attr_name)) { + return; + } + if (attr_dels_.find(attr_name) != attr_dels_.end()) { + attr_dels_[attr_name](); + attr_dels_.erase(attr_name); + } + attrs_.erase(attr_name); + } + + // Set a pointer to the attribute. Pass takes ownership of the attribute. + template + void Set(const std::string& attr_name, AttrType* attr) { + VLOG(3) << "Setting the attribute " << attr_name << " for the pass " + << name(); + attrs_[attr_name] = attr; + attr_dels_[attr_name] = [attr, attr_name]() { + VLOG(8) << "deleting " << attr_name; + delete attr; + }; + } + + // Set a pointer to the attribute. Pass doesn't take ownership. Caller + // should delete the attribute. + template + void SetNotOwned(const std::string& attr_name, AttrType* attr) { + IR_ENFORCE(0 == attrs_.count(attr_name), + "Attribute %s already set in the pass.", + attr_name); + attrs_[attr_name] = attr; + } + protected: virtual void Run(Operation* op) = 0; @@ -108,6 +182,9 @@ class IR_API Pass { friend class PassManager; friend class detail::PassAdaptor; + + std::unordered_map attrs_; + std::unordered_map> attr_dels_; }; class PatternRewritePass : public Pass { diff --git a/test/cpp/pir/pattern_rewrite/drr_attention_fuse_test.cc b/test/cpp/pir/pattern_rewrite/drr_attention_fuse_test.cc index 4fabb500bea5b..8485f493c794c 100644 --- a/test/cpp/pir/pattern_rewrite/drr_attention_fuse_test.cc +++ b/test/cpp/pir/pattern_rewrite/drr_attention_fuse_test.cc @@ -145,8 +145,13 @@ TEST(DrrTest, AttentionFuse) { pir::PassManager pm(ctx); pm.AddPass(pir::CreateAttentionFusePass()); - paddle::framework::Scope scope; - pm.AddPass(pir::CreateConstantFoldingPass(phi::CPUPlace{}, &scope)); + std::unique_ptr constant_folding_pass = + pir::CreateConstantFoldingPass(); + phi::Place place = phi::CPUPlace(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place); + constant_folding_pass->Set(pir::kParamScopeAttr, + new paddle::framework::Scope()); + pm.AddPass(std::move(constant_folding_pass)); pm.EnableIRPrinting(); CHECK_EQ(pm.Run(&program), true); diff --git a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc index de91323b041eb..93156a9d697ce 100644 --- a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc +++ b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc @@ -402,8 +402,13 @@ TEST(pattern_rewrite, Patterns) { pm.AddPass(pir::CreateConv2dBnFusePass()); pm.AddPass(pir::CreateConv2dAddActFusePass()); pm.AddPass(pir::CreateConv2dAddFusePass()); - paddle::framework::Scope scope; - pm.AddPass(pir::CreateConstantFoldingPass(phi::CPUPlace{}, &scope)); + std::unique_ptr constant_folding_pass = + pir::CreateConstantFoldingPass(); + phi::Place place = phi::CPUPlace(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place); + constant_folding_pass->Set(pir::kParamScopeAttr, + new paddle::framework::Scope()); + pm.AddPass(std::move(constant_folding_pass)); pm.AddPass(pir::CreateDeadCodeEliminationPass()); // pm.EnablePassTiming(); pm.EnableIRPrinting(); @@ -475,7 +480,12 @@ TEST(constant_folding, ConstantFolding) { BuildConstantFoldingProgram(&program, ctx, &scope); pir::PassManager pm(ctx); - pm.AddPass(pir::CreateConstantFoldingPass(phi::CPUPlace{}, &scope)); + std::unique_ptr constant_folding_pass = + pir::CreateConstantFoldingPass(); + phi::Place place = phi::CPUPlace(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place); + constant_folding_pass->SetNotOwned(pir::kParamScopeAttr, &scope); + pm.AddPass(std::move(constant_folding_pass)); pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.EnableIRPrinting(); @@ -537,8 +547,13 @@ TEST(constant_folding, ConstantFolding_Combine) { BuildConcatProgram(&program, ctx); pir::PassManager pm(ctx); - paddle::framework::Scope scope; - pm.AddPass(pir::CreateConstantFoldingPass(phi::CPUPlace{}, &scope)); + std::unique_ptr constant_folding_pass = + pir::CreateConstantFoldingPass(); + phi::Place place = phi::CPUPlace(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place); + constant_folding_pass->Set(pir::kParamScopeAttr, + new paddle::framework::Scope()); + pm.AddPass(std::move(constant_folding_pass)); pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.EnableIRPrinting(); @@ -573,8 +588,13 @@ TEST(constant_folding, ConstantFolding_MultiOutput) { BuildMultiOutputProgram(&program, ctx); pir::PassManager pm(ctx); - paddle::framework::Scope scope; - pm.AddPass(pir::CreateConstantFoldingPass(phi::CPUPlace{}, &scope)); + std::unique_ptr constant_folding_pass = + pir::CreateConstantFoldingPass(); + phi::Place place = phi::CPUPlace(); + constant_folding_pass->SetNotOwned(pir::kPlaceAttr, &place); + constant_folding_pass->Set(pir::kParamScopeAttr, + new paddle::framework::Scope()); + pm.AddPass(std::move(constant_folding_pass)); pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.EnableIRPrinting(); From a726569128c33fcc854b76dff39f7136640cd16a Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Tue, 26 Dec 2023 14:23:27 +0800 Subject: [PATCH 041/146] [Dy2St] Enable `test_lstm` in PIR mode (#60343) --- test/dygraph_to_static/test_lstm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/dygraph_to_static/test_lstm.py b/test/dygraph_to_static/test_lstm.py index 41f473b2f89d4..990dab4d3f21f 100644 --- a/test/dygraph_to_static/test_lstm.py +++ b/test/dygraph_to_static/test_lstm.py @@ -60,14 +60,14 @@ def tearDown(self): def run_lstm(self, to_static): with enable_to_static_guard(to_static): - paddle.static.default_main_program().random_seed = 1001 - paddle.static.default_startup_program().random_seed = 1001 + paddle.seed(1001) net = paddle.jit.to_static(Net(12, 2)) x = paddle.zeros((2, 10, 12)) y = net(x) return y.numpy() + @test_legacy_and_pt_and_pir def test_lstm_to_static(self): dygraph_out = self.run_lstm(to_static=False) static_out = self.run_lstm(to_static=True) From 3d2b2deb14d19f0627c8423bd3deb6984a807ce8 Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Tue, 26 Dec 2023 14:29:06 +0800 Subject: [PATCH 042/146] fix the limitation of fthenb schedule (#60134) (#60315) --- .../fleet/meta_parallel/pipeline_parallel.py | 10 +++------- python/paddle/distributed/fleet/model.py | 13 ++++++------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py index c7605b18c30ae..962df0c43ae72 100644 --- a/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py +++ b/python/paddle/distributed/fleet/meta_parallel/pipeline_parallel.py @@ -1566,13 +1566,9 @@ def forward_backward_pipeline( self._forward_only = forward_only assert ( - self.accumulate_steps >= self.num_stages - ), "accumulate_steps({}) should be larger than num_stages({}) for pipeline with interleave".format( - self.accumulate_steps, self.num_stages - ) - assert ( - self.accumulate_steps < 2 * self.num_stages - ), "accumulate_steps({}) should be smaller than 2 * num_stages({}) for pipeline with interleave".format( + self.accumulate_steps == self.num_stages + or self.accumulate_steps % self.num_stages != 0 + ), "accumulate_steps({}) and num_stages({}) should be a multiple or accumulate_steps % num_stages == 0".format( self.accumulate_steps, self.num_stages ) diff --git a/python/paddle/distributed/fleet/model.py b/python/paddle/distributed/fleet/model.py index c54b63ff17d9e..c83039f2fe6b3 100755 --- a/python/paddle/distributed/fleet/model.py +++ b/python/paddle/distributed/fleet/model.py @@ -162,17 +162,16 @@ def distributed_model(model): accumulate_steps = strategy.pipeline_configs['accumulate_steps'] pp_degree = fleet_env._hcg.get_pipe_parallel_world_size() if ( - accumulate_steps >= pp_degree - and accumulate_steps < pp_degree * 2 + accumulate_steps > pp_degree + and accumulate_steps % pp_degree == 0 ): - # NOTE(shenliang03): Hacky for unbalanced pipeline parallel with interleave - # Currently, we only support pp_degree <= accumulate_steps < 2 * pp_degree - model = PipelineParallelWithInterleaveFthenB( + # interleave pipeline + model = PipelineParallelWithInterleave( model, fleet_env._hcg, strategy=strategy ) else: - # interleave pipeline - model = PipelineParallelWithInterleave( + # NOTE(shenliang03): Hacky for unbalanced pipeline parallel with interleave + model = PipelineParallelWithInterleaveFthenB( model, fleet_env._hcg, strategy=strategy ) From 4f84c586162840dd5bcd59f6a3a9a9c0d4c3ff61 Mon Sep 17 00:00:00 2001 From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:58:54 +0800 Subject: [PATCH 043/146] =?UTF-8?q?=E3=80=90CMake=20opt=20No.10=E3=80=91Re?= =?UTF-8?q?move=20all=20paddle=5Ftest=20DEPS=20in=20test/cpp/pir/cinn/CMak?= =?UTF-8?q?eLists.txt=20(#60239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * rm deps in pir/cinn * fix * fix --------- Co-authored-by: 张春乔 <83450930+Liyulingyue@users.noreply.github.com> --- test/cpp/pir/cinn/CMakeLists.txt | 77 +++++--------------------------- 1 file changed, 12 insertions(+), 65 deletions(-) diff --git a/test/cpp/pir/cinn/CMakeLists.txt b/test/cpp/pir/cinn/CMakeLists.txt index 80b35ae6b589f..b38edcbb62041 100644 --- a/test/cpp/pir/cinn/CMakeLists.txt +++ b/test/cpp/pir/cinn/CMakeLists.txt @@ -1,82 +1,29 @@ add_subdirectory(adt) if(WITH_TESTING AND WITH_CINN) - paddle_test(test_pir_compiler SRCS pir_compiler_test.cc DEPS pir_compiler - cinn_runtime_dialect) + paddle_test(test_pir_compiler SRCS pir_compiler_test.cc) - paddle_test(test_jit_instruction SRCS jit_instruction_test.cc DEPS - cinn_runtime_dialect pir_compiler) + paddle_test(test_jit_instruction SRCS jit_instruction_test.cc) - paddle_test( - test_dialect_convert - SRCS - dialect_convert_test.cc - DEPS - drr - pd_to_cinn_pass - op_dialect_vjp - cinn_op_dialect - pir) + paddle_test(test_dialect_convert SRCS dialect_convert_test.cc) - paddle_test( - test_add_broadcast_to_elementwise - SRCS - add_broadcast_to_elementwise_test.cc - DEPS - drr - pd_to_cinn_pass - op_dialect_vjp - cinn_op_dialect - add_broadcast_to_elementwise_pass - pir) + paddle_test(test_add_broadcast_to_elementwise SRCS + add_broadcast_to_elementwise_test.cc) - paddle_test( - test_sub_graph_extract - SRCS - sub_graph_extract_test.cc - DEPS - drr - pd_to_cinn_pass - cinn_op_dialect - op_dialect_vjp - pir_transforms - pir) + paddle_test(test_sub_graph_extract SRCS sub_graph_extract_test.cc DEPS + pir_transforms) paddle_test(test_ir_op_fusion SRCS ir_op_fusion_test.cc) - paddle_test( - test_pir_all_path - SRCS - pir_all_path_test.cc - DEPS - pir_transforms - cinn_op_dialect - pd_to_cinn_pass - add_broadcast_to_elementwise_pass) + paddle_test(test_pir_all_path SRCS pir_all_path_test.cc) - paddle_test( - test_group_op - SRCS - group_op_test.cc - DEPS - pd_to_cinn_pass - add_broadcast_to_elementwise_pass - cinn_op_dialect - pir_transforms) + paddle_test(test_group_op SRCS group_op_test.cc) - paddle_test( - dynamic_reshape_test - SRCS - dynamic_reshape_test.cc - DEPS - pd_to_cinn_pass - cinn_op_dialect - pir_transforms) + paddle_test(dynamic_reshape_test SRCS dynamic_reshape_test.cc) - paddle_test(test_pir_build_cinn_pass SRCS build_cinn_pass_test.cc DEPS - pir_transforms pir) + paddle_test(test_pir_build_cinn_pass SRCS build_cinn_pass_test.cc) - paddle_test(test_compilation_task SRCS compilation_task_test.cc DEPS pir) + paddle_test(test_compilation_task SRCS compilation_task_test.cc) # DO NOT forget add test name here, otherwise it will not be executed in # CINN CI. From fae1352d04e72abbfa17199b8fc4d5971472563a Mon Sep 17 00:00:00 2001 From: ooo oo <106524776+ooooo-create@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:16:54 +0800 Subject: [PATCH 044/146] [doc] fix system message in `Fleet_en.html` (#60306) * fix system message * update * fix indentation --- python/paddle/distributed/fleet/fleet.py | 1 - python/paddle/tensor/manipulation.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/fleet/fleet.py b/python/paddle/distributed/fleet/fleet.py index 5d03552d6d98f..81547d24878d5 100755 --- a/python/paddle/distributed/fleet/fleet.py +++ b/python/paddle/distributed/fleet/fleet.py @@ -584,7 +584,6 @@ def collective_perf(self, comm_type, round=50, size_and_time={}): Examples: .. code-block:: python - :name: code-init-example1 >>> import paddle.distributed.fleet as fleet >>> fleet.init(is_collective=True) diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 853af843a8120..167411500bee5 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -3661,6 +3661,7 @@ def scatter(x, index, updates, overwrite=True, name=None): Examples: .. code-block:: python + :name: scatter-example-2 >>> import paddle From 20273340ff9037ffe8dc6f4dcd10a6bbd4680154 Mon Sep 17 00:00:00 2001 From: xingmingyyj <135400902+xingmingyyj@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:32:51 +0800 Subject: [PATCH 045/146] fix test_lars_momentum (#60287) --- paddle/fluid/pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 12 ++++++++++++ paddle/phi/api/yaml/op_compat.yaml | 6 ++++++ test/white_list/pir_op_test_white_list | 1 + 4 files changed, 20 insertions(+) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 0225bd45f2700..d379bedaab643 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -126,6 +126,7 @@ 'fused_scale_bias_add_relu', 'fused_dconv_drelu_dbn', 'fused_dot_product_attention', + 'lars_momentum', 'recv_v2', 'rnn_', 'row_conv', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 4872f701bd795..57d7857a2498c 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1443,6 +1443,18 @@ optional: dropout1_seed, dropout2_seed, linear1_bias, linear2_bias, ln1_scale, ln1_bias, ln2_scale, ln2_bias, ln2_mean, ln2_variance, ln1_mean, ln1_variance, ln1_out backward: fused_feedforward_grad +- op: lars_momentum + args: (Tensor param, Tensor velocity, Tensor grad, Tensor learning_rate, Tensor master_param, float mu, float lars_coeff=0.001f, float[] lars_weight_decay={0.0005}, float epsilon=0, bool multi_precision=false, float rescale_grad=1.0f) + output: Tensor(param_out), Tensor(velocity_out), Tensor(master_param_out) + infer_meta: + func: SparseMomentumInferMeta + param: [param, learning_rate, velocity] + kernel: + func: lars_momentum + param: [param, velocity, learning_rate, grad, master_param, lars_weight_decay, mu, lars_coeff, epsilon, multi_precision, rescale_grad] + data_type: param + optional: master_param, master_param_out + - op: number_count args: (Tensor numbers, int upper_range) output: Tensor(out) diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 8150e6cdd55cd..f0e87043d965d 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -3460,6 +3460,12 @@ outputs : out : Out +- op: lars_momentum + inputs: + {param : Param, grad : Grad, velocity : Velocity, learning_rate : LearningRate, master_param : MasterParam} + outputs : + {param_out: ParamOut, velocity_out: VelocityOut, master_param_out: MasterParamOut} + - op: lod_array_length inputs : {x: X} diff --git a/test/white_list/pir_op_test_white_list b/test/white_list/pir_op_test_white_list index 9e4de5ccffcfc..1155fce81f300 100644 --- a/test/white_list/pir_op_test_white_list +++ b/test/white_list/pir_op_test_white_list @@ -203,6 +203,7 @@ test_mean_op test_memcpy_op test_meshgrid_op test_mode_op +test_momentum_op test_mul_int8_mkldnn_op test_mul_op test_multi_dot_op From 4af8ecca447eba12cf57597d95935b0b5f4311b1 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:34:54 +0800 Subject: [PATCH 046/146] Migrate ReshapeTransform to pir (#60341) --- python/paddle/distribution/transform.py | 18 +++++++++++++----- .../test_distribution_transform_static.py | 4 ++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/python/paddle/distribution/transform.py b/python/paddle/distribution/transform.py index 39e98a910499b..9e850da2b2fbc 100644 --- a/python/paddle/distribution/transform.py +++ b/python/paddle/distribution/transform.py @@ -166,7 +166,9 @@ def forward(self, x): Returns: Tensor: Outcome of forward transformation. """ - if not isinstance(x, paddle.base.framework.Variable): + if not isinstance( + x, (paddle.base.framework.Variable, paddle.pir.Value) + ): raise TypeError( f"Expected 'x' is a Tensor or Real, but got {type(x)}." ) @@ -187,7 +189,9 @@ def inverse(self, y): Returns: Tensor: Outcome of inverse transform. """ - if not isinstance(y, paddle.base.framework.Variable): + if not isinstance( + y, (paddle.base.framework.Variable, paddle.pir.Value) + ): raise TypeError( f"Expected 'y' is a Tensor or Real, but got {type(y)}." ) @@ -209,12 +213,14 @@ def forward_log_det_jacobian(self, x): Returns: Tensor: The log of the absolute value of Jacobian determinant. """ - if not isinstance(x, paddle.base.framework.Variable): + if not isinstance( + x, (paddle.base.framework.Variable, paddle.pir.Value) + ): raise TypeError( f"Expected 'y' is a Tensor or Real, but got {type(x)}." ) if ( - isinstance(x, paddle.base.framework.Variable) + isinstance(x, (paddle.base.framework.Variable, paddle.pir.Value)) and x.dim() < self._domain.event_rank ): raise ValueError( @@ -241,7 +247,9 @@ def inverse_log_det_jacobian(self, y): Returns: Tensor: The value of :math:`log|det J_{f^{-1}}(y)|`. """ - if not isinstance(y, paddle.base.framework.Variable): + if not isinstance( + y, (paddle.base.framework.Variable, paddle.pir.Value) + ): raise TypeError(f"Expected 'y' is a Tensor, but got {type(y)}.") if y.dim() < self._codomain.event_rank: raise ValueError( diff --git a/test/distribution/test_distribution_transform_static.py b/test/distribution/test_distribution_transform_static.py index 45de7bda08f41..3d128df5acb84 100644 --- a/test/distribution/test_distribution_transform_static.py +++ b/test/distribution/test_distribution_transform_static.py @@ -20,6 +20,7 @@ import paddle from paddle.distribution import transform, variable +from paddle.pir_utils import test_with_pir_api np.random.seed(2022) paddle.seed(2022) @@ -1157,6 +1158,7 @@ def test_domain(self): def test_codomain(self): self.assertTrue(isinstance(self._t._codomain, variable.Independent)) + @test_with_pir_api def test_forward(self): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -1177,6 +1179,7 @@ def test_forward(self): atol=config.ATOL.get(str(expected.dtype)), ) + @test_with_pir_api def test_inverse(self): exe = paddle.static.Executor() sp = paddle.static.Program() @@ -1198,6 +1201,7 @@ def test_inverse(self): atol=config.ATOL.get(str(expected.dtype)), ) + @test_with_pir_api def test_forward_log_det_jacobian(self): exe = paddle.static.Executor() sp = paddle.static.Program() From 9fab1fe754744eaaee8c829b89bbfc9ce230ab19 Mon Sep 17 00:00:00 2001 From: yangguohao <70266361+yangguohao@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:40:27 +0800 Subject: [PATCH 047/146] =?UTF-8?q?=E3=80=90Hackathon=20No.7=E3=80=91?= =?UTF-8?q?=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20apply=20API=20-part?= =?UTF-8?q?=20(#59374)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add tensor apply * fix * fix 2023-11-27 * fix * fix V2 * add apply in Variable * add apply in newir * add test * fix * fix2 * fix example code * change shape * fix docs * fix docs --- paddle/fluid/pybind/eager_method.cc | 32 ++++++ paddle/fluid/pybind/pir.cc | 35 +++++- .../base/dygraph/tensor_patch_methods.py | 100 +++++++++++++++++ python/paddle/base/framework.py | 10 ++ test/legacy_test/test_apply.py | 105 ++++++++++++++++++ test/legacy_test/test_inplace.py | 31 ++++++ 6 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 test/legacy_test/test_apply.py diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 5effab997848d..5c35e41eab0c9 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -1754,6 +1754,30 @@ static PyObject* tensor__setitem_dygraph(TensorObject* self, EAGER_CATCH_AND_THROW_RETURN_NULL } +static PyObject* tensor_apply(TensorObject* self, + PyObject* args, + PyObject* kwargs) { + EAGER_TRY + PyObject* apply_func = PyTuple_GET_ITEM(args, 0); + PyTensorHook func = PyTensorHook(apply_func); + paddle::Tensor out = func(self->tensor); + return ToPyObject(out); + EAGER_CATCH_AND_THROW_RETURN_NULL +} + +static PyObject* tensor_apply_(TensorObject* self, + PyObject* args, + PyObject* kwargs) { + EAGER_TRY + PyObject* apply_func = PyTuple_GET_ITEM(args, 0); + PyTensorHook func = PyTensorHook(apply_func); + paddle::Tensor out = func(self->tensor); + self->tensor.set_impl(out.impl()); + Py_INCREF(self); + return reinterpret_cast(self); + EAGER_CATCH_AND_THROW_RETURN_NULL +} + static PyObject* tensor_register_grad_hook(TensorObject* self, PyObject* args, PyObject* kwargs) { @@ -3167,6 +3191,14 @@ PyMethodDef variable_methods[] = { // NOLINT (PyCFunction)(void (*)())tensor__setitem_dygraph, METH_VARARGS | METH_KEYWORDS, nullptr}, + {"_apply", + (PyCFunction)(void (*)())tensor_apply, + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {"_apply_", + (PyCFunction)(void (*)())tensor_apply_, + METH_VARARGS | METH_KEYWORDS, + nullptr}, {"_register_grad_hook", (PyCFunction)(void (*)())tensor_register_grad_hook, METH_VARARGS | METH_KEYWORDS, diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 2af4d5eb55c02..bbd389c4886a3 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -60,7 +60,7 @@ #include "paddle/fluid/pir/transforms/inplace_pass.h" #include "paddle/fluid/pir/transforms/replace_fetch_with_shadow_output_pass.h" #include "paddle/fluid/pir/transforms/shape_optimization_pass.h" - +#include "paddle/fluid/pybind/eager_utils.h" #include "paddle/phi/core/enforce.h" #include "paddle/pir/core/attribute.h" #include "paddle/pir/core/block.h" @@ -581,6 +581,38 @@ const phi::DDim &GetValueDims(Value value) { } } +pir::OpResult apply(Value self, py::object func) { + py::gil_scoped_acquire gil; + auto stop_gradient = self.attribute(kAttrStopGradients); + if (stop_gradient && !stop_gradient.data()) { + PADDLE_THROW(phi::errors::Unavailable( + "Cannot apply function on a tensor that required gradient.")); + } + PyObject *py_func = func.release().ptr(); + Py_INCREF(py_func); + PyObject *res = nullptr; + try { + py::object obj = py::cast(self); + PyObject *tmp_self = obj.release().ptr(); + Py_INCREF(tmp_self); + res = PyObject_CallFunctionObjArgs(py_func, tmp_self, nullptr); + Py_DECREF(tmp_self); + } catch (std::exception &e) { + PADDLE_THROW(phi::errors::Unavailable( + "Apply function of Tensor raises an exception: %s.", e.what())); + } catch (...) { + PADDLE_THROW(phi::errors::Fatal( + "Apply function of Tensor raises an unknown exception.")); + } + if (res == Py_None) { + return self.dyn_cast(); + } + auto out = CastPyArg2Value(res, "", 0); + Py_DECREF(py_func); + Py_DECREF(res); + return out.dyn_cast(); +} + void BindValue(py::module *m) { py::class_ value(*m, "Value", R"DOC( Value class represents the SSA value in the IR system. It is a directed edge @@ -738,6 +770,7 @@ void BindValue(py::module *m) { print_stream << ")"; return print_stream.str(); }) + .def("apply", &apply) .def("is_same", &Value::operator==) .def("hash", [](Value self) { return std::hash{}(self); }) .def("__repr__", &Value2String); diff --git a/python/paddle/base/dygraph/tensor_patch_methods.py b/python/paddle/base/dygraph/tensor_patch_methods.py index 771fa4a178104..a6d1f90df4fa4 100644 --- a/python/paddle/base/dygraph/tensor_patch_methods.py +++ b/python/paddle/base/dygraph/tensor_patch_methods.py @@ -383,6 +383,104 @@ def gradient(self): return (np.array(self.grad), np.array(self.grad.rows())) return np.array(self.grad) + @framework.dygraph_only + def apply_(self, func): + """ + Inplace apply the python function to the tensor. + + Returns: + None + + Examples: + .. code-block:: python + + >>> import paddle + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "float64") + >>> f = lambda x: 3*x+2 + >>> x.apply_(f) + >>> print(x) + Tensor(shape=[3, 3], dtype=float64, place=Place(cpu), stop_gradient=True, + [[2.90000004, 3.50000000, 2.30000000], + [4.69999993, 4.69999993, 4.09999996], + [3.20000002, 4.40000004, 2.60000001]]) + + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "float16") + >>> x.apply_(f) + + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "bfloat16") + >>> x.apply_(f) + + + >>> if paddle.is_compiled_with_cuda(): + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("gpu", "float32") + >>> x.apply_(f) + """ + if not self.stop_gradient: + raise RuntimeError( + "Cannot apply function on a tensor that required gradient." + ) + return self._apply_(func) + + def apply(self, func): + """ + Apply the python function to the tensor. + + Returns: + None + + Examples: + .. code-block:: python + + >>> import paddle + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "float64") + >>> f = lambda x: 3*x+2 + >>> y = x.apply(f) + >>> print(y) + Tensor(shape=[3, 3], dtype=float64, place=Place(cpu), stop_gradient=True, + [[2.90000004, 3.50000000, 2.30000000], + [4.69999993, 4.69999993, 4.09999996], + [3.20000002, 4.40000004, 2.60000001]]) + + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "float16") + >>> y = x.apply(f) + + + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("cpu", "bfloat16") + >>> y = x.apply(f) + + + >>> if paddle.is_compiled_with_cuda(): + >>> x = paddle.to_tensor([[0.3, 0.5, 0.1], + >>> [0.9, 0.9, 0.7], + >>> [0.4, 0.8, 0.2]]).to("gpu", "float32") + >>> y = x.apply(f) + + """ + if not self.stop_gradient: + raise RuntimeError( + "Cannot apply function on a tensor that required gradient." + ) + return self._apply(func) + @framework.dygraph_only def register_hook(self, hook): """ @@ -1142,6 +1240,8 @@ def coalesce(self, name=None): ("clear_grad", clear_grad), ("inplace_version", inplace_version), ("gradient", gradient), + ("apply_", apply_), + ("apply", apply), ("register_hook", register_hook), ("__str__", __str__), ("__repr__", __str__), diff --git a/python/paddle/base/framework.py b/python/paddle/base/framework.py index e44e8d157623f..1225eba4e4242 100644 --- a/python/paddle/base/framework.py +++ b/python/paddle/base/framework.py @@ -1855,6 +1855,16 @@ def forward_hook_wrapper(x): skip_vars_in_backward_input=[self], ) + def apply(self, func): + if not self.stop_gradient: + raise RuntimeError( + "Cannot apply function on a tensor that required gradient." + ) + try: + return func(self) + except: + raise ValueError(f"The PyFunc {func.__name__} could not be applied") + def __str__(self): return self._to_readable_code() diff --git a/test/legacy_test/test_apply.py b/test/legacy_test/test_apply.py new file mode 100644 index 0000000000000..2c11bd26e932c --- /dev/null +++ b/test/legacy_test/test_apply.py @@ -0,0 +1,105 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +import paddle + + +class TestTensorApplyAPI(unittest.TestCase): + def setUp(self): + self.x = paddle.to_tensor([1, 2, 3, 4, 5], stop_gradient=True) + self.function = lambda x: 3 * x + 2 + + def test_dtype(self): + for dtype in ["float64", "float16", "bfloat16"]: + self.x.to(dtype) + self.test_dygraph() + + @unittest.skipIf( + not paddle.is_compiled_with_cuda(), + "only support cuda", + ) + def test_on_gpu(self): + self.x.to("gpu") + self.test_dygraph() + + def test_dygraph(self): + y = self.x.apply(self.function) + np.testing.assert_allclose( + self.function(self.x).numpy(), y.numpy(), rtol=1e-05 + ) + + def test_error(self): + self.x.stop_gradient = False + + def fn_inplace(x): + x.apply_(self.function) + + def fn_outplace(x, func): + x.apply(func) + + def function(x, y, z): + return x + y + z + + self.assertRaises(RuntimeError, fn_inplace, self.x) + self.assertRaises(RuntimeError, fn_outplace, self.x, self.function) + with paddle.jit.api.sot_mode_guard(False): + self.assertRaises( + RuntimeError, + paddle.jit.to_static(fn_outplace), + self.x, + self.function, + ) + self.x.stop_gradient = True + self.assertRaises( + ValueError, + paddle.jit.to_static(fn_outplace), + self.x, + function, + ) + self.x.stop_gradient = False + with paddle.pir_utils.IrGuard(): + paddle.disable_static() + self.assertRaises( + RuntimeError, + paddle.jit.to_static(fn_outplace), + self.x, + self.function, + ) + + def test_to_static(self): + def fn(x, func): + y = x.apply(func) + return y + + with paddle.jit.api.sot_mode_guard(False): + jit_g = paddle.jit.to_static(fn) + out_legacy_ir = jit_g(self.x, self.function) + with paddle.pir_utils.IrGuard(): + paddle.disable_static() + jit_g = paddle.jit.to_static(fn) + out_pir = jit_g(self.x, self.function) + np.testing.assert_allclose( + self.function(self.x).numpy(), out_legacy_ir.numpy(), rtol=1e-05 + ) + np.testing.assert_allclose( + self.function(self.x).numpy(), out_pir.numpy(), rtol=1e-05 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/legacy_test/test_inplace.py b/test/legacy_test/test_inplace.py index f06edfd83206c..42f9a46cfb910 100644 --- a/test/legacy_test/test_inplace.py +++ b/test/legacy_test/test_inplace.py @@ -1680,5 +1680,36 @@ def test_backward_error(self): loss.backward() +class TestDygraphTensorApplyInplace(unittest.TestCase): + def setUp(self): + self.init_data() + self.set_np_compare_func() + + def init_data(self): + self.input_var_numpy = np.random.uniform(-5, 5, [10, 20, 1]) + self.dtype = "float32" + + def set_np_compare_func(self): + self.np_compare = np.array_equal + + def non_inplace_api_processing(self, var, f): + return var.apply(f) + + def inplace_api_processing(self, var, f): + return var.apply_(f) + + def test_inplace_api(self): + var = paddle.to_tensor(self.input_var_numpy, stop_gradient=True).astype( + self.dtype + ) + f = lambda x: 3 * x + 2 + non_inplace_var = self.non_inplace_api_processing(var, f) + inplace_var = self.inplace_api_processing(var, f) + self.assertTrue(id(var) == id(inplace_var)) + np.testing.assert_array_equal( + non_inplace_var.numpy(), inplace_var.numpy() + ) + + if __name__ == '__main__': unittest.main() From a3a346639ffcc939f7751b654ea940c2dfb559e9 Mon Sep 17 00:00:00 2001 From: Ryan <44900829+DrRyanHuang@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:43:31 +0800 Subject: [PATCH 048/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.1?= =?UTF-8?q?=E3=80=8120=E3=80=81103=E3=80=81104=E3=80=81120=E3=80=91=20Migr?= =?UTF-8?q?ate=20L1Loss/BCELoss/HSigmoidLoss/SmoothL1Loss/KLDivLoss=20into?= =?UTF-8?q?=20pir=20(#58708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/nn/functional/loss.py | 10 +- test/legacy_test/test_bce_loss.py | 14 +- test/legacy_test/test_hsigmoid_op.py | 35 ++-- test/legacy_test/test_kldiv_loss_op.py | 6 +- test/legacy_test/test_l1_loss.py | 136 ++++++++------- test/legacy_test/test_smooth_l1_loss.py | 222 +++++++++++++----------- 6 files changed, 237 insertions(+), 186 deletions(-) diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index a770902faa108..40fea46157f27 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -659,7 +659,7 @@ def binary_cross_entropy( % reduction ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): out = _C_ops.bce_loss(input, label) if weight is not None: out = _C_ops.multiply(out, weight, 'axis', -1) @@ -984,7 +984,7 @@ def hsigmoid_loss( if num_classes < 2: raise ValueError(f'Expected num_classes >= 2 (got {num_classes})') - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): out, _, _ = _C_ops.hsigmoid_loss( input, label, @@ -1103,7 +1103,7 @@ def smooth_l1_loss(input, label, reduction='mean', delta=1.0, name=None): """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): out = _C_ops.huber_loss(input, label, delta) else: check_variable_and_dtype( @@ -1329,7 +1329,7 @@ def l1_loss(input, label, reduction='mean', name=None): "received %s, which is not allowed." % reduction ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): unreduced = _C_ops.abs(_C_ops.subtract(input, label)) if reduction == 'mean': @@ -1688,7 +1688,7 @@ def kl_div(input, label, reduction='mean', name=None): ): label = paddle.cast(label, 'float64') - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): out = _C_ops.kldiv_loss(input, label, 'none') if reduction == 'mean': out = paddle.mean(out) diff --git a/test/legacy_test/test_bce_loss.py b/test/legacy_test/test_bce_loss.py index a9fe9cfa030d9..007bdffad0288 100644 --- a/test/legacy_test/test_bce_loss.py +++ b/test/legacy_test/test_bce_loss.py @@ -20,6 +20,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api def test_static_layer( @@ -152,6 +153,7 @@ def calc_bceloss(input_np, label_np, reduction='mean', weight_np=None): class TestBCELoss(unittest.TestCase): + @test_with_pir_api def test_BCELoss(self): input_np = np.random.uniform(0.1, 0.8, size=(20, 30)).astype(np.float64) label_np = np.random.randint(0, 2, size=(20, 30)).astype(np.float64) @@ -185,6 +187,7 @@ def test_BCELoss(self): ) np.testing.assert_allclose(dy_functional, expected, rtol=1e-05) + @test_with_pir_api def test_BCELoss_weight(self): input_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( np.float64 @@ -262,10 +265,10 @@ def setUp(self): self.outputs = {'Out': output_np} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', check_pir=True) def init_test_case(self): self.shape = [10, 10] @@ -286,17 +289,20 @@ def init_test_cast(self): class TestBceLossOpFP16(TestBceLossOp): def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', check_pir=True) def init_test_dtype(self): self.dtype = np.float16 class TestBceLossOpStaticFP16(unittest.TestCase): + @test_with_pir_api def test_fp16(self): + if not core.is_compiled_with_cuda(): + return paddle.enable_static() shape = [2, 3, 20] x_data = np.random.uniform(0.1, 0.8, shape).astype("float16") diff --git a/test/legacy_test/test_hsigmoid_op.py b/test/legacy_test/test_hsigmoid_op.py index 65cb8548e9eb8..9659b5e3b77d3 100644 --- a/test/legacy_test/test_hsigmoid_op.py +++ b/test/legacy_test/test_hsigmoid_op.py @@ -21,6 +21,7 @@ import paddle import paddle.nn.functional as F from paddle import base +from paddle.pir_utils import test_with_pir_api paddle.enable_static() np.random.seed(100) @@ -218,13 +219,14 @@ def setUp(self): self.user_grads = hsigmoid_grad(x, w, label, bias, num_classes) def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): self.check_grad( ['X', 'W', 'Bias'], ['Out'], user_defined_grads=self.user_grads, + check_pir=True, ) @@ -278,7 +280,7 @@ def setUp(self): self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) class TestHSigmoidOpWithSparseGrad(unittest.TestCase): @@ -323,9 +325,11 @@ def hs_net_conf(self, is_sparse): return avg_cost, data_list def training_test(self, is_sparse): - with base.program_guard(base.Program(), base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): paddle.seed(1) - start_up = base.default_startup_program() + start_up = paddle.static.default_startup_program() x = np.arange(6).reshape(6) path_table = np.array([(1, 2, -1), (1, 2, -1)]).astype('int64') path_code = np.array([(1, 0, -1), (0, 0, -1)]).astype('int64') @@ -335,10 +339,10 @@ def training_test(self, is_sparse): optimizer = paddle.optimizer.SGD(learning_rate=1e-3) optimizer.minimize(loss) - main_program = base.default_main_program() + main_program = paddle.static.default_main_program() place = base.CPUPlace() feeder = base.DataFeeder(feed_list=data_list, place=place) - exe = base.Executor(place) + exe = paddle.static.Executor(place) exe.run(start_up) result = [] @@ -414,13 +418,14 @@ def setUp(self): self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): self.check_grad( ['Bias', 'X', 'W'], ['Out'], no_grad_set=set('Label'), + check_pir=True, ) @@ -479,10 +484,12 @@ def setUp(self): self.outputs = {'PreOut': pre_output, 'Out': out} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X', 'W'], ['Out'], no_grad_set=set('Label')) + self.check_grad( + ['X', 'W'], ['Out'], no_grad_set=set('Label'), check_pir=True + ) class TestHSigmoidLossAPI(unittest.TestCase): @@ -564,6 +571,7 @@ def test_dygraph_api(self): np.testing.assert_allclose(self.out_np, out.numpy(), rtol=1e-05) paddle.enable_static() + @test_with_pir_api def test_static_api(self): train_program = paddle.static.Program() startup_program = paddle.static.Program() @@ -619,10 +627,11 @@ def test_static_api(self): for ret in [ret1, ret2]: np.testing.assert_allclose(self.out_np, ret, rtol=1e-05) + @test_with_pir_api def test_base_api(self): - train_program = base.Program() - startup_program = base.Program() - with base.program_guard(train_program, startup_program): + train_program = paddle.static.Program() + startup_program = paddle.static.Program() + with paddle.static.program_guard(train_program, startup_program): x = paddle.static.data('x', [-1, self.feature_size]) labels = paddle.static.data('labels', [-1, 1], 'int64') path_table = None @@ -647,7 +656,7 @@ def test_base_api(self): path_code=path_code, ) - exe = base.Executor(self.place) + exe = paddle.static.Executor(self.place) exe.run(startup_program) feed_dict = {'x': self.x_np, 'labels': self.labels_np} if self.is_custom: diff --git a/test/legacy_test/test_kldiv_loss_op.py b/test/legacy_test/test_kldiv_loss_op.py index ea93d0e4dd607..599b9764c984d 100644 --- a/test/legacy_test/test_kldiv_loss_op.py +++ b/test/legacy_test/test_kldiv_loss_op.py @@ -18,6 +18,7 @@ import paddle from paddle.nn.functional import kl_div +from paddle.pir_utils import test_with_pir_api def kldiv_loss(x, target, reduction): @@ -55,10 +56,10 @@ def setUp(self): self.outputs = {'Loss': loss.astype('float64')} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X'], 'Loss', no_grad_set={"Target"}) + self.check_grad(['X'], 'Loss', no_grad_set={"Target"}, check_pir=True) def initTestCase(self): self.x_shape = (4, 5, 5) @@ -111,6 +112,7 @@ def test_kl_loss_sum(self): def test_kl_loss_none(self): self.run_kl_loss('none') + @test_with_pir_api def test_kl_loss_static_api(self): with paddle_static_guard(): input = paddle.static.data(name='input', shape=[5, 20]) diff --git a/test/legacy_test/test_l1_loss.py b/test/legacy_test/test_l1_loss.py index 651d55977b34c..3a21e7ff97e48 100644 --- a/test/legacy_test/test_l1_loss.py +++ b/test/legacy_test/test_l1_loss.py @@ -18,6 +18,8 @@ import paddle from paddle import base +from paddle.framework import in_pir_mode +from paddle.pir_utils import test_with_pir_api class TestFunctionalL1Loss(unittest.TestCase): @@ -43,42 +45,48 @@ def run_imperative(self): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertEqual(dy_result.shape, [10, 10, 5]) + @test_with_pir_api def run_static(self, use_gpu=False): - input = paddle.static.data( - name='input', shape=[10, 10, 5], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[10, 10, 5], dtype='float32' - ) - result0 = paddle.nn.functional.l1_loss(input, label) - result1 = paddle.nn.functional.l1_loss(input, label, reduction='sum') - result2 = paddle.nn.functional.l1_loss(input, label, reduction='none') - y = paddle.nn.functional.l1_loss(input, label, name='aaa') - - place = base.CUDAPlace(0) if use_gpu else base.CPUPlace() - exe = base.Executor(place) - exe.run(base.default_startup_program()) - static_result = exe.run( - feed={"input": self.input_np, "label": self.label_np}, - fetch_list=[result0, result1, result2], - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input = paddle.static.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) + result0 = paddle.nn.functional.l1_loss(input, label) + result1 = paddle.nn.functional.l1_loss( + input, label, reduction='sum' + ) + result2 = paddle.nn.functional.l1_loss( + input, label, reduction='none' + ) + y = paddle.nn.functional.l1_loss(input, label, name='aaa') - expected = np.mean(np.abs(self.input_np - self.label_np)) - np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) - expected = np.sum(np.abs(self.input_np - self.label_np)) - np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) - expected = np.abs(self.input_np - self.label_np) - np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) + place = base.CUDAPlace(0) if use_gpu else base.CPUPlace() + exe = paddle.static.Executor(place) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) - self.assertTrue('aaa' in y.name) + expected = np.mean(np.abs(self.input_np - self.label_np)) + np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) + expected = np.sum(np.abs(self.input_np - self.label_np)) + np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) + expected = np.abs(self.input_np - self.label_np) + np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) + if not in_pir_mode(): + self.assertTrue('aaa' in y.name) def test_cpu(self): paddle.disable_static(place=paddle.base.CPUPlace()) self.run_imperative() paddle.enable_static() - with base.program_guard(base.Program()): - self.run_static() + self.run_static() def test_gpu(self): if not base.core.is_compiled_with_cuda(): @@ -88,11 +96,11 @@ def test_gpu(self): self.run_imperative() paddle.enable_static() - with base.program_guard(base.Program()): - self.run_static(use_gpu=True) + self.run_static(use_gpu=True) # test case the raise message def test_errors(self): + @test_with_pir_api def test_value_error(): input = paddle.static.data( name='input', shape=[10, 10, 5], dtype='float32' @@ -133,45 +141,49 @@ def run_imperative(self): np.testing.assert_allclose(dy_result.numpy(), expected, rtol=1e-05) self.assertEqual(dy_result.shape, [10, 10, 5]) + @test_with_pir_api def run_static(self, use_gpu=False): - input = paddle.static.data( - name='input', shape=[10, 10, 5], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[10, 10, 5], dtype='float32' - ) - l1_loss = paddle.nn.loss.L1Loss() - result0 = l1_loss(input, label) - l1_loss = paddle.nn.loss.L1Loss(reduction='sum') - result1 = l1_loss(input, label) - l1_loss = paddle.nn.loss.L1Loss(reduction='none') - result2 = l1_loss(input, label) - l1_loss = paddle.nn.loss.L1Loss(name='aaa') - result3 = l1_loss(input, label) - - place = base.CUDAPlace(0) if use_gpu else base.CPUPlace() - exe = base.Executor(place) - exe.run(base.default_startup_program()) - static_result = exe.run( - feed={"input": self.input_np, "label": self.label_np}, - fetch_list=[result0, result1, result2], - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input = paddle.static.data( + name='input', shape=[10, 10, 5], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[10, 10, 5], dtype='float32' + ) + l1_loss = paddle.nn.loss.L1Loss() + result0 = l1_loss(input, label) + l1_loss = paddle.nn.loss.L1Loss(reduction='sum') + result1 = l1_loss(input, label) + l1_loss = paddle.nn.loss.L1Loss(reduction='none') + result2 = l1_loss(input, label) + l1_loss = paddle.nn.loss.L1Loss(name='aaa') + result3 = l1_loss(input, label) + + place = base.CUDAPlace(0) if use_gpu else base.CPUPlace() + exe = paddle.static.Executor(place) + static_result = exe.run( + feed={"input": self.input_np, "label": self.label_np}, + fetch_list=[result0, result1, result2], + ) - expected = np.mean(np.abs(self.input_np - self.label_np)) - np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) - expected = np.sum(np.abs(self.input_np - self.label_np)) - np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) - expected = np.abs(self.input_np - self.label_np) - np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) - self.assertTrue('aaa' in result3.name) + expected = np.mean(np.abs(self.input_np - self.label_np)) + np.testing.assert_allclose(static_result[0], expected, rtol=1e-05) + expected = np.sum(np.abs(self.input_np - self.label_np)) + np.testing.assert_allclose(static_result[1], expected, rtol=1e-05) + expected = np.abs(self.input_np - self.label_np) + np.testing.assert_allclose(static_result[2], expected, rtol=1e-05) + + if not in_pir_mode(): + self.assertTrue('aaa' in result3.name) def test_cpu(self): paddle.disable_static(place=paddle.base.CPUPlace()) self.run_imperative() paddle.enable_static() - with base.program_guard(base.Program()): - self.run_static() + self.run_static() def test_gpu(self): if not base.core.is_compiled_with_cuda(): @@ -181,11 +193,11 @@ def test_gpu(self): self.run_imperative() paddle.enable_static() - with base.program_guard(base.Program()): - self.run_static(use_gpu=True) + self.run_static(use_gpu=True) # test case the raise message def test_errors(self): + @test_with_pir_api def test_value_error(): loss = paddle.nn.loss.L1Loss(reduction="reduce_mean") diff --git a/test/legacy_test/test_smooth_l1_loss.py b/test/legacy_test/test_smooth_l1_loss.py index f070b747aeb5e..d9c1b3d4fcb13 100644 --- a/test/legacy_test/test_smooth_l1_loss.py +++ b/test/legacy_test/test_smooth_l1_loss.py @@ -18,6 +18,7 @@ import paddle from paddle import base +from paddle.pir_utils import test_with_pir_api def smooth_l1_loss_forward(val, delta): @@ -46,33 +47,40 @@ def setUp(self): def test_smooth_l1_loss_mean(self): input_np = np.random.random([100, 200]).astype(np.float32) label_np = np.random.random([100, 200]).astype(np.float32) - prog = base.Program() - startup_prog = base.Program() + place = ( base.CUDAPlace(0) if base.core.is_compiled_with_cuda() else base.CPUPlace() ) - with base.program_guard(prog, startup_prog): - input = paddle.static.data( - name='input', shape=[100, 200], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[100, 200], dtype='float32' - ) - smooth_l1_loss = paddle.nn.loss.SmoothL1Loss() - ret = smooth_l1_loss(input, label) - - exe = base.Executor(place) - (static_ret,) = exe.run( - prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret], - ) - self.assertIsNotNone(static_ret) + + expected = smooth_l1_loss_np(input_np, label_np, reduction='mean') + + @test_with_pir_api + def test_dynamic_or_pir_mode(): + prog = paddle.static.Program() + startup_prog = paddle.static.Program() + with paddle.static.program_guard(prog, startup_prog): + input = paddle.static.data( + name='input', shape=[100, 200], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[100, 200], dtype='float32' + ) + smooth_l1_loss = paddle.nn.loss.SmoothL1Loss() + ret = smooth_l1_loss(input, label) + + exe = paddle.static.Executor(place) + (static_ret,) = exe.run( + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) + self.assertIsNotNone(static_ret) + np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + with base.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss() dy_ret = smooth_l1_loss( @@ -81,41 +89,46 @@ def test_smooth_l1_loss_mean(self): ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = smooth_l1_loss_np(input_np, label_np, reduction='mean') - np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) - np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + + test_dynamic_or_pir_mode() np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_smooth_l1_loss_sum(self): input_np = np.random.random([100, 200]).astype(np.float32) label_np = np.random.random([100, 200]).astype(np.float32) - prog = base.Program() - startup_prog = base.Program() + place = ( base.CUDAPlace(0) if base.core.is_compiled_with_cuda() else base.CPUPlace() ) - with base.program_guard(prog, startup_prog): - input = paddle.static.data( - name='input', shape=[100, 200], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[100, 200], dtype='float32' - ) - smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='sum') - ret = smooth_l1_loss(input, label) - - exe = base.Executor(place) - (static_ret,) = exe.run( - prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret], - ) - self.assertIsNotNone(static_ret) + expected = smooth_l1_loss_np(input_np, label_np, reduction='sum') + + @test_with_pir_api + def test_dynamic_or_pir_mode(): + prog = paddle.static.Program() + startup_prog = paddle.static.Program() + with paddle.static.program_guard(prog, startup_prog): + input = paddle.static.data( + name='input', shape=[100, 200], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[100, 200], dtype='float32' + ) + smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='sum') + ret = smooth_l1_loss(input, label) + + exe = paddle.static.Executor(place) + (static_ret,) = exe.run( + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) + self.assertIsNotNone(static_ret) + np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + with base.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='sum') dy_ret = smooth_l1_loss( @@ -124,41 +137,46 @@ def test_smooth_l1_loss_sum(self): ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = smooth_l1_loss_np(input_np, label_np, reduction='sum') - np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) - np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + + test_dynamic_or_pir_mode() np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_smooth_l1_loss_none(self): input_np = np.random.random([100, 200]).astype(np.float32) label_np = np.random.random([100, 200]).astype(np.float32) - prog = base.Program() - startup_prog = base.Program() + place = ( base.CUDAPlace(0) if base.core.is_compiled_with_cuda() else base.CPUPlace() ) - with base.program_guard(prog, startup_prog): - input = paddle.static.data( - name='input', shape=[100, 200], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[100, 200], dtype='float32' - ) - smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='none') - ret = smooth_l1_loss(input, label) - - exe = base.Executor(place) - (static_ret,) = exe.run( - prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret], - ) - self.assertIsNotNone(static_ret) + expected = smooth_l1_loss_np(input_np, label_np, reduction='none') + + @test_with_pir_api + def test_dynamic_or_pir_mode(): + prog = paddle.static.Program() + startup_prog = paddle.static.Program() + with paddle.static.program_guard(prog, startup_prog): + input = paddle.static.data( + name='input', shape=[100, 200], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[100, 200], dtype='float32' + ) + smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='none') + ret = smooth_l1_loss(input, label) + + exe = paddle.static.Executor(place) + (static_ret,) = exe.run( + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) + self.assertIsNotNone(static_ret) + np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + with base.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(reduction='none') dy_ret = smooth_l1_loss( @@ -167,42 +185,47 @@ def test_smooth_l1_loss_none(self): ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = smooth_l1_loss_np(input_np, label_np, reduction='none') - np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) - np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + + test_dynamic_or_pir_mode() np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) def test_smooth_l1_loss_delta(self): input_np = np.random.random([100, 200]).astype(np.float32) label_np = np.random.random([100, 200]).astype(np.float32) delta = np.random.rand() - prog = base.Program() - startup_prog = base.Program() + place = ( base.CUDAPlace(0) if base.core.is_compiled_with_cuda() else base.CPUPlace() ) - with base.program_guard(prog, startup_prog): - input = paddle.static.data( - name='input', shape=[100, 200], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[100, 200], dtype='float32' - ) - smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(delta=delta) - ret = smooth_l1_loss(input, label) - - exe = base.Executor(place) - (static_ret,) = exe.run( - prog, - feed={ - 'input': input_np, - 'label': label_np, - }, - fetch_list=[ret], - ) - self.assertIsNotNone(static_ret) + expected = smooth_l1_loss_np(input_np, label_np, delta=delta) + + @test_with_pir_api + def test_dynamic_or_pir_mode(): + prog = paddle.static.Program() + startup_prog = paddle.static.Program() + with paddle.static.program_guard(prog, startup_prog): + input = paddle.static.data( + name='input', shape=[100, 200], dtype='float32' + ) + label = paddle.static.data( + name='label', shape=[100, 200], dtype='float32' + ) + smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(delta=delta) + ret = smooth_l1_loss(input, label) + + exe = paddle.static.Executor(place) + (static_ret,) = exe.run( + feed={ + 'input': input_np, + 'label': label_np, + }, + fetch_list=[ret], + ) + self.assertIsNotNone(static_ret) + np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + with base.dygraph.guard(): smooth_l1_loss = paddle.nn.loss.SmoothL1Loss(delta=delta) dy_ret = smooth_l1_loss( @@ -211,9 +234,8 @@ def test_smooth_l1_loss_delta(self): ) dy_ret_value = dy_ret.numpy() self.assertIsNotNone(dy_ret_value) - expected = smooth_l1_loss_np(input_np, label_np, delta=delta) - np.testing.assert_allclose(static_ret, dy_ret_value, rtol=1e-05) - np.testing.assert_allclose(static_ret, expected, rtol=1e-05) + + test_dynamic_or_pir_mode() np.testing.assert_allclose(dy_ret_value, expected, rtol=1e-05) From fdf9d71ed6b9596a096b6d3d9390d0cf016ce258 Mon Sep 17 00:00:00 2001 From: ooo oo <106524776+ooooo-create@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:00:25 +0800 Subject: [PATCH 049/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.177?= =?UTF-8?q?=E3=80=91Migrate=20paddle.geometric.reindex=5Fgraph=20into=20pi?= =?UTF-8?q?r=20(#60323)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/geometric/reindex.py | 4 ++-- test/legacy_test/test_graph_reindex.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/paddle/geometric/reindex.py b/python/paddle/geometric/reindex.py index 0ea5efac9c97a..69d0e7070e09a 100644 --- a/python/paddle/geometric/reindex.py +++ b/python/paddle/geometric/reindex.py @@ -17,7 +17,7 @@ from paddle.base.data_feeder import check_variable_and_dtype from paddle.base.framework import Variable from paddle.base.layer_helper import LayerHelper -from paddle.framework import in_dynamic_mode, in_dynamic_or_pir_mode +from paddle.framework import in_dynamic_or_pir_mode __all__ = [] @@ -90,7 +90,7 @@ def reindex_graph( True if value_buffer is not None and index_buffer is not None else False ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): reindex_src, reindex_dst, out_nodes = _C_ops.reindex_graph( x, neighbors, diff --git a/test/legacy_test/test_graph_reindex.py b/test/legacy_test/test_graph_reindex.py index f505a00937cb7..85ad07f86af0f 100644 --- a/test/legacy_test/test_graph_reindex.py +++ b/test/legacy_test/test_graph_reindex.py @@ -370,6 +370,7 @@ def test_heter_reindex_result_v3(self): np.testing.assert_allclose(reindex_dst, reindex_dst_, rtol=1e-05) np.testing.assert_allclose(out_nodes, out_nodes_, rtol=1e-05) + @test_with_pir_api def test_reindex_result_static(self): paddle.enable_static() with paddle.static.program_guard(paddle.static.Program()): From 4a72678a921a81a9eb8e07ee69fa2b9c96f76a80 Mon Sep 17 00:00:00 2001 From: xuxinyi389 <104957571+xuxinyi389@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:11:19 +0800 Subject: [PATCH 050/146] disable_trt_uts (#59907) * disable_trt_uts * add --- tools/windows/run_unittests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index 03e802750b8df..3ab9fb83adfdc 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -293,6 +293,11 @@ disable_wingpu_cuda12_test="^test_cholesky_op$|\ ^test_trt_convert_transpose$|\ ^test_trt_convert_unsqueeze2$|\ ^test_simplify_with_basic_ops_pass_autoscan$|\ +^test_trt_convert_nearest_interp$|\ +^test_trt_pool_op$|\ +^test_trt_convert_clip$|\ +^test_trt_convert_grid_sampler$|\ +^test_trt_convert_p_norm$|\ ^disable_wingpu_cuda12_test$" # /*=================Fixed Disabled Windows TRT MKL unittests=======================*/ From b4ff023c2653e995d4a82ac55429d5871f8c7183 Mon Sep 17 00:00:00 2001 From: hjyp <53164956+Tomoko-hjf@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:17:53 +0800 Subject: [PATCH 051/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.143?= =?UTF-8?q?=E3=80=81144=E3=80=91=20Migrate=20margin=5Fcross=5Fentropy?= =?UTF-8?q?=E3=80=81masked=5Fmultihead=5Fattention=20(#58762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../functional/masked_multihead_attention.py | 4 +-- python/paddle/nn/functional/loss.py | 2 +- .../test_margin_cross_entropy_op.py | 34 ++++++++++++++----- .../test_masked_multihead_attention_op.py | 7 +++- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/python/paddle/incubate/nn/functional/masked_multihead_attention.py b/python/paddle/incubate/nn/functional/masked_multihead_attention.py index 9b1f3d464ab48..f8131e2910461 100644 --- a/python/paddle/incubate/nn/functional/masked_multihead_attention.py +++ b/python/paddle/incubate/nn/functional/masked_multihead_attention.py @@ -13,7 +13,7 @@ # limitations under the License. from paddle import _C_ops -from paddle.framework import LayerHelper, in_dynamic_mode +from paddle.framework import LayerHelper, in_dynamic_or_pir_mode def masked_multihead_attention( @@ -90,7 +90,7 @@ def masked_multihead_attention( """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.masked_multihead_attention_( x, cache_kv, diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index 40fea46157f27..d1611106b7c52 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -2314,7 +2314,7 @@ def margin_cross_entropy( if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=-1) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): softmax, loss = _C_ops.margin_cross_entropy( logits, label, diff --git a/test/legacy_test/test_margin_cross_entropy_op.py b/test/legacy_test/test_margin_cross_entropy_op.py index 59d965456554c..1fbda0faeb7eb 100644 --- a/test/legacy_test/test_margin_cross_entropy_op.py +++ b/test/legacy_test/test_margin_cross_entropy_op.py @@ -18,7 +18,9 @@ from op_test import OpTest, convert_float_to_uint16, paddle_static_guard import paddle -from paddle.base import Program, core, program_guard +from paddle.base import core +from paddle.pir_utils import test_with_pir_api +from paddle.static import Program, program_guard def stable_softmax_comm(x): @@ -148,10 +150,14 @@ def setUp(self): } def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), atol=1e-5) + self.check_output_with_place( + core.CUDAPlace(0), atol=1e-5, check_pir=True + ) def test_check_grad(self): - self.check_grad_with_place(core.CUDAPlace(0), ["Logits"], "Loss") + self.check_grad_with_place( + core.CUDAPlace(0), ["Logits"], "Loss", check_pir=True + ) @unittest.skipIf( @@ -168,6 +174,7 @@ def test_check_grad(self): "Loss", numeric_grad_delta=5e-2, max_relative_error=5e-2, + check_pir=True, ) @@ -179,7 +186,9 @@ def init_dtype(self): self.dtype = np.float16 def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), atol=5e-2) + self.check_output_with_place( + core.CUDAPlace(0), atol=5e-2, check_pir=True + ) def test_check_grad(self): self.check_grad_with_place( @@ -188,6 +197,7 @@ def test_check_grad(self): "Loss", numeric_grad_delta=6e-1, max_relative_error=6e-1, + check_pir=True, ) @@ -264,7 +274,9 @@ def setUp(self): } def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), atol=5e-2) + self.check_output_with_place( + core.CUDAPlace(0), atol=5e-2, check_pir=True + ) def test_check_grad(self): self.check_grad_with_place( @@ -273,6 +285,7 @@ def test_check_grad(self): "Loss", numeric_grad_delta=6e-1, max_relative_error=6e-1, + check_pir=True, ) @@ -301,13 +314,17 @@ def init_loss_params(self): class TestMarginCrossEntropyOpCPU(TestMarginCrossEntropyOp): def test_check_output(self): try: - self.check_output_with_place(core.CPUPlace(), atol=1e-5) + self.check_output_with_place( + core.CPUPlace(), atol=1e-5, check_pir=True + ) except RuntimeError: pass def test_check_grad(self): try: - self.check_grad_with_place(core.CPUPlace(), ["Logits"], "Loss") + self.check_grad_with_place( + core.CPUPlace(), ["Logits"], "Loss", check_pir=True + ) except RuntimeError: pass @@ -347,6 +364,7 @@ def init_dtype(self): def init_reduction(self): self.reduction = None + @test_with_pir_api def test_static(self): for place in self.places: self.check_static_result(place=place) @@ -404,7 +422,7 @@ def check_static_result(self, place): exe = paddle.base.Executor(place) [loss_res, softmax_res] = exe.run( - paddle.base.default_main_program(), + paddle.static.default_main_program(), feed={'logits': logits_np, 'label': labels_np}, fetch_list=[loss, softmax], ) diff --git a/test/legacy_test/test_masked_multihead_attention_op.py b/test/legacy_test/test_masked_multihead_attention_op.py index 8eecbe5d7befc..d0954c79f3e5c 100644 --- a/test/legacy_test/test_masked_multihead_attention_op.py +++ b/test/legacy_test/test_masked_multihead_attention_op.py @@ -19,6 +19,7 @@ import paddle from paddle.framework import core from paddle.incubate.nn.functional import masked_multihead_attention +from paddle.pir_utils import test_with_pir_api @unittest.skipIf( @@ -213,6 +214,7 @@ def check_main( paddle.enable_static() return paddle_naive_mmha_out, paddle_mmha_out + @test_with_pir_api def test_mmha_fp16(self): if not paddle.is_compiled_with_cuda(): return @@ -234,6 +236,7 @@ def test_mmha_fp16(self): atol=1e-3, ) + @test_with_pir_api def test_mmha_qkv_out_scale(self): if not paddle.is_compiled_with_cuda(): return @@ -255,6 +258,7 @@ def test_mmha_qkv_out_scale(self): atol=1e-3, ) + @test_with_pir_api def test_mmha_outlinear_in_scale(self): if not paddle.is_compiled_with_cuda(): return @@ -463,11 +467,12 @@ def check_main( "bias_static": bias.astype(dtype), "src_mask_static": src_mask.astype(dtype), }, - fetch_list=[outs], + fetch_list=[outs[0], outs[1]], ) return paddle_naive_mmha_out, out_s + @test_with_pir_api def test_mmha_fp16(self): if not paddle.is_compiled_with_cuda(): return From 42ab2d523fc300be866d67ae9d9c29b143c9d93a Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Tue, 26 Dec 2023 16:37:23 +0800 Subject: [PATCH 052/146] [dy2s] speed up interpreter cache key computation (#60218) --- .../eager/to_static/run_program_op_func.h | 28 +++----- .../eager/to_static/run_program_op_node.h | 66 +++++++++---------- paddle/fluid/framework/executor_cache.cc | 12 ++-- paddle/fluid/framework/executor_cache.h | 31 ++++----- 4 files changed, 62 insertions(+), 75 deletions(-) diff --git a/paddle/fluid/eager/to_static/run_program_op_func.h b/paddle/fluid/eager/to_static/run_program_op_func.h index d97675fa41acc..519b0772f1164 100644 --- a/paddle/fluid/eager/to_static/run_program_op_func.h +++ b/paddle/fluid/eager/to_static/run_program_op_func.h @@ -129,8 +129,8 @@ static std::vector Trans2ContiguousTensors( return res; } -inline int64_t hash_int_value(int64_t value) { - return value + 0x9e3779b9 + (value << 6) + (value >> 2); +int64_t hash_with_seed(int64_t value, int64_t seed) { + return seed + 0x9e3779b9 + (value << 6) + (value >> 2); } inline void run_program_ad_func( @@ -159,14 +159,10 @@ inline void run_program_ad_func( auto params_tmp = Trans2ContiguousTensors(params); // Call forward function // if require_any_grad is False, don't save any middle vars. - std::vector place_hash_keys = std::vector(); + int64_t place_hash_key = 0; for (const paddle::Tensor& tensor : x) { int64_t device_type = static_cast(tensor.place().GetType()); - place_hash_keys.emplace_back(hash_int_value(device_type)); - } - for (const paddle::Tensor& tensor : params) { - int64_t device_type = static_cast(tensor.place().GetType()); - place_hash_keys.emplace_back(hash_int_value(device_type)); + place_hash_key = hash_with_seed(place_hash_key, device_type); } RunProgramAPI(x_tmp, params_tmp, @@ -174,7 +170,7 @@ inline void run_program_ad_func( step_scope, require_any_grad, attrs, - place_hash_keys); + place_hash_key); VLOG(2) << "start run run_program grad"; auto is_test = false; if (attrs.count("is_test")) { @@ -188,7 +184,7 @@ inline void run_program_ad_func( auto grad_node = std::make_shared(1, 2); // Set place hash keys for backward - grad_node->SetPlaceHashKeys(place_hash_keys); + grad_node->SetPlaceHashKey(place_hash_key); // Set Attributes grad_node->SetAttrMap(attrs); @@ -288,14 +284,10 @@ inline void pir_run_program_ad_func( // Call forward function // if require_any_grad is False, don't save any middle vars. - std::vector place_hash_keys = std::vector(); + int64_t place_hash_key = 0x9e3779b9; for (const paddle::Tensor& tensor : x) { int64_t device_type = static_cast(tensor.place().GetType()); - place_hash_keys.emplace_back(hash_int_value(device_type)); - } - for (const paddle::Tensor& tensor : params) { - int64_t device_type = static_cast(tensor.place().GetType()); - place_hash_keys.emplace_back(hash_int_value(device_type)); + place_hash_key = hash_with_seed(place_hash_key, device_type); } PirRunProgramAPI(x, params, @@ -304,10 +296,10 @@ inline void pir_run_program_ad_func( step_scope, require_any_grad, attrs, - place_hash_keys); + place_hash_key); if (!is_test && require_any_grad) { // Set place hash keys for backward - grad_node->SetPlaceHashKeys(place_hash_keys); + grad_node->SetPlaceHashKey(place_hash_key); // Set Attributes grad_node->SetAttrMap(attrs); diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index 1fc63942a7669..257b249e51600 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -423,7 +423,7 @@ inline void PirRunProgramAPI( std::vector &step_scope, // NOLINT bool require_any_grad, const paddle::framework::AttributeMap &attrs, - const std::vector &place_hash_keys) { + const int64_t &place_hash_key) { VLOG(2) << "RunProgramOpKernel Compute"; // In the original run_program OP, the default value of the is_test // attribute is false, we should check if there is is_test parameter @@ -489,7 +489,7 @@ inline void PirRunProgramAPI( std::shared_ptr interpreter_core = nullptr; if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/false)) { + program_id, global_inner_scope, place_hash_key, /*is_grad=*/false)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -517,7 +517,7 @@ inline void PirRunProgramAPI( /*is_grad=*/false, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); // Step 3. get all eager gc vars // std::set skip_eager_delete_vars = // paddle::framework::details::ParseSafeEagerDeletionSkipVarsSet( @@ -556,7 +556,7 @@ inline void PirRunProgramAPI( VLOG(2) << "Get interpretercore cache by program:" << program_id; // Step 1. get cache interpretercore auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/false); + program_id, global_inner_scope, place_hash_key, /*is_grad=*/false); interpreter_core = cached_value.core_; // Step 2. update scope for cache interpretercore details::ShareTensorsIntoScopeByValue( @@ -618,7 +618,7 @@ inline void RunProgramAPI( std::vector &step_scope, // NOLINT bool require_any_grad, const paddle::framework::AttributeMap &attrs, - const std::vector &place_hash_keys) { + const int64_t &place_hash_key) { VLOG(2) << "RunProgramOpKernel Compute"; // In the original run_program OP, the default value of the is_test // attribute is false, we should check if there is is_test parameter @@ -689,7 +689,7 @@ inline void RunProgramAPI( std::shared_ptr interpreter_core = nullptr; if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/false)) { + program_id, global_inner_scope, place_hash_key, /*is_grad=*/false)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -723,7 +723,7 @@ inline void RunProgramAPI( /*is_grad=*/false, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); } else { interpreter_core = paddle::framework::CreateProgramInterpreterCoreInfoToCache( @@ -732,7 +732,7 @@ inline void RunProgramAPI( /*is_grad=*/false, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); } // Step 3. get all eager gc vars std::set skip_eager_delete_vars; @@ -763,7 +763,7 @@ inline void RunProgramAPI( interpretercore_info_cache.UpdateSkipEagerDeleteVars( program_id, global_inner_scope, - place_hash_keys, + place_hash_key, false, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); @@ -775,7 +775,7 @@ inline void RunProgramAPI( VLOG(2) << "Get interpretercore cahce by program:" << program_id; // Step 1. get cache interpretercore auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/false); + program_id, global_inner_scope, place_hash_key, /*is_grad=*/false); interpreter_core = cached_value.core_; // Step 2. update scope for cache interpretercore details::ShareTensorsIntoScopeWithName(x, input_names, global_inner_scope); @@ -827,7 +827,7 @@ inline void RunProgramGradAPI( const paddle::framework::AttributeMap &attrs, std::vector &x_grad, // NOLINT std::vector ¶ms_grad, // NOLINT - const std::vector &place_hash_keys) { + const int64_t &place_hash_key) { // if all output vars are set to stop_gradient, grad op no need to executed if (x_grad.empty() && params_grad.empty()) return; auto *out_scope_vec = &step_scope; @@ -859,7 +859,7 @@ inline void RunProgramGradAPI( std::shared_ptr interpreter_core = nullptr; if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/true)) { + program_id, global_inner_scope, place_hash_key, /*is_grad=*/true)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -889,7 +889,7 @@ inline void RunProgramGradAPI( /*is_grad=*/true, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); } else { interpreter_core = paddle::framework::CreateProgramInterpreterCoreInfoToCache( @@ -898,18 +898,18 @@ inline void RunProgramGradAPI( /*is_grad=*/true, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); } // share threadpool // NOTE(zhiqiu): this only works interpreter_core is executed strictly // after the related fwd_interpreter_core. if (interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, false)) { + program_id, global_inner_scope, place_hash_key, false)) { auto fwd_interpreter_core = interpretercore_info_cache .GetMutable(program_id, global_inner_scope, - place_hash_keys, + place_hash_key, /*is_grad=*/false) .core_; interpreter_core->ShareWorkQueueFrom(fwd_interpreter_core); @@ -936,7 +936,7 @@ inline void RunProgramGradAPI( interpretercore_info_cache.UpdateSkipEagerDeleteVars( program_id, global_inner_scope, - place_hash_keys, + place_hash_key, /*is_grad=*/true, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); @@ -947,7 +947,7 @@ inline void RunProgramGradAPI( 1); VLOG(2) << "Get interpretercore cahce by program:" << program_id; auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/true); + program_id, global_inner_scope, place_hash_key, /*is_grad=*/true); interpreter_core = cached_value.core_; // update scope @@ -998,7 +998,7 @@ inline void PirRunProgramGradAPI( const paddle::framework::AttributeMap &attrs, std::vector &x_grad, // NOLINT std::vector ¶ms_grad, // NOLINT - const std::vector &place_hash_keys) { + const int64_t &place_hash_key) { // if all output vars are set to stop_gradient, grad op no need to executed if (x_grad.empty() && params_grad.empty()) return; auto *out_scope_vec = &step_scope; @@ -1055,7 +1055,7 @@ inline void PirRunProgramGradAPI( std::shared_ptr interpreter_core = nullptr; if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/true)) { + program_id, global_inner_scope, place_hash_key, /*is_grad=*/true)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -1076,16 +1076,16 @@ inline void PirRunProgramGradAPI( /*is_grad=*/true, program_id, global_inner_scope, - place_hash_keys); + place_hash_key); // share threadpool // NOTE(zhiqiu): this only works interpreter_core is executed strictly // after the related fwd_interpreter_core. if (interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_keys, false)) { + program_id, global_inner_scope, place_hash_key, false)) { auto fwd_interpreter_core = interpretercore_info_cache .GetMutable(program_id, global_inner_scope, - place_hash_keys, + place_hash_key, /*is_grad=*/false) .core_; interpreter_core->ShareWorkQueueFrom(fwd_interpreter_core); @@ -1105,7 +1105,7 @@ inline void PirRunProgramGradAPI( interpretercore_info_cache.UpdateSkipEagerDeleteVars( program_id, global_inner_scope, - place_hash_keys, + place_hash_key, /*is_grad=*/true, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); @@ -1117,7 +1117,7 @@ inline void PirRunProgramGradAPI( 1); VLOG(2) << "Get interpretercore cahce by program:" << program_id; auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_keys, /*is_grad=*/true); + program_id, global_inner_scope, place_hash_key, /*is_grad=*/true); interpreter_core = cached_value.core_; if (interpreter_core->GetVariableScope()->GetMutableScope() != @@ -1229,7 +1229,7 @@ class GradNodeRunProgram : public egr::GradNodeBase { attrs_, x_grad_ptr, params_grad_ptr, - place_hash_keys_); + place_hash_key_); VLOG(3) << "End Eager Backward Node: GradNodeRunProgram: Ptr " << this; *executed_ = true; @@ -1261,8 +1261,8 @@ class GradNodeRunProgram : public egr::GradNodeBase { step_scope_ = scopes; } - void SetPlaceHashKeys(const std::vector &place_hash_keys) { - place_hash_keys_ = place_hash_keys; + void SetPlaceHashKey(const int64_t &place_hash_key) { + place_hash_key_ = place_hash_key; } protected: @@ -1334,7 +1334,7 @@ class GradNodeRunProgram : public egr::GradNodeBase { // Attribute Map paddle::framework::AttributeMap attrs_; - std::vector place_hash_keys_; + int64_t place_hash_key_; // why use shared_ptr. because paddle.grad will copy GradNode, if // we use bool, the copied node have different executed states. @@ -1417,7 +1417,7 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { attrs_, x_grad_ptr, params_grad_ptr, - place_hash_keys_); + place_hash_key_); VLOG(3) << "End Eager Backward Node: PirGradNodeRunProgram"; *executed_ = true; @@ -1451,8 +1451,8 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { step_scope_ = scopes; } - void SetPlaceHashKeys(const std::vector &place_hash_keys) { - place_hash_keys_ = place_hash_keys; + void SetPlaceHashKey(const int64_t &place_hash_key) { + place_hash_key_ = place_hash_key; } protected: @@ -1523,7 +1523,7 @@ class PirGradNodeRunProgram : public egr::GradNodeBase { // Attribute Map paddle::framework::AttributeMap attrs_; - std::vector place_hash_keys_; + int64_t place_hash_key_; std::shared_ptr executed_ = std::make_shared(false); }; diff --git a/paddle/fluid/framework/executor_cache.cc b/paddle/fluid/framework/executor_cache.cc index e1d1610619b6d..6af7443358361 100644 --- a/paddle/fluid/framework/executor_cache.cc +++ b/paddle/fluid/framework/executor_cache.cc @@ -308,7 +308,7 @@ std::shared_ptr CreateProgramInterpreterCoreInfoToCache( bool is_grad, int64_t program_id, framework::Scope *scope, - const std::vector &seeds) { + const int64_t &place_hash_key) { auto &interpretercore_info_cache = framework::InterpreterCoreInfoCache::Instance(); if (interpretercore_info_cache.Size() > 256000u /* max_cached_size*/) { @@ -325,8 +325,8 @@ std::shared_ptr CreateProgramInterpreterCoreInfoToCache( core.reset(new InterpreterCore( place, program_desc.Block(0), scope, execution_config)); - auto &cached_value = - interpretercore_info_cache.GetMutable(program_id, scope, seeds, is_grad); + auto &cached_value = interpretercore_info_cache.GetMutable( + program_id, scope, place_hash_key, is_grad); cached_value.core_ = core; return core; } @@ -337,7 +337,7 @@ std::shared_ptr CreatePirInterpreterCoreInfoToCache( bool is_grad, int64_t program_id, framework::Scope *scope, - const std::vector &seeds) { + const int64_t &place_hash_key) { auto &interpretercore_info_cache = framework::InterpreterCoreInfoCache::Instance(); if (interpretercore_info_cache.Size() > 256000u /* max_cached_size*/) { @@ -354,8 +354,8 @@ std::shared_ptr CreatePirInterpreterCoreInfoToCache( core.reset(new InterpreterCore( place, {}, ir_program->block(), scope, execution_config)); - auto &cached_value = - interpretercore_info_cache.GetMutable(program_id, scope, seeds, is_grad); + auto &cached_value = interpretercore_info_cache.GetMutable( + program_id, scope, place_hash_key, is_grad); cached_value.core_ = core; cached_value.ir_prog_ = std::move(ir_program); return core; diff --git a/paddle/fluid/framework/executor_cache.h b/paddle/fluid/framework/executor_cache.h index e095a75ddb1f0..57d9b06d92b0e 100644 --- a/paddle/fluid/framework/executor_cache.h +++ b/paddle/fluid/framework/executor_cache.h @@ -195,14 +195,12 @@ class InterpreterCoreInfoCache { bool Has(int64_t program_id, const framework::Scope* scope, - const std::vector& seeds, + const int64_t& place_hash_key, bool is_grad) { if (FLAGS_enable_pir_in_executor || FLAGS_enable_pir_with_pt_in_dy2st) { int64_t scope_i = reinterpret_cast(scope); program_id = hash_with_seed(program_id, scope_i); - for (int64_t seed : seeds) { - program_id = hash_with_seed(program_id, seed); - } + program_id = hash_with_seed(program_id, place_hash_key); } return info_map_.find(program_id) != info_map_.end() && info_map_[program_id].IsAvailable(is_grad); @@ -210,33 +208,30 @@ class InterpreterCoreInfoCache { InterpreterCoreInfo::CacheValue& GetMutable(int64_t program_id, const framework::Scope* scope, - const std::vector& seeds, + const int64_t& place_hash_key, bool is_grad) { if (FLAGS_enable_pir_in_executor || FLAGS_enable_pir_with_pt_in_dy2st) { int64_t scope_i = reinterpret_cast(scope); program_id = hash_with_seed(program_id, scope_i); - for (int64_t seed : seeds) { - program_id = hash_with_seed(program_id, seed); - } + program_id = hash_with_seed(program_id, place_hash_key); } return info_map_[program_id].GetMutable(is_grad); } void UpdateSkipEagerDeleteVars(int64_t program_id, const framework::Scope* scope, - const std::vector& seeds, + const int64_t& place_hash_key, bool is_grad, const std::set& skip_vars) { - auto& cached_value = GetMutable(program_id, scope, seeds, is_grad); + auto& cached_value = GetMutable(program_id, scope, place_hash_key, is_grad); cached_value.skip_eager_delete_vars_ = std::move(skip_vars); } - std::set& GetSkipEagerDeleteVars( - int64_t program_id, - const framework::Scope* scope, - const std::vector& seeds, - bool is_grad) { - auto& cached_value = GetMutable(program_id, scope, seeds, is_grad); + std::set& GetSkipEagerDeleteVars(int64_t program_id, + const framework::Scope* scope, + const int64_t& place_hash_key, + bool is_grad) { + auto& cached_value = GetMutable(program_id, scope, place_hash_key, is_grad); return cached_value.skip_eager_delete_vars_; } @@ -259,7 +254,7 @@ std::shared_ptr CreateProgramInterpreterCoreInfoToCache( bool is_grad, int64_t program_id, framework::Scope* scope, - const std::vector& seeds); + const int64_t& place_hash_key); std::shared_ptr CreatePirInterpreterCoreInfoToCache( std::unique_ptr<::pir::Program> ir_prog, @@ -267,7 +262,7 @@ std::shared_ptr CreatePirInterpreterCoreInfoToCache( bool is_grad, int64_t program_id, framework::Scope* scope, - const std::vector& seeds); + const int64_t& place_hash_key); std::unique_ptr<::pir::Program> ApplyIrPass(::pir::Program* program, phi::Place place); From 0bffc435f826842218ec2fcc6b9675a77366195b Mon Sep 17 00:00:00 2001 From: kangguangli Date: Tue, 26 Dec 2023 16:40:16 +0800 Subject: [PATCH 053/146] [PIR] fix property overwrite (#60225) --- test/CMakeLists.txt | 19 ++++++++++------ test/legacy_test/op_test.py | 31 ++++++++++++++++++++------ test/white_list/pir_op_test_white_list | 6 ----- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a61b75dd75af4..fd305ce6e8955 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -96,8 +96,10 @@ function(set_pit_tests_properties) PIR_OP_TESTS) foreach(IR_OP_TEST ${PIR_OP_TESTS}) if(TEST ${IR_OP_TEST}) - set_tests_properties( - ${IR_OP_TEST} PROPERTIES ENVIRONMENT "FLAGS_PIR_OPTEST_WHITE_LIST=True") + set_property( + TEST ${IR_OP_TEST} + APPEND + PROPERTY ENVIRONMENT "FLAGS_PIR_OPTEST_WHITE_LIST=True") endif() endforeach() @@ -105,8 +107,10 @@ function(set_pit_tests_properties) PIR_OP_NO_CHECK_TESTS) foreach(IR_OP_TEST ${PIR_OP_NO_CHECK_TESTS}) if(TEST ${IR_OP_TEST}) - set_tests_properties(${IR_OP_TEST} PROPERTIES ENVIRONMENT - "FLAGS_PIR_NO_CHECK=True") + set_property( + TEST ${IR_OP_TEST} + APPEND + PROPERTY ENVIRONMENT "FLAGS_PIR_NO_CHECK=True") endif() endforeach() @@ -115,9 +119,10 @@ function(set_pit_tests_properties) PIR_OP_RELAXED_TESTS) foreach(IR_OP_TEST ${PIR_OP_RELAXED_TESTS}) if(TEST ${IR_OP_TEST}) - set_tests_properties( - ${IR_OP_TEST} PROPERTIES ENVIRONMENT - "FLAGS_PIR_OPTEST_RELAX_CHECK=True") + set_property( + TEST ${IR_OP_TEST} + APPEND + PROPERTY ENVIRONMENT "FLAGS_PIR_OPTEST_RELAX_CHECK=True") endif() endforeach() diff --git a/test/legacy_test/op_test.py b/test/legacy_test/op_test.py index 2222638fb339b..8510e24a3b855 100644 --- a/test/legacy_test/op_test.py +++ b/test/legacy_test/op_test.py @@ -1489,9 +1489,13 @@ def _check_ir_output(self, place, program, feed_map, fetch_list, outs): check_method = np.testing.assert_array_equal if os.getenv("FLAGS_PIR_OPTEST_RELAX_CHECK", None) == "True": - check_method = lambda x, y, z: np.testing.assert_allclose( - x, y, err_msg=z, atol=1e-6, rtol=1e-6 - ) + + def relaxed_check(x, y, err_msg=""): + np.testing.assert_allclose( + x, y, err_msg=err_msg, atol=1e-6, rtol=1e-6 + ) + + check_method = relaxed_check if os.getenv("FLAGS_PIR_NO_CHECK", None) == "True": check_method = lambda x, y, err_msg: None @@ -3537,12 +3541,25 @@ def _check_ir_grad_output( check_method = np.testing.assert_array_equal if os.getenv("FLAGS_PIR_OPTEST_RELAX_CHECK", None) == "True": - check_method = lambda x, y, z: np.testing.assert_allclose( - x, y, err_msg=z, atol=1e-6, rtol=1e-6 - ) + + def relaxed_check_method(x, y, err_msg): + atol = 1e-6 + rtol = 1e-6 + if x.dtype == np.float16: + atol = 1e-5 + rtol = 1e-3 + np.testing.assert_allclose( + x, y, err_msg=err_msg, atol=atol, rtol=rtol + ) + + check_method = relaxed_check_method if os.getenv("FLAGS_PIR_NO_CHECK", None) == "True": - check_method = lambda x, y, err_msg: None + + def no_check_method(x, y, err_msg): + pass + + check_method = no_check_method for i in range(len(new_gradients)): check_method( diff --git a/test/white_list/pir_op_test_white_list b/test/white_list/pir_op_test_white_list index 1155fce81f300..2bf69d7d82faf 100644 --- a/test/white_list/pir_op_test_white_list +++ b/test/white_list/pir_op_test_white_list @@ -6,10 +6,7 @@ test_adadelta_op test_adagrad_op test_adagrad_op_static_build test_adamax_op -test_adamw_op -test_adamw_op_static_build test_addmm_op -test_affine_grid_op test_allclose_op test_amp_check_finite_and_scale_op test_angle_op @@ -60,14 +57,12 @@ test_conv2d_bf16_mkldnn_op test_conv2d_int8_mkldnn_op test_conv2d_mkldnn_op test_conv2d_op -test_conv2d_op_depthwise_conv test_conv2d_transpose_bf16_mkldnn_op test_conv2d_transpose_mkldnn_op test_conv2d_transpose_op test_conv2d_transpose_op_depthwise_conv test_conv3d_mkldnn_op test_conv3d_op -test_conv3d_transpose_op test_conv3d_transpose_part2_op test_crop_tensor_op test_cross_op @@ -234,7 +229,6 @@ test_polygamma_op test_pool2d_int8_mkldnn_op test_pool2d_mkldnn_op test_pool2d_op -test_pool3d_op test_pool_max_op test_prelu_mkldnn_op test_prelu_op From 5937fb04eb5f7c44372a573a09f2ab603e211ab5 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:52:23 +0800 Subject: [PATCH 054/146] =?UTF-8?q?=E3=80=90pir=E3=80=91add=20array=5Fwrit?= =?UTF-8?q?e/read=20vjp=20and=20api=20(#60338)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimize backward * [PIR] add vjp interface for while op * [PIR] fix ci error. * modify while stopgradient * merge * modify while grad bug * modify while grad op * modify * increment vp * [PIR] add get_used_external_value interface for block. * while case * delete print * delete print * Update python/paddle/autograd/ir_backward.py * [PIR] add unit_test for get_used_external_value * modify while_loop * code_style * modofy ci bug * modify while api * modify ci * modify array * Update python/paddle/autograd/ir_backward.py * Update test/legacy_test/test_cond.py * update --------- Co-authored-by: winter-wang <1030748926@qq.com> --- .../pir/dialect/op_generator/op_build_gen.py | 6 +- .../pir/dialect/operator/ir/manual_op.cc | 3 +- .../fluid/pir/dialect/operator/ir/manual_op.h | 19 ++++- .../pir/dialect/operator/ir/manual_op_vjp.cc | 75 ++++++++++++++++++- paddle/fluid/pybind/eager_utils.cc | 39 ++++++++++ paddle/fluid/pybind/eager_utils.h | 2 + .../fluid/pybind/manual_static_op_function.h | 4 +- python/paddle/tensor/array.py | 10 +-- test/ir/pir/test_while_api.py | 14 +++- test/legacy_test/test_while_loop_op.py | 28 ++++--- 10 files changed, 173 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py index 09b84e089e75a..6ef23da7135ef 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_build_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_build_gen.py @@ -386,7 +386,7 @@ def GenBuildOutputs( """ CREATE_INTARRAY_MUTABLE_ATTRIBUE_WITH_UNKONW_DATA_TEMPLATE = """ phi::IntArray {name}; - if ({name}_.dyn_cast().owner()->isa()) {{ + if ({name}_.dyn_cast() && {name}_.dyn_cast().owner()->isa()) {{ {name} = std::move(phi::IntArray(paddle::dialect::GetInt64Vector( {name}_.dyn_cast().owner() ->dyn_cast() @@ -408,7 +408,7 @@ def GenBuildOutputs( }}\n""" CREATE_VECTOR_INT_MUTABLE_ATTRIBUE_WITH_UNKONW_DATA_TEMPLATE = """ std::vector {name}; - if ({name}_.dyn_cast().owner()->isa()) {{ + if ({name}_.dyn_cast() && {name}_.dyn_cast().owner()->isa()) {{ {name} = paddle::dialect::GetInt64Vector( {name}_.dyn_cast().owner() ->dyn_cast() @@ -428,7 +428,7 @@ def GenBuildOutputs( }}\n""" CREATE_SCALAR_MUTABLE_ATTRIBUE_WITH_UNKONW_DATA_TEMPLATE = """ phi::Scalar {name}; - if ({name}_.dyn_cast().owner()->isa()) {{ + if ({name}_.dyn_cast() && {name}_.dyn_cast().owner()->isa()) {{ {name} = std::move(phi::Scalar({name}_.dyn_cast().owner() ->dyn_cast() .attribute("value") diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index dad8c36e2f358..b068db2e70837 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -1340,7 +1340,8 @@ void ArrayReadOp::Build(pir::Builder &builder, paddle::dialect::IrMetaTensor meta_array(&dense_array); phi::Scalar i_scalar; - if (i.dyn_cast().owner()->isa()) { + if (i.dyn_cast() && + i.dyn_cast().owner()->isa()) { i_scalar = std::move(phi::Scalar(i.dyn_cast() .owner() diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.h b/paddle/fluid/pir/dialect/operator/ir/manual_op.h index 1f367b4319d8c..43b4935b0ffcd 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.h @@ -215,8 +215,10 @@ class ArrayLengthOp static void InferMeta(phi::InferMetaContext *infer_meta); }; -class ArrayReadOp - : public pir::Op { +class ArrayReadOp : public pir::Op { public: using Op::Op; static const char *name() { return "pd_op.array_read"; } @@ -236,10 +238,17 @@ class ArrayReadOp pir::Value i() { return operand_source(1); } pir::OpResult out() { return result(0); } static void InferMeta(phi::InferMetaContext *infer_meta); + static std::vector> Vjp( + pir::Operation *op, + const std::vector> &inputs_, + const std::vector> &outputs, + const std::vector> &out_grads, + const std::vector> &stop_gradients); }; class ArrayWrite_Op : public pir::Op { public: @@ -259,6 +268,12 @@ class ArrayWrite_Op : public pir::Op> Vjp( + pir::Operation *op, + const std::vector> &inputs_, + const std::vector> &outputs, + const std::vector> &out_grads, + const std::vector> &stop_gradients); }; class ArrayToTensorOp diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc index f35ab01117d2a..b59a16ea5ff6e 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc @@ -170,7 +170,7 @@ std::vector> Increment_Op::Vjp( inputs_.size(), 1, platform::errors::InvalidArgument( - "Increment_ op's inputs size should be 2, but now is %d.", + "Increment_ op's inputs size should be 1, but now is %d.", inputs_.size())); PADDLE_ENFORCE_EQ( outputs.size(), @@ -192,5 +192,78 @@ std::vector> Increment_Op::Vjp( return res; } +std::vector> ArrayWrite_Op::Vjp( + pir::Operation* op, + const std::vector>& inputs_, + const std::vector>& outputs, + const std::vector>& out_grads, + const std::vector>& stop_gradients) { + PADDLE_ENFORCE_EQ( + inputs_.size(), + 3, + platform::errors::InvalidArgument( + "ArrayWrite_ op's inputs size should be 3, but now is %d.", + inputs_.size())); + PADDLE_ENFORCE_EQ( + outputs.size(), + 1, + platform::errors::InvalidArgument( + "ArrayWrite_ op's outputs size should be 1, but now is %d.", + outputs.size())); + + PADDLE_ENFORCE_EQ( + out_grads.size(), + 1, + platform::errors::InvalidArgument( + "ArrayWrite_ op's outputs size should be 1, but now is %d.", + outputs.size())); + + VLOG(6) << "Vjp prepare call ArrayWrite_'s vjp inteface"; + pir::OpResult tensor_res = + paddle::dialect::array_read(out_grads[0][0], inputs_[2][0]); + + std::vector> res{{tensor_res}}; + if (stop_gradients[0][0]) { + res = {{}}; + } + return res; +} + +std::vector> ArrayReadOp::Vjp( + pir::Operation* op, + const std::vector>& inputs_, + const std::vector>& outputs, + const std::vector>& out_grads, + const std::vector>& stop_gradients) { + PADDLE_ENFORCE_EQ( + inputs_.size(), + 2, + platform::errors::InvalidArgument( + "Array_read op's inputs size should be 2, but now is %d.", + inputs_.size())); + PADDLE_ENFORCE_EQ( + outputs.size(), + 1, + platform::errors::InvalidArgument( + "Array_read op's outputs size should be 1, but now is %d.", + outputs.size())); + + PADDLE_ENFORCE_EQ( + out_grads.size(), + 1, + platform::errors::InvalidArgument( + "Array_read op's outputs size should be 1, but now is %d.", + outputs.size())); + + VLOG(6) << "Vjp prepare call Array_read's vjp inteface"; + pir::OpResult tensor_res = paddle::dialect::array_write_( + inputs_[0][0], out_grads[0][0], inputs_[1][0]); + + std::vector> res{{tensor_res}}; + if (stop_gradients[0][0]) { + res = {{}}; + } + return res; +} } // namespace dialect } // namespace paddle diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 4c030c1ff7d7d..9889d9511b4d0 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -188,6 +188,45 @@ bool PyObject_CheckIROpResult(PyObject* obj) { return PyObject_TypeCheck(obj, g_ir_opresult_pytype); } +bool PyObject_CheckIRValue(PyObject* obj) { + return PyObject_TypeCheck(obj, g_ir_value_pytype); +} + +bool PyObject_CheckIRVectorOfValue(PyObject* obj) { + if (PyList_Check(obj)) { + Py_ssize_t len = PyList_Size(obj); + PyObject* item = nullptr; + // if obj is [], parse it as std::vector + if (len == 0) { + return false; + } + for (Py_ssize_t i = 0; i < len; i++) { + item = PyList_GetItem(obj, i); + if (!PyObject_CheckIRValue(item)) { + return false; + } + } + return true; + } else if (PyTuple_Check(obj)) { + Py_ssize_t len = PyTuple_Size(obj); + PyObject* item = nullptr; + if (len == 0) { + return false; + } + for (Py_ssize_t i = 0; i < len; i++) { + item = PyTuple_GetItem(obj, i); + if (!PyObject_CheckIRValue(item)) { + return false; + } + } + return true; + } else if (PyObject_TypeCheck(obj, g_ir_value_pytype)) { + return true; + } else { + return false; + } +} + bool PyObject_CheckIRVectorOfOpResult(PyObject* obj) { if (PyList_Check(obj)) { Py_ssize_t len = PyList_Size(obj); diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 060d40227b9c8..fd3be0bdf2fcc 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -72,7 +72,9 @@ bool PyObject_CheckLongOrConvertToLong(PyObject** obj); bool PyObject_CheckFloatOrConvertToFloat(PyObject** obj); bool PyObject_CheckStr(PyObject* obj); bool PyObject_CheckIROpResult(PyObject* obj); +bool PyObject_CheckIRValue(PyObject* obj); bool PyObject_CheckIRVectorOfOpResult(PyObject* obj); +bool PyObject_CheckIRVectorOfValue(PyObject* obj); bool CastPyArg2AttrBoolean(PyObject* obj, ssize_t arg_pos); int CastPyArg2AttrInt(PyObject* obj, ssize_t arg_pos); int64_t CastPyArg2AttrLong(PyObject* obj, ssize_t arg_pos); diff --git a/paddle/fluid/pybind/manual_static_op_function.h b/paddle/fluid/pybind/manual_static_op_function.h index 247c2c105633f..21285163dd64f 100644 --- a/paddle/fluid/pybind/manual_static_op_function.h +++ b/paddle/fluid/pybind/manual_static_op_function.h @@ -181,7 +181,7 @@ static PyObject *static_api_array_read(PyObject *self, PyObject *i_obj = PyTuple_GET_ITEM(args, 1); pir::Value i; - if (PyObject_CheckIROpResult(i_obj)) { + if (PyObject_CheckIRValue(i_obj)) { i = CastPyArg2Value(i_obj, "array_read", 1); } else { int64_t i_tmp = CastPyArg2Int(i_obj, "array_read", 1); @@ -215,7 +215,7 @@ static PyObject *static_api_array_write_(PyObject *self, auto x = CastPyArg2Value(x_obj, "array_write_", 1); PyObject *i_obj = PyTuple_GET_ITEM(args, 2); pir::Value i; - if (PyObject_CheckIROpResult(i_obj)) { + if (PyObject_CheckIRValue(i_obj)) { i = CastPyArg2Value(i_obj, "array_write_", 2); } else { int64_t i_tmp = CastPyArg2Int(i_obj, "array_write_", 2); diff --git a/python/paddle/tensor/array.py b/python/paddle/tensor/array.py index 75aa9ad8b1cce..0259200118e7c 100644 --- a/python/paddle/tensor/array.py +++ b/python/paddle/tensor/array.py @@ -56,7 +56,7 @@ def array_length(array): return len(array) elif in_pir_mode(): if ( - not isinstance(array, paddle.pir.OpResult) + not isinstance(array, paddle.pir.Value) or not array.is_dense_tensor_array_type() ): raise TypeError( @@ -137,7 +137,7 @@ def array_read(array, i): return array[i] elif in_pir_mode(): if ( - not isinstance(array, paddle.pir.OpResult) + not isinstance(array, paddle.pir.Value) or not array.is_dense_tensor_array_type() ): raise TypeError( @@ -219,13 +219,13 @@ def array_write(x, i, array=None): return array elif in_pir_mode(): check_variable_and_dtype(i, 'i', ['int64'], 'array_write') - if not isinstance(x, paddle.pir.OpResult): + if not isinstance(x, paddle.pir.Value): raise TypeError( f"x should be pir.OpResult, but recevied {type(x)}." ) if array is not None: if ( - not isinstance(array, paddle.pir.OpResult) + not isinstance(array, paddle.pir.Value) or not array.is_dense_tensor_array_type() ): raise TypeError("array should be tensor array vairable") @@ -302,7 +302,7 @@ def create_array(dtype, initialized_list=None): # NOTE: Only support plain list like [x, y,...], not support nested list in static graph mode. for val in array: - if not isinstance(val, (Variable, paddle.pir.OpResult)): + if not isinstance(val, (Variable, paddle.pir.Value)): raise TypeError( "All values in `initialized_list` should be Variable or pir.OpResult, but recevied {}.".format( type(val) diff --git a/test/ir/pir/test_while_api.py b/test/ir/pir/test_while_api.py index 45b68b9fcf125..cc07cdbb58ad6 100644 --- a/test/ir/pir/test_while_api.py +++ b/test/ir/pir/test_while_api.py @@ -62,16 +62,13 @@ def test_while_base(self): def test_get_used_external_value(self): main_program = paddle.static.Program() with paddle.pir.core.program_guard(main_program): - print(main_program) i = paddle.full(shape=[1], fill_value=0) - print(main_program) x = paddle.full(shape=[1], fill_value=10) y = paddle.full(shape=[1], fill_value=5) # i, x = paddle.static.nn.while_loop(cond, body, [i, ten]) paddle.static.nn.while_loop( lambda p, q: p < q, lambda p, q: [p + y, q + i], [i, x] ) - print(main_program) while_op = main_program.global_block().ops[-1] self.assertEqual(while_op.name(), "pd_op.while") body_block = while_op.as_while_op().body() @@ -175,7 +172,6 @@ def test_backward(self): out, [i, j], ) - self.assertEqual( grad_outs[0].get_defining_op().name(), "pd_op.while" ) @@ -189,6 +185,16 @@ def test_backward(self): "cf.has_elements", ) + self.assertEqual( + main_program.global_block() + .ops[-1] + .as_while_op() + .body() + .ops[-3] + .name(), + "pd_op.add_grad", + ) + def test_backward_with_loop_var_same_to_extral_var(self): main_program = paddle.static.Program() with paddle.pir.core.program_guard(main_program): diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index 4675c2b30e73e..534d5fa42e7e3 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -374,7 +374,7 @@ def body(i, x): class TestApiWhileLoop_NestedWithBackwardAndLoDTensorArray(unittest.TestCase): # TODO(zhangbo): Support while grad exe for pir - + # @test_with_pir_api def test_nested_net_with_backward_and_lodtensor(self): def external_cond(i, j, x, mem_array): return paddle.less_than(i, array_len) @@ -411,6 +411,7 @@ def internal_body(j, x, mem_array): d2 = paddle.static.data(name='d2', shape=[10], dtype='float32') x = paddle.static.data(name='x', shape=[10], dtype='float32') x.stop_gradient = False + x.persistable = True i = paddle.zeros(shape=[1], dtype='int64') i.stop_gradient = True init = paddle.zeros(shape=[10], dtype='float32') @@ -436,10 +437,9 @@ def internal_body(j, x, mem_array): external_cond, external_body, [i, j, x, mem_array] ) - sum_result = paddle.tensor.array_read(array=mem_array, i=j) + sum_result = paddle.tensor.array_read(array=out[3], i=j) mean = paddle.mean(sum_result) - append_backward(mean) - + grad_list = append_backward(mean) place = ( base.CUDAPlace(0) if core.is_compiled_with_cuda() @@ -453,11 +453,21 @@ def internal_body(j, x, mem_array): feed_x = np.ones(10).astype('float32') data_sum = d[0] + d[1] + d[2] + 3 * feed_x x_grad = [0.3] * 10 - res = exe.run( - main_program, - feed={'d0': d[0], 'd1': d[1], 'd2': d[2], 'x': feed_x}, - fetch_list=[sum_result.name, x.grad_name], - ) + if paddle.framework.in_pir_mode(): + for p, g in grad_list: + if p.is_same(x): + dx = g + res = exe.run( + main_program, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2], 'x': feed_x}, + fetch_list=[sum_result, dx], + ) + else: + res = exe.run( + main_program, + feed={'d0': d[0], 'd1': d[1], 'd2': d[2], 'x': feed_x}, + fetch_list=[sum_result.name, x.grad_name], + ) np.testing.assert_allclose(res[0], data_sum, rtol=1e-05) np.testing.assert_allclose(res[1], x_grad, rtol=1e-05) From 96b9068fdc3c7f8fa2252ebeed80d4f3a0184661 Mon Sep 17 00:00:00 2001 From: Haohongxiang <86215757+haohongxiang@users.noreply.github.com> Date: Tue, 26 Dec 2023 17:16:55 +0800 Subject: [PATCH 055/146] [Auto Parallel] Add target path for semi-auto and remove llama uts into a new file (#60273) --- .../hybrid_strategy/CMakeLists.txt | 15 ++ .../test_cross_mesh_reshard.py | 67 ++++++++ ...test_semi_auto_parallel_hybrid_strategy.py | 145 ------------------ .../test_semi_auto_parallel_llama_model.py | 120 +++++++++++++++ .../hybrid_strategy/testslist.csv | 2 + tools/auto_parallel/ci_auto_parallel.sh | 4 +- tools/auto_parallel/target_path_lists.sh | 5 + 7 files changed, 212 insertions(+), 146 deletions(-) create mode 100644 test/auto_parallel/hybrid_strategy/test_cross_mesh_reshard.py create mode 100644 test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py diff --git a/test/auto_parallel/hybrid_strategy/CMakeLists.txt b/test/auto_parallel/hybrid_strategy/CMakeLists.txt index f9cf3eeaea90b..71716897874b2 100644 --- a/test/auto_parallel/hybrid_strategy/CMakeLists.txt +++ b/test/auto_parallel/hybrid_strategy/CMakeLists.txt @@ -12,6 +12,14 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_semi_auto_parallel_hybrid_strategy PROPERTIES TIMEOUT "180" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_semi_auto_parallel_llama_model MODULES + test_semi_auto_parallel_llama_model ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_semi_auto_parallel_llama_model + PROPERTIES TIMEOUT "180" LABELS "RUN_TYPE=HYBRID") +endif() if((WITH_GPU) AND (LINUX)) py_test_modules( test_save_load_state_dict MODULES test_save_load_state_dict ENVS @@ -27,3 +35,10 @@ if((WITH_GPU) AND (LINUX)) set_tests_properties(test_semi_auto_parallel_c_cross_entropy PROPERTIES TIMEOUT "120" LABELS "RUN_TYPE=HYBRID") endif() +if((WITH_GPU) AND (LINUX)) + py_test_modules( + test_cross_mesh_reshard MODULES test_cross_mesh_reshard ENVS + "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python") + set_tests_properties(test_cross_mesh_reshard PROPERTIES TIMEOUT "120" LABELS + "RUN_TYPE=HYBRID") +endif() diff --git a/test/auto_parallel/hybrid_strategy/test_cross_mesh_reshard.py b/test/auto_parallel/hybrid_strategy/test_cross_mesh_reshard.py new file mode 100644 index 0000000000000..d0f2439f61474 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_cross_mesh_reshard.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelCrossMeshReshard(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp( + num_of_devices=4, + timeout=120, + nnode=1, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_simple_net_cross_mesh_reshard(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_cross_mesh_reshard.py", + user_defined_envs=envs, + ) + + +class TestSemiAutoParallelNdCrossMeshReshard( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp(num_of_devices=8, timeout=200, nnode=1) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_simple_net_bybrid_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_nd_cross_mesh_reshard.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_strategy.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_strategy.py index 947a66d184140..a41be97193713 100644 --- a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_strategy.py +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_hybrid_strategy.py @@ -17,8 +17,6 @@ import collective.test_communication_api_base as test_base -import paddle - class TestSemiAutoParallelDPMPStrategy(test_base.CommunicationTestDistBase): def setUp(self): @@ -128,148 +126,5 @@ def test_simple_net_dp_mp_pp_sp(self): ckpt_path.cleanup() -class TestSemiAutoParallelCrossMeshReshard(test_base.CommunicationTestDistBase): - def setUp(self): - super().setUp( - num_of_devices=4, - timeout=120, - nnode=1, - ) - self._default_envs = { - "dtype": "float32", - "seed": "2023", - } - self._changeable_envs = {"backend": ["gpu"]} - - def test_simple_net_cross_mesh_reshard(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - for envs in envs_list: - self.run_test_case( - "semi_auto_parallel_cross_mesh_reshard.py", - user_defined_envs=envs, - ) - - -class TestSemiAutoParallelNdCrossMeshReshard( - test_base.CommunicationTestDistBase -): - def setUp(self): - super().setUp(num_of_devices=8, timeout=200, nnode=1) - self._default_envs = { - "dtype": "float32", - "seed": "2023", - } - self._changeable_envs = {"backend": ["gpu"]} - - def test_simple_net_bybrid_strategy(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - for envs in envs_list: - self.run_test_case( - "semi_auto_parallel_nd_cross_mesh_reshard.py", - user_defined_envs=envs, - ) - - -class TestSemiAutoParallelLlamaDPMPStrategy( - test_base.CommunicationTestDistBase -): - def setUp(self): - super().setUp(num_of_devices=4, timeout=200, nnode=1) - self._default_envs = { - "dtype": "float32", - "seed": "2023", - } - self._changeable_envs = {"backend": ["gpu"]} - - def test_simple_net_hybrid_strategy(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - cuda_version_main = int(paddle.version.cuda().split(".")[0]) - device_prop_main = paddle.device.cuda.get_device_capability()[0] - if cuda_version_main >= 11 and device_prop_main >= 8: - for envs in envs_list: - self.run_test_case( - "semi_auto_parallel_for_llama_decoder_dp_mp.py", - user_defined_envs=envs, - ) - - -class TestSemiAutoParallelLlama2D(test_base.CommunicationTestDistBase): - def setUp(self): - super().setUp(num_of_devices=4, timeout=400, nnode=1) - self._default_envs = {"dp": "2", "mp": "2", "pp": "1", "acc_step": "2"} - self._changeable_envs = { - "backend": ["gpu"], - "use_sp": ["true", "false"], - "recompute": ["true", "false"], - "recompute_granularity": ["full", "full_attn", "core_attn"], - } - - def test_simple_net_hybrid_strategy(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - for envs in envs_list: - self.run_test_case( - "semi_auto_llama.py", - user_defined_envs=envs, - ) - - -class TestSemiAutoParallelLlama3D(test_base.CommunicationTestDistBase): - def setUp(self): - super().setUp(num_of_devices=8, timeout=200, nnode=1) - self._default_envs = {"dp": "2", "mp": "2", "pp": "2", "acc_step": "2"} - self._changeable_envs = { - "backend": ["gpu"], - "use_sp": ["true", "false"], - "use_param_group": ["false", "true"], - # TODO(Yuang Liu): add recompute ut to pp after fixing pp probs - # "recompute": ["true", "false"], - # "recompute_granularity": ["full", "full_attn", "core_attn"], - } - - def test_simple_net_hybrid_strategy(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - for envs in envs_list: - self.run_test_case( - "semi_auto_llama.py", - user_defined_envs=envs, - ) - - -class TestSemiAutoParallelLlamaACC(test_base.CommunicationTestDistBase): - def setUp(self): - super().setUp(num_of_devices=8, timeout=200, nnode=1) - self._default_envs = { - "dp": "2", - "mp": "2", - "pp": "2", - "acc_step": "1", - "FLAGS_embedding_deterministic": "1", - "FLAGS_cudnn_deterministic": "1", - } - self._changeable_envs = { - "backend": ["gpu"], - } - - def test_simple_net_hybrid_strategy_acc(self): - envs_list = test_base.gen_product_envs_list( - self._default_envs, self._changeable_envs - ) - for envs in envs_list: - self.run_test_case( - "semi_auto_llama.py", - user_defined_envs=envs, - ) - - if __name__ == "__main__": unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py new file mode 100644 index 0000000000000..36b6c1d5d0e97 --- /dev/null +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py @@ -0,0 +1,120 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + +import paddle + + +class TestSemiAutoParallelLlamaDPMPStrategy( + test_base.CommunicationTestDistBase +): + def setUp(self): + super().setUp(num_of_devices=4, timeout=200, nnode=1) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["gpu"]} + + def test_simple_net_hybrid_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + cuda_version_main = int(paddle.version.cuda().split(".")[0]) + device_prop_main = paddle.device.cuda.get_device_capability()[0] + if cuda_version_main >= 11 and device_prop_main >= 8: + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_for_llama_decoder_dp_mp.py", + user_defined_envs=envs, + ) + + +class TestSemiAutoParallelLlama2D(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp(num_of_devices=4, timeout=400, nnode=1) + self._default_envs = {"dp": "2", "mp": "2", "pp": "1", "acc_step": "2"} + self._changeable_envs = { + "backend": ["gpu"], + "use_sp": ["true", "false"], + "recompute": ["true", "false"], + "recompute_granularity": ["full", "full_attn", "core_attn"], + } + + def test_simple_net_hybrid_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_llama.py", + user_defined_envs=envs, + ) + + +class TestSemiAutoParallelLlama3D(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp(num_of_devices=8, timeout=200, nnode=1) + self._default_envs = {"dp": "2", "mp": "2", "pp": "2", "acc_step": "2"} + self._changeable_envs = { + "backend": ["gpu"], + "use_sp": ["true", "false"], + "use_param_group": ["false", "true"], + # TODO(Yuang Liu): add recompute ut to pp after fixing pp probs + # "recompute": ["true", "false"], + # "recompute_granularity": ["full", "full_attn", "core_attn"], + } + + def test_simple_net_hybrid_strategy(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_llama.py", + user_defined_envs=envs, + ) + + +class TestSemiAutoParallelLlamaACC(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp(num_of_devices=8, timeout=200, nnode=1) + self._default_envs = { + "dp": "2", + "mp": "2", + "pp": "2", + "acc_step": "1", + "FLAGS_embedding_deterministic": "1", + "FLAGS_cudnn_deterministic": "1", + } + self._changeable_envs = { + "backend": ["gpu"], + } + + def test_simple_net_hybrid_strategy_acc(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_llama.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/auto_parallel/hybrid_strategy/testslist.csv b/test/auto_parallel/hybrid_strategy/testslist.csv index 7cc4a4b9e987b..765743aeb4ec9 100644 --- a/test/auto_parallel/hybrid_strategy/testslist.csv +++ b/test/auto_parallel/hybrid_strategy/testslist.csv @@ -1,4 +1,6 @@ name,os,arch,timeout,run_type,launcher,num_port,run_serial,envs,conditions test_semi_auto_parallel_hybrid_strategy,LINUX,GPU,180,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_semi_auto_parallel_llama_model,LINUX,GPU,180,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_save_load_state_dict,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., test_semi_auto_parallel_c_cross_entropy,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., +test_cross_mesh_reshard,LINUX,GPU,120,HYBRID,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../.., diff --git a/tools/auto_parallel/ci_auto_parallel.sh b/tools/auto_parallel/ci_auto_parallel.sh index 16268333596ce..09095d7f6122b 100644 --- a/tools/auto_parallel/ci_auto_parallel.sh +++ b/tools/auto_parallel/ci_auto_parallel.sh @@ -53,7 +53,9 @@ for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{pri dir2=${arr_file_name[1]} dir3=${arr_file_name[2]} dir4=${arr_file_name[3]} - file_item=$dir1/$dir2/$dir3/$dir4 + dir5=${arr_file_name[4]} + dir6=${arr_file_name[5]} + file_item=$dir1/$dir2/$dir3/$dir4/$dir5/$dir6 echo "file_name:"${file_name}, "path:"${file_item} if [ ! -f ${file_name} ];then # deleting files for PR continue diff --git a/tools/auto_parallel/target_path_lists.sh b/tools/auto_parallel/target_path_lists.sh index bf0d127c44b77..fb1c943ff3a79 100644 --- a/tools/auto_parallel/target_path_lists.sh +++ b/tools/auto_parallel/target_path_lists.sh @@ -21,6 +21,9 @@ target_lists_for_semi_auto_ci=( "paddle/fluid/pybind/auto_parallel_py.h" "paddle/phi/infermeta/spmd_rules" "paddle/phi/core/distributed" + "paddle/phi/api/yaml/generator/dist_api_gen.py" + "paddle/phi/api/yaml/generator/dist_bw_api_gen.py" + "tools/auto_parallel/target_path_lists.sh" "test/auto_parallel" ) @@ -30,6 +33,7 @@ target_lists_for_pir_ci=( "paddle/fluid/pir/dialect" "paddle/fluid/pir/transforms" "paddle/pir" + "tools/auto_parallel/target_path_lists.sh" ) target_lists_for_dygraph_ci=( @@ -38,5 +42,6 @@ target_lists_for_dygraph_ci=( "python/paddle/distributed/sharding" "paddle/fluid/distributed/collective" "paddle/phi/core/distributed" + "tools/auto_parallel/target_path_lists.sh" "test/collective/hybrid_strategy" ) From 0b88eef8f1307d7758729477302b90f36c41e29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= Date: Tue, 26 Dec 2023 18:23:41 +0800 Subject: [PATCH 056/146] =?UTF-8?q?=E3=80=90CINN=E3=80=91Add=20compute=5Fi?= =?UTF-8?q?nline=5Ftactics=20for=20dynamic=20group=20schedule=20(#60342)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add compute_inline_tactics for dynamic group schedule * add comment for func args --- .../dy_shape_group_scheduler.cc | 11 ++-- .../st_shape_group_scheduler.cc | 5 +- .../ir/group_schedule/tactic/CMakeLists.txt | 1 + .../tactic/arrange_storage_tactic.h | 4 +- .../tactic/compute_inline_tactic.cc | 54 +++++++++++++++++++ .../tactic/compute_inline_tactic.h | 39 ++++++++++++++ .../group_schedule/tactic/schedule_tactic.h | 1 + paddle/cinn/pybind/schedule.cc | 24 ++++----- 8 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc create mode 100644 paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h diff --git a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc index f0804e16aee36..04e7afa8760f6 100644 --- a/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.cc @@ -14,12 +14,14 @@ #include "paddle/cinn/ir/group_schedule/dy_shape_group_scheduler.h" #include "paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h" +#include "paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h" namespace cinn { namespace ir { void DynamicShapeGroupScheduler::Init() { std::unordered_set output_names = OutputTensorNames(); + tactics_.emplace_back(new ComputeInlineTactic(output_names, target_)); tactics_.emplace_back(new ArrangeStorageTactic(output_names)); } @@ -31,20 +33,13 @@ void DynamicShapeGroupScheduler::Schedule() { ir_sch_->Fuse(loops); } - for (all_blocks = ir_sch_->GetAllBlocks(); all_blocks.size() > 1;) { - auto block0 = all_blocks[0]; - ir_sch_->ComputeInline(block0); - all_blocks = ir_sch_->GetAllBlocks(); - } - + ApplyTactics(); all_blocks = ir_sch_->GetAllBlocks(); auto block0_loops = ir_sch_->GetLoops(all_blocks[0]); auto splited_loops1 = ir_sch_->Split(block0_loops[0], {1024, -1}); ir_sch_->Bind(splited_loops1[0], "threadIdx.x"); - ApplyTactics(); - ir::Expr predicate1 = ir::LE::Make(Expr(1023), Expr(1024)); std::unique_ptr new_ir_sch1 = std::make_unique(*ir_sch_); diff --git a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc index a5cb17dc5a2a7..86f114def4146 100644 --- a/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc +++ b/paddle/cinn/ir/group_schedule/st_shape_group_scheduler.cc @@ -47,9 +47,12 @@ static const std::unordered_set GEN_FUNC_NAME(GEN_FUNC_NAME_WITH_TYPE, CINN_NVGPU_FUNC_TYPE) #undef GEN_FUNC_NAME +#undef GEN_FUNC_NAME_WITH_TYPE +#undef CINN_NVGPU_FUNC_TYPE +#undef CINN_NVGPU_FUNC2STRING }; -bool IsProhibitScheduleExternCallBlock(ir::Expr block) { +static bool IsProhibitScheduleExternCallBlock(ir::Expr block) { ir::ScheduleBlockRealize* sch_block_realize = block.As(); CHECK_NOTNULL(sch_block_realize); diff --git a/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt index 50e8500ae38bc..6ed979ece476b 100644 --- a/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt +++ b/paddle/cinn/ir/group_schedule/tactic/CMakeLists.txt @@ -1,3 +1,4 @@ core_gather_headers() gather_srcs(cinnapi_src SRCS arrange_storage_tactic.cc) +gather_srcs(cinnapi_src SRCS compute_inline_tactic.cc) diff --git a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h index 05c9e67225a8f..0371aead7e163 100644 --- a/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/arrange_storage_tactic.h @@ -21,12 +21,12 @@ namespace cinn { namespace ir { -class ArrangeStorageTactic : public ScheduleTactic { +class ArrangeStorageTactic final : public ScheduleTactic { public: explicit ArrangeStorageTactic( const std::unordered_set& output_names); - void Apply(ir::IRSchedule* sch, const std::string& block_id) final; + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; private: std::unordered_set output_names_; diff --git a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc new file mode 100644 index 0000000000000..81bf65366a968 --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.cc @@ -0,0 +1,54 @@ +// Copyright (c) 2023 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h" + +#include +#include + +#include "paddle/cinn/auto_schedule/search_space/auto_gen_rule/auto_inline.h" +#include "paddle/cinn/ir/ir.h" +#include "paddle/cinn/ir/ir_printer.h" +#include "paddle/cinn/ir/schedule/ir_schedule.h" + +namespace cinn { +namespace ir { + +ComputeInlineTactic::ComputeInlineTactic( + const std::unordered_set& output_names, const Target& target) + : output_names_(output_names), target_(target) {} + +void ComputeInlineTactic::Apply(ir::IRSchedule* sch, + const std::string& block_id) { + VLOG(5) << "[Start DoComputeInline] func body: " + << sch->GetModule().GetExprs().front(); + + // TODO(LiuYang): Compute of ops will be rewrited so that we + // don't use it in dynamic group_schedule rules temporarily. + // if (IsProhibitScheduleExternCallBlock(node->Block())) { + // return; + // } + auto_schedule::AutoInline inliner(target_, output_names_); + VLOG(6) << "try ComputeInline on: " << block_id + << ", before ComputeInline, func body: " + << sch->GetModule().GetExprs().front(); + ir::Expr schedule_block = sch->GetBlock(block_id); + inliner.Apply(sch, schedule_block); + VLOG(6) << "try ComputeInline on: " << block_id + << ", after ComputeInline, func body: " + << sch->GetModule().GetExprs().front(); +} + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h new file mode 100644 index 0000000000000..71754fdc4adcd --- /dev/null +++ b/paddle/cinn/ir/group_schedule/tactic/compute_inline_tactic.h @@ -0,0 +1,39 @@ +// Copyright (c) 2023 CINN Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include "paddle/cinn/common/target.h" +#include "paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h" + +namespace cinn { +namespace ir { + +class ComputeInlineTactic final : public ScheduleTactic { + public: + explicit ComputeInlineTactic( + const std::unordered_set& output_names, + const cinn::common::Target& target); + + void Apply(ir::IRSchedule* sch, const std::string& block_id) override; + + private: + std::unordered_set output_names_; + cinn::common::Target target_; +}; + +} // namespace ir +} // namespace cinn diff --git a/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h index 49c4d8b623f45..bc2c88c7d5ccd 100644 --- a/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h +++ b/paddle/cinn/ir/group_schedule/tactic/schedule_tactic.h @@ -15,6 +15,7 @@ #pragma once #include + #include "paddle/cinn/ir/schedule/ir_schedule.h" namespace cinn { diff --git a/paddle/cinn/pybind/schedule.cc b/paddle/cinn/pybind/schedule.cc index d9a8e418cabdb..a00eb5e7032f6 100644 --- a/paddle/cinn/pybind/schedule.cc +++ b/paddle/cinn/pybind/schedule.cc @@ -35,18 +35,18 @@ void BindSchedule(py::module *m) { py::arg("debug_flag") = false, py::arg("err_msg_level") = utils::ErrorMessageLevel::kGeneral, py::arg("is_dynamic_shape") = false) - .def_static("make", - [](ir::LoweredFunc &ir_func) { - ir::ModuleExpr *module_expr = - new ir::ModuleExpr({ir_func->body}); - auto scheduler = std::make_unique( - *module_expr, - -1, - false, - utils::ErrorMessageLevel::kGeneral, - true); - return scheduler; - }) + .def_static( + "make", + [](ir::LoweredFunc &ir_func) { + ir::ModuleExpr *module_expr = new ir::ModuleExpr({ir_func->body}); + auto scheduler = std::make_unique( + *module_expr, + /* rand_seed = */ -1, + /* debug_flag = */ false, + /* err_msg_level = */ utils::ErrorMessageLevel::kGeneral, + /* is_dynamic_shape = */ true); + return scheduler; + }) .def("fuse", py::overload_cast &>(&ir::IRSchedule::Fuse)) .def("split", From dd7a7be3a893f846201cebfa3c4f486eb32fa8d8 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Tue, 26 Dec 2023 18:35:55 +0800 Subject: [PATCH 057/146] fix (#60327) --- .../pir_adaptor/pir_adaptor_util.cc | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 8717c7d4fd2e1..eba12327d10a0 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -626,18 +626,26 @@ void HandleForInplaceOp(pir::Operation* op, pir::Value inplace_value = op->operand_source(yaml_parser.InputName2Id().at(inplace_name)); std::string var_name = value_exe_info->GetVarName(inplace_value); - VLOG(4) << "inplace: " << value_name << " -> " << inplace_name - << " (var: " << var_name << ")"; - value_exe_info->AddValue2VarName(value, var_name); + if (var_name != "") { + VLOG(4) << "inplace: " << value_name << " -> " << inplace_name + << " (var: " << var_name << ")"; + value_exe_info->AddValue2VarName(value, var_name); + } else { + BuildValue(value, var_name_prefix, value_exe_info); + } } else if (yaml_parser.HasView(value_name)) { const std::string& view_name = yaml_parser.ViewName(value_name); pir::Value view_value = op->operand_source(yaml_parser.InputName2Id().at(view_name)); // const std::string& var_name = value_2_var_name->at(view_value); std::string var_name = value_exe_info->GetVarName(view_value); - VLOG(4) << "view: " << value_name << " -> " << view_name - << " (var: " << var_name << ")"; - value_exe_info->AddValue2VarName(value, var_name); + if (var_name != "") { + VLOG(4) << "view: " << value_name << " -> " << view_name + << " (var: " << var_name << ")"; + value_exe_info->AddValue2VarName(value, var_name); + } else { + BuildValue(value, var_name_prefix, value_exe_info); + } } else { BuildValue(value, var_name_prefix, value_exe_info); } From 9765ba805b40db5b00c8003d24cf45013ebf2420 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Tue, 26 Dec 2023 18:43:41 +0800 Subject: [PATCH 058/146] Fit vpp for allreduce_matmul_grad_overlapping pass (#60317) * Fit vpp for allreduce_matmul_grad_overlapping pass * Fix * Fix * Fix --- .../auto_parallel/static/parallelizer_v2.py | 5 +- .../allreduce_matmul_grad_overlapping.py | 71 +++++++++++++++++-- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py index 115f260873d62..73dd1de8508bf 100644 --- a/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py +++ b/python/paddle/distributed/auto_parallel/static/parallelizer_v2.py @@ -412,8 +412,11 @@ def _apply_post_optimization( "loss. Try to export CUDA_DEVICE_MAX_CONNECTIONS=1 for better performance." ) + config = { + "dist_context": self._dist_context, + } allreduce_matmul_grad_overlapping_pass = new_pass( - "allreduce_matmul_grad_overlapping", {} + "allreduce_matmul_grad_overlapping", config ) allreduce_matmul_grad_overlapping_pass.apply( [main_program], [startup_program], self._pass_context diff --git a/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py b/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py index c6457b612ff81..b5b05ad09c524 100644 --- a/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py +++ b/python/paddle/distributed/passes/allreduce_matmul_grad_overlapping.py @@ -13,10 +13,17 @@ # limitations under the License. import collections +import logging +from ..auto_parallel.static.utils import ( + get_logger, + naive_set_dist_op_attr_for_program_by_mesh_and_mapping, +) from .pass_base import PassBase, register_pass from .pass_utils import AutoParallelStreamType +logger = get_logger(logging.INFO) + # For allreduce pattern in the backward phase of column parallel linear: # dX, dY = matmul_grad(X, Y, dOut) @@ -31,19 +38,28 @@ class AllreduceMatmulGradOverlappingPass(PassBase): def __init__(self): super().__init__() - self.set_attr("allreduce_stream", None) + self.op_namescope = "/auto_parallel/allreduce_matmul_grad_overlapping" + self.set_attr("dist_context", None) def _check_self(self): + if self.get_attr("dist_context") is None: + return False return True def _check_conflict(self, other_pass): return True def _apply_single_impl(self, main_program, startup_program, context): + self.dist_context = self.get_attr("dist_context") block = main_program.global_block() + matmul_grad_id_to_allreduce_id = ( self._get_all_matmul_grad_and_allreduce_pairs(block) ) + logger.info( + f"overlap matmul_grad and allreduce: {matmul_grad_id_to_allreduce_id}" + ) + self._split_matmul_grad_and_multi_streaming_allreduce( block, matmul_grad_id_to_allreduce_id ) @@ -70,22 +86,38 @@ def _get_all_matmul_grad_and_allreduce_pairs(self, block): def _insert_reshape_op(self, block, index, x, shape, op_role, out=None): var_x = block.var(x[0]) + x_dist_attr = self.dist_context.get_tensor_dist_attr_for_program(var_x) + if out is None: out = block.create_var( name=f"{x[0]}@reshape.out", dtype=var_x.dtype, persistable=False, ) + self.dist_context.set_tensor_dist_attr_for_program(out, x_dist_attr) + x_shape = block.create_var( name=f"{x[0]}@reshape.xshape", dtype=var_x.dtype ) + self.dist_context.set_tensor_dist_attr_for_program(x_shape, x_dist_attr) - block._insert_op_without_sync( + reshape_op = block._insert_op_without_sync( index=index, type="reshape2", inputs={"X": x}, outputs={"Out": out, "XShape": x_shape}, - attrs={"shape": shape, "op_role": op_role}, + attrs={ + "shape": shape, + "op_role": op_role, + 'op_namescope': self.op_namescope, + }, + ) + naive_set_dist_op_attr_for_program_by_mesh_and_mapping( + reshape_op, + process_mesh=x_dist_attr.process_mesh, + ref_mapping=x_dist_attr.dims_mapping, + ctx=self.dist_context, + chunk_id=x_dist_attr.chunk_id, ) block._sync_with_cpp() @@ -161,13 +193,30 @@ def _split_matmul_grad_and_multi_streaming_allreduce( dtype=var_y_grad.dtype, persistable=False, ) - block._insert_op_without_sync( + self.dist_context.set_tensor_dist_attr_for_program( + new_y_grad, + self.dist_context.get_tensor_dist_attr_for_program(var_y_grad), + ) + + matmul_grad_dist_attr = ( + self.dist_context.get_op_dist_attr_for_program(matmul_grad_op) + ) + matmul_op = block._insert_op_without_sync( index=allreduce_id + 3, type="matmul_v2", inputs={"X": new_x, "Y": new_out_grad}, outputs={"Out": new_y_grad}, - attrs={"trans_x": True, "trans_y": False, "op_role": op_role}, + attrs={ + "trans_x": True, + "trans_y": False, + "op_role": op_role, + 'op_namescope': self.op_namescope, + }, ) + self.dist_context.set_op_dist_attr_for_program( + matmul_op, matmul_grad_dist_attr + ) + self._insert_reshape_op( block, allreduce_id + 4, @@ -177,12 +226,20 @@ def _split_matmul_grad_and_multi_streaming_allreduce( y_grad, ) - block._insert_op_without_sync( + matmul_op = block._insert_op_without_sync( index=matmul_grad_id + 1, type="matmul_v2", inputs={"X": out_grad, "Y": y}, outputs={"Out": x_grad}, - attrs={"trans_x": False, "trans_y": True, "op_role": op_role}, + attrs={ + "trans_x": False, + "trans_y": True, + "op_role": op_role, + 'op_namescope': self.op_namescope, + }, + ) + self.dist_context.set_op_dist_attr_for_program( + matmul_op, matmul_grad_dist_attr ) block._remove_op(matmul_grad_id) From c52aec73b2a39283be80dbc99a69681a651e6ebe Mon Sep 17 00:00:00 2001 From: megemini Date: Tue, 26 Dec 2023 19:04:21 +0800 Subject: [PATCH 059/146] =?UTF-8?q?=E3=80=90Hackathon=205th=20No.28?= =?UTF-8?q?=E3=80=91=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20slice=5Fscat?= =?UTF-8?q?ter=20API=20(#59973)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Add] hack5 28 api * [Change] use set_value op * [Change] values to value * [Fix] resolve conflict * [Update] add test cases * [Fix] code example * [Add] dtype test cases * [Change] start/stop docstring * [Change] fix start/stop docstring * [Change] broadcast value to exp_shape * [Change] axes with list of int * [Add] as tensor test case * [Change] code style --- python/paddle/__init__.py | 2 + python/paddle/tensor/__init__.py | 2 + python/paddle/tensor/manipulation.py | 106 +++++++- test/legacy_test/test_slice_scatter.py | 347 +++++++++++++++++++++++++ 4 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 test/legacy_test/test_slice_scatter.py diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index ef4c7c96c4c38..fc7b2a3533f89 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -309,6 +309,7 @@ select_scatter, shard_index, slice, + slice_scatter, split, squeeze, squeeze_, @@ -627,6 +628,7 @@ 'amin', 'any', 'slice', + 'slice_scatter', 'normal', 'normal_', 'logsumexp', diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index 9172d8c7fbf1d..b26798892a2b2 100644 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -186,6 +186,7 @@ select_scatter, shard_index, slice, + slice_scatter, split, squeeze, squeeze_, @@ -613,6 +614,7 @@ 'scatter_nd', 'shard_index', 'slice', + 'slice_scatter', 'split', 'tensor_split', 'hsplit', diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 167411500bee5..d5f8833a21662 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -27,7 +27,7 @@ check_variable_and_dtype, convert_dtype, ) -from ..base.framework import Variable +from ..base.framework import Variable, default_main_program from ..framework import ( LayerHelper, _current_expected_place, @@ -6753,3 +6753,107 @@ def select_scatter(x, values, axis, index, name=None): ) return output + + +def slice_scatter(x, value, axes, starts, ends, strides, name=None): + """ + Embeds the `value` tensor into `x` along multiple axes. Returns a new tensor instead of a view. + The size of `axes` must be equal to `starts` , `ends` and `strides`. + + Args: + x (Tensor) : The input Tensor. Supported data types are `bool`, `float16`, `float32`, `float64`, `uint8`, `int8`, `int16`, `int32`, `int64`, `bfloat16`, `complex64`, `complex128`. + value (Tensor) : The tensor to embed into x. Supported data types are `bool`, `float16`, `float32`, `float64`, `uint8`, `int8`, `int16`, `int32`, `int64`, `bfloat16`, `complex64`, `complex128`. + axes (list|tuple) : the dimensions to insert the value. + starts (list|tuple) : the start indices of where to insert. + ends (list|tuple) : the stop indices of where to insert. + strids (list|tuple) : the steps for each insert. + name (str, optional): Name for the operation (optional, default is None). + + Returns: + Tensor, same dtype and shape with x + + Examples: + .. code-block:: python + + >>> import paddle + + >>> x = paddle.zeros((3, 9)) + >>> value = paddle.ones((3, 2)) + >>> res = paddle.slice_scatter(x, value, axes=[1], starts=[2], ends=[6], strides=[2]) + >>> print(res) + Tensor(shape=[3, 9], dtype=float32, place=Place(cpu), stop_gradient=True, + [[0., 0., 1., 0., 1., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0., 0.]]) + + >>> # broadcast `value` got the same result + >>> x = paddle.zeros((3, 9)) + >>> value = paddle.ones((3, 1)) + >>> res = paddle.slice_scatter(x, value, axes=[1], starts=[2], ends=[6], strides=[2]) + >>> print(res) + Tensor(shape=[3, 9], dtype=float32, place=Place(cpu), stop_gradient=True, + [[0., 0., 1., 0., 1., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0., 0.]]) + + >>> # broadcast `value` along multiple axes + >>> x = paddle.zeros((3, 3, 5)) + >>> value = paddle.ones((1, 3, 1)) + >>> res = paddle.slice_scatter(x, value, axes=[0, 2], starts=[1, 0], ends=[3, 4], strides=[1, 2]) + >>> print(res) + Tensor(shape=[3, 3, 5], dtype=float32, place=Place(cpu), stop_gradient=True, + [[[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]], + [[1., 0., 1., 0., 0.], + [1., 0., 1., 0., 0.], + [1., 0., 1., 0., 0.]], + [[1., 0., 1., 0., 0.], + [1., 0., 1., 0., 0.], + [1., 0., 1., 0., 0.]]]) + + """ + none_axes = [] + decrease_axes = [] + dtype = x.dtype + value = value.astype(dtype) + + if in_dynamic_or_pir_mode(): + return _C_ops.set_value_with_tensor( + x, + value, + starts, + ends, + strides, + axes, + decrease_axes, + none_axes, + ) + else: + attrs = { + 'axes': axes, + 'starts': starts, + 'ends': ends, + 'steps': strides, + 'decrease_axes': decrease_axes, + 'none_axes': none_axes, + 'dtype': dtype, + } + + inputs = { + 'Input': x, + 'ValueTensor': value, + } + + helper = LayerHelper('slice_scatter', **locals()) + output = helper.create_variable_for_type_inference(dtype=x.dtype) + cur_block = default_main_program().current_block() + cur_block.append_op( + type="set_value", + inputs=inputs, + outputs={'Out': output}, + attrs=attrs, + inplace_map={"Input": "Out"}, + ) + + return output diff --git a/test/legacy_test/test_slice_scatter.py b/test/legacy_test/test_slice_scatter.py new file mode 100644 index 0000000000000..075b5a5741886 --- /dev/null +++ b/test/legacy_test/test_slice_scatter.py @@ -0,0 +1,347 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import unittest + +import numpy as np + +import paddle +from paddle.framework import core +from paddle.pir_utils import test_with_pir_api + +paddle.enable_static() + + +def numpy_ref(_x, value, axes, starts, ends, strides): + x = np.copy(_x) + + try: + value = np.broadcast_to(value, x.shape) + except: + pass + + indices_x = [] + indices_v = [] + for ndim_idx in range(x.ndim): + if ndim_idx not in axes: + ind = list(range(x.shape[ndim_idx])) + indices_x.append(ind) + indices_v.append(ind) + else: + _idx = list(axes).index(ndim_idx) + ind_x = list(range(starts[_idx], ends[_idx], strides[_idx])) + ind_v = list(range(len(ind_x))) + indices_x.append(ind_x) + indices_v.append(ind_v) + + for index_x, index_v in zip( + itertools.product(*indices_x), itertools.product(*indices_v) + ): + x[index_x] = value[index_v] + + return x + + +class TestSliceScatterApi(unittest.TestCase): + def setUp(self): + np.random.seed(2023) + + self.init_dtype() + self.init_shape() + + self.x_np = np.random.random(self.x_shape).astype( + 'uint16' if self.dtype == 'bfloat16' else self.dtype + ) + self.value_np = np.random.random(self.value_shape).astype( + 'uint16' if self.dtype == 'bfloat16' else self.dtype + ) + self.place = [paddle.CPUPlace()] + if core.is_compiled_with_cuda(): + self.place.append(paddle.CUDAPlace(0)) + + def init_dtype(self): + self.dtype = 'float64' + + def init_shape(self): + self.x_shape = [8, 6] + self.value_shape = [8, 2] + self.axes = [1] + self.starts = [2] + self.ends = [6] + self.strides = [2] + + @test_with_pir_api + def test_api_static(self): + paddle.enable_static() + + for place in self.place: + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('x', self.x_shape, self.dtype) + value = paddle.static.data( + 'value', self.value_shape, self.dtype + ) + + out = paddle.slice_scatter( + x, + value, + axes=self.axes, + starts=self.starts, + ends=self.ends, + strides=self.strides, + ) + exe = paddle.static.Executor(place) + res = exe.run( + feed={ + 'x': self.x_np, + 'value': self.value_np, + }, + fetch_list=[out], + )[0] + + out_ref = numpy_ref( + self.x_np, + self.value_np, + axes=self.axes, + starts=self.starts, + ends=self.ends, + strides=self.strides, + ) + + np.testing.assert_allclose(res, out_ref) + + def test_api_dygraph(self): + for place in self.place: + paddle.disable_static(place) + x_tensor = paddle.to_tensor(self.x_np) + value_tensor = paddle.to_tensor(self.value_np) + out = paddle.slice_scatter( + x_tensor, + value_tensor, + axes=self.axes, + starts=self.starts, + ends=self.ends, + strides=self.strides, + ) + out_ref = numpy_ref( + self.x_np, + self.value_np, + axes=self.axes, + starts=self.starts, + ends=self.ends, + strides=self.strides, + ) + + np.testing.assert_allclose(out.numpy(), out_ref) + + paddle.enable_static() + + +class TestSliceScatterApiIntComplex128(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'complex128' + + +class TestSliceScatterApiIntComplex64(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'complex64' + + +class TestSliceScatterApiInt64(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'int64' + + +class TestSliceScatterApiInt32(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'int32' + + +class TestSliceScatterApiInt16(TestSliceScatterApi): + def init_dtype(self): + # old ir `set_value` not support this dtype + if paddle.framework.in_dynamic_or_pir_mode(): + self.dtype = 'int16' + else: + self.dtype = 'float64' + + +class TestSliceScatterApiInt8(TestSliceScatterApi): + def init_dtype(self): + # old ir `set_value` not support this dtype + if paddle.framework.in_dynamic_or_pir_mode(): + self.dtype = 'int8' + else: + self.dtype = 'float64' + + +class TestSliceScatterApiUint8(TestSliceScatterApi): + def init_dtype(self): + # old ir `set_value` not support this dtype + if paddle.framework.in_dynamic_or_pir_mode(): + self.dtype = 'uint8' + else: + self.dtype = 'float64' + + +class TestSliceScatterApiBool(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'bool' + + +class TestSliceScatterApiBfloat16(TestSliceScatterApi): + def init_dtype(self): + # old ir `set_value` not support this dtype + if paddle.framework.in_dynamic_or_pir_mode(): + self.dtype = 'bfloat16' + else: + self.dtype = 'float64' + + +class TestSliceScatterApiFloat16(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'float16' + + +class TestSliceScatterApiFloat32(TestSliceScatterApi): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterApi3D(TestSliceScatterApi): + def init_shape(self): + self.x_shape = [8, 6, 3] + self.value_shape = [8, 2, 3] + self.axes = [1] + self.starts = [2] + self.ends = [6] + self.strides = [2] + + +class TestSliceScatterApi3DFloat32(TestSliceScatterApi3D): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterApi4D(TestSliceScatterApi): + def init_shape(self): + self.x_shape = [8, 6, 3, 5] + self.value_shape = [8, 2, 3, 5] + self.axes = [1] + self.starts = [2] + self.ends = [6] + self.strides = [2] + + +class TestSliceScatterApi4DFloat32(TestSliceScatterApi4D): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterApi4DAxis3(TestSliceScatterApi): + def init_shape(self): + self.x_shape = [8, 6, 3, 9] + self.value_shape = [8, 6, 3, 2] + self.axes = [3] + self.starts = [2] + self.ends = [6] + self.strides = [2] + + +class TestSliceScatterApi4DAxis3Float32(TestSliceScatterApi4DAxis3): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterApiBroadcase2D(TestSliceScatterApi): + def init_shape(self): + self.x_shape = [8, 9] + self.value_shape = [8, 1] + self.axes = [1] + self.starts = [2] + self.ends = [6] + self.strides = [2] + + +class TestSliceScatterApiBroadcase2DFloat32(TestSliceScatterApiBroadcase2D): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterApiBroadcase3D(TestSliceScatterApi): + def init_shape(self): + self.x_shape = [8, 9, 6] + self.value_shape = [1, 9, 1] + self.axes = [0, 2] + self.starts = [2, 3] + self.ends = [7, 5] + self.strides = [3, 2] + + +class TestSliceScatterApiBroadcase3DFloat32(TestSliceScatterApiBroadcase3D): + def init_dtype(self): + self.dtype = 'float32' + + +class TestSliceScatterTensorApi(unittest.TestCase): + def test_tensor(self): + paddle.disable_static() + _x = np.random.rand(8, 6) + _value = np.random.rand(8, 3) + + x = paddle.to_tensor(_x) + value = paddle.to_tensor(_value) + + axes = [1] + starts = [0] + ends = [6] + strides = [2] + + out = x.slice_scatter(value, axes, starts, ends, strides) + out_ref = numpy_ref(_x, _value, axes, starts, ends, strides) + + np.testing.assert_allclose(out.numpy(), out_ref) + + paddle.enable_static() + + +class TestSliceScatterApiError(unittest.TestCase): + def test_error_ndim(self): + paddle.disable_static() + with self.assertRaises(ValueError): + x = paddle.to_tensor(np.random.rand(8, 6, 3)) + value = paddle.to_tensor(np.random.rand(8, 3)) + _ = paddle.slice_scatter( + x, value, axes=[0], starts=[0], ends=[8], strides=[1] + ) + + def test_error_index(self): + paddle.disable_static() + with self.assertRaises(ValueError): + x = paddle.to_tensor(np.random.rand(8, 6)) + value = paddle.to_tensor(np.random.rand(8, 3)) + _ = paddle.slice_scatter( + x, value, axes=[1], starts=[0], ends=[6], strides=[1] + ) + + with self.assertRaises(ValueError): + x = paddle.to_tensor(np.random.rand(8, 6)) + value = paddle.to_tensor(np.random.rand(2, 6)) + _ = paddle.slice_scatter( + x, value, axes=[0], starts=[0], ends=[8], strides=[1] + ) + + +if __name__ == '__main__': + unittest.main() From 6f02c440078310f96726a597c6269f2cf84c6ee3 Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Tue, 26 Dec 2023 19:30:09 +0800 Subject: [PATCH 060/146] [SOT] support call_AST stmt (#59562) --- python/paddle/jit/sot/infer_meta.py | 31 +++- .../jit/sot/opcode_translator/__init__.py | 2 +- .../{transform.py => eval_frame_callback.py} | 8 +- .../executor/function_graph.py | 143 +++++++++++++----- .../executor/opcode_executor.py | 74 +++++---- .../executor/opcode_inline_executor.py | 4 +- .../executor/variables/callable.py | 8 + .../paddle/jit/sot/symbolic/compile_cache.py | 20 ++- python/paddle/jit/sot/symbolic/interpreter.py | 4 + .../paddle/jit/sot/symbolic/statement_ir.py | 58 +++++-- .../jit/sot/symbolic/symbolic_context.py | 16 +- python/paddle/jit/sot/utils/__init__.py | 4 + python/paddle/jit/sot/utils/call_ast_utils.py | 96 ++++++++++++ python/paddle/jit/sot/utils/envs.py | 9 ++ .../test_custom_cpu_to_static.py | 4 +- test/sot/test_03_tuple.py | 2 + test/sot/test_call_ast.py | 61 ++++++++ 17 files changed, 456 insertions(+), 88 deletions(-) rename python/paddle/jit/sot/opcode_translator/{transform.py => eval_frame_callback.py} (90%) create mode 100644 python/paddle/jit/sot/utils/call_ast_utils.py create mode 100644 test/sot/test_call_ast.py diff --git a/python/paddle/jit/sot/infer_meta.py b/python/paddle/jit/sot/infer_meta.py index ea2cbff8b1cc1..7f90468bdf4b0 100644 --- a/python/paddle/jit/sot/infer_meta.py +++ b/python/paddle/jit/sot/infer_meta.py @@ -261,14 +261,39 @@ def infer_meta_for_layer(layer, *args, **kwargs): ) = layer.forward.get_concrete_program(*args_, **kwargs_) out = partial_program_layer._restore_out( - paddle.utils.flatten( - convert_variable_to_meta_info(concrete_program.outputs) - ) + [ + x + for x in paddle.utils.flatten( + convert_variable_to_meta_info(concrete_program.outputs) + ) + if isinstance(x, MetaInfo) + ] ) layer.forward.rollback() return out +def ast_infer_meta(static_function, *args, **kwargs): + args_, kwargs_ = convert_meta_to_input_spec((args, kwargs)) + + ( + concrete_program, + partial_program_layer, + ) = static_function.get_concrete_program(*args_, **kwargs_) + + out = partial_program_layer._restore_out( + [ + x + for x in paddle.utils.flatten( + convert_variable_to_meta_info(concrete_program.outputs) + ) + if isinstance(x, MetaInfo) + ] + ) + + return out + + @Singleton class SpecialInferMeta: """ diff --git a/python/paddle/jit/sot/opcode_translator/__init__.py b/python/paddle/jit/sot/opcode_translator/__init__.py index 64fda66a2747d..dec41c8bba172 100644 --- a/python/paddle/jit/sot/opcode_translator/__init__.py +++ b/python/paddle/jit/sot/opcode_translator/__init__.py @@ -13,6 +13,6 @@ # limitations under the License. from .skip_files import setup_skip_files -from .transform import eval_frame_callback # noqa: F401 +from .eval_frame_callback import eval_frame_callback # noqa: F401 setup_skip_files() diff --git a/python/paddle/jit/sot/opcode_translator/transform.py b/python/paddle/jit/sot/opcode_translator/eval_frame_callback.py similarity index 90% rename from python/paddle/jit/sot/opcode_translator/transform.py rename to python/paddle/jit/sot/opcode_translator/eval_frame_callback.py index 4f6ad8e43e90c..d454bb43aa035 100644 --- a/python/paddle/jit/sot/opcode_translator/transform.py +++ b/python/paddle/jit/sot/opcode_translator/eval_frame_callback.py @@ -58,7 +58,9 @@ def eval_frame_callback(frame, **kwargs) -> CustomCode: ) log_do(4, partial(print_locals, frame)) - log_format(3, "[transform] OriginCode: {}\n", frame.f_code.co_name) + log_format( + 3, "[eval_frame_callback] OriginCode: {}\n", frame.f_code.co_name + ) log_do(3, lambda: dis.dis(frame.f_code)) custom_code = OpcodeExecutorCache()(frame, **kwargs) @@ -66,13 +68,13 @@ def eval_frame_callback(frame, **kwargs) -> CustomCode: if custom_code.code is None: log_format( 3, - "[transform] NewCode (same as origin code): {}\n", + "[eval_frame_callback] NewCode (same as origin code): {}\n", frame.f_code.co_name, ) else: log_format( 3, - "[transform] NewCode: {}\n", + "[eval_frame_callback] NewCode: {}\n", custom_code.code.co_name, ) log_do(3, lambda: dis.dis(custom_code.code)) diff --git a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py index a188f56154a85..ab525a71e360d 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py +++ b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py @@ -24,7 +24,12 @@ from functools import cached_property from typing import Any, Callable -from ...infer_meta import InferMetaCache, LayerInferMetaCache, MetaInfo +from ...infer_meta import ( + InferMetaCache, + LayerInferMetaCache, + MetaInfo, + ast_infer_meta, +) from ...profiler import EventGuard, event_register from ...symbolic.statement_ir import Reference, Symbol from ...symbolic.symbolic_context import SymbolicTraceContext @@ -56,6 +61,7 @@ ) from .tracker import BuiltinTracker, DummyTracker from .variables import ( + ConstantVariable, DictVariable, GlobalVariable, ListVariable, @@ -99,6 +105,18 @@ def func(x): return map_variables(func, inputs) +def get_symbol_meta_map(inputs): + output = {} + + def func(x): + if isinstance(x, TensorVariable): + output[x.get_symbol()] = x.meta + return x + + map_variables(func, inputs) + return output + + class FunctionGraph: """ A Graph representation corresponding to each FunctionFrame @@ -129,7 +147,6 @@ def __init__(self, frame, **kwargs): self._global_guarded_variables: OrderedSet[VariableBase] = OrderedSet() self._print_variables = [] self._inplace_tensors = OrderedSet() - self.build_strategy = kwargs.get('build_strategy', None) self._kwargs = kwargs @cached_property @@ -292,7 +309,7 @@ def load(self, var): return VariableLoader(store_var_info, self.pycode_gen) - def _build_compile_fn_with_name_store(self, ret_vars, to_store_vars): + def _build_compile_fn_with_name_store(self, to_store_vars): class VariableLoader: def __init__(self, index_for_load, pycode_gen): self._index_for_load = index_for_load @@ -309,7 +326,7 @@ def load(self, var, allow_push_null=True): to_store_vars = list( filter(lambda x: not isinstance(x, NullVariable), to_store_vars) ) - self.start_compile(*(ret_vars + to_store_vars)) + self.start_compile(*to_store_vars) name_gen = NameGenerator("__start_compile_saved_") for var in to_store_vars[::-1]: @@ -327,6 +344,22 @@ def _log_fn(): return VariableLoader(index_for_load, self.pycode_gen) + def get_compiled_fn(self, *ret_vars): + ret_items = [ + ret_item + for ret_var in ret_vars + for ret_item in ret_var.flatten_items() + ] + + tensor_items = self._find_tensor_outputs(ret_items) + + compiled_fn, _ = self.sir_ctx.compile_fn( + [Symbol(tensor_var.var_name) for tensor_var in tensor_items], + **self._kwargs, + ) + + return compiled_fn + @event_register("start_compile", event_level=2) def start_compile(self, *ret_vars: VariableBase): """ @@ -441,36 +474,6 @@ def message_handler(*args, **kwargs): **kwargs, ) - @staticmethod - def get_opcode_executor_stack(): - # NOTE: only for debug. - # dependent on OpcodeExecutor. - from .opcode_executor import OpcodeExecutorBase - - if len(OpcodeExecutorBase.call_stack) == 0: - # In test case, we can meet this senario. - return [] - current_executor = OpcodeExecutorBase.call_stack[-1] - current_line = current_executor._current_line - filename = current_executor._code.co_filename - source_lines, start_line = inspect.getsourcelines( - current_executor._code - ) - # TODO(SigureMo): In 3.11, lineno maybe changed after multiple breakgraph, - # We need to find a way to fix this. - line_idx = min(current_line - start_line, len(source_lines) - 1) - code_line = source_lines[line_idx] - stack = [] - stack.append( - ' File "{}", line {}, in {}'.format( - filename, - current_line, - current_executor._code.co_name, - ) - ) - stack.append(f' {code_line}') - return stack - def call_layer( self, layer: PaddleLayerVariable, @@ -504,14 +507,46 @@ def message_handler(*args, **kwargs): infer_meta_fn, compute_fn, layer, *args, **kwargs ) + def call_ast( + self, + static_function: tuple, + *args: VariableBase, + **kwargs: VariableBase, + ): + """ + call paddle layer, start symbolic trace. + + Args: + layer: paddle layer + """ + + def compute_fn(static_function, inputs, outputs, stacks): + self.sir_ctx.call_AST( + static_function, + inputs=inputs, + outputs=outputs, + stacks=stacks, + ) + + def message_handler(*args, **kwargs): + return "Call ast faild" + + try: + return inner_error_default_handler( + self.symbolic_call, message_handler + )(ast_infer_meta, compute_fn, static_function, *args, **kwargs) + except Exception as e: + log(3, f"[call AST] {e}") + return None + def symbolic_call(self, infer_meta_fn, compute_fn, func, *args, **kwargs): """ Using infer_meta_fn and compute_fn convert func to symbolic function. Args: infer_meta_fn: function for infer meta, (func, metas, kwmetas) -> output_metas - compute_fn : function for sir compile, (func, input_symbols, outputs_symbols) -> None - func : symbolic function + compute_fn : function for add stmt to sir, (func, input_symbols, outputs_symbols, stacks) -> None + func : the logical function which will be represent as a stmt """ self.collect_input_variables(list(args)) self.collect_input_variables(list(kwargs.values())) @@ -523,6 +558,10 @@ def symbolic_call(self, infer_meta_fn, compute_fn, func, *args, **kwargs): convert_to_symbol(args), convert_to_symbol(kwargs), ) + + self.sir_ctx.TOS.set_symbol_meta_map(get_symbol_meta_map(args)) + self.sir_ctx.TOS.set_symbol_meta_map(get_symbol_meta_map(kwargs)) + log(3, f" inputs : {inputs_symbols}", "\n") outputs = map_if( @@ -565,7 +604,37 @@ def symbolic_call(self, infer_meta_fn, compute_fn, func, *args, **kwargs): outputs, self, DummyTracker(list(args) + list(kwargs.values())) ) else: - return None + return ConstantVariable.wrap_literal(None, self) + + @staticmethod + def get_opcode_executor_stack(): + # NOTE: only for debug. + # dependent on OpcodeExecutor. + from .opcode_executor import OpcodeExecutorBase + + if len(OpcodeExecutorBase.call_stack) == 0: + # In test case, we can meet this senario. + return [] + current_executor = OpcodeExecutorBase.call_stack[-1] + current_line = current_executor._current_line + filename = current_executor._code.co_filename + source_lines, start_line = inspect.getsourcelines( + current_executor._code + ) + # TODO(SigureMo): In 3.11, lineno maybe changed after multiple breakgraph, + # We need to find a way to fix this. + line_idx = max(min(current_line - start_line, len(source_lines) - 1), 0) + code_line = source_lines[line_idx] + stack = [] + stack.append( + ' File "{}", line {}, in {}'.format( + filename, + current_line, + current_executor._code.co_name, + ) + ) + stack.append(f' {code_line}') + return stack def _put_inner(self, vars: VariableBase): """ diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 17e74c9bfb0be..66efe59674234 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -36,6 +36,7 @@ InnerError, OrderedSet, SotUndefinedVar, + get_static_function, log, log_do, ) @@ -180,7 +181,7 @@ def pop_jump_if_op_wrapper(fns: list[Callable[[Any], Any]]): """ - @jump_break_graph_decorator + @if_break_graph_decorator def inner(self: OpcodeExecutorBase, instr: Instruction): """ Inner function that represents the wrapped POP_JUMP_IF opcode operation. @@ -214,7 +215,7 @@ def inner(self: OpcodeExecutorBase, instr: Instruction): return inner -def jump_break_graph_decorator(normal_jump: Callable): +def if_break_graph_decorator(normal_jump: Callable): """ A decorator function that breaks off the graph when a JUMP-related instruction is encountered. @@ -231,8 +232,8 @@ def inner(self: OpcodeExecutor, instr: Instruction): if isinstance(result, TensorVariable): # fallback when in OpcodeExecutor # raise error in OpcodeInlineExecutor - log(3, "[BreakGraph] jump break graph, because if tensor\n") - self._break_graph_in_jump(result, instr) + log(3, "[BreakGraph] break graph for if jump tensor\n") + self._break_graph_when_if(result, instr) return Stop(state="BreakGraph") else: return normal_jump(self, instr) @@ -265,7 +266,7 @@ def wrapper(self: OpcodeExecutor, instr: Instruction): ) if isinstance(self, OpcodeExecutor): log(3, f"[BreakGraph] call function Break graph: {e}\n") - self._break_graph_in_call(origin_stack, instr, push_n) + self._break_graph_when_call(origin_stack, instr, push_n) return Stop(state="BreakGraph") else: raise e @@ -390,7 +391,7 @@ def _prepare_virtual_env(self): """ raise NotImplementedError("Please implement virtual_env.") - def _break_graph_in_jump(self, result, instr: Instruction): + def _break_graph_when_if(self, result, instr: Instruction): """ Breaks the graph in JUMP instructions. @@ -512,7 +513,7 @@ def run(self): Executes the opcode. """ - log(3, f"start execute opcode: {self._code}\n") + log(3, f"[EXECUTOR RUN] Start execute opcode: {self._code}\n") self._lasti = 0 while True: if self._lasti >= len(self._instructions): @@ -524,6 +525,7 @@ def run(self): self.stop_state = is_stop.state self.pop_call_stack_until_self() break + log(3, f"[EXECUTOR RUN] End execute opcode: {self._code}\n") def step(self, instr: Instruction): """ @@ -1269,7 +1271,7 @@ def CONTAINS_OP(self, instr: Instruction): )(left, right) ) - @jump_break_graph_decorator + @if_break_graph_decorator def JUMP_IF_FALSE_OR_POP(self, instr: Instruction): pred_obj = self.stack.top if isinstance(pred_obj, (ConstantVariable, ContainerVariable)): @@ -1285,7 +1287,7 @@ def JUMP_IF_FALSE_OR_POP(self, instr: Instruction): "Currently don't support predicate a non-const / non-tensor obj." ) - @jump_break_graph_decorator + @if_break_graph_decorator def JUMP_IF_TRUE_OR_POP(self, instr: Instruction): pred_obj = self.stack.top if isinstance(pred_obj, (ConstantVariable, ContainerVariable)): @@ -1549,22 +1551,25 @@ def gen_compute_in_break_with_name_store(self, restore_names, instr_idx): instr_idx: the index for branch 1 to find the boundary and copy origin opcode """ - if self._graph.sir_ctx.TOS.graph_size() < ENV_MIN_GRAPH_SIZE.get(): - store_var_info = {} - for name in restore_names: - _var = self.get_var(name) - if _var not in self.stack: - store_var_info[_var.id] = name + # if we want get compiled fn, and do not do ast twice, + # we must give retval to get_compiled_fn which strictly same as start_compile + store_vars = list(self.stack) + store_var_info = {} + + for name in restore_names: + _var = self.get_var(name) + if _var not in self.stack: + store_vars.append(_var) + store_var_info[_var.id] = name + + compile_fn = self._graph.get_compiled_fn(*store_vars) + + if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): return self._graph._restore_origin_opcode( list(self.stack), store_var_info, instr_idx ) else: - store_vars = list(self.stack) - for name in restore_names: - _var = self.get_var(name) - if _var not in self.stack: - store_vars.append(_var) - return self._graph._build_compile_fn_with_name_store([], store_vars) + return self._graph._build_compile_fn_with_name_store(store_vars) def _create_resume_fn(self, index, stack_size): """ @@ -1583,7 +1588,7 @@ def _create_resume_fn(self, index, stack_size): return fn, inputs @fallback_when_occur_error - def _break_graph_in_jump(self, result: TensorVariable, instr: Instruction): + def _break_graph_when_if(self, result: TensorVariable, instr: Instruction): """ Break the graph at a JUMP instruction. @@ -1658,7 +1663,7 @@ def _break_graph_in_jump(self, result: TensorVariable, instr: Instruction): self.guard_fn = self._graph.guard_fn @fallback_when_occur_error - def _break_graph_in_call( + def _break_graph_when_call( self, origin_stack: VariableStack, instr: Instruction, @@ -1729,6 +1734,22 @@ def _break_graph_in_call( self.guard_fn = self._graph.guard_fn def transform(self): + static_function = get_static_function(self._frame, "eval_frame") + if static_function is not None: + code = self._frame.f_code + inputs = [] + for i in range(code.co_argcount): + arg_name = code.co_varnames[i] + value = self._locals[arg_name] + inputs.append(value) + output = self._graph.call_ast(static_function, *inputs) + if output is not None: + self.stack.push(output) + self.RETURN_VALUE(None) + return ( + CustomCode(self.new_code, self.new_code is None), + self.guard_fn, + ) self.run() if self.new_code is self.empty_code: raise InnerError("OpExecutor return a empty new_code.") @@ -1790,7 +1811,7 @@ def _gen_loop_body_between( return pycode_gen.create_fn_with_inputs(inputs) @fallback_when_occur_error - def _break_graph_in_for_loop( + def _break_graph_when_for_loop( self, iterator: VariableBase, for_iter: Instruction ): ''' @@ -2061,7 +2082,7 @@ def FOR_ITER(self, instr): iterator.idx = backup_iter_idx self._graph.remove_global_guarded_variable(iterator) self.stack.push(iterator) - self._break_graph_in_for_loop(iterator, instr) + self._break_graph_when_for_loop(iterator, instr) return Stop(state="BreakGraph") def RETURN_VALUE(self, instr: Instruction): @@ -2069,7 +2090,8 @@ def RETURN_VALUE(self, instr: Instruction): len(self.stack) == 1 ), f"Stack must have one element, but get {len(self.stack)} elements." ret_val = self.stack.pop() - if self._graph.sir_ctx.TOS.graph_size() < ENV_MIN_GRAPH_SIZE.get(): + compile_fn = self._graph.get_compiled_fn(ret_val) + if compile_fn.graph_size() < ENV_MIN_GRAPH_SIZE.get(): self.new_code = None else: self._graph.start_compile(ret_val) diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py index c24e94b07ffb2..9d6488dc4447a 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_inline_executor.py @@ -283,7 +283,7 @@ def RETURN_VALUE(self, instr: Instruction): self.return_value = self.stack.pop() return Stop(state="Return") - def _break_graph_in_jump(self, result, instr: Instruction): + def _break_graph_when_if(self, result, instr: Instruction): """ Helper method to raise a BreakGraphError when breaking the graph in a jump operation. @@ -292,7 +292,7 @@ def _break_graph_in_jump(self, result, instr: Instruction): instr (Instruction): The jump instruction. """ raise BreakGraphError( - "OpcodeInlineExecutor want call _break_graph_in_jump." + "OpcodeInlineExecutor want break graph when simulate `if`." ) def _create_resume_fn(self, index: int, stack_size: int = 0): diff --git a/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py b/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py index 1e28a9402b6ab..4edf14e5ca0d9 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py +++ b/python/paddle/jit/sot/opcode_translator/executor/variables/callable.py @@ -25,6 +25,7 @@ from .... import psdb from ....profiler import EventGuard from ....utils import ( + get_static_function, is_break_graph_api, is_break_graph_tensor_methods, is_builtin_fn, @@ -177,6 +178,13 @@ def call_function(self, /, *args, **kwargs) -> VariableBase: return result checkpoint = self.graph.save_memo() + + static_function = get_static_function(self.value, "inline_call") + if static_function is not None: + output = self.graph.call_ast(static_function, *args, **kwargs) + if output is not None: + return output + try: inline_executor = OpcodeInlineExecutor(self, *args, **kwargs) with EventGuard( diff --git a/python/paddle/jit/sot/symbolic/compile_cache.py b/python/paddle/jit/sot/symbolic/compile_cache.py index 465de3f6adf50..90cdf1bc36699 100644 --- a/python/paddle/jit/sot/symbolic/compile_cache.py +++ b/python/paddle/jit/sot/symbolic/compile_cache.py @@ -20,8 +20,9 @@ import paddle from paddle.amp.auto_cast import amp_state from paddle.base.data_feeder import convert_dtype -from paddle.framework import _dygraph_tracer +from paddle.framework import _dygraph_tracer, use_pir_api +from ..infer_meta import convert_meta_to_input_spec from ..profiler import EventGuard from ..utils import ( Cache, @@ -86,6 +87,23 @@ def amp_cast_inputs(self, args, kwargs): false_fn=lambda x: x, ) + def graph_size(self): + if self.partial_program is None: + input_spec = convert_meta_to_input_spec( + [self.SIR.symbol_meta_map[symbol] for symbol in self.SIR.inputs] + ) + ( + self.concrete_program, + self.partial_program, + ) = self.compiled_fn.get_concrete_program(input_spec) + self.partial_program.training = self.is_training + if use_pir_api(): + return len(self.partial_program.program.program.global_block().ops) + else: + if self.partial_program.program.num_blocks > 1: + return -1 + return len(self.partial_program.program.block(0).ops) + def __call__(self, *args, **kwargs): with EventGuard(f"FallbackWrapper: {self.SIR.name}"): if StepInfoManager().need_back_trace: diff --git a/python/paddle/jit/sot/symbolic/interpreter.py b/python/paddle/jit/sot/symbolic/interpreter.py index ac243e98ec41f..ec49ecaec39a6 100644 --- a/python/paddle/jit/sot/symbolic/interpreter.py +++ b/python/paddle/jit/sot/symbolic/interpreter.py @@ -155,6 +155,10 @@ def layer(self, stmt, inputs): assert layer is not None, "SIR bound layer is None." return layer(*args, **kwargs) + def AST(self, stmt, inputs): + args, kwargs = inputs + return stmt.converted_func(*args, **kwargs) + def compile_sir(context: SymbolicTraceContext, name: str): """ diff --git a/python/paddle/jit/sot/symbolic/statement_ir.py b/python/paddle/jit/sot/symbolic/statement_ir.py index 1e0ab465e0bd8..edf2ab4aed16d 100644 --- a/python/paddle/jit/sot/symbolic/statement_ir.py +++ b/python/paddle/jit/sot/symbolic/statement_ir.py @@ -19,10 +19,12 @@ """ from __future__ import annotations +import functools import weakref from typing import Any, Callable -from paddle.utils import is_sequence, map_structure +import paddle +from paddle.utils import flatten, map_structure from ..utils import NameGenerator, OrderedSet, Singleton, flatten_extend @@ -85,7 +87,7 @@ def __init__( outputs: list[Symbol], stacks: list[str], ): - assert type in ["call", "api", "method", "layer"] + assert type in ["call", "api", "method", "layer", "AST"] self.name = name self.inputs = inputs # (list of Symbols, dict of Symbols) self.outputs = outputs # list of Symbol | PythonObj @@ -96,9 +98,9 @@ def __init__( def __str__(self): def to_string(inps): - if isinstance(inps, str) or not is_sequence(inps): - return inps.__str__() - inps = (x.__str__() for x in inps) + inps = [x.__str__() for x in flatten(inps) if isinstance(x, Symbol)] + if len(inps) == 0: + return "(Empty)" return ", ".join(inps) return "{} || {} = {} ({}) ".format( @@ -158,12 +160,44 @@ def __init__( outputs: list[Symbol], stacks: list[str], ): + if isinstance(layer, Reference): + name = layer().__class__.__name__ + else: + name = layer.__class__.__name__ super().__init__( - "layer", layer.__class__.__name__, inputs, outputs, stacks + "layer", + name, + inputs, + outputs, + stacks, ) self.layer = layer +class ASTStatement(Statement): + def __init__( + self, + static_function, + inputs: list[Symbol], + outputs: list[Symbol], + stacks: list[str], + ): + # this dygraph_function always has attr __code__, which is checked before + dygraph_func = static_function.dygraph_function + super().__init__( + "AST", + dygraph_func.__code__.co_name, + inputs, + outputs, + stacks, + ) + converted_func = paddle.jit.dy2static.convert_to_static(dygraph_func) + func_self = getattr(dygraph_func, '__self__', None) + if func_self is not None: + converted_func = functools.partial(converted_func, func_self) + self.converted_func = converted_func + + class StatementIR: """ StatementIR is the carrier that records the code for building the neural network model.It is @@ -181,6 +215,8 @@ def __init__(self, name: str): self.outputs = [] # list of Symbol | PythonObj self.statements = [] # list of Statement + self.symbol_meta_map = {} + def __len__(self): return len(self.statements) @@ -189,8 +225,14 @@ def __deepcopy__(self, memo=None): new_sir.inputs = list(self.inputs) new_sir.outputs = list(self.outputs) new_sir.statements = list(self.statements) + new_sir.symbol_meta_map = dict(self.symbol_meta_map.items()) return new_sir + def set_symbol_meta_map(self, meta_map): + # if the meta of a input symbol inplace changed, we should get the origin meta as input of SIR + meta_map.update(self.symbol_meta_map) + self.symbol_meta_map = meta_map + def add_input(self, input): self.inputs.append(input) @@ -230,10 +272,6 @@ def __str__(self): def __repr__(self): return self.__str__() - def graph_size(self): - call_layers = [x for x in self.statements if x.type == "layer"] - return len(self.statements) + len(call_layers) - @Singleton class StatementIRFactory: diff --git a/python/paddle/jit/sot/symbolic/symbolic_context.py b/python/paddle/jit/sot/symbolic/symbolic_context.py index 47f40bbcc9ec7..931586645149a 100644 --- a/python/paddle/jit/sot/symbolic/symbolic_context.py +++ b/python/paddle/jit/sot/symbolic/symbolic_context.py @@ -18,6 +18,7 @@ from .compile_cache import CompileSIRCache from .statement_ir import ( ApiStatement, + ASTStatement, CallStatement, LayerStatement, MethodStatement, @@ -69,7 +70,6 @@ def call_API(self, api, inputs, outputs, stacks): """ Call a paddle api. """ - assert callable(api), "call_API must receive a paddle api." stmt = ApiStatement(api, inputs, outputs, stacks) self.TOS.add_statement(stmt) @@ -94,6 +94,10 @@ def call_LAYER(self, layer, inputs, outputs, stacks): stmt = LayerStatement(layer, inputs, outputs, stacks) self.TOS.add_statement(stmt) + def call_AST(self, static_function, inputs, outputs, stacks): + stmt = ASTStatement(static_function, inputs, outputs, stacks) + self.TOS.add_statement(stmt) + def get_sir(self, name: str): """ Get a SIR from statement_factory. @@ -130,14 +134,18 @@ def compile_do_nothing(self, ret_vals): ret_vals (list[Symbol]): the return values of the function. """ - def dummy_func(*args, **kwargs): - return [] + class DummyFunc: + def __call__(*args, **kwargs): + return [] + + def graph_size(self): + return 0 # return None function dummy_stmt_ir = StatementIR("dummy_func") dummy_stmt_ir.outputs = [] dummy_stmt_ir.inputs = [] - return dummy_func, dummy_stmt_ir + return DummyFunc(), dummy_stmt_ir def compile_fn(self, ret_vals, **kwargs): """ diff --git a/python/paddle/jit/sot/utils/__init__.py b/python/paddle/jit/sot/utils/__init__.py index 307ef1c21b800..16e2cd5b1afe5 100644 --- a/python/paddle/jit/sot/utils/__init__.py +++ b/python/paddle/jit/sot/utils/__init__.py @@ -19,9 +19,11 @@ ENV_SHOW_TRACKERS, ENV_SOT_LOG_LEVEL, ENV_STRICT_MODE, + ENV_SOT_WITH_CONTROL_FLOW, cost_model_guard, min_graph_size_guard, strict_mode_guard, + with_control_flow_guard, ) from .exceptions import ( # noqa: F401 BreakGraphError, @@ -50,6 +52,7 @@ current_tmp_name_records, execute_time, flatten_extend, + flatten, get_unbound_method, hashable, in_paddle_module, @@ -69,3 +72,4 @@ no_eval_frame, tmp_name_guard, ) +from .call_ast_utils import get_static_function, try_ast_func diff --git a/python/paddle/jit/sot/utils/call_ast_utils.py b/python/paddle/jit/sot/utils/call_ast_utils.py new file mode 100644 index 0000000000000..612334287b0a5 --- /dev/null +++ b/python/paddle/jit/sot/utils/call_ast_utils.py @@ -0,0 +1,96 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import types + +import paddle + +from .envs import ENV_SOT_WITH_CONTROL_FLOW +from .exceptions import InnerError +from .utils import Singleton + +try_ast_codes = set() + + +def try_ast_func(func): + def _is_wrapped(f): + return hasattr(f, '__wrapped__') + + unwrapped_f = func + if hasattr(unwrapped_f, "__code__"): + try_ast_codes.add(func.__code__) + + while _is_wrapped(unwrapped_f): + unwrapped_f = unwrapped_f.__wrapped__ + if hasattr(unwrapped_f, "__code__"): + try_ast_codes.add(func.__code__) + + return func + + +@Singleton +class StaticFunctionManager: + def __init__(self): + self.code_map = {} + + def ast_transform_with_frame(self, frame): + code = frame.f_code + if code not in try_ast_codes: + return None + if code not in self.code_map: + if code.co_name.startswith("#") or code.co_name.startswith("$"): + self.code_map[code] = None + elif len(code.co_cellvars) + len(code.co_freevars) != 0: + self.code_map[code] = None + else: + function = types.FunctionType( + code, + frame.f_globals, + code.co_name, + (), + (), + ) + function = paddle.jit.to_static(function, full_graph=True) + self.code_map[code] = function + + return self.code_map[code] + + def ast_transform_with_callable(self, fn): + if not inspect.isfunction(fn) or not hasattr(fn, "__code__"): + return None + + code = fn.__code__ + if code not in try_ast_codes: + return None + if code not in self.code_map: + if code.co_name.startswith("#") or code.co_name.startswith("$"): + self.code_map[code] = None + elif len(code.co_cellvars) + len(code.co_freevars) != 0: + self.code_map[code] = None + else: + self.code_map[code] = paddle.jit.to_static(fn, full_graph=True) + + return self.code_map[code] + + +def get_static_function(obj, type_): + if ENV_SOT_WITH_CONTROL_FLOW.get(): + if type_ == "eval_frame": + return StaticFunctionManager().ast_transform_with_frame(obj) + elif type_ == "inline_call": + return StaticFunctionManager().ast_transform_with_callable(obj) + else: + raise InnerError(f"Can not get static function with type {type_}.") + return None diff --git a/python/paddle/jit/sot/utils/envs.py b/python/paddle/jit/sot/utils/envs.py index a7d8ceafb7f0c..bc6879664890e 100644 --- a/python/paddle/jit/sot/utils/envs.py +++ b/python/paddle/jit/sot/utils/envs.py @@ -29,6 +29,9 @@ ENV_STRICT_MODE = BooleanEnvironmentVariable("STRICT_MODE", False) ENV_SHOW_TRACKERS = StringEnvironmentVariable("SHOW_TRACKERS", "") ENV_CLEAN_CODE = BooleanEnvironmentVariable("CLEAN_CODE", False) +ENV_SOT_WITH_CONTROL_FLOW = BooleanEnvironmentVariable( + "SOT_WITH_CONTROL_FLOW", True +) @contextmanager @@ -47,3 +50,9 @@ def strict_mode_guard(value: bool): def min_graph_size_guard(value: int): with EnvironmentVariableGuard(ENV_MIN_GRAPH_SIZE, value): yield + + +@contextmanager +def with_control_flow_guard(value: bool): + with EnvironmentVariableGuard(ENV_SOT_WITH_CONTROL_FLOW, value): + yield diff --git a/test/custom_runtime/test_custom_cpu_to_static.py b/test/custom_runtime/test_custom_cpu_to_static.py index b365f8ab39811..78978e9175310 100644 --- a/test/custom_runtime/test_custom_cpu_to_static.py +++ b/test/custom_runtime/test_custom_cpu_to_static.py @@ -164,7 +164,9 @@ def forward(self, x): # convert to static model build_strategy = paddle.static.BuildStrategy() - mnist = paddle.jit.to_static(model, build_strategy=build_strategy) + mnist = paddle.jit.to_static( + model, build_strategy=build_strategy, full_graph=True + ) # data loader transform = paddle.vision.transforms.Compose( diff --git a/test/sot/test_03_tuple.py b/test/sot/test_03_tuple.py index 797d54384714d..d0db1d100a42c 100644 --- a/test/sot/test_03_tuple.py +++ b/test/sot/test_03_tuple.py @@ -24,6 +24,7 @@ import paddle from paddle.jit.sot.psdb import check_no_breakgraph +from paddle.jit.sot.utils import with_control_flow_guard @check_no_breakgraph @@ -80,6 +81,7 @@ def test_tuple_methods_int(self): self.assert_results(tuple_count_int, 1, paddle.to_tensor(2)) self.assert_results(tuple_index_int, 1, paddle.to_tensor(2)) + @with_control_flow_guard(False) def test_tuple_methods_tensor(self): a = paddle.to_tensor(1) b = paddle.to_tensor(2) diff --git a/test/sot/test_call_ast.py b/test/sot/test_call_ast.py new file mode 100644 index 0000000000000..e893af485e4f1 --- /dev/null +++ b/test/sot/test_call_ast.py @@ -0,0 +1,61 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from test_case_base import TestCaseBase + +import paddle +from paddle.jit.sot.utils import try_ast_func, with_control_flow_guard + + +@try_ast_func +def calc(x, y, z): + if x < 5: + a = x + y + b = y - z + c = a * b + return c + else: + a = x - y + b = y + z + c = a * b + return c + + +def inline_call_ast(x, y): + a = x - y + 3 + b = x + y + c = x * y + z = calc(a, b, c) + return z + a + + +class TestNumpyAdd(TestCaseBase): + @with_control_flow_guard(True) + def test_full_graph_ast(self): + x = paddle.to_tensor([2]) + y = paddle.to_tensor([3]) + z = paddle.to_tensor([4]) + self.assert_results(calc, x, y, z) + + @with_control_flow_guard(True) + def test_inline_ast(self): + x = paddle.to_tensor([2]) + y = paddle.to_tensor([3]) + self.assert_results(inline_call_ast, x, y) + + +if __name__ == "__main__": + unittest.main() From 7a91864ba581c024e06fee93725633d4367ead42 Mon Sep 17 00:00:00 2001 From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com> Date: Tue, 26 Dec 2023 20:10:04 +0800 Subject: [PATCH 061/146] [PIR]Enable grad dtypecheck (#60048) * enable grad --- paddle/fluid/pir/dialect/op_generator/api_gen.py | 2 +- paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc | 4 +++- paddle/phi/kernels/gpu/reduce_kernel.cu | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/api_gen.py b/paddle/fluid/pir/dialect/op_generator/api_gen.py index 7fbfa092cc011..39324fe9b1a99 100644 --- a/paddle/fluid/pir/dialect/op_generator/api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/api_gen.py @@ -593,7 +593,7 @@ def _gen_check_data_type(self, op_info, op_name): ) if ( - op_name.endswith(('_grad', '_grad_', '_grad_dense', '_grad_sparse')) + op_name in ["real_grad", "imag_grad"] or len(mapping_name_to_type) == 0 ): return "" diff --git a/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc b/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc index 52d13a402301f..6abe072314d9e 100644 --- a/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/reduce_mean_grad_kernel.cc @@ -49,4 +49,6 @@ PD_REGISTER_KERNEL(mean_grad, float, double, phi::dtype::complex, - phi::dtype::complex) {} + phi::dtype::complex, + int, + int64_t) {} diff --git a/paddle/phi/kernels/gpu/reduce_kernel.cu b/paddle/phi/kernels/gpu/reduce_kernel.cu index 51b50ed6e0024..9bcff72e56706 100644 --- a/paddle/phi/kernels/gpu/reduce_kernel.cu +++ b/paddle/phi/kernels/gpu/reduce_kernel.cu @@ -348,7 +348,9 @@ PD_REGISTER_KERNEL(mean_grad, phi::dtype::float16, phi::dtype::bfloat16, phi::dtype::complex, - phi::dtype::complex) {} + phi::dtype::complex, + int, + int64_t) {} PD_REGISTER_KERNEL(min_grad, GPU, From 3285173ab03e64b82a837ed43767c4fcbd31a9b2 Mon Sep 17 00:00:00 2001 From: QingshuChen Date: Tue, 26 Dec 2023 20:19:51 +0800 Subject: [PATCH 062/146] suport fc bf16 xpu (#60223) --- CMakeLists.txt | 3 +- cmake/external/xpu.cmake | 2 +- paddle/fluid/operators/matmul_op_xpu.cc | 3 + paddle/phi/backends/xpu/xpu3_op_list.cc | 16 +- paddle/phi/kernels/xpu/matmul_grad_kernel.cc | 2 + paddle/phi/kernels/xpu/matmul_kernel.cc | 10 +- paddle/phi/kernels/xpu/xpu_api_wrapper.h | 180 ++++++++++++++++++- python/setup.py.in | 2 +- setup.py | 2 +- test/xpu/test_matmul_op_xpu.py | 38 +++- 10 files changed, 239 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 916b56751868d..806314977745c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,7 +54,8 @@ option(WITH_XPU "Compile PaddlePaddle with BAIDU KUNLUN XPU" OFF) option(WITH_XPU_KP "Compile PaddlePaddle with BAIDU XPU compiler " OFF) option(WITH_XPU_XFT "Compile PaddlePaddle with BAIDU XPU-XFT" OFF) option(WITH_XPU_PLUGIN "Compile PaddlePaddle with BAIDU XPU plugin" OFF) -option(WITH_XPU_XHPC "Compile PaddlePaddle with BAIDU XPU-HPC library" OFF) +option(WITH_XPU_XHPC "Compile PaddlePaddle with BAIDU XPU-HPC library" + ${WITH_XPU}) option(WITH_WIN_DUMP_DBG "Compile with windows core dump debug mode" OFF) option(WITH_ROCM "Compile PaddlePaddle with ROCM platform" OFF) option(WITH_IPU "Compile PaddlePaddle with Graphcore IPU" OFF) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index d45d0ad2a7245..f871ae810a6c8 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE) set(XPU_BASE_DATE "20231203") endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) - set(XPU_XHPC_BASE_DATE "20231215") + set(XPU_XHPC_BASE_DATE "20231225") endif() set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) diff --git a/paddle/fluid/operators/matmul_op_xpu.cc b/paddle/fluid/operators/matmul_op_xpu.cc index 5f9e9459800da..de2aa41d971df 100644 --- a/paddle/fluid/operators/matmul_op_xpu.cc +++ b/paddle/fluid/operators/matmul_op_xpu.cc @@ -155,10 +155,13 @@ namespace plat = paddle::platform; REGISTER_OP_XPU_KERNEL( matmul, ops::MatMulXPUKernel, + ops::MatMulXPUKernel, ops::MatMulXPUKernel); REGISTER_OP_XPU_KERNEL( matmul_grad, ops::MatMulGradXPUKernel, + ops::MatMulGradXPUKernel, ops::MatMulGradXPUKernel); #endif diff --git a/paddle/phi/backends/xpu/xpu3_op_list.cc b/paddle/phi/backends/xpu/xpu3_op_list.cc index 24a35b7029aae..623f63444c308 100644 --- a/paddle/phi/backends/xpu/xpu3_op_list.cc +++ b/paddle/phi/backends/xpu/xpu3_op_list.cc @@ -598,13 +598,21 @@ XPUOpMap& get_kl3_ops() { {"max_pool2d_with_index_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"matmul_grad", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::BFLOAT16, + phi::DataType::FLOAT16})}, {"matmul_v2_grad", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::BFLOAT16, + phi::DataType::FLOAT16})}, {"matmul_v2", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::BFLOAT16, + phi::DataType::FLOAT16})}, {"matmul", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::BFLOAT16, + phi::DataType::FLOAT16})}, {"mean_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"mean", diff --git a/paddle/phi/kernels/xpu/matmul_grad_kernel.cc b/paddle/phi/kernels/xpu/matmul_grad_kernel.cc index f94abe6300017..9c5424d9b3e5c 100644 --- a/paddle/phi/kernels/xpu/matmul_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/matmul_grad_kernel.cc @@ -179,6 +179,7 @@ PD_REGISTER_KERNEL(matmul_grad, ALL_LAYOUT, phi::MatmulGradKernel, float, + phi::dtype::bfloat16, phi::dtype::float16) {} PD_REGISTER_KERNEL(matmul_with_flatten_grad, @@ -186,4 +187,5 @@ PD_REGISTER_KERNEL(matmul_with_flatten_grad, ALL_LAYOUT, phi::MatmulWithFlattenGradKernel, float, + phi::dtype::bfloat16, phi::dtype::float16) {} diff --git a/paddle/phi/kernels/xpu/matmul_kernel.cc b/paddle/phi/kernels/xpu/matmul_kernel.cc index d703f10a248fb..7714199ff474b 100644 --- a/paddle/phi/kernels/xpu/matmul_kernel.cc +++ b/paddle/phi/kernels/xpu/matmul_kernel.cc @@ -76,12 +76,18 @@ void MatmulWithFlattenKernel(const Context& dev_ctx, } // namespace phi -PD_REGISTER_KERNEL( - matmul, XPU, ALL_LAYOUT, phi::MatmulKernel, float, phi::dtype::float16) {} +PD_REGISTER_KERNEL(matmul, + XPU, + ALL_LAYOUT, + phi::MatmulKernel, + float, + phi::dtype::bfloat16, + phi::dtype::float16) {} PD_REGISTER_KERNEL(matmul_with_flatten, XPU, ALL_LAYOUT, phi::MatmulWithFlattenKernel, float, + phi::dtype::bfloat16, phi::dtype::float16) {} diff --git a/paddle/phi/kernels/xpu/xpu_api_wrapper.h b/paddle/phi/kernels/xpu/xpu_api_wrapper.h index 70ee326500e1c..d4d29aa7a4ad7 100644 --- a/paddle/phi/kernels/xpu/xpu_api_wrapper.h +++ b/paddle/phi/kernels/xpu/xpu_api_wrapper.h @@ -25,12 +25,14 @@ namespace phi { using XPUTypeFP16 = typename XPUTypeTrait::Type; +using XPUTypeBF16 = typename XPUTypeTrait::Type; enum XPUFCCalcType { FC_INT16 = 0, FC_INT32, FC_FLOAT, FC_INT32_WITH_LL, + FC_TF32, }; template @@ -44,7 +46,11 @@ XPUFCCalcType FCCalcType() { return XPUFCCalcType::FC_FLOAT; } else if (std::getenv("XPU_PADDLE_FC_INT32_WITH_LL") != nullptr) { return XPUFCCalcType::FC_INT32_WITH_LL; + } else if (std::is_same::value || + std::is_same::value) { + return XPUFCCalcType::FC_TF32; } + return XPUFCCalcType::FC_INT16; } @@ -272,6 +278,78 @@ static void xpu_fc_wrapper(xpu::Context* ctx, } } +template <> +void xpu_fc_wrapper(xpu::Context* ctx, + const XPUTypeBF16* x, + const XPUTypeBF16* w, + XPUTypeBF16* y, + int m, + int n, + int k, + bool x_trans, + bool w_trans, + const float* x_maxptr, + const float* w_maxptr, + float* y_maxptr, + int ldx, + int ldw, + int ldy, + float alpha, + float beta, + const float* bias, + const xpu::Activation_t& act) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_wrapper"); +} + +template <> +void xpu_fc_wrapper(xpu::Context* ctx, + const XPUTypeBF16* x, + const XPUTypeBF16* w, + XPUTypeBF16* y, + int m, + int n, + int k, + bool x_trans, + bool w_trans, + const float* x_maxptr, + const float* w_maxptr, + float* y_maxptr, + int ldx, + int ldw, + int ldy, + float alpha, + float beta, + const float* bias, + const xpu::Activation_t& act) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_wrapper"); +} + +template <> +void xpu_fc_wrapper(xpu::Context* ctx, + const XPUTypeBF16* x, + const XPUTypeBF16* w, + XPUTypeBF16* y, + int m, + int n, + int k, + bool x_trans, + bool w_trans, + const float* x_maxptr, + const float* w_maxptr, + float* y_maxptr, + int ldx, + int ldw, + int ldy, + float alpha, + float beta, + const float* bias, + const xpu::Activation_t& act) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_wrapper"); +} + template <> void xpu_fc_wrapper(xpu::Context* ctx, const XPUTypeFP16* x, @@ -336,7 +414,95 @@ static void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, } template <> -void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, +void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, + int bs, + bool trans_x, + bool trans_w, + int m, + int n, + int k, + float alpha, + const XPUTypeBF16* x, + int stride_x, + const XPUTypeBF16* w, + int stride_w, + float beta, + XPUTypeBF16* y, + int stride_y, + const float* x_maxptr, + const float* w_maxptr) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_batch_wrapper"); +} + +template <> +void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, + int bs, + bool trans_x, + bool trans_w, + int m, + int n, + int k, + float alpha, + const XPUTypeBF16* x, + int stride_x, + const XPUTypeBF16* w, + int stride_w, + float beta, + XPUTypeBF16* y, + int stride_y, + const float* x_maxptr, + const float* w_maxptr) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_batch_wrapper"); +} + +template <> +void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, + int bs, + bool trans_x, + bool trans_w, + int m, + int n, + int k, + float alpha, + const XPUTypeBF16* x, + int stride_x, + const XPUTypeBF16* w, + int stride_w, + float beta, + XPUTypeBF16* y, + int stride_y, + const float* x_maxptr, + const float* w_maxptr) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_batch_wrapper"); +} + +template <> +void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, + int bs, + bool trans_x, + bool trans_w, + int m, + int n, + int k, + float alpha, + const XPUTypeBF16* x, + int stride_x, + const XPUTypeBF16* w, + int stride_w, + float beta, + XPUTypeBF16* y, + int stride_y, + const float* x_maxptr, + const float* w_maxptr) { + int r = xpu::Error_t::INVALID_PARAM; + PADDLE_ENFORCE_XDNN_SUCCESS(r, "xpu_fc_batch_wrapper"); +} + +template <> +void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, int bs, bool trans_x, bool trans_w, @@ -344,12 +510,12 @@ void xpu_fc_batch_wrapper(xpu::Context* xpu_ctx, int n, int k, float alpha, - const XPUTypeFP16* x, + const XPUTypeBF16* x, int stride_x, - const XPUTypeFP16* w, + const XPUTypeBF16* w, int stride_w, float beta, - XPUTypeFP16* y, + XPUTypeBF16* y, int stride_y, const float* x_maxptr, const float* w_maxptr) { @@ -390,17 +556,19 @@ static void MatMulXPUFunction(xpu::Context* xpu_ctx, using XPUType = typename XPUTypeTrait::Type; int fccal_type = FCCalcType(); - decltype(&xpu_fc_wrapper) fc_api_list[4] = { + decltype(&xpu_fc_wrapper) fc_api_list[5] = { &xpu_fc_wrapper, &xpu_fc_wrapper, &xpu_fc_wrapper, &xpu_fc_wrapper, + &xpu_fc_wrapper, }; - decltype(&xpu_fc_batch_wrapper) fc_batch_api_list[4] = { + decltype(&xpu_fc_batch_wrapper) fc_batch_api_list[5] = { &xpu_fc_batch_wrapper, &xpu_fc_batch_wrapper, &xpu_fc_batch_wrapper, &xpu_fc_batch_wrapper, + &xpu_fc_batch_wrapper, }; auto fc_api = fc_api_list[fccal_type]; diff --git a/python/setup.py.in b/python/setup.py.in index f8581129642c2..772d9f77aca62 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -89,7 +89,7 @@ def get_xpu_xccl_version(): return 'False' def get_xpu_xhpc_version(): - if '@WITH_XPU_XHPC@' == 'ON': + if '@WITH_XPU@' == 'ON': return '@XPU_XHPC_BASE_DATE@' else: return 'False' diff --git a/setup.py b/setup.py index 1b688f15e9885..72ca9b1bce723 100644 --- a/setup.py +++ b/setup.py @@ -400,7 +400,7 @@ def get_xpu_xccl_version(): def get_xpu_xhpc_version(): - with_xpu_xhpc = env_dict.get("WITH_XPU_XHPC") + with_xpu_xhpc = env_dict.get("WITH_XPU") if with_xpu_xhpc == 'ON': return env_dict.get("XPU_XHPC_BASE_DATE") else: diff --git a/test/xpu/test_matmul_op_xpu.py b/test/xpu/test_matmul_op_xpu.py index 24369b79f8cab..38d72fd83ab30 100644 --- a/test/xpu/test_matmul_op_xpu.py +++ b/test/xpu/test_matmul_op_xpu.py @@ -354,11 +354,43 @@ def dynamic_create_class(self): return base_class, classes +class XPUTestMatmulOpBF16(XPUOpTestWrapper): + def __init__(self): + self.op_name = "matmul" + self.use_dynamic_create_class = True + + def dynamic_create_class(self): + base_class = TestMatmulBaseGenerator + classes = [] + for dim in [2]: + for transpose_X in [False, True]: + for transpose_Y in [False, True]: + class_name = 'TestMatMulOp2_dimX_{}_dim_Y_{}_transX_{}_transY_{}'.format( + dim, dim, transpose_X, transpose_Y + ) + shape_X, shape_Y = generate_compatible_shapes_2( + dim, transpose_X, transpose_Y + ) + attr_dict = { + 'shape_X': shape_X, + 'shape_Y': shape_Y, + 'transpose_X': transpose_X, + 'transpose_Y': transpose_Y, + 'op_type': "matmul", + } + classes.append([class_name, attr_dict]) + return base_class, classes + + support_types = get_xpu_op_support_types('matmul') for stype in support_types: - create_test_class(globals(), XPUTestMatmulOpErr, stype) - create_test_class(globals(), XPUTestMatmulOp1, stype) - create_test_class(globals(), XPUTestMatmulOp3, stype) + if "bfloat16" in str(stype): + # only support fc_fusion now + create_test_class(globals(), XPUTestMatmulOpBF16, stype) + else: + create_test_class(globals(), XPUTestMatmulOpErr, stype) + create_test_class(globals(), XPUTestMatmulOp1, stype) + create_test_class(globals(), XPUTestMatmulOp3, stype) if __name__ == "__main__": paddle.enable_static() From 48c21b50fa2ecce7e48753da9b3e9b1cdac60e36 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Tue, 26 Dec 2023 20:39:18 +0800 Subject: [PATCH 063/146] fix windows bug for common lib (#60340) * fix windows bug * Update inference_lib.cmake --- cmake/inference_lib.cmake | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 06dc5d6173794..517ac24cccc72 100755 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -237,6 +237,16 @@ copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR}) set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid") +if(WIN32) + set(paddle_common_lib ${PADDLE_BINARY_DIR}/paddle/common/common.*) +else() + set(paddle_common_lib ${PADDLE_BINARY_DIR}/paddle/common/libcommon.*) +endif() +copy( + inference_lib_dist + SRCS ${paddle_common_lib} + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib) + if(WIN32) if(WITH_STATIC_LIB) set(paddle_inference_lib @@ -268,11 +278,6 @@ else() SRCS ${paddle_phi_lib} DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib) endif() - set(paddle_common_lib ${PADDLE_BINARY_DIR}/paddle/common/libcommon.*) - copy( - inference_lib_dist - SRCS ${paddle_common_lib} - DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib) endif() copy( From 3055fac26c5c3e1c61b12d147c46311b09410f17 Mon Sep 17 00:00:00 2001 From: Ryan <44900829+DrRyanHuang@users.noreply.github.com> Date: Wed, 27 Dec 2023 00:21:17 +0800 Subject: [PATCH 064/146] Remove AstVarScope && AstVarEnv (#60373) --- .../paddle/jit/dy2static/static_analysis.py | 140 +----------------- .../dygraph_to_static/test_static_analysis.py | 21 --- 2 files changed, 4 insertions(+), 157 deletions(-) diff --git a/python/paddle/jit/dy2static/static_analysis.py b/python/paddle/jit/dy2static/static_analysis.py index da6006a4e503c..81bfa589b018f 100644 --- a/python/paddle/jit/dy2static/static_analysis.py +++ b/python/paddle/jit/dy2static/static_analysis.py @@ -40,110 +40,6 @@ def __init__(self, node): self.node_var_type = {NodeVarType.UNKNOWN} -class AstVarScope: - """ - AstVarScope is a class holding the map from current scope variable to its - type. - """ - - SCOPE_TYPE_SCRIPT = 0 - SCOPE_TYPE_FUNCTION = 1 - SCOPE_TYPE_CLASS = 2 - - def __init__( - self, scope_name='', scope_type=SCOPE_TYPE_SCRIPT, parent_scope=None - ): - self.sub_scopes = [] - self.name_to_id = {} - self.id_to_type = {} - self.cur_id = 0 - - self.scope_name = scope_name - self.scope_type = scope_type - self.parent_scope = parent_scope - if parent_scope is not None: - parent_scope.sub_scopes.append(self) - - def add_var_type(self, var_name, node_var_type): - var_type = self.get_var_type(var_name) - if var_type == {NodeVarType.UNKNOWN}: - self.set_var_type(var_name, node_var_type) - else: - if isinstance(node_var_type, set): - var_type.update(node_var_type) - else: - var_type.add(node_var_type) - - def set_var_type(self, var_name, node_var_type): - if var_name in self.name_to_id: - num_id = self.name_to_id[var_name] - else: - num_id = self.cur_id - self.cur_id += 1 - self.name_to_id[var_name] = num_id - self.id_to_type[num_id] = ( - node_var_type if isinstance(node_var_type, set) else {node_var_type} - ) - - def get_var_type(self, var_name): - if var_name in self.name_to_id: - num_id = self.name_to_id[var_name] - return self.id_to_type[num_id] - if self.parent_scope is None: - return {NodeVarType.UNKNOWN} - return self.parent_scope.get_var_type(var_name) - - -class AstVarEnv: - """ - A class maintains scopes and mapping from name strings to type. - """ - - def __init__(self): - self.cur_scope = AstVarScope() - - def enter_scope(self, scope_name, scope_type): - self.cur_scope = AstVarScope( - scope_name, scope_type, parent_scope=self.cur_scope - ) - return self.cur_scope - - def exit_scope(self): - assert self.cur_scope.parent_scope is not None, ( - "Call exit_scope in " - "AstVarEnv when current scope doesn't have parent scope." - ) - self.cur_scope = self.cur_scope.parent_scope - return self.cur_scope - - def get_parent_scope(self): - assert self.cur_scope.parent_scope is not None, ( - "Call parent_scope in " - "AstVarEnv when current scope doesn't have parent scope." - ) - return self.cur_scope.parent_scope - - def add_var_type(self, var_name, node_var_type): - self.cur_scope.add_var_type(var_name, node_var_type) - - def set_var_type(self, var_name, node_var_type): - self.cur_scope.set_var_type(var_name, node_var_type) - - def get_var_type(self, var_name): - return self.cur_scope.get_var_type(var_name) - - def get_scope_var_type(self): - ''' - Returns a dict mapping from variable name to type. Used for debug and - test. - ''' - cur_scope_dict = {} - for name in self.cur_scope.name_to_id: - node_var_type = self.cur_scope.get_var_type(name) - cur_scope_dict[name] = node_var_type - return cur_scope_dict - - class StaticAnalysisVisitor: """ A class that does static analysis @@ -157,7 +53,6 @@ def run(self, ast_root): self.node_wrapper_root = None self.ancestor_wrappers = [] self.node_to_wrapper_map = {} - self.var_env = AstVarEnv() self.dfs_visit(ast_root) @@ -179,16 +74,7 @@ def dfs_visit(self, node): self.ancestor_wrappers.append(cur_wrapper) for child in gast.iter_child_nodes(node): - if isinstance(child, (gast.FunctionDef, gast.AsyncFunctionDef)): - # TODO: current version is function name mapping to its type - # consider complex case involving parameters - self.var_env.enter_scope( - child.name, AstVarScope.SCOPE_TYPE_FUNCTION - ) - func_type = self.dfs_visit(child) - self.var_env.exit_scope() - else: - self.dfs_visit(child) + self.dfs_visit(child) self.ancestor_wrappers.pop() cur_wrapper.node_var_type = self._get_node_var_type(cur_wrapper) @@ -200,9 +86,6 @@ def get_node_wrapper_root(self): def get_node_to_wrapper_map(self): return self.node_to_wrapper_map - def get_var_env(self): - return self.var_env - def is_tensor_node(self, node): tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES} node_wrapper = self.node_to_wrapper_map.get(node, None) @@ -262,7 +145,6 @@ def _get_node_var_type(self, cur_wrapper): for target in node.targets: if isinstance(target, gast.Name): self.node_to_wrapper_map[target].node_var_type = ret_type - self.var_env.set_var_type(target.id, ret_type) # Handle statements like `a, b = paddle.shape(x)` elif isinstance(target, gast.Tuple): for sub_target in target.elts: @@ -270,7 +152,6 @@ def _get_node_var_type(self, cur_wrapper): self.node_to_wrapper_map[ sub_target ].node_var_type = ret_type - self.var_env.set_var_type(sub_target.id, ret_type) return ret_type if isinstance(node, gast.AnnAssign): @@ -289,7 +170,6 @@ def _get_node_var_type(self, cur_wrapper): ret_type = node_value_type if isinstance(node.target, gast.Name): self.node_to_wrapper_map[node.target].node_var_type = ret_type - self.var_env.set_var_type(node.target.id, ret_type) return ret_type if isinstance(node, gast.Name): @@ -304,22 +184,14 @@ def _get_node_var_type(self, cur_wrapper): ): return self._get_func_argument_type(parent_node_wrapper, node) - return self.var_env.get_var_type(node.id) + return {NodeVarType.UNKNOWN} if isinstance(node, gast.Return): # If return nothing: if node.value is None: return {NodeVarType.NONE} - return_type = self.node_to_wrapper_map[node.value].node_var_type - assert ( - self.var_env.cur_scope.scope_type - == AstVarScope.SCOPE_TYPE_FUNCTION - ), "Return at non-function scope" - func_name = self.var_env.cur_scope.scope_name - parent_scope = self.var_env.get_parent_scope() - parent_scope.add_var_type(func_name, return_type) - return return_type + return {NodeVarType.UNKNOWN} if isinstance(node, gast.Call): if is_dygraph_api(node): @@ -333,7 +205,7 @@ def _get_node_var_type(self, cur_wrapper): return {NodeVarType.NUMPY_NDARRAY} if isinstance(node.func, gast.Name): - return self.var_env.get_var_type(node.func.id) + return {NodeVarType.UNKNOWN} if isinstance(node, gast.Subscript): if self.is_tensor_node(node.value): return {NodeVarType.TENSOR} @@ -363,7 +235,6 @@ def _get_func_argument_type(self, parent_node_wrapper, node): var_type = {NodeVarType.UNKNOWN} if node.annotation is not None: var_type = {NodeVarType.type_from_annotation(node.annotation)} - self.var_env.set_var_type(node.id, var_type) # if annotation and value(Constant) are diffent type, we use value type if parent_node.defaults: @@ -374,7 +245,4 @@ def _get_func_argument_type(self, parent_node_wrapper, node): if isinstance(defaults_node, gast.Constant): var_type = self._get_constant_node_type(defaults_node) - # Add node with identified type into cur_env. - self.var_env.set_var_type(node.id, var_type) - return var_type diff --git a/test/dygraph_to_static/test_static_analysis.py b/test/dygraph_to_static/test_static_analysis.py index e4e5afb574417..ea44992a04844 100644 --- a/test/dygraph_to_static/test_static_analysis.py +++ b/test/dygraph_to_static/test_static_analysis.py @@ -200,27 +200,6 @@ def test_construct_node_wrapper(self): node_to_wrapper_map = visitor.get_node_to_wrapper_map() self._check_wrapper(wrapper_root, node_to_wrapper_map) - def test_var_env(self): - for i, func in enumerate(test_funcs): - var_type = result_var_type[i] - test_source_code = inspect.getsource(func) - ast_root = gast.parse(test_source_code) - print(gast.dump(ast_root)) - visitor = StaticAnalysisVisitor(ast_root) - var_env = visitor.get_var_env() - - # There must be 1 sub scope for the test function - self.assertEqual(1, len(var_env.cur_scope.sub_scopes)) - var_env.cur_scope = var_env.cur_scope.sub_scopes[0] - - scope_var_type = var_env.get_scope_var_type() - print(scope_var_type) - self.assertEqual(len(scope_var_type), len(var_type)) - for name in scope_var_type: - print("Test var name %s" % (name)) - self.assertTrue(name in var_type) - self.assertEqual(scope_var_type[name], var_type[name]) - if __name__ == '__main__': unittest.main() From 887ef4ac9dd5f92731fa9be8038deaa024615593 Mon Sep 17 00:00:00 2001 From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com> Date: Wed, 27 Dec 2023 09:23:17 +0800 Subject: [PATCH 065/146] [compilation opt]Change cc test old (#60307) * change_cc_test_old * change_cc_test_old * change_cc_test_old * update * update * update * update * update * change_cc_test_old * opt cmake * update * update --- .../fluid/pir/dialect/op_generator/op_gen.py | 11 +- paddle/phi/common/data_type.h | 2 +- paddle/phi/common/int_array.h | 9 +- paddle/phi/core/infermeta_utils.cc | 2 +- paddle/phi/core/infermeta_utils.h | 24 +-- paddle/phi/core/meta_tensor.h | 2 +- paddle/phi/infermeta/binary.h | 6 +- paddle/phi/infermeta/nullary.h | 4 +- paddle/phi/kernels/elementwise_add_kernel.h | 8 +- test/cpp/pir/core/CMakeLists.txt | 17 +- test/cpp/pir/core/ir_program_test.cc | 165 ------------------ 11 files changed, 43 insertions(+), 207 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 8e56406583385..e3fbba6ed7bf7 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -39,6 +39,10 @@ ) import gen as vjp_gen +# Note(Galaxy1458) The need_export_symbol_op_list is used +# for some unittests these need to export symbol op compiled with dynamic lib. +need_export_symbol_op_list = ['AbsOp', 'FullOp'] + # ===================================== # String Template for h file code gen # ===================================== @@ -89,7 +93,7 @@ """ OP_DECLARE_TEMPLATE = """ -class {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ +class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ public: using Op::Op; static const char *name() {{ return "{dialect_op_name}"; }} @@ -1351,8 +1355,12 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): ) # gen op_declare_str/op_defined_str + TEST_API = "" + if op_class_name in need_export_symbol_op_list: + TEST_API = "TEST_API" if len(op_non_mutable_attribute_name_list) == 0: op_declare_str = OP_DECLARE_TEMPLATE.format( + TEST_API=TEST_API, op_name=op_class_name, dialect_op_name=op_dialect_name, interfaces=op_interfaces_str, @@ -1372,6 +1380,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): op_defined_str = "" else: op_declare_str = OP_DECLARE_TEMPLATE.format( + TEST_API=TEST_API, op_name=op_class_name, dialect_op_name=op_dialect_name, interfaces=op_interfaces_str, diff --git a/paddle/phi/common/data_type.h b/paddle/phi/common/data_type.h index 4c4555fd74bb0..f28dd7e1c6ef1 100644 --- a/paddle/phi/common/data_type.h +++ b/paddle/phi/common/data_type.h @@ -35,7 +35,7 @@ using bfloat16 = ::phi::dtype::bfloat16; using pstring = ::phi::dtype::pstring; // The enum value are consistent with jit/property.proto -enum class DataType { +enum class TEST_API DataType { UNDEFINED = 0, BOOL, diff --git a/paddle/phi/common/int_array.h b/paddle/phi/common/int_array.h index 6eab8609e54b2..29e411104c68c 100644 --- a/paddle/phi/common/int_array.h +++ b/paddle/phi/common/int_array.h @@ -32,7 +32,7 @@ template class IntArrayBase { public: // Constructor support implicit - IntArrayBase() = default; + TEST_API IntArrayBase() = default; IntArrayBase(const std::vector& vec) : array_(vec) {} // NOLINT @@ -58,12 +58,13 @@ class IntArrayBase { explicit IntArrayBase(const common::DDim& dims); // The Tensor must have one dim - IntArrayBase(const T& tensor); // NOLINT + TEST_API IntArrayBase(const T& tensor); // NOLINT // The Tensor in vec must have only one element - IntArrayBase(const std::vector& tensor_list); // NOLINT + TEST_API IntArrayBase(const std::vector& tensor_list); // NOLINT - explicit IntArrayBase(const std::vector& tensor_ref_list); + TEST_API explicit IntArrayBase( + const std::vector& tensor_ref_list); template IntArrayBase(const IntArrayBase& other) : array_(other.GetData()) {} diff --git a/paddle/phi/core/infermeta_utils.cc b/paddle/phi/core/infermeta_utils.cc index 18f3042bbf9c2..b644cf8021e43 100644 --- a/paddle/phi/core/infermeta_utils.cc +++ b/paddle/phi/core/infermeta_utils.cc @@ -153,7 +153,7 @@ template const std::vector& InferMetaContext::AttrAt( template const Scalar& InferMetaContext::AttrAt(size_t idx) const; template const std::vector& InferMetaContext::AttrAt(size_t idx) const; template const IntArray& InferMetaContext::AttrAt(size_t idx) const; -template const DataType& InferMetaContext::AttrAt(size_t idx) const; +template TEST_API const DataType& InferMetaContext::AttrAt(size_t idx) const; template const DataLayout& InferMetaContext::AttrAt(size_t idx) const; template const Place& InferMetaContext::AttrAt(size_t idx) const; template const TensorRef& InferMetaContext::AttrAt(size_t idx) const; diff --git a/paddle/phi/core/infermeta_utils.h b/paddle/phi/core/infermeta_utils.h index 06036b2c13894..494fe160696ff 100644 --- a/paddle/phi/core/infermeta_utils.h +++ b/paddle/phi/core/infermeta_utils.h @@ -41,32 +41,32 @@ class InferMetaContext { const MetaConfig& GetMetaConfig() const; void EmplaceBackInput(MetaTensor input); - void EmplaceBackOutput(MetaTensor output); - void EmplaceBackAttr(Attribute attr); + TEST_API void EmplaceBackOutput(MetaTensor output); + TEST_API void EmplaceBackAttr(Attribute attr); void EmplaceBackInputs( paddle::small_vector inputs); void EmplaceBackOutputs( paddle::small_vector outputs); - virtual const MetaTensor& InputAt(size_t idx) const; + TEST_API virtual const MetaTensor& InputAt(size_t idx) const; - virtual std::vector InputsBetween(size_t start, - size_t end) const; - virtual paddle::optional> + TEST_API virtual std::vector InputsBetween( + size_t start, size_t end) const; + TEST_API virtual paddle::optional> OptionalInputsBetween(size_t start, size_t end) const; - virtual MetaTensor* MutableOutputAt(size_t idx); - virtual std::vector MutableOutputBetween(size_t start, - size_t end); + TEST_API virtual MetaTensor* MutableOutputAt(size_t idx); + TEST_API virtual std::vector MutableOutputBetween(size_t start, + size_t end); template - const AttrType& AttrAt(size_t idx) const; + TEST_API const AttrType& AttrAt(size_t idx) const; - const Attribute& AttrAt(size_t idx) const; + TEST_API const Attribute& AttrAt(size_t idx) const; const std::pair& InputRangeAt(size_t idx) const; - const std::pair& OutputRangeAt(size_t idx) const; + TEST_API const std::pair& OutputRangeAt(size_t idx) const; virtual ~InferMetaContext() = default; diff --git a/paddle/phi/core/meta_tensor.h b/paddle/phi/core/meta_tensor.h index b28081c8d4ef7..626cd238109b4 100644 --- a/paddle/phi/core/meta_tensor.h +++ b/paddle/phi/core/meta_tensor.h @@ -34,7 +34,7 @@ struct MetaConfig { is_run_mkldnn_kernel(is_run_mkldnn_kernel) {} // NOLINT }; -class MetaTensor { +class TEST_API MetaTensor { public: typedef void (*unspecified_bool_type)(); diff --git a/paddle/phi/infermeta/binary.h b/paddle/phi/infermeta/binary.h index d082caea28636..c081c1690c28d 100644 --- a/paddle/phi/infermeta/binary.h +++ b/paddle/phi/infermeta/binary.h @@ -221,9 +221,9 @@ void DropoutNdInferMeta(const MetaTensor& x, MetaTensor* out, MetaTensor* mask); -void ElementwiseInferMeta(const MetaTensor& x, - const MetaTensor& y, - MetaTensor* out); +TEST_API void ElementwiseInferMeta(const MetaTensor& x, + const MetaTensor& y, + MetaTensor* out); void ElementwiseRawInferMeta(const MetaTensor& x_meta, const MetaTensor& y_meta, diff --git a/paddle/phi/infermeta/nullary.h b/paddle/phi/infermeta/nullary.h index c424bba6f8a8b..5eda8fc1a8461 100644 --- a/paddle/phi/infermeta/nullary.h +++ b/paddle/phi/infermeta/nullary.h @@ -48,7 +48,9 @@ void CreateVecShapeInferMeta(const std::vector& shape, void CreateArrayInferMeta(DataType dtype, MetaTensor* out); -void CreateInferMeta(const IntArray& shape, DataType dtype, MetaTensor* out); +TEST_API void CreateInferMeta(const IntArray& shape, + DataType dtype, + MetaTensor* out); void CreateInferMetaBase(const std::vector& shape, DataType dtype, diff --git a/paddle/phi/kernels/elementwise_add_kernel.h b/paddle/phi/kernels/elementwise_add_kernel.h index 05145863e8beb..eef77a50eeae3 100644 --- a/paddle/phi/kernels/elementwise_add_kernel.h +++ b/paddle/phi/kernels/elementwise_add_kernel.h @@ -19,10 +19,10 @@ namespace phi { template -void AddKernel(const Context& dev_ctx, - const DenseTensor& x, - const DenseTensor& y, - DenseTensor* out); +TEST_API void AddKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& y, + DenseTensor* out); template DenseTensor Add(const Context& dev_ctx, diff --git a/test/cpp/pir/core/CMakeLists.txt b/test/cpp/pir/core/CMakeLists.txt index 66389b600f156..eab172b2054a0 100644 --- a/test/cpp/pir/core/CMakeLists.txt +++ b/test/cpp/pir/core/CMakeLists.txt @@ -4,20 +4,9 @@ paddle_test(ir_value_test SRCS ir_value_test.cc) paddle_test(ir_op_test SRCS ir_op_test.cc DEPS test_dialect) paddle_test(ir_region_test SRCS ir_region_test.cc) paddle_test(ir_builder_test SRCS ir_builder_test.cc) -cc_test_old( - ir_program_test - SRCS - ir_program_test.cc - DEPS - common - gtest - pir - op_dialect_vjp - phi) - -cc_test_old(ir_infershape_test SRCS ir_infershape_test.cc DEPS common gtest) - -cc_test_old(scalar_attribute_test SRCS scalar_attribute_test.cc DEPS gtest) +paddle_test(ir_program_test SRCS ir_program_test.cc) +paddle_test(ir_infershape_test SRCS ir_infershape_test.cc) +paddle_test(scalar_attribute_test SRCS scalar_attribute_test.cc) file( DOWNLOAD https://paddle-ci.gz.bcebos.com/ir_translator_test/resnet50_main.prog diff --git a/test/cpp/pir/core/ir_program_test.cc b/test/cpp/pir/core/ir_program_test.cc index 4c28a5a0cf22b..b4c7c89ae85ce 100644 --- a/test/cpp/pir/core/ir_program_test.cc +++ b/test/cpp/pir/core/ir_program_test.cc @@ -69,142 +69,6 @@ void AddOp::Build(pir::Builder &, IR_DECLARE_EXPLICIT_TEST_TYPE_ID(AddOp) IR_DEFINE_EXPLICIT_TYPE_ID(AddOp) -TEST(program_test, program) { - // (1) Init environment. - pir::IrContext *ctx = pir::IrContext::Instance(); - pir::Dialect *builtin_dialect = - ctx->GetOrRegisterDialect(); - builtin_dialect->RegisterOp(); - pir::Dialect *paddle_dialect = - ctx->GetOrRegisterDialect(); - - // (2) Create an empty program object - pir::Program program(ctx); - - // (3) Create a float32 DenseTensor Parameter and save into Program - pir::Type fp32_dtype = pir::Float32Type::get(ctx); - phi::DDim dims = {2, 2}; - phi::DataLayout data_layout = phi::DataLayout::NCHW; - phi::LoD lod = {{0, 1, 2}}; - size_t offset = 0; - pir::Type dense_tensor_dtype = paddle::dialect::DenseTensorType::get( - ctx, fp32_dtype, dims, data_layout, lod, offset); - - std::vector data_a = {1, 2, 3, 4}; - std::unique_ptr parameter_a = - std::make_unique(reinterpret_cast(data_a.data()), - 4 * sizeof(float), - dense_tensor_dtype); - program.SetParameter("a", std::move(parameter_a)); - EXPECT_EQ(program.parameters_num() == 1, true); - - std::vector data_b = {5, 6, 7, 8}; - std::unique_ptr parameter_b = - std::make_unique(reinterpret_cast(data_b.data()), - 4 * sizeof(float), - dense_tensor_dtype); - program.SetParameter("b", std::move(parameter_b)); - EXPECT_EQ(program.parameters_num() == 2, true); - - // (4) Def a = ParameterOp("a"), and create DenseTensor for a. - pir::Builder builder(ctx, program.block()); - auto op1 = builder.Build("a", dense_tensor_dtype); - - EXPECT_EQ(&program, op1->GetParentProgram()); - EXPECT_EQ(op1->result_type(0).dialect().id(), paddle_dialect->id()); - using Interface = paddle::dialect::ParameterConvertInterface; - Interface *a_interface = - op1->result_type(0).dialect().GetRegisteredInterface(); - std::shared_ptr a_var = - a_interface->ParameterToVariable(program.GetParameter("a")); - const phi::DenseTensor &a_tensor = a_var->Get(); - EXPECT_EQ(a_tensor.numel(), 4); - EXPECT_EQ(a_tensor.dims(), dims); - EXPECT_EQ(a_tensor.dtype(), paddle::dialect::TransToPhiDataType(fp32_dtype)); - EXPECT_EQ(a_tensor.layout(), data_layout); - EXPECT_EQ(a_tensor.lod(), lod); - EXPECT_EQ(a_tensor.offset(), offset); - for (int64_t i = 0; i < a_tensor.numel(); i++) { - EXPECT_EQ(*(a_tensor.data() + i), data_a[i]); - } - - // (5) Def b = ParameterOp("b"), and create DenseTensor for b. - auto op2 = builder.Build("b", dense_tensor_dtype); - - EXPECT_EQ(op2->result_type(0).dialect().id(), paddle_dialect->id()); - Interface *b_interface = - op2->result_type(0).dialect().GetRegisteredInterface(); - std::shared_ptr b_var = - b_interface->ParameterToVariable(program.GetParameter("b")); - const phi::DenseTensor &b_tensor = b_var->Get(); - EXPECT_EQ(b_tensor.numel(), 4); - EXPECT_EQ(b_tensor.dims(), dims); - EXPECT_EQ(b_tensor.dtype(), paddle::dialect::TransToPhiDataType(fp32_dtype)); - EXPECT_EQ(b_tensor.layout(), data_layout); - EXPECT_EQ(b_tensor.lod(), lod); - EXPECT_EQ(b_tensor.offset(), offset); - for (int64_t i = 0; i < b_tensor.numel(); i++) { - EXPECT_EQ(*(b_tensor.data() + i), data_b[i]); - } - - // (6) Def c = AddOp(a, b), execute this op. - auto op3 = - builder.Build(op1->result(0), op2->result(0), dense_tensor_dtype); - phi::CPUContext *dev_ctx = static_cast( - paddle::platform::DeviceContextPool::Instance().Get( - paddle::platform::CPUPlace())); - phi::DenseTensor c_tensor = - phi::Add(*dev_ctx, a_tensor, b_tensor); - std::shared_ptr variable_c = - std::make_shared(); - auto *dst_tensor = variable_c->GetMutable(); - *dst_tensor = c_tensor; - EXPECT_EQ(dst_tensor->numel(), b_tensor.numel()); - EXPECT_EQ(dst_tensor->dims(), b_tensor.dims()); - EXPECT_EQ(dst_tensor->dtype(), b_tensor.dtype()); - EXPECT_EQ(dst_tensor->layout(), b_tensor.layout()); - EXPECT_EQ(dst_tensor->lod(), b_tensor.lod()); - EXPECT_EQ(dst_tensor->offset(), b_tensor.offset()); - for (int64_t i = 0; i < dst_tensor->numel(); i++) { - EXPECT_EQ(*(dst_tensor->data() + i), data_a[i] + data_b[i]); - } - - // (7) Def AbsOp(b) - auto abs_op = builder.Build(op1->result(0)); - paddle::dialect::OpYamlInfoInterface interface = - abs_op->dyn_cast(); - EXPECT_EQ(std::get<0>(interface.GetOpInfo())[0].name == "x", true); - - // (8) Def SetParameterOp(c, "c") - auto op4 = builder.Build(op3->result(0), "c"); - - EXPECT_EQ(op4->operand(0).type().dialect().id(), paddle_dialect->id()); - Interface *c_interface = - op4->operand(0).type().dialect().GetRegisteredInterface(); - // pir::Parameter *parameter_c = - // c_interface->VariableToParameter(variable_c.get()); - std::unique_ptr parameter_c = - c_interface->VariableToParameter(variable_c.get()); - EXPECT_EQ(parameter_c->type(), dense_tensor_dtype); - for (int64_t i = 0; i < dst_tensor->numel(); i++) { - EXPECT_EQ(*(dst_tensor->data() + i), - *(static_cast(parameter_c->data()) + i)); - } - program.SetParameter("c", std::move(parameter_c)); - - // (8) Traverse Program - EXPECT_EQ(program.block()->size() == 5, true); - EXPECT_EQ(program.parameters_num() == 3, true); - - std::stringstream ss; - program.Print(ss); - - std::stringstream ss_ostram; - ss_ostram << program; - - EXPECT_EQ(ss.str(), ss_ostram.str()); -} - TEST(program_test, slice_combine_test) { // (1) Init environment. pir::IrContext *ctx = pir::IrContext::Instance(); @@ -261,32 +125,3 @@ TEST(program_test, slice_combine_test) { // (8) Traverse Program EXPECT_EQ(program.block()->size() == 4, true); } - -TEST(program_test, builder) { - pir::IrContext *ctx = pir::IrContext::Instance(); - ctx->GetOrRegisterDialect(); - pir::Program program(ctx); - pir::Builder builder = pir::Builder(ctx, program.block()); - - paddle::dialect::FullOp full_op = builder.Build( - std::vector{2, 2}, 1.5, phi::DataType::FLOAT32, phi::CPUPlace()); - pir::Type full_op_output = full_op->result_type(0); - EXPECT_EQ(program.block()->size(), 1u); - EXPECT_EQ(program.block()->back(), *full_op.operation()); - EXPECT_EQ(full_op.num_operands(), 0u); - EXPECT_EQ(full_op.num_results(), 1u); - EXPECT_EQ(full_op.attributes().size(), 5u); - EXPECT_EQ( - full_op_output.dyn_cast().offset() == 0, - true); - for (auto dim : common::vectorize( - full_op_output.dyn_cast() - .dims())) { - EXPECT_EQ(dim == 2, true); - } - - pir::ConstantOp constant = builder.Build( - pir::Int32Attribute::get(ctx, 2), pir::Int32Type::get(ctx)); - EXPECT_EQ(program.block()->size() == 2, true); - EXPECT_EQ(constant.value().dyn_cast().data() == 2, true); -} From f479d98dae208b29f717f590045cf75496988535 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Wed, 27 Dec 2023 09:34:24 +0800 Subject: [PATCH 066/146] [PIR+CINN]Fix Bool ArrayAttribute Convert Bug (#60339) --- paddle/cinn/hlir/framework/pir/utils.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/paddle/cinn/hlir/framework/pir/utils.cc b/paddle/cinn/hlir/framework/pir/utils.cc index eddc7a54c4878..8833ac496e32c 100644 --- a/paddle/cinn/hlir/framework/pir/utils.cc +++ b/paddle/cinn/hlir/framework/pir/utils.cc @@ -164,11 +164,18 @@ utils::Attribute CompatibleInfo::ConvertAttribute( vec_int64.push_back( vec_element.dyn_cast<::pir::Int64Attribute>().data()); } - dst_attr = vec_int64; + } else if (attr_vec[0].isa<::pir::BoolAttribute>()) { + std::vector vec_bool; + int index = 0; + for (auto vec_element : attr_vec) { + vec_bool.push_back( + vec_element.dyn_cast<::pir::BoolAttribute>().data()); + } + dst_attr = vec_bool; } else { LOG(FATAL) - << "only suuport int32 and int64 attribute in ArrayAttribute"; + << "only support bool/int32/int64 attribute in ArrayAttribute"; } } } else { From edcbe330574ea8aea18426af453e6faf67557fcf Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Wed, 27 Dec 2023 09:55:25 +0800 Subject: [PATCH 067/146] [tests] clean `paddle.paddle` (#60380) --- python/paddle/incubate/optimizer/functional/utils.py | 2 +- test/auto_parallel/test_auto_conditional_block.py | 2 +- test/ipu/test_dy2static_fp16_ipu.py | 2 +- test/ipu/test_dy2static_ipu.py | 4 ++-- test/ipu/test_modelruntime_ipu.py | 2 +- test/ipu/test_print_op_ipu.py | 2 +- test/legacy_test/test_conv2d_transpose_op.py | 4 ++-- .../test_dist_fleet_a_sync_optimizer_auto_async.py | 2 +- test/legacy_test/test_fused_multi_transformer_op.py | 10 +++++----- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/paddle/incubate/optimizer/functional/utils.py b/python/paddle/incubate/optimizer/functional/utils.py index 52edef7177ae9..c6a6f1c6b405a 100644 --- a/python/paddle/incubate/optimizer/functional/utils.py +++ b/python/paddle/incubate/optimizer/functional/utils.py @@ -72,7 +72,7 @@ def false_fn(): paddle.static.nn.cond(is_symmetric, None, false_fn) # eigvals only support cpu paddle.set_device("cpu") - eigvals = paddle.paddle.linalg.eigvals(H0) + eigvals = paddle.linalg.eigvals(H0) is_positive = paddle.all(eigvals.real() > 0.0) and paddle.all( eigvals.imag() == 0.0 ) diff --git a/test/auto_parallel/test_auto_conditional_block.py b/test/auto_parallel/test_auto_conditional_block.py index d34f788a3c392..5d7eeb94430a1 100644 --- a/test/auto_parallel/test_auto_conditional_block.py +++ b/test/auto_parallel/test_auto_conditional_block.py @@ -80,7 +80,7 @@ def forward(self, input): def loss_func(pred, label): - error_cost = paddle.paddle.nn.functional.square_error_cost(pred, label) + error_cost = paddle.nn.functional.square_error_cost(pred, label) error_cost = error_cost[error_cost > 0].astype("float32") loss = paddle.mean(error_cost) return loss diff --git a/test/ipu/test_dy2static_fp16_ipu.py b/test/ipu/test_dy2static_fp16_ipu.py index 3b37073bb0a73..bc3e5342ef47d 100644 --- a/test/ipu/test_dy2static_fp16_ipu.py +++ b/test/ipu/test_dy2static_fp16_ipu.py @@ -34,7 +34,7 @@ def forward(self, x, target=None): x = paddle.flatten(x, 1, -1) if target is not None: x = paddle.nn.functional.softmax(x) - loss = paddle.paddle.nn.functional.cross_entropy( + loss = paddle.nn.functional.cross_entropy( x, target, reduction='none', use_softmax=False ) if self.use_ipu: diff --git a/test/ipu/test_dy2static_ipu.py b/test/ipu/test_dy2static_ipu.py index b98bdc0351400..eaca14de6a398 100644 --- a/test/ipu/test_dy2static_ipu.py +++ b/test/ipu/test_dy2static_ipu.py @@ -49,7 +49,7 @@ def forward(self, x, target=None): if target is not None: if self.use_softmax: x = paddle.nn.functional.softmax(x) - loss = paddle.paddle.nn.functional.cross_entropy( + loss = paddle.nn.functional.cross_entropy( x, target, reduction='none', use_softmax=False ) if self.use_reduction: @@ -219,7 +219,7 @@ def create_model(self, use_ipu=False): class TestWithoutIdentityLoss2(TestBase): def set_op_attrs(self): - self.loss_op = paddle.paddle.nn.functional.softmax_with_cross_entropy + self.loss_op = paddle.nn.functional.softmax_with_cross_entropy def set_data_feed(self): self.data = paddle.uniform((8, 3, 10, 10), dtype='float32') diff --git a/test/ipu/test_modelruntime_ipu.py b/test/ipu/test_modelruntime_ipu.py index c66f7abdfa4fe..44124fa70c336 100644 --- a/test/ipu/test_modelruntime_ipu.py +++ b/test/ipu/test_modelruntime_ipu.py @@ -33,7 +33,7 @@ def forward(self, x, target=None): x = paddle.flatten(x, 1, -1) if target is not None: x = paddle.nn.functional.softmax(x) - loss = paddle.paddle.nn.functional.cross_entropy( + loss = paddle.nn.functional.cross_entropy( x, target, reduction='none', use_softmax=False ) return x, loss diff --git a/test/ipu/test_print_op_ipu.py b/test/ipu/test_print_op_ipu.py index 10449cd48ae83..442077009fc48 100644 --- a/test/ipu/test_print_op_ipu.py +++ b/test/ipu/test_print_op_ipu.py @@ -120,7 +120,7 @@ def forward(self, x, target=None): x = paddle.flatten(x, 1, -1) if target is not None: x = paddle.nn.functional.softmax(x) - loss = paddle.paddle.nn.functional.cross_entropy( + loss = paddle.nn.functional.cross_entropy( x, target, reduction='none', use_softmax=False ) loss = paddle.incubate.identity_loss(loss, 1) diff --git a/test/legacy_test/test_conv2d_transpose_op.py b/test/legacy_test/test_conv2d_transpose_op.py index 339ef086d7b81..b598d51de3fc3 100644 --- a/test/legacy_test/test_conv2d_transpose_op.py +++ b/test/legacy_test/test_conv2d_transpose_op.py @@ -1349,7 +1349,7 @@ def var_prefix(self): def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = paddle.assign([17]) - out = paddle.paddle.nn.functional.conv2d_transpose( + out = paddle.nn.functional.conv2d_transpose( x, w_var, stride=2, output_size=output_size ) return out @@ -1388,7 +1388,7 @@ def path_prefix(self): def call_func(self, x): w_var = paddle.randn((3, 6, 3, 3), dtype='float32') output_size = [17, paddle.assign([17])] - out = paddle.paddle.nn.functional.conv2d_transpose( + out = paddle.nn.functional.conv2d_transpose( x, w_var, stride=2, output_size=output_size ) return out diff --git a/test/legacy_test/test_dist_fleet_a_sync_optimizer_auto_async.py b/test/legacy_test/test_dist_fleet_a_sync_optimizer_auto_async.py index 67807c6673701..186e7ac5ad497 100644 --- a/test/legacy_test/test_dist_fleet_a_sync_optimizer_auto_async.py +++ b/test/legacy_test/test_dist_fleet_a_sync_optimizer_auto_async.py @@ -58,7 +58,7 @@ def test_a_sync_optimizer3(self): size=[1000000000, 100000], param_attr=paddle.base.ParamAttr( name="embedding", - initializer=paddle.paddle.nn.initializer.Constant(value=0.01), + initializer=paddle.nn.initializer.Constant(value=0.01), ), is_sparse=True, ) diff --git a/test/legacy_test/test_fused_multi_transformer_op.py b/test/legacy_test/test_fused_multi_transformer_op.py index 577957e8b0e41..4e52cd2eb5e56 100644 --- a/test/legacy_test/test_fused_multi_transformer_op.py +++ b/test/legacy_test/test_fused_multi_transformer_op.py @@ -60,7 +60,7 @@ def setUp(self): self.__class__.no_need_check_grad = False bias_attr = paddle.base.ParamAttr( - initializer=paddle.paddle.nn.initializer.Constant(value=0.0005) + initializer=paddle.nn.initializer.Constant(value=0.0005) ) self.q_proj = Linear( self.embed_dim, @@ -1383,16 +1383,16 @@ def config(self): self.has_attn_mask = False self.x_type = np.float32 self.weight_attr = paddle.ParamAttr( - initializer=paddle.paddle.nn.initializer.Constant(0.0) + initializer=paddle.nn.initializer.Constant(0.0) ) self.bias_attr = paddle.ParamAttr( - initializer=paddle.paddle.nn.initializer.Constant(0.0005) + initializer=paddle.nn.initializer.Constant(0.0005) ) self.ln_w_attr = paddle.ParamAttr( - initializer=paddle.paddle.nn.initializer.Constant(1.0) + initializer=paddle.nn.initializer.Constant(1.0) ) self.ln_b_attr = paddle.ParamAttr( - initializer=paddle.paddle.nn.initializer.Constant(0.0) + initializer=paddle.nn.initializer.Constant(0.0) ) def test_fused_multi_transformer_op(self): From a65530681a75d0db61e3f89baf79150e0ddd0253 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Wed, 27 Dec 2023 10:34:31 +0800 Subject: [PATCH 068/146] test=document_fix (#60378) --- tools/gpups_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/gpups_test.sh b/tools/gpups_test.sh index b390c7f056957..883604ef6685e 100644 --- a/tools/gpups_test.sh +++ b/tools/gpups_test.sh @@ -27,6 +27,8 @@ function collect_failed_tests() { done } +# disable test: test_dist_fuse_resunit_pass + serial_list="^test_conv2d_op$|\ ^test_conv2d_transpose_op$|\ ^test_dygraph_dataparallel_bf16$|\ @@ -62,7 +64,6 @@ parallel_list="^init_phi_test$|\ ^test_custom_kernel$|\ ^test_dist_fleet_ps11$|\ ^test_dist_fleet_ps12$|\ -^test_dist_fuse_resunit_pass$|\ ^test_executor_feed_non_tensor$|\ ^test_flash_attention$|\ ^test_fuse_resunit_pass$|\ From a3db642eaf26185e4e4a767ca1d2bb6e56a7c13b Mon Sep 17 00:00:00 2001 From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com> Date: Wed, 27 Dec 2023 10:38:49 +0800 Subject: [PATCH 069/146] opt cmake (#60358) * opt cmake * update * update --- paddle/fluid/ir_adaptor/translator/translate.h | 2 +- paddle/fluid/ir_adaptor/translator/utils.h | 2 +- paddle/fluid/pir/dialect/operator/ir/control_flow_op.h | 6 +++--- test/cpp/pir/core/CMakeLists.txt | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/translate.h b/paddle/fluid/ir_adaptor/translator/translate.h index 47ad12003f807..fa3dcc7892bff 100644 --- a/paddle/fluid/ir_adaptor/translator/translate.h +++ b/paddle/fluid/ir_adaptor/translator/translate.h @@ -22,7 +22,7 @@ namespace paddle { -std::unique_ptr<::pir::Program> TranslateLegacyProgramToProgram( +TEST_API std::unique_ptr<::pir::Program> TranslateLegacyProgramToProgram( const ::paddle::framework::ProgramDesc& legacy_program); } // namespace paddle diff --git a/paddle/fluid/ir_adaptor/translator/utils.h b/paddle/fluid/ir_adaptor/translator/utils.h index a4765940d0a78..600c06edac84c 100644 --- a/paddle/fluid/ir_adaptor/translator/utils.h +++ b/paddle/fluid/ir_adaptor/translator/utils.h @@ -59,7 +59,7 @@ pir::Operation* InsertSliceOperationForTarget( std::ostream& operator<<(std::ostream& os, const std::vector& vec_str); -std::vector CheckUnregisteredOperation( +TEST_API std::vector CheckUnregisteredOperation( pir::IrContext* ctx, const framework::ProgramDesc& legacy_program); inline DataType VarTypeToDataType( diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h index f3b9b30c2e400..baffcadc12718 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h @@ -41,8 +41,8 @@ class IfOp : public pir::Op { std::unique_ptr &&false_block); pir::Value cond() { return operand_source(0); } - pir::Block &true_block(); - pir::Block &false_block(); + TEST_API pir::Block &true_block(); + TEST_API pir::Block &false_block(); pir::Region &true_region() { return (*this)->region(0); } pir::Region &false_region() { return (*this)->region(1); } void Print(pir::IrPrinter &printer); // NOLINT @@ -78,7 +78,7 @@ class WhileOp : public pir::Op { pir::OperationArgument &argument, // NOLINT pir::Value cond, const std::vector &inputs); - pir::Block &body(); + TEST_API pir::Block &body(); pir::Value cond(); const pir::Block::ArgListType &block_args() { return body().args(); } void Print(pir::IrPrinter &printer); // NOLINT diff --git a/test/cpp/pir/core/CMakeLists.txt b/test/cpp/pir/core/CMakeLists.txt index eab172b2054a0..c53dbb3a71666 100644 --- a/test/cpp/pir/core/CMakeLists.txt +++ b/test/cpp/pir/core/CMakeLists.txt @@ -33,8 +33,7 @@ file( copy_if_different(${CMAKE_CURRENT_SOURCE_DIR}/TestParserText.txt ${CMAKE_CURRENT_BINARY_DIR}/TestParserText.txt) -cc_test_old(program_translator_test SRCS program_translator_test.cc DEPS - program_translator gtest) +paddle_test(program_translator_test SRCS program_translator_test.cc) paddle_test(add_dialect_parser_test SRCS add_dialect_parser_test.cc DEPS gtest) From 3dd6eabe35661a33ab404b4062b7de410ea29976 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Wed, 27 Dec 2023 10:45:54 +0800 Subject: [PATCH 070/146] suport pir in laplace, sequence_mask L1Decay (#60345) --- .../pir/dialect/op_generator/ops_api_gen.py | 1 - paddle/fluid/pybind/pir.cc | 7 +++- paddle/phi/api/yaml/legacy_ops.yaml | 9 +++++ python/paddle/distribution/laplace.py | 8 +++- python/paddle/nn/functional/extension.py | 13 ++----- python/paddle/pir/core.py | 4 ++ python/paddle/tensor/random.py | 2 +- test/distribution/parameterize.py | 9 ++++- .../test_distribution_laplace_static.py | 39 +++++++++++++++++-- test/legacy_test/test_regularizer.py | 34 ++++++++++++++++ test/sequence/test_sequence_mask.py | 4 +- 11 files changed, 107 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index d379bedaab643..9fd6bd4bfbd98 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -100,7 +100,6 @@ 'self_dp_attention', 'get_tensor_from_selected_rows', 'print', - 'sequence_mask', 'number_count', 'assign_value', ] diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index bbd389c4886a3..65e3f69bc05b6 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -614,7 +614,9 @@ pir::OpResult apply(Value self, py::object func) { } void BindValue(py::module *m) { - py::class_ value(*m, "Value", R"DOC( + py::class_ value(*m, + "Value", + R"DOC( Value class represents the SSA value in the IR system. It is a directed edge and a base class. @@ -622,7 +624,8 @@ void BindValue(py::module *m) { The constructor of Value should not be invoked directly. Value can be automatically constructed when build network. - )DOC"); + )DOC", + pybind11::dynamic_attr()); g_ir_value_pytype = reinterpret_cast(value.ptr()); value.def(py::init<>()) .def_property_readonly( diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 7cd2b4b6e3f32..dc582641b769e 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -980,6 +980,15 @@ intermediate : noise backward : rrelu_grad +- op : sequence_mask + args: (Tensor x, Scalar(int) max_len, int out_dtype) + output: Tensor(y) + infer_meta: + func: SequenceMaskScalarInferMeta + kernel: + func: sequence_mask_scalar + data_type : x + - op : set_value args : (Tensor x, IntArray starts, IntArray ends, IntArray steps, int64_t[] axes, int64_t[] decrease_axes, int64_t[] none_axes, int64_t[] shape, Scalar[] values) output : Tensor(out) diff --git a/python/paddle/distribution/laplace.py b/python/paddle/distribution/laplace.py index fc4b57eeba79c..481ab3f51f1e8 100644 --- a/python/paddle/distribution/laplace.py +++ b/python/paddle/distribution/laplace.py @@ -54,12 +54,16 @@ class Laplace(distribution.Distribution): """ def __init__(self, loc, scale): - if not isinstance(loc, (numbers.Real, framework.Variable)): + if not isinstance( + loc, (numbers.Real, framework.Variable, paddle.pir.Value) + ): raise TypeError( f"Expected type of loc is Real|Variable, but got {type(loc)}" ) - if not isinstance(scale, (numbers.Real, framework.Variable)): + if not isinstance( + scale, (numbers.Real, framework.Variable, paddle.pir.Value) + ): raise TypeError( f"Expected type of scale is Real|Variable, but got {type(scale)}" ) diff --git a/python/paddle/nn/functional/extension.py b/python/paddle/nn/functional/extension.py index b4c0847778bf5..dac7ba30d93fd 100644 --- a/python/paddle/nn/functional/extension.py +++ b/python/paddle/nn/functional/extension.py @@ -15,7 +15,7 @@ # TODO: define the extention functions -from paddle import _C_ops, _legacy_C_ops, in_dynamic_mode, tensor +from paddle import _C_ops, tensor from paddle.utils import deprecated from ...base.data_feeder import check_type, check_variable_and_dtype @@ -100,16 +100,11 @@ def sequence_mask(x, maxlen=None, dtype='int64', name=None): """ - if in_dynamic_mode(): - if not isinstance(dtype, core.VarDesc.VarType): + if in_dynamic_or_pir_mode(): + if not isinstance(dtype, (core.VarDesc.VarType, core.DataType)): dtype = convert_np_dtype_to_dtype_(dtype) if maxlen is not None: - if isinstance(maxlen, core.eager.Tensor): - attrs = ('out_dtype', dtype) - out = _legacy_C_ops.sequence_mask(x, maxlen, *attrs) - else: - attrs = ('out_dtype', dtype, 'maxlen', maxlen) - out = _legacy_C_ops.sequence_mask(x, None, *attrs) + out = _C_ops.sequence_mask(x, maxlen, dtype) out.stop_gradient = True return out diff --git a/python/paddle/pir/core.py b/python/paddle/pir/core.py index be5273f513afb..1933ef7fabf84 100644 --- a/python/paddle/pir/core.py +++ b/python/paddle/pir/core.py @@ -273,10 +273,13 @@ def create_parameter( name=None, **kwargs, ): + regularizer = None if 'initializer' not in kwargs: raise ValueError( "initializer is None, if you want to create parameter, please pass its initializer." ) + if 'regularizer' in kwargs: + regularizer = kwargs['regularizer'] if dtype is not None: if not isinstance(dtype, DataType): dtype = convert_np_dtype_to_dtype_(dtype) @@ -302,6 +305,7 @@ def create_parameter( param.stop_gradient = not trainable param.persistable = True + param.regularizer = regularizer return param diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index 6829bc84b045c..149541e0d94bd 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -858,7 +858,7 @@ def uniform(shape, dtype=None, min=-1.0, max=1.0, seed=0, name=None): ) ) - if not isinstance(dtype, core.VarDesc.VarType): + if not isinstance(dtype, (core.VarDesc.VarType, core.DataType)): dtype = convert_np_dtype_to_dtype_(dtype) if in_dynamic_mode(): diff --git a/test/distribution/parameterize.py b/test/distribution/parameterize.py index 5b32e871ac0a6..4488553fdec57 100644 --- a/test/distribution/parameterize.py +++ b/test/distribution/parameterize.py @@ -47,7 +47,7 @@ def decorate(cls): return decorate -def parameterize_cls(fields, values=None): +def parameterize_cls(fields, values=None, test_pir=False): fields = [fields] if isinstance(fields, str) else fields params = [dict(zip(fields, vals)) for vals in values] @@ -56,10 +56,15 @@ def decorate(cls): for k, v in enumerate(params): test_cls = dict(cls.__dict__) test_cls.update(v) + test_cls["test_pir"] = False name = cls.__name__ + str(k) name = name + '.' + v.get('suffix') if v.get('suffix') else name - test_cls_module[name] = type(name, (cls,), test_cls) + if test_pir: + name = name + ".pir" + test_cls["test_pir"] = True + pir_type = type(name, (cls,), test_cls) + test_cls_module[name] = pir_type for m in list(cls.__dict__): if m.startswith("test"): diff --git a/test/distribution/test_distribution_laplace_static.py b/test/distribution/test_distribution_laplace_static.py index d4ad34d995bb5..1987749abe559 100644 --- a/test/distribution/test_distribution_laplace_static.py +++ b/test/distribution/test_distribution_laplace_static.py @@ -31,9 +31,10 @@ ('one-dim', parameterize.xrand((2,)), parameterize.xrand((2,))), ('multi-dim', parameterize.xrand((5, 5)), parameterize.xrand((5, 5))), ], + test_pir=True, ) class TestLaplace(unittest.TestCase): - def setUp(self): + def build_program(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) @@ -61,6 +62,13 @@ def setUp(self): self.samples, ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.build_program() + else: + self.build_program() + def test_mean(self): self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) np.testing.assert_allclose( @@ -155,9 +163,10 @@ def _np_entropy(self): np.array([[4.0, 6], [8, 2]]), ), ], + test_pir=True, ) class TestLaplacePDF(unittest.TestCase): - def setUp(self): + def build_program(self): startup_program = paddle.static.Program() main_program = paddle.static.Program() executor = paddle.static.Executor(self.place) @@ -183,6 +192,13 @@ def setUp(self): main_program, feed=self.feeds, fetch_list=fetch_list ) + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.build_program() + else: + self.build_program() + def test_prob(self): np.testing.assert_allclose( self.prob, @@ -228,9 +244,10 @@ def test_icdf(self): np.array([0.5]), ) ], + test_pir=True, ) class TestLaplaceAndLaplaceKL(unittest.TestCase): - def setUp(self): + def build_program(self): self.mp = paddle.static.Program() self.sp = paddle.static.Program() self.executor = paddle.static.Executor(self.place) @@ -253,7 +270,14 @@ def setUp(self): 'scale2': self.scale2, } - def test_kl_divergence(self): + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.build_program() + else: + self.build_program() + + def add_kl_divergence(self): with paddle.static.program_guard(self.mp, self.sp): out = paddle.distribution.kl_divergence(self._dist_1, self._dist_2) self.executor.run(self.sp) @@ -262,6 +286,13 @@ def test_kl_divergence(self): ) np.testing.assert_allclose(out, self._np_kl(), atol=0, rtol=0.50) + def test_kl_divergence(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.add_kl_divergence() + else: + self.add_kl_divergence() + def _np_kl(self): x = np.linspace( scipy.stats.laplace.ppf(0.01), scipy.stats.laplace.ppf(0.99), 1000 diff --git a/test/legacy_test/test_regularizer.py b/test/legacy_test/test_regularizer.py index a0edf81058b9e..bb01e80c08a48 100644 --- a/test/legacy_test/test_regularizer.py +++ b/test/legacy_test/test_regularizer.py @@ -111,6 +111,40 @@ def test_l2decay_regularizer(self): self.assertEqual(block.ops[-2].type, 'scale') self.assertEqual(block.ops[-3].type, 'sign') + def test_l1decay_regularizer(self): + with paddle.pir_utils.IrGuard(): + main_program = paddle.static.Program() + with paddle.static.program_guard(main_program): + block = main_program.global_block() + mul_x = paddle.pir.core.create_parameter( + dtype="float32", + shape=[5, 10], + name="mul.x", + regularizer=regularizer.L1Decay(0.5), + initializer=paddle.nn.initializer.Constant(1), + ) + self.assertIsNotNone(mul_x.regularizer) + self.assertTrue( + isinstance(mul_x.regularizer, regularizer.L1Decay) + ) + + mul_y = paddle.static.data( + dtype="float32", shape=[10, 8], name="mul.y" + ) + mul_out = paddle.matmul(mul_x, mul_y) + mean_out = paddle.mean(mul_out) + grads = paddle.autograd.ir_backward.grad(mean_out, [mul_x]) + params_grads = [(mul_x, grads[0])] + self.assertEqual(len(params_grads), 1) + count_ops = len(block.ops) + optimizer = paddle.optimizer.Adam() + params_grads = optimizer.append_regularization_ops(params_grads) + self.assertEqual(len(params_grads), 1) + self.assertEqual(len(block.ops), count_ops + 5) + self.assertEqual(block.ops[-1].name(), 'pd_op.add_n') + self.assertEqual(block.ops[-3].name(), 'pd_op.scale') + self.assertEqual(block.ops[-5].name(), 'pd_op.sign') + def bow_net( data, diff --git a/test/sequence/test_sequence_mask.py b/test/sequence/test_sequence_mask.py index 9cd14490f43c0..57dee2e13bade 100644 --- a/test/sequence/test_sequence_mask.py +++ b/test/sequence/test_sequence_mask.py @@ -71,7 +71,7 @@ def calc_ground_truth_mask(self): return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) class SequenceMaskTest1(SequenceMaskTestBase): @@ -139,7 +139,7 @@ def calc_ground_truth_mask(self): return (index_broadcast < x_broadcast).astype(self.mask_dtype) def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) class SequenceMaskTest1_tensor_attr(SequenceMaskTestBase_tensor_attr): From 04ac74e79c4ee1f42ea9e585e1a681aaed0aace5 Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:01:38 +0800 Subject: [PATCH 071/146] fix (#60347) --- .../pir/dialect/operator/ir/manual_op.cc | 36 ++++++++++--------- .../fluid/pir/dialect/operator/ir/manual_op.h | 4 +-- test/dygraph_to_static/test_for_enumerate.py | 8 ++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index b068db2e70837..2196ab411b3ff 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -1784,8 +1784,6 @@ phi::DataType SliceArrayOp::GetKernelTypeForVar( return expected_kernel_dtype; } -const char *SliceArrayDenseOp::attributes_name[1] = {"starts"}; - OpInfoTuple SliceArrayDenseOp::GetOpInfo() { std::vector inputs = { paddle::dialect::OpInputInfo("input", @@ -1793,11 +1791,16 @@ OpInfoTuple SliceArrayDenseOp::GetOpInfo() { false, false, false, + false), + paddle::dialect::OpInputInfo("starts", + "paddle::dialect::IntArrayAttribute", + false, + false, + true, false)}; - std::vector attributes = { - paddle::dialect::OpAttributeInfo("starts", - "paddle::dialect::IntArrayAttribute", - "std::vector")}; + + std::vector attributes = {}; + std::vector outputs = { paddle::dialect::OpOutputInfo( "out", "paddle::dialect::DenseTensorType", false, false)}; @@ -1806,8 +1809,8 @@ OpInfoTuple SliceArrayDenseOp::GetOpInfo() { {"input", "starts"}, "slice_array_dense", {"input", "starts"}, - {}, - {}, + {"input"}, + {"input"}, {}, {}); return std::make_tuple( @@ -1820,7 +1823,7 @@ void SliceArrayDenseOp::VerifySig() { VLOG(4) << "Verifying inputs:"; { auto input_size = num_operands(); - IR_ENFORCE(input_size == 1u, + IR_ENFORCE(input_size == 2u, "The size %d of inputs must be equal to 1.", input_size); IR_ENFORCE((*this) @@ -1829,14 +1832,13 @@ void SliceArrayDenseOp::VerifySig() { .isa(), "Type validation failed for the 0th input, got %s.", (*this)->operand_source(0).type()); - } - VLOG(4) << "Verifying attributes:"; - { - auto &attributes = this->attributes(); - IR_ENFORCE(attributes.count("starts") > 0, "starts does not exist."); - IR_ENFORCE( - attributes.at("starts").isa(), - "Type of attribute: starts is not paddle::dialect::IntArrayAttribute."); + IR_ENFORCE((*this)->operand_source(1).type().isa() || + (*this) + ->operand_source(1) + .type() + .isa(), + "Type validation failed for the 1st input, got %s.", + (*this)->operand_source(1).type()); } VLOG(4) << "Verifying outputs:"; { diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.h b/paddle/fluid/pir/dialect/operator/ir/manual_op.h index 43b4935b0ffcd..4d00120640951 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.h @@ -329,8 +329,8 @@ class SliceArrayDenseOp public: using Op::Op; static const char *name() { return "pd_op.slice_array_dense"; } - static const char *attributes_name[1]; - static constexpr uint32_t attributes_num = 1; + static constexpr const char **attributes_name = nullptr; + static constexpr uint32_t attributes_num = 0; static OpInfoTuple GetOpInfo(); void VerifySig(); diff --git a/test/dygraph_to_static/test_for_enumerate.py b/test/dygraph_to_static/test_for_enumerate.py index f35ed405b1faa..a540cef2e387b 100644 --- a/test/dygraph_to_static/test_for_enumerate.py +++ b/test/dygraph_to_static/test_for_enumerate.py @@ -19,9 +19,7 @@ import numpy as np from dygraph_to_static_utils import ( Dy2StTestBase, - IrMode, - ToStaticMode, - disable_test_case, + compare_legacy_with_pt, enable_to_static_guard, test_legacy_and_pt_and_pir, ) @@ -497,7 +495,7 @@ class TestForIterVarList(TestForInRangeConfig): def set_test_func(self): self.dygraph_func = for_iter_var_list - @disable_test_case((ToStaticMode.AST, IrMode.PT)) + @compare_legacy_with_pt def test_transformed_result_compare(self): self.set_test_func() self.transformed_result_compare() @@ -507,7 +505,7 @@ class TestForEnumerateVarList(TestForInRangeConfig): def set_test_func(self): self.dygraph_func = for_enumerate_var_list - @disable_test_case((ToStaticMode.AST, IrMode.PT)) + @compare_legacy_with_pt def test_transformed_result_compare(self): self.set_test_func() self.transformed_result_compare() From bc60ffa754f7bd2ca36854abd3046b27096aff3a Mon Sep 17 00:00:00 2001 From: Chen Zhiyang <1792266893@qq.com> Date: Wed, 27 Dec 2023 11:01:58 +0800 Subject: [PATCH 072/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.300?= =?UTF-8?q?=E3=80=81305=E3=80=81306=E3=80=91Migrate=20SigmoidTransform?= =?UTF-8?q?=E3=80=81StackTransform=E3=80=81StickBreakingTransform=20into?= =?UTF-8?q?=20pir=20(#60325)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * SigmoidTransform pir passed * StackTransform and StickBreakingTransform pir passed * move tests to _static.py --- .../test_distribution_transform_static.py | 291 ++++++++++++++++++ 1 file changed, 291 insertions(+) diff --git a/test/distribution/test_distribution_transform_static.py b/test/distribution/test_distribution_transform_static.py index 3d128df5acb84..0306127261ed7 100644 --- a/test/distribution/test_distribution_transform_static.py +++ b/test/distribution/test_distribution_transform_static.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import typing import unittest import numpy as np @@ -1223,5 +1224,295 @@ def test_forward_log_det_jacobian(self): ) +def _np_softplus(x, beta=1.0, threshold=20.0): + if np.any(beta * x > threshold): + return x + return 1.0 / beta * np.log1p(np.exp(beta * x)) + + +class TestSigmoidTransform(unittest.TestCase): + def setUp(self): + self._t = transform.SigmoidTransform() + + def test_is_injective(self): + self.assertTrue(self._t._is_injective()) + + def test_domain(self): + self.assertTrue(isinstance(self._t._domain, variable.Real)) + + def test_codomain(self): + self.assertTrue(isinstance(self._t._codomain, variable.Variable)) + + @param.param_func( + ((np.ones((5, 10)), 1 / (1 + np.exp(-np.ones((5, 10))))),) + ) + @test_with_pir_api + def test_forward(self, input, expected): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.SigmoidTransform() + out = model.forward(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + np.testing.assert_allclose( + result, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + ((np.ones(10), np.log(np.ones(10)) - np.log1p(-np.ones(10))),) + ) + @test_with_pir_api + def test_inverse(self, input, expected): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.SigmoidTransform() + out = model.inverse(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + np.testing.assert_allclose( + result, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func( + ( + ( + np.ones(10), + -_np_softplus(-np.ones(10)) - _np_softplus(np.ones(10)), + ), + ) + ) + @test_with_pir_api + def test_forward_log_det_jacobian(self, input, expected): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.SigmoidTransform() + out = model.forward_log_det_jacobian(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + np.testing.assert_allclose( + result, + expected, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) + def test_forward_shape(self, shape, expected_shape): + self.assertEqual(self._t.forward_shape(shape), expected_shape) + + @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) + def test_inverse_shape(self, shape, expected_shape): + self.assertEqual(self._t.forward_shape(shape), expected_shape) + + @param.param_func([(np.array(1.0), np.array(1.0))]) + @test_with_pir_api + def test_zerodim(self, input, expected): + shape = () + if paddle.framework.in_pir_mode(): + shape = [] + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, 'float32') + model = transform.SigmoidTransform() + self.assertEqual(model.forward(x).shape, shape) + self.assertEqual(model.inverse(x).shape, shape) + self.assertEqual(model.forward_log_det_jacobian(x).shape, shape) + self.assertEqual(model.inverse_log_det_jacobian(x).shape, shape) + self.assertEqual(model.forward_shape(x.shape), shape) + self.assertEqual(model.inverse_shape(x.shape), shape) + + +class TestStickBreakingTransform(unittest.TestCase): + def setUp(self): + self._t = transform.StickBreakingTransform() + + def test_is_injective(self): + self.assertTrue(self._t._is_injective()) + + def test_domain(self): + self.assertTrue(isinstance(self._t._domain, variable.Independent)) + + def test_codomain(self): + self.assertTrue(isinstance(self._t._codomain, variable.Variable)) + + @param.param_func(((np.random.random(10),),)) + @test_with_pir_api + def test_forward(self, input): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.StickBreakingTransform() + fwd = model.forward(x) + out = model.inverse(fwd) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + np.testing.assert_allclose( + result, + input, + rtol=config.RTOL.get(str(input.dtype)), + atol=config.ATOL.get(str(input.dtype)), + ) + + @param.param_func([((2, 3, 5), (2, 3, 6))]) + def test_forward_shape(self, shape, expected_shape): + self.assertEqual(self._t.forward_shape(shape), expected_shape) + + @param.param_func([((2, 3, 5), (2, 3, 4))]) + def test_inverse_shape(self, shape, expected_shape): + self.assertEqual(self._t.inverse_shape(shape), expected_shape) + + @param.param_func(((np.random.random(10),),)) + @test_with_pir_api + def test_forward_log_det_jacobian(self, input): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.StickBreakingTransform() + out = model.forward_log_det_jacobian(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + self.assertEqual(result.shape, ()) + + +@param.place(config.DEVICES) +@param.param_cls( + (param.TEST_CASE_NAME, 'transforms', 'axis'), + [ + ('simple_one_transform', [transform.ExpTransform()], 0), + ], +) +class TestStackTransform(unittest.TestCase): + def setUp(self): + self._t = transform.StackTransform(self.transforms, self.axis) + + def test_is_injective(self): + self.assertTrue(self._t._is_injective()) + + def test_domain(self): + self.assertTrue(isinstance(self._t._domain, variable.Stack)) + + def test_codomain(self): + self.assertTrue(isinstance(self._t._codomain, variable.Stack)) + + @param.param_func( + [ + (np.array([[0.0, 1.0, 2.0, 3.0]]),), + (np.array([[-5.0, 6.0, 7.0, 8.0]]),), + ] + ) + @test_with_pir_api + def test_forward(self, input): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.StackTransform(self.transforms, self.axis) + out = model.forward(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + self.assertEqual(tuple(result.shape), input.shape) + + @param.param_func( + [ + (np.array([[1.0, 2.0, 3.0]]),), + ( + np.array( + [[6.0, 7.0, 8.0]], + ), + ), + ] + ) + @test_with_pir_api + def test_inverse(self, input): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.StackTransform(self.transforms, self.axis) + out = model.inverse(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + self.assertEqual(tuple(result.shape), input.shape) + + @param.param_func( + [(np.array([[1.0, 2.0, 3.0]]),), (np.array([[6.0, 7.0, 8.0]]),)] + ) + @test_with_pir_api + def test_forward_log_det_jacobian(self, input): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('X', input.shape, input.dtype) + model = transform.StackTransform(self.transforms, self.axis) + out = model.forward_log_det_jacobian(x) + place = ( + paddle.CUDAPlace(0) + if paddle.core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + exe = paddle.static.Executor(place) + (result,) = exe.run(feed={'X': input}, fetch_list=[out]) + self.assertEqual(tuple(result.shape), input.shape) + + @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) + def test_forward_shape(self, shape, expected_shape): + self.assertEqual(self._t.forward_shape(shape), expected_shape) + + @param.param_func([((), ()), ((2, 3, 5), (2, 3, 5))]) + def test_inverse_shape(self, shape, expected_shape): + self.assertEqual(self._t.forward_shape(shape), expected_shape) + + def test_axis(self): + self.assertEqual(self._t.axis, self.axis) + + @param.param_func( + [ + (0, 0, TypeError), + ([0], 0, TypeError), + ([paddle.distribution.ExpTransform()], 'axis', TypeError), + ] + ) + @test_with_pir_api + def test_init_exception(self, transforms, axis, exc): + with self.assertRaises(exc): + paddle.distribution.StackTransform(transforms, axis) + + @test_with_pir_api + def test_transforms(self): + self.assertIsInstance((self._t.transforms), typing.Sequence) + + if __name__ == '__main__': unittest.main() From 2515e7c5c1dc460843d907279e7070168612ea5d Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:05:01 +0800 Subject: [PATCH 073/146] =?UTF-8?q?=E3=80=90pir=E3=80=91=20add=20array=5Fw?= =?UTF-8?q?rite=20and=20array=5Fread=20grad=20case=20(#60360)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimize backward * [PIR] add vjp interface for while op * [PIR] fix ci error. * modify while stopgradient * merge * modify while grad bug * modify while grad op * modify * increment vp * [PIR] add get_used_external_value interface for block. * while case * delete print * delete print * Update python/paddle/autograd/ir_backward.py * [PIR] add unit_test for get_used_external_value * modify while_loop * code_style * modofy ci bug * modify while api * modify ci * modify array * Update python/paddle/autograd/ir_backward.py * Update test/legacy_test/test_cond.py * update * modify array_write grad info --------- Co-authored-by: winter-wang <1030748926@qq.com> --- .../pir/dialect/operator/ir/manual_op.cc | 6 ++- .../pir/dialect/operator/ir/manual_op_vjp.cc | 9 +++- .../fluid/pir/dialect/operator/utils/utils.cc | 3 ++ paddle/fluid/pybind/pir.cc | 3 ++ python/paddle/tensor/array.py | 6 +-- test/legacy_test/test_array_read_write_op.py | 51 +++++++++++++++++++ test/legacy_test/test_while_loop_op.py | 1 + 7 files changed, 71 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc index 2196ab411b3ff..d3d8c46111bbb 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op.cc @@ -1260,7 +1260,7 @@ OpInfoTuple ArrayReadOp::GetOpInfo() { false, false, false, - false), + true), OpInputInfo( "i", "paddle::dialect::ScalarAttribute", false, false, true, false)}; @@ -1370,6 +1370,7 @@ void ArrayReadOp::Build(pir::Builder &builder, dense_out.lod()); argument_outputs.push_back(out_type); argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); + ::pir::PassStopGradientsDefaultly(argument); } void ArrayReadOp::VerifySig() { @@ -1428,7 +1429,7 @@ OpInfoTuple ArrayWrite_Op::GetOpInfo() { false, false), OpInputInfo( - "x", "paddle::dialect::DenseTensorType", false, false, false, false), + "x", "paddle::dialect::DenseTensorType", false, false, false, true), OpInputInfo( "i", "paddle::dialect::ScalarAttribute", false, false, true, false)}; @@ -1493,6 +1494,7 @@ void ArrayWrite_Op::Build(pir::Builder &builder, dense_out.layout()); argument_outputs.push_back(out_type); argument.AddOutputs(argument_outputs.begin(), argument_outputs.end()); + ::pir::PassStopGradientsDefaultly(argument); } void ArrayWrite_Op::VerifySig() { diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc index b59a16ea5ff6e..2ce536aa3d1d7 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc @@ -256,8 +256,13 @@ std::vector> ArrayReadOp::Vjp( outputs.size())); VLOG(6) << "Vjp prepare call Array_read's vjp inteface"; - pir::OpResult tensor_res = paddle::dialect::array_write_( - inputs_[0][0], out_grads[0][0], inputs_[1][0]); + + paddle::dialect::DenseTensorType outgrad_type = + out_grads[0][0].type().dyn_cast(); + pir::Value new_array = paddle::dialect::create_array( + paddle::dialect::TransToPhiDataType(outgrad_type.dtype())); + pir::OpResult tensor_res = + paddle::dialect::array_write_(new_array, out_grads[0][0], inputs_[1][0]); std::vector> res{{tensor_res}}; if (stop_gradients[0][0]) { diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 783ecbd567554..6782b2f8bfd7c 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -273,6 +273,9 @@ std::string GetValueDataType(const pir::Value& value) { } else if (value.type().isa()) { return phi::DataTypeToString(dialect::TransToPhiDataType( value.type().dyn_cast().dtype())); + } else if (value.type().isa()) { + return phi::DataTypeToString(dialect::TransToPhiDataType( + value.type().dyn_cast().dtype())); } else { PADDLE_THROW( phi::errors::InvalidType("Currently, we can only get dtype for " diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 65e3f69bc05b6..330f5650caf1a 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -562,6 +562,9 @@ phi::DataType GetValueDtype(Value value) { } else if (value.type().isa()) { return paddle::dialect::TransToPhiDataType( value.type().dyn_cast().dtype()); + } else if (value.type().isa()) { + return paddle::dialect::TransToPhiDataType( + value.type().dyn_cast().dtype()); } else { PADDLE_THROW(phi::errors::InvalidArgument( "Currently, we can only get phi::DataType from DenseTensorType and " diff --git a/python/paddle/tensor/array.py b/python/paddle/tensor/array.py index 0259200118e7c..b618c7a0f85c6 100644 --- a/python/paddle/tensor/array.py +++ b/python/paddle/tensor/array.py @@ -220,9 +220,7 @@ def array_write(x, i, array=None): elif in_pir_mode(): check_variable_and_dtype(i, 'i', ['int64'], 'array_write') if not isinstance(x, paddle.pir.Value): - raise TypeError( - f"x should be pir.OpResult, but recevied {type(x)}." - ) + raise TypeError(f"x should be pir.Value, but recevied {type(x)}.") if array is not None: if ( not isinstance(array, paddle.pir.Value) @@ -304,7 +302,7 @@ def create_array(dtype, initialized_list=None): for val in array: if not isinstance(val, (Variable, paddle.pir.Value)): raise TypeError( - "All values in `initialized_list` should be Variable or pir.OpResult, but recevied {}.".format( + "All values in `initialized_list` should be Variable or pir.Value, but recevied {}.".format( type(val) ) ) diff --git a/test/legacy_test/test_array_read_write_op.py b/test/legacy_test/test_array_read_write_op.py index 4a6d1aa90f03e..5125ec16cf70d 100644 --- a/test/legacy_test/test_array_read_write_op.py +++ b/test/legacy_test/test_array_read_write_op.py @@ -22,6 +22,7 @@ from paddle.base.backward import append_backward from paddle.base.executor import Executor from paddle.base.framework import default_main_program +from paddle.pir_utils import test_with_pir_api def _test_read_write(x): @@ -191,6 +192,56 @@ def test_array(self): fetched_out1, np.ones([1, 3], dtype="float32") * 6 ) + @test_with_pir_api + def test_array_backward(self): + np.random.seed(2013) + main_program = paddle.static.Program() + startup_program = paddle.static.Program() + with paddle.static.program_guard(main_program, startup_program): + d0 = paddle.static.data(name='d0', shape=[10], dtype='float32') + d0.stop_gradient = False + d0.persistable = True + i = paddle.zeros(shape=[1], dtype='int64') + mem_array = paddle.tensor.array_write(x=d0, i=i) + mem_array.stop_gradient = False + mem_array.persistable = True + out = paddle.tensor.array_read(array=mem_array, i=i) + mean = paddle.mean(out) + grad_list = append_backward(mean) + + place = ( + base.CUDAPlace(0) + if core.is_compiled_with_cuda() + else base.CPUPlace() + ) + d = np.random.random(size=[10]).astype('float32') + exe = base.Executor(place) + + if paddle.framework.in_pir_mode(): + for p, g in grad_list: + if p.is_same(d0): + dd0 = g + if p.is_same(mem_array): + dmem_array = g + res = exe.run( + main_program, + feed={'d0': d}, + fetch_list=[mean, dd0], # dmem_array + ) + # pir not support fetch tensorarray + else: + res = exe.run( + main_program, + feed={'d0': d}, + fetch_list=[mean.name, d0.grad_name, mem_array.grad_name], + ) + np.testing.assert_allclose(res[2], [[0.1] * 10], rtol=1e-05) + + mean = 0.6097253 + x_grad = [0.1] * 10 + np.testing.assert_allclose(res[0], mean, rtol=1e-05) + np.testing.assert_allclose(res[1], x_grad, rtol=1e-05) + if __name__ == '__main__': unittest.main() diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index 534d5fa42e7e3..ca874defb6b0d 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -418,6 +418,7 @@ def internal_body(j, x, mem_array): mem_array = paddle.tensor.array_write(x=init, i=i) data_array = paddle.tensor.array_write(x=d0, i=i) mem_array.stop_gradient = False + mem_array.persistable = True i = paddle.increment(i) paddle.tensor.array_write(d1, i, array=data_array) i = paddle.increment(i) From aec353c92695404e832d417fb5191fb186b45b7f Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Wed, 27 Dec 2023 11:05:41 +0800 Subject: [PATCH 074/146] [XPU] update XHPC to 20231226 (#60377) - opt rms_norm_grad bf16 n 4096 - xblas fc_fusion remove cublasLtDestroy calls --- cmake/external/xpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index f871ae810a6c8..64e9154f9f8e3 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE) set(XPU_BASE_DATE "20231203") endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) - set(XPU_XHPC_BASE_DATE "20231225") + set(XPU_XHPC_BASE_DATE "20231226") endif() set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) From 2dfa0f7983d7e63e92915e1e2a18c2d91e473ce5 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:05:55 +0800 Subject: [PATCH 075/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.289?= =?UTF-8?q?=E3=80=91Migrate=20pca=5Flowrank=20to=20pir=20(#60320)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/tensor/random.py | 2 +- test/legacy_test/test_pca_lowrank.py | 102 +++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index 149541e0d94bd..945cc8ba00fb7 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -429,7 +429,7 @@ def gaussian(shape, mean=0.0, std=1.0, seed=0, dtype=None, name=None): op_type_for_check, supported_dtypes, dtype ) ) - if not isinstance(dtype, core.VarDesc.VarType): + if not isinstance(dtype, (core.VarDesc.VarType, core.DataType)): dtype = convert_np_dtype_to_dtype_(dtype) if in_dynamic_or_pir_mode(): diff --git a/test/legacy_test/test_pca_lowrank.py b/test/legacy_test/test_pca_lowrank.py index 68f0005b36823..07d6c652b3232 100644 --- a/test/legacy_test/test_pca_lowrank.py +++ b/test/legacy_test/test_pca_lowrank.py @@ -133,5 +133,107 @@ def test_niter_range(): self.assertRaises(ValueError, test_niter_range) +class TestStaticPcaLowrankAPI(unittest.TestCase): + def transpose(self, x): + shape = x.shape + perm = list(range(0, len(shape))) + perm = perm[:-2] + [perm[-1]] + [perm[-2]] + return paddle.transpose(x, perm) + + def random_matrix(self, rows, columns, *batch_dims, **kwargs): + dtype = kwargs.get('dtype', 'float64') + + x = paddle.randn(batch_dims + (rows, columns), dtype=dtype) + u, _, vh = paddle.linalg.svd(x, full_matrices=False) + k = min(rows, columns) + s = paddle.linspace(1 / (k + 1), 1, k, dtype=dtype) + return (u * s.unsqueeze(-2)) @ vh + + def random_lowrank_matrix(self, rank, rows, columns, *batch_dims, **kwargs): + B = self.random_matrix(rows, rank, *batch_dims, **kwargs) + C = self.random_matrix(rank, columns, *batch_dims, **kwargs) + return B.matmul(C) + + def run_subtest( + self, guess_rank, actual_rank, matrix_size, batches, pca, **options + ): + main = paddle.static.Program() + startup = paddle.static.Program() + with paddle.static.program_guard(main, startup): + if isinstance(matrix_size, int): + rows = columns = matrix_size + else: + rows, columns = matrix_size + a_input = self.random_lowrank_matrix( + actual_rank, rows, columns, *batches + ) + a = a_input + + u, s, v = pca(a_input, q=guess_rank, **options) + + self.assertEqual(s.shape[-1], guess_rank) + self.assertEqual(u.shape[-2], rows) + self.assertEqual(u.shape[-1], guess_rank) + self.assertEqual(v.shape[-1], guess_rank) + self.assertEqual(v.shape[-2], columns) + + A1 = u.matmul(paddle.diag_embed(s)).matmul(self.transpose(v)) + ones_m1 = paddle.ones(batches + (rows, 1), dtype=a.dtype) + c = a.sum(axis=-2) / rows + c = c.reshape(batches + (1, columns)) + A2 = a - ones_m1.matmul(c) + detect_rank = (s.abs() > 1e-5).sum(axis=-1) + left1 = actual_rank * paddle.ones(batches, dtype=paddle.int64) + S = paddle.linalg.svd(A2, full_matrices=False)[1] + left2 = s[..., :actual_rank] + right = S[..., :actual_rank] + + exe = paddle.static.Executor() + exe.run(startup) + A1, A2, left1, detect_rank, left2, right = exe.run( + main, + feed={}, + fetch_list=[A1, A2, left1, detect_rank, left2, right], + ) + + np.testing.assert_allclose(A1, A2, atol=1e-5) + if not left1.shape: + np.testing.assert_allclose(int(left1), int(detect_rank)) + else: + np.testing.assert_allclose(left1, detect_rank) + np.testing.assert_allclose(left2, right) + + def test_forward(self): + with paddle.pir_utils.IrGuard(): + pca_lowrank = paddle.linalg.pca_lowrank + all_batches = [(), (1,), (3,), (2, 3)] + for actual_rank, size in [ + (2, (17, 4)), + (2, (100, 4)), + (6, (100, 40)), + ]: + for batches in all_batches: + for guess_rank in [ + actual_rank, + actual_rank + 2, + actual_rank + 6, + ]: + if guess_rank <= min(*size): + self.run_subtest( + guess_rank, + actual_rank, + size, + batches, + pca_lowrank, + ) + self.run_subtest( + guess_rank, + actual_rank, + size[::-1], + batches, + pca_lowrank, + ) + + if __name__ == "__main__": unittest.main() From 5cbf32f3ec9d7e932d14470f968794942b2cab75 Mon Sep 17 00:00:00 2001 From: Zero Rains Date: Wed, 27 Dec 2023 11:54:21 +0800 Subject: [PATCH 076/146] =?UTF-8?q?=E3=80=90Hackathon=205th=20No.103?= =?UTF-8?q?=E3=80=91=20fix=20the=20bug=20in=20moving=20=20fc=5Fmkldnn=20to?= =?UTF-8?q?=20phi=20-part=20(#59531)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * try to fix the bug in fc_mkldnn * fix the missing attr bug * fix the parameters bug * remove the paramars in pir * roback and add attr * add the scale_out --------- Co-authored-by: zeroRains --- paddle/fluid/framework/operator.h | 2 +- paddle/fluid/operators/compat/fc.pbtxt | 2 +- .../fc_elementwise_layernorm_fuse_pass.cc | 14 --- .../pir/transforms/fusion/fc_fuse_pass.cc | 49 ++------ .../fusion/fc_with_special_op_fuse_pass.cc | 105 +++++------------- paddle/phi/api/yaml/fused_ops.yaml | 2 +- paddle/phi/api/yaml/op_compat.yaml | 6 +- paddle/phi/infermeta/fusion.cc | 29 +---- paddle/phi/infermeta/fusion.h | 7 -- paddle/phi/kernels/fusion/onednn/fc_kernel.cc | 60 ++++++++-- paddle/phi/kernels/impl/fc_kernel_impl.h | 7 -- 11 files changed, 93 insertions(+), 190 deletions(-) diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index 7f47ef640c19c..31c6bb4ef8806 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -103,7 +103,7 @@ constexpr char kEnableCacheRuntimeContext[] = "@ENABLE_CACHE_RUNTIME_CONTEXT@"; /// TODO(luotao): Note that this temporal attribute would be deleted after all /// ops contain it. constexpr char kAllKernelsMustComputeRuntimeShape[] = - "@ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE@"; + "ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE"; // define some kernel priority /* Define multiple kernel type fallback order*/ diff --git a/paddle/fluid/operators/compat/fc.pbtxt b/paddle/fluid/operators/compat/fc.pbtxt index b7b9fe7acda73..babd80260d771 100644 --- a/paddle/fluid/operators/compat/fc.pbtxt +++ b/paddle/fluid/operators/compat/fc.pbtxt @@ -27,7 +27,7 @@ extra { type: BOOLEAN } attrs { - name: "@ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE@" + name: "ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE" type: BOOLEAN } attrs { diff --git a/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc index c3bef294a8db9..fdb4621fb350b 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc @@ -31,14 +31,7 @@ class FcElementwiseLayerNormFusePattern { {"in_num_col_dims", pat.Attr("in_num_col_dims")}, {"activation_type", pat.Attr("activation_type")}, - {"use_mkldnn", pat.Attr("use_mkldnn")}, {"padding_weights", pat.Attr("padding_weights")}, - {"use_quantizer", pat.Attr("use_quantizer")}, - {"mkldnn_data_type", pat.Attr("mkldnn_data_type")}, - {"scale_in", pat.Attr("scale_in")}, - {"scale_weights", pat.Attr("scale_weights")}, - {"scale_out", pat.Attr("scale_out")}, - {"force_fp32_output", pat.Attr("force_fp32_output")}, }); const auto &add = pat.Op(paddle::dialect::AddOp::name()); const auto &layernorm = @@ -104,14 +97,7 @@ class FcElementwiseLayerNormFuse2Pattern { {"in_num_col_dims", pat.Attr("in_num_col_dims")}, {"activation_type", pat.Attr("activation_type")}, - {"use_mkldnn", pat.Attr("use_mkldnn")}, {"padding_weights", pat.Attr("padding_weights")}, - {"use_quantizer", pat.Attr("use_quantizer")}, - {"mkldnn_data_type", pat.Attr("mkldnn_data_type")}, - {"scale_in", pat.Attr("scale_in")}, - {"scale_weights", pat.Attr("scale_weights")}, - {"scale_out", pat.Attr("scale_out")}, - {"force_fp32_output", pat.Attr("force_fp32_output")}, }); const auto &add = pat.Op(paddle::dialect::AddOp::name()); const auto &layernorm = diff --git a/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc index 269ffd8633da8..2a320b75d6cc3 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc @@ -65,32 +65,15 @@ class MatmulAddPattern : public pir::drr::DrrPatternBase { const auto &false_attr = res.Attr( [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); - const auto &fc = res.Op( - paddle::dialect::FcOp::name(), - {{ - {"in_num_col_dims", in_num_col_dims_attr}, - {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return ""; })}, - {"use_mkldnn", false_attr}, - {"padding_weights", false_attr}, - {"use_quantizer", false_attr}, - {"mkldnn_data_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return "float32"; })}, - {"scale_in", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"scale_weights", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::vector { return {1.0f}; })}, - {"scale_out", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"force_fp32_output", false_attr}, - }}); + const auto &fc = + res.Op(paddle::dialect::FcOp::name(), + {{ + {"in_num_col_dims", in_num_col_dims_attr}, + {"activation_type", + res.Attr([](const pir::drr::MatchContext &match_ctx) + -> std::string { return ""; })}, + {"padding_weights", false_attr}, + }}); fc({&res.Tensor("x"), &res.Tensor("w"), &res.Tensor("y")}, {&res.Tensor("add_out")}); } @@ -105,14 +88,7 @@ class FcWithReluPattern : public pir::drr::DrrPatternBase { {{ {"in_num_col_dims", pat.Attr("in_num_col_dims")}, {"activation_type", pat.Attr("activation_type")}, - {"use_mkldnn", pat.Attr("use_mkldnn")}, {"padding_weights", pat.Attr("padding_weights")}, - {"use_quantizer", pat.Attr("use_quantizer")}, - {"mkldnn_data_type", pat.Attr("mkldnn_data_type")}, - {"scale_in", pat.Attr("scale_in")}, - {"scale_weights", pat.Attr("scale_weights")}, - {"scale_out", pat.Attr("scale_out")}, - {"force_fp32_output", pat.Attr("force_fp32_output")}, }}); fc({&pat.Tensor("x"), &pat.Tensor("w"), &pat.Tensor("y")}, {&pat.Tensor("fc_out")}); @@ -133,14 +109,7 @@ class FcWithReluPattern : public pir::drr::DrrPatternBase { {"activation_type", res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::string { return "relu"; })}, - {"use_mkldnn", pat.Attr("use_mkldnn")}, {"padding_weights", pat.Attr("padding_weights")}, - {"use_quantizer", pat.Attr("use_quantizer")}, - {"mkldnn_data_type", pat.Attr("mkldnn_data_type")}, - {"scale_in", pat.Attr("scale_in")}, - {"scale_weights", pat.Attr("scale_weights")}, - {"scale_out", pat.Attr("scale_out")}, - {"force_fp32_output", pat.Attr("force_fp32_output")}, }}); fc_with_relu({&res.Tensor("x"), &res.Tensor("w"), &res.Tensor("y")}, {&res.Tensor("relu_out")}); diff --git a/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc index 59994c5e5d924..6bb2b3a6d512d 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc @@ -94,32 +94,15 @@ class SqueezeFcFusePattern const auto &false_attr = res.Attr( [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); - const auto &fc = res.Op( - paddle::dialect::FcOp::name(), - {{ - {"in_num_col_dims", in_num_col_dims_attr}, - {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return ""; })}, - {"use_mkldnn", false_attr}, - {"padding_weights", false_attr}, - {"use_quantizer", false_attr}, - {"mkldnn_data_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return "float32"; })}, - {"scale_in", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"scale_weights", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::vector { return {1.0f}; })}, - {"scale_out", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"force_fp32_output", false_attr}, - }}); + const auto &fc = + res.Op(paddle::dialect::FcOp::name(), + {{ + {"in_num_col_dims", in_num_col_dims_attr}, + {"activation_type", + res.Attr([](const pir::drr::MatchContext &match_ctx) + -> std::string { return ""; })}, + {"padding_weights", false_attr}, + }}); fc({&res.Tensor("x"), &res.Tensor("w"), &res.Tensor("bias")}, {&res.Tensor("add_out")}); } @@ -248,32 +231,15 @@ class ReshapeFcFusePattern const auto &false_attr = res.Attr( [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); - const auto &fc = res.Op( - paddle::dialect::FcOp::name(), - {{ - {"in_num_col_dims", in_num_col_dims_attr}, - {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return ""; })}, - {"use_mkldnn", false_attr}, - {"padding_weights", false_attr}, - {"use_quantizer", false_attr}, - {"mkldnn_data_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return "float32"; })}, - {"scale_in", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"scale_weights", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::vector { return {1.0f}; })}, - {"scale_out", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"force_fp32_output", false_attr}, - }}); + const auto &fc = + res.Op(paddle::dialect::FcOp::name(), + {{ + {"in_num_col_dims", in_num_col_dims_attr}, + {"activation_type", + res.Attr([](const pir::drr::MatchContext &match_ctx) + -> std::string { return ""; })}, + {"padding_weights", false_attr}, + }}); fc({&res.Tensor("x"), &res.Tensor("w"), &res.Tensor("bias")}, {&res.Tensor("add_out")}); } @@ -336,32 +302,15 @@ class FlattenFcFusePattern const auto &false_attr = res.Attr( [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); - const auto &fc = res.Op( - paddle::dialect::FcOp::name(), - {{ - {"in_num_col_dims", in_num_col_dims_attr}, - {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return ""; })}, - {"use_mkldnn", false_attr}, - {"padding_weights", false_attr}, - {"use_quantizer", false_attr}, - {"mkldnn_data_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::string { return "float32"; })}, - {"scale_in", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"scale_weights", - res.Attr([](const pir::drr::MatchContext &match_ctx) - -> std::vector { return {1.0f}; })}, - {"scale_out", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 1.0f; - })}, - {"force_fp32_output", false_attr}, - }}); + const auto &fc = + res.Op(paddle::dialect::FcOp::name(), + {{ + {"in_num_col_dims", in_num_col_dims_attr}, + {"activation_type", + res.Attr([](const pir::drr::MatchContext &match_ctx) + -> std::string { return ""; })}, + {"padding_weights", false_attr}, + }}); fc({&res.Tensor("x"), &res.Tensor("w"), &res.Tensor("bias")}, {&res.Tensor("add_out")}); } diff --git a/paddle/phi/api/yaml/fused_ops.yaml b/paddle/phi/api/yaml/fused_ops.yaml index 366e3564aff3e..a31dee6a4c27d 100644 --- a/paddle/phi/api/yaml/fused_ops.yaml +++ b/paddle/phi/api/yaml/fused_ops.yaml @@ -122,7 +122,7 @@ data_type : x - op : fc - args : (Tensor input, Tensor w, Tensor bias, int in_num_col_dims = 1, str activation_type = "", bool use_mkldnn = false, bool padding_weights = false, bool use_quantizer = false, str mkldnn_data_type = "float32", float scale_in = 1.0f, float[] scale_weights = {1.0f}, float scale_out = 1.0f, bool force_fp32_output = false) + args : (Tensor input, Tensor w, Tensor bias, int in_num_col_dims = 1, str activation_type = "", bool padding_weights = false) output : Tensor(out) infer_meta : func : FCInferMeta diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index f0e87043d965d..556a713fdac30 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -1050,12 +1050,8 @@ bias : Bias outputs : out : Out - attrs : - scale_in : Scale_in - scale_weights : Scale_weights - scale_out : Scale_out extra : - [bool @ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE@ = true] + attrs : [bool ALL_KERNELS_MUST_COMPUTE_RUNTIME_SHAPE = true, bool use_mkldnn = false, bool use_quantizer = false, str mkldnn_data_type = "float32", float Scale_in = 1.0f, "float[] Scale_weights = {1.0f}", float Scale_out = 1.0f, bool force_fp32_output = false] - op : feed outputs: {out: Out} diff --git a/paddle/phi/infermeta/fusion.cc b/paddle/phi/infermeta/fusion.cc index 7847a5bbb7805..f38ffe0f1fc9d 100644 --- a/paddle/phi/infermeta/fusion.cc +++ b/paddle/phi/infermeta/fusion.cc @@ -3425,14 +3425,7 @@ void FCInferMeta(const MetaTensor& input, const MetaTensor& bias, const int in_num_col_dims, const std::string& activation_type, - const bool use_mkldnn, const bool padding_weights, - const bool use_quantizer, - const std::string& mkldnn_data_type, - const float scale_in, - const std::vector& sclae_weights, - const float scale_out, - const bool force_fp32_output, MetaTensor* out) { PADDLE_ENFORCE_GE( in_num_col_dims, @@ -3441,15 +3434,7 @@ void FCInferMeta(const MetaTensor& input, "The in_num_col_dims is expected to equal or greater than 1. " "But received the in_num_col_dims is %d. ", in_num_col_dims)); - std::string mkldnn_data_type_list[] = {"float32", "int8", "bfloat16"}; - PADDLE_ENFORCE_EQ( - std::find(std::begin(mkldnn_data_type_list), - std::end(mkldnn_data_type_list), - mkldnn_data_type) != std::end(mkldnn_data_type_list), - true, - phi::errors::InvalidArgument("The mkldnn_data_type shoule be [float32, " - "int8, bfloat16], but found %s.", - mkldnn_data_type.c_str())); + auto w_dims = w.dims(); PADDLE_ENFORCE_EQ( w_dims.size(), @@ -3522,18 +3507,6 @@ void FCInferMeta(const MetaTensor& input, activation_type.c_str())); } - if (use_mkldnn) { - PADDLE_ENFORCE_EQ( - in_dims.size() >= 2 && in_dims.size() <= 4, - true, - phi::errors::Unimplemented( - "The Input of fc is expected to be a 2-D, 3-D or 4-D tensor when " - "use_mkldnn is set. But received the number of Input's " - "dimensions is %d, Input's shape is %s.", - in_dims.size(), - in_dims)); - } - std::vector output_dims; phi::funcs::FCOutputSize( in_dims, w_dims, output_dims, in_num_col_dims, padding_weights); diff --git a/paddle/phi/infermeta/fusion.h b/paddle/phi/infermeta/fusion.h index 002cc96eab4fe..ade4e38d457a6 100644 --- a/paddle/phi/infermeta/fusion.h +++ b/paddle/phi/infermeta/fusion.h @@ -807,14 +807,7 @@ void FCInferMeta(const MetaTensor& input, const MetaTensor& bias, const int in_num_col_dims, const std::string& activation_type, - const bool use_mkldnn, const bool padding_weights, - const bool use_quantizer, - const std::string& mkldnn_data_type, - const float scale_in, - const std::vector& sclae_weights, - const float scale_out, - const bool force_fp32_output, MetaTensor* out); void VariableLengthMemoryEfficientAttentionInferMeta( diff --git a/paddle/phi/kernels/fusion/onednn/fc_kernel.cc b/paddle/phi/kernels/fusion/onednn/fc_kernel.cc index 6eed95b9b1c9a..0d39677276ead 100644 --- a/paddle/phi/kernels/fusion/onednn/fc_kernel.cc +++ b/paddle/phi/kernels/fusion/onednn/fc_kernel.cc @@ -567,17 +567,61 @@ void FCKernel(const Context& dev_ctx, const paddle::optional& bias, const int in_num_col_dims, const std::string& activation_type, - const bool use_mkldnn, const bool padding_weights, - const bool use_quantizer, - const std::string& mkldnn_data_type, - const float scale_in, - const std::vector& scale_weights, - const float scale_out, - const bool force_fp32_output, DenseTensor* out) { + const bool use_mkldnn = + dev_ctx.HasDnnAttr("use_mkldnn") + ? PADDLE_GET_CONST(bool, dev_ctx.GetDnnAttr("use_mkldnn")) + : false; + const bool use_quantizer = + dev_ctx.HasDnnAttr("use_quantizer") + ? PADDLE_GET_CONST(bool, dev_ctx.GetDnnAttr("use_quantizer")) + : false; + const std::string mkldnn_data_type = + dev_ctx.HasDnnAttr("mkldnn_data_type") + ? PADDLE_GET_CONST(std::string, + dev_ctx.GetDnnAttr("mkldnn_data_type")) + : "float32"; + const float scale_in = + dev_ctx.HasDnnAttr("Scale_in") + ? PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("Scale_in")) + : 1.0f; + std::vector tmp_scale_weights = {1.0f}; + const std::vector scale_weights = + dev_ctx.HasDnnAttr("Scale_weights") + ? PADDLE_GET_CONST(std::vector, + dev_ctx.GetDnnAttr("Scale_weights")) + : tmp_scale_weights; + const float scale_out = + dev_ctx.HasDnnAttr("Scale_out") + ? PADDLE_GET_CONST(float, dev_ctx.GetDnnAttr("Scale_out")) + : 1.0f; + const bool force_fp32_output = + dev_ctx.HasDnnAttr("force_fp32_output") + ? PADDLE_GET_CONST(bool, dev_ctx.GetDnnAttr("force_fp32_output")) + : false; + std::string mkldnn_data_type_list[] = {"float32", "int8", "bfloat16"}; + PADDLE_ENFORCE_EQ( + std::find(std::begin(mkldnn_data_type_list), + std::end(mkldnn_data_type_list), + mkldnn_data_type) != std::end(mkldnn_data_type_list), + true, + phi::errors::InvalidArgument("The mkldnn_data_type shoule be [float32, " + "int8, bfloat16], but found %s.", + mkldnn_data_type.c_str())); + auto in_dims = input.dims(); + if (use_mkldnn) { + PADDLE_ENFORCE_EQ( + in_dims.size() >= 2 && in_dims.size() <= 4, + true, + phi::errors::Unimplemented( + "The Input of fc is expected to be a 2-D, 3-D or 4-D tensor when " + "use_mkldnn is set. But received the number of Input's " + "dimensions is %d, Input's shape is %s.", + in_dims.size(), + in_dims)); + } bool fuse_relu = activation_type == "relu"; - IF_CHANGE_FC_TW_TYPENAME((std::is_same::value), ([&] { if (force_fp32_output) { // NOLINT RunKernel(dev_ctx, diff --git a/paddle/phi/kernels/impl/fc_kernel_impl.h b/paddle/phi/kernels/impl/fc_kernel_impl.h index c30da9d4e5000..3709a15880b4c 100644 --- a/paddle/phi/kernels/impl/fc_kernel_impl.h +++ b/paddle/phi/kernels/impl/fc_kernel_impl.h @@ -30,14 +30,7 @@ void FCKernel(const Context& dev_ctx, const paddle::optional& bias, const int in_num_col_dims, const std::string& activation_type, - const bool use_mkldnn, const bool padding_weights, - const bool use_quantizer, - const std::string& mkldnn_data_type, - const float scale_in, - const std::vector& scale_weights, - const float scale_out, - const bool force_fp32_output, DenseTensor* out) { bool with_relu = (activation_type == "relu") ? true : false; From bf4b4b73eec4d93a13b7f05a229515811b5179f6 Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Wed, 27 Dec 2023 12:08:22 +0800 Subject: [PATCH 077/146] [DimExpr] DimExpr support print (#60146) * DimExpr support print * ToTxtString * Fix ASSERT_EQ bug * Fix typo * Fix unittest * ToTxtString->ToString * Fix windows CI * ShapeOrData cannot use IR_API --- paddle/pir/dialect/shape/utils/dim_expr.cc | 60 +++++++++++++++++++ paddle/pir/dialect/shape/utils/dim_expr.h | 5 ++ .../pir/shape_dialect/symbol_dim_expr_test.cc | 13 ++++ 3 files changed, 78 insertions(+) diff --git a/paddle/pir/dialect/shape/utils/dim_expr.cc b/paddle/pir/dialect/shape/utils/dim_expr.cc index 9c46a8841c1e1..0d9b6ece23245 100644 --- a/paddle/pir/dialect/shape/utils/dim_expr.cc +++ b/paddle/pir/dialect/shape/utils/dim_expr.cc @@ -124,4 +124,64 @@ bool DimExpr::operator!=(const DimExpr& other) const { return !(*this == other); } +namespace { + +std::string ToStringImpl(std::int64_t dim_expr) { + return std::to_string(dim_expr); +} + +std::string ToStringImpl(const std::string& dim_expr) { return dim_expr; } + +std::string ToStringImpl(const Negative& dim_expr) { + return "-" + ToString(dim_expr->data); +} + +std::string ToStringImpl(const Reciprocal& dim_expr) { + return "1 / (" + ToString(dim_expr->data) + ")"; +} + +std::string ListDimExprToString(const List& dim_exprs, + const std::string& delim = ", ") { + std::string ret; + for (std::size_t i = 0; i < dim_exprs->size(); ++i) { + if (i > 0) { + ret += delim; + } + ret += ToString(dim_exprs->at(i)); + } + return ret; +} + +std::string ToStringImpl(const Add& dim_expr) { + return "Add(" + ListDimExprToString(dim_expr.operands, ", ") + ")"; +} + +std::string ToStringImpl(const Mul& dim_expr) { + return "Mul(" + ListDimExprToString(dim_expr.operands, ", ") + ")"; +} + +std::string ToStringImpl(const Max& dim_expr) { + return "Max(" + ListDimExprToString(dim_expr.operands, ", ") + ")"; +} + +std::string ToStringImpl(const Min& dim_expr) { + return "Min(" + ListDimExprToString(dim_expr.operands, ", ") + ")"; +} + +std::string ToStringImpl(const Broadcast& dim_expr) { + return "Broadcast(" + ListDimExprToString(dim_expr.operands, ", ") + ")"; +} + +} // namespace + +std::string ToString(const DimExpr& dim_expr) { + return std::visit([](const auto& impl) { return ToStringImpl(impl); }, + dim_expr.variant()); +} + +std::ostream& operator<<(std::ostream& stream, const DimExpr& dim_expr) { + stream << ToString(dim_expr); + return stream; +} + } // namespace symbol diff --git a/paddle/pir/dialect/shape/utils/dim_expr.h b/paddle/pir/dialect/shape/utils/dim_expr.h index 50f992ba75880..277a6febe66ed 100644 --- a/paddle/pir/dialect/shape/utils/dim_expr.h +++ b/paddle/pir/dialect/shape/utils/dim_expr.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -248,4 +249,8 @@ class ShapeOrData { using ShapeOrDataDimExprs = ShapeOrData; +IR_API std::string ToString(const DimExpr& dim_expr); + +IR_API std::ostream& operator<<(std::ostream&, const DimExpr& dim_expr); + } // namespace symbol diff --git a/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc b/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc index ef5fe03069e4a..6157850e3842c 100644 --- a/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc +++ b/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc @@ -111,4 +111,17 @@ TEST(DimExpr, equal) { builder.Broadcast(DimExpr("S0"), constant1)); } +TEST(DimExpr, print) { + DimExprBuilder builder{nullptr}; + DimExpr sym0 = DimExpr("S0"); + DimExpr sym1 = DimExpr("S1"); + ASSERT_EQ((ToString(sym0 + sym1)), "Add(S0, S1)"); + ASSERT_EQ((ToString(sym0 - sym1)), "Add(S0, -S1)"); + ASSERT_EQ((ToString(sym0 * sym1)), "Mul(S0, S1)"); + ASSERT_EQ((ToString(sym0 / sym1)), "Mul(S0, 1 / (S1))"); + ASSERT_EQ((ToString(builder.Max(sym0, sym1))), "Max(S0, S1)"); + ASSERT_EQ((ToString(builder.Min(sym0, sym1))), "Min(S0, S1)"); + ASSERT_EQ((ToString(builder.Broadcast(sym0, sym1))), "Broadcast(S0, S1)"); +} + } // namespace symbol::test From a681d98f5eb110b1a14776c6154784898d76ccde Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Wed, 27 Dec 2023 12:24:03 +0800 Subject: [PATCH 078/146] [DimExpr] DimExpr to ir::Expr (#60344) * DimExprConverter * Add Unittest --- paddle/cinn/common/CMakeLists.txt | 5 +- paddle/cinn/common/dim_expr_converter.cc | 101 ++++++++++++++++++ paddle/cinn/common/dim_expr_converter.h | 26 +++++ paddle/cinn/common/dim_expr_converter_test.cc | 79 ++++++++++++++ 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 paddle/cinn/common/dim_expr_converter.cc create mode 100644 paddle/cinn/common/dim_expr_converter.h create mode 100644 paddle/cinn/common/dim_expr_converter_test.cc diff --git a/paddle/cinn/common/CMakeLists.txt b/paddle/cinn/common/CMakeLists.txt index f528813b47ac6..b71055169945c 100644 --- a/paddle/cinn/common/CMakeLists.txt +++ b/paddle/cinn/common/CMakeLists.txt @@ -22,7 +22,8 @@ gather_srcs( python_interpreter_guard.cc nvgpu_dev_info.cc integer_set.cc - dim_expr_simplify.cc) + dim_expr_simplify.cc + dim_expr_converter.cc) cinn_cc_test(test_equation_graph_topo_walker SRCS equation_graph_topo_walker_test.cc DEPS gtest glog) @@ -49,4 +50,6 @@ endif() if(NOT CINN_ONLY) cinn_cc_test(dim_expr_simplify_test SRCS dim_expr_simplify_test.cc DEPS cinncore) + cinn_cc_test(dim_expr_converter_test SRCS dim_expr_converter_test.cc DEPS + cinncore) endif() diff --git a/paddle/cinn/common/dim_expr_converter.cc b/paddle/cinn/common/dim_expr_converter.cc new file mode 100644 index 0000000000000..e17b961689b29 --- /dev/null +++ b/paddle/cinn/common/dim_expr_converter.cc @@ -0,0 +1,101 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/common/dim_expr_converter.h" +#include "paddle/cinn/common/ir_util.h" + +namespace cinn::common { +using namespace symbol; // NOLINT + +namespace { + +struct DimExprToIrExprVisitor { + ir::Expr ConvertToIrExpr(const DimExpr& dim_expr) { + return std::visit(*this, dim_expr.variant()); + } + + ir::Expr operator()(const int64_t& dim) { return ir::Expr(dim); } + + ir::Expr operator()(const std::string& dim_expr) { + Var x = ir::_Var_::Make(dim_expr, Int(64)); + return x; + } + + ir::Expr operator()(const Negative& dim_expr) { + const auto& [operand] = *dim_expr; + return ir::Sub::Make(ir::Expr(std::int64_t(0)), ConvertToIrExpr(operand)); + } + + ir::Expr operator()(const Reciprocal& dim_expr) { + const auto& [operand] = *dim_expr; + return ir::Div::Make(ir::Expr(std::int64_t(1)), ConvertToIrExpr(operand)); + } + + ir::Expr operator()(const Add& dim_expr) { + const auto& [operands] = dim_expr; + if (operands->empty()) { + return ir::Expr(std::int64_t(0)); + } + ir::Expr sum = ConvertToIrExpr(operands->at(0)); + for (std::size_t i = 1; i < operands->size(); ++i) { + sum = ir::Add::Make(sum, ConvertToIrExpr(operands->at(i))); + } + return sum; + } + + ir::Expr operator()(const Mul& dim_expr) { + const auto& [operands] = dim_expr; + if (operands->empty()) { + return ir::Expr(std::int64_t(1)); + } + ir::Expr product = ConvertToIrExpr(operands->at(0)); + for (std::size_t i = 1; i < operands->size(); ++i) { + product = ir::Mul::Make(product, ConvertToIrExpr(operands->at(i))); + } + return product; + } + + ir::Expr operator()(const Max& dim_expr) { + const auto& [operands] = dim_expr; + CHECK(!operands->empty()); + ir::Expr max = ConvertToIrExpr(operands->at(0)); + for (std::size_t i = 1; i < operands->size(); ++i) { + max = ir::Max::Make(max, ConvertToIrExpr(operands->at(i))); + } + return max; + } + + ir::Expr operator()(const Min& dim_expr) { + const auto& [operands] = dim_expr; + CHECK(!operands->empty()); + ir::Expr min = ConvertToIrExpr(operands->at(0)); + for (std::size_t i = 1; i < operands->size(); ++i) { + min = ir::Min::Make(min, ConvertToIrExpr(operands->at(i))); + } + return min; + } + + ir::Expr operator()(const Broadcast& dim_expr) { + LOG(FATAL) + << "no support for converting from Broadcast to ir::Expr"; + } +}; + +} // namespace + +ir::Expr DimExprConverter::ConvertToIrExpr(const DimExpr& dim_expr) const { + return DimExprToIrExprVisitor().ConvertToIrExpr(dim_expr); +} + +} // namespace cinn::common diff --git a/paddle/cinn/common/dim_expr_converter.h b/paddle/cinn/common/dim_expr_converter.h new file mode 100644 index 0000000000000..cbe782e05ab07 --- /dev/null +++ b/paddle/cinn/common/dim_expr_converter.h @@ -0,0 +1,26 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/cinn/ir/ir.h" +#include "paddle/pir/dialect/shape/utils/dim_expr.h" + +namespace cinn::common { + +struct DimExprConverter final { + ir::Expr ConvertToIrExpr(const symbol::DimExpr&) const; +}; + +} // namespace cinn::common diff --git a/paddle/cinn/common/dim_expr_converter_test.cc b/paddle/cinn/common/dim_expr_converter_test.cc new file mode 100644 index 0000000000000..a2313e7297798 --- /dev/null +++ b/paddle/cinn/common/dim_expr_converter_test.cc @@ -0,0 +1,79 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "gtest/gtest.h" + +#include "paddle/cinn/common/dim_expr_converter.h" +#include "paddle/cinn/common/ir_util.h" +#include "paddle/cinn/ir/ir_printer.h" + +namespace cinn::common::test { + +using namespace symbol; // NOLINT + +TEST(Convert, AddExpr) { + List num_lists{DimExpr(4), DimExpr(5), DimExpr("sym_0")}; + DimExpr dim_expr{Add{num_lists}}; + ir::Expr src_expr = DimExprConverter().ConvertToIrExpr(dim_expr); + + ir::Expr expr1 = + ir::Add::Make(ir::Expr(std::int64_t(4)), ir::Expr(std::int64_t(5))); + ir::Expr dst_expr = ir::Add::Make(expr1, ir::_Var_::Make("sym_0", Int(64))); + ASSERT_TRUE(MathEqual(src_expr, dst_expr)); +} + +TEST(Convert, SubExpr) { + DimExpr dim_expr = DimExpr(4) - DimExpr("sym_0"); + ir::Expr src_expr = DimExprConverter().ConvertToIrExpr(dim_expr); + + ir::Expr expr1 = ir::Sub::Make(ir::Expr(std::int64_t(0)), + ir::_Var_::Make("sym_0", Int(64))); + ir::Expr dst_expr = ir::Add::Make(ir::Expr(std::int64_t(4)), expr1); + ASSERT_TRUE(MathEqual(src_expr, dst_expr)); +} + +TEST(Convert, MulExpr) { + List num_lists{DimExpr(4), DimExpr(5), DimExpr("sym_0")}; + DimExpr dim_expr{Mul{num_lists}}; + ir::Expr src_expr = DimExprConverter().ConvertToIrExpr(dim_expr); + + ir::Expr expr1 = + ir::Mul::Make(ir::Expr(std::int64_t(4)), ir::Expr(std::int64_t(5))); + ir::Expr dst_expr = ir::Mul::Make(expr1, ir::_Var_::Make("sym_0", Int(64))); + ASSERT_TRUE(MathEqual(src_expr, dst_expr)); +} + +TEST(Convert, MaxExpr) { + List num_lists{DimExpr(4), DimExpr(5), DimExpr("sym_0")}; + DimExpr dim_expr{Max{num_lists}}; + ir::Expr src_expr = DimExprConverter().ConvertToIrExpr(dim_expr); + + std::ostringstream stream; + stream << src_expr; + ASSERT_EQ(stream.str(), "cinn_max(cinn_max(4ll, 5ll), sym_0)"); +} + +TEST(Convert, MinExpr) { + List num_lists{DimExpr(4), DimExpr(5), DimExpr("sym_0")}; + DimExpr dim_expr{Min{num_lists}}; + ir::Expr src_expr = DimExprConverter().ConvertToIrExpr(dim_expr); + + std::ostringstream stream; + stream << src_expr; + ASSERT_EQ(stream.str(), "cinn_min(cinn_min(4ll, 5ll), sym_0)"); +} + +} // namespace cinn::common::test From 22b49df19a3afec176b3a877c47fa56139bf2f23 Mon Sep 17 00:00:00 2001 From: freeliuzc Date: Wed, 27 Dec 2023 13:53:08 +0800 Subject: [PATCH 079/146] [inference] Support groupwise mode of gemv kernel (#60204) * support gemv-groupwise func && weightQuanter-groupwise && weightDeQuanter-groupwise * fix build bug * add unit_test && fix bug * delete useless code * fix ci build bug * fix ci && optimize * fix merge conflict * add op change info * fix weight_only_linear_pass * fix format * solve ci unit_test --- .../fusion/fused_weight_only_linear_pass.cc | 11 +- paddle/phi/api/yaml/backward.yaml | 4 +- paddle/phi/api/yaml/op_version.yaml | 24 + paddle/phi/api/yaml/ops.yaml | 6 +- paddle/phi/infermeta/backward.cc | 7 + paddle/phi/infermeta/backward.h | 1 + paddle/phi/infermeta/binary.cc | 51 +- paddle/phi/infermeta/binary.h | 1 + paddle/phi/infermeta/multiary.cc | 36 +- paddle/phi/infermeta/multiary.h | 1 + paddle/phi/infermeta/unary.cc | 17 +- paddle/phi/infermeta/unary.h | 1 + .../phi/kernels/cpu/weight_quantize_kernel.cc | 29 +- .../kernels/funcs/weight_dequant_functor.h | 142 ++- paddle/phi/kernels/funcs/weight_only_gemv.cu | 949 ++++++++++++------ paddle/phi/kernels/funcs/weight_only_gemv.h | 23 +- .../kernels/gpu/weight_dequantize_kernel.cu | 3 +- .../gpu/weight_only_linear_grad_kernel.cu | 16 +- .../kernels/gpu/weight_only_linear_kernel.cu | 49 +- .../phi/kernels/gpu/weight_quantize_kernel.cu | 7 + .../impl/weight_quantize_kernel_impl.h | 70 ++ paddle/phi/kernels/weight_dequantize_kernel.h | 1 + .../kernels/weight_only_linear_grad_kernel.h | 1 + .../phi/kernels/weight_only_linear_kernel.h | 1 + paddle/phi/kernels/weight_quantize_kernel.h | 1 + python/paddle/nn/quant/quantized_linear.py | 47 +- test/quantization/test_weight_only_linear.py | 263 ++++- 27 files changed, 1331 insertions(+), 431 deletions(-) diff --git a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc index 57485355ad22d..fa83418e562ba 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc @@ -96,9 +96,14 @@ class FusedWeightOnlyLinearPattern return getSMVersion(); }); + const auto &group_size_attr = res.Attr( + [](const pir::drr::MatchContext &match_ctx) -> int { return -1; }); + const auto &weight_quantize = res.Op(paddle::dialect::WeightQuantizeOp::name(), - {{"algo", weight_only_int8_attr}, {"arch", arch_attr}}); + {{"algo", weight_only_int8_attr}, + {"arch", arch_attr}, + {"group_size", group_size_attr}}); weight_quantize({&res.Tensor("w")}, {&res.Tensor("quanted_weight_tensor"), &res.Tensor("weight_scale_tensor")}); @@ -110,7 +115,9 @@ class FusedWeightOnlyLinearPattern const auto &weight_only_linear = res.Op(paddle::dialect::WeightOnlyLinearOp::name(), - {{"weight_dtype", weight_dtype_attr}, {"arch", arch_attr}}); + {{"weight_dtype", weight_dtype_attr}, + {"arch", arch_attr}, + {"group_size", group_size_attr}}); weight_only_linear({&res.Tensor("x"), &res.Tensor("quanted_weight_tensor"), &res.Tensor("bias"), diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index 21ec2126c8f94..938ea9d500046 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -2603,8 +2603,8 @@ no_need_buffer : input - backward_op : weight_only_linear_grad - forward : weight_only_linear(Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, str weight_dtype, int arch) -> Tensor(out) - args : (Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, Tensor out_grad, str weight_dtype, int arch) + forward : weight_only_linear(Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, str weight_dtype, int arch, int group_size) -> Tensor(out) + args : (Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, Tensor out_grad, str weight_dtype, int arch, int group_size) output : Tensor(x_grad) infer_meta : func : WeightOnlyLinearGradInferMeta diff --git a/paddle/phi/api/yaml/op_version.yaml b/paddle/phi/api/yaml/op_version.yaml index bd296a6191de3..7c9618f52b17b 100644 --- a/paddle/phi/api/yaml/op_version.yaml +++ b/paddle/phi/api/yaml/op_version.yaml @@ -472,6 +472,30 @@ comment : The axis to apply unique. If None, the input will be flattened. default : std::vector{} +- op : weight_dequantize + version : + - checkpoint : Upgrade weight_dequantize, add a new attribute [group_size] + action : + - add_attr : group_size + comment : The group size of the dequantization scales. + default : -1 + +- op : weight_only_linear + version : + - checkpoint : Upgrade weight_only_linear, add a new attribute [group_size] + action : + - add_attr : group_size + comment : The group size of the dequantization scales. + default : -1 + +- op : weight_quantize + version : + - checkpoint : Upgrade weight_quantize, add a new attribute [group_size] + action : + - add_attr : group_size + comment : The group size of the quantization scales. + default : -1 + - op : yolo_box version : - checkpoint : Upgrade yolo box to add new attribute [iou_aware, iou_aware_factor]. diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index c15fb2fdb1998..de7c49250ea16 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -2833,7 +2833,7 @@ backward : warprnnt_grad - op : weight_dequantize - args : (Tensor x, Tensor scale, str algo="weight_only_int8", DataType out_dtype=DataType::FLOAT16) + args : (Tensor x, Tensor scale, str algo = "weight_only_int8", DataType out_dtype = DataType::FLOAT16, int group_size = -1) output : Tensor(out) infer_meta : func : WeightDequantizeInferMeta @@ -2842,7 +2842,7 @@ data_type : out_dtype - op : weight_only_linear - args : (Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, str weight_dtype, int arch = 80) + args : (Tensor x, Tensor weight, Tensor bias, Tensor weight_scale, str weight_dtype, int arch = 80, int group_size = -1) output : Tensor(out) infer_meta : func : WeightOnlyLinearInferMeta @@ -2853,7 +2853,7 @@ backward: weight_only_linear_grad - op : weight_quantize - args : (Tensor x, str algo = "weight_only_int8", int arch = 80) + args : (Tensor x, str algo = "weight_only_int8", int arch = 80, int group_size = -1) output : Tensor(out), Tensor(scale) infer_meta : func : WeightQuantizeInferMeta diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 6d6eab8097337..ee2388762668b 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -1191,12 +1191,19 @@ void WeightOnlyLinearGradInferMeta(const MetaTensor& x, const MetaTensor& out_grad, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, MetaTensor* x_grad) { PADDLE_ENFORCE_EQ( ((arch == 80) || (arch == 86)), true, phi::errors::InvalidArgument( "Currently weightonly linear grad only support arch = 80 or 86. ")); + PADDLE_ENFORCE_EQ( + group_size, + -1, + phi::errors::InvalidArgument( + "Currently weightonly linear grad only support per-channel mode. ")); + x_grad->set_dims(x.dims()); x_grad->set_dtype(x.dtype()); } diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h index 86878c5feb082..922bafed0add8 100644 --- a/paddle/phi/infermeta/backward.h +++ b/paddle/phi/infermeta/backward.h @@ -469,6 +469,7 @@ void WeightOnlyLinearGradInferMeta(const MetaTensor& x, const MetaTensor& out_grad, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, MetaTensor* x_grad); void YoloLossGradInferMeta(const MetaTensor& x, diff --git a/paddle/phi/infermeta/binary.cc b/paddle/phi/infermeta/binary.cc index 8b85a3efc4dd8..b771fba031317 100644 --- a/paddle/phi/infermeta/binary.cc +++ b/paddle/phi/infermeta/binary.cc @@ -3381,6 +3381,7 @@ void WeightDequantizeInferMeta(const MetaTensor& x, const MetaTensor& scale, const std::string& algo, DataType out_dtype, + const int32_t group_size, MetaTensor* out) { PADDLE_ENFORCE_EQ(x.dims().size(), 2UL, @@ -3388,18 +3389,44 @@ void WeightDequantizeInferMeta(const MetaTensor& x, "The x tensor of dequantize op must be 2D, but got[%d]", x.dims().size())); PADDLE_ENFORCE_EQ( - scale.dims().size(), - 1UL, - phi::errors::InvalidArgument( - "The scale tensor of dequantize op must be 1D, but got[%d]", - scale.dims().size())); - PADDLE_ENFORCE_EQ(scale.dims()[0], - x.dims()[0], - phi::errors::InvalidArgument( - "The scale tensor's shape must be equal to the x " - "tensor's shape, but got [%d] not equal to [%d]", - scale.dims()[0], - x.dims()[0])); + (group_size == -1 || group_size == 64 || group_size == 128), + true, + phi::errors::InvalidArgument("group_size must be -1, 64 or 128.")); + + auto dim_scale = scale.dims(); + + // per-channel dequantization + if (group_size == -1) { + PADDLE_ENFORCE_EQ( + dim_scale.size(), + 1UL, + phi::errors::InvalidArgument("The scale tensor of dequantize op must " + "be 1D in per-channel mode, but got[%d]", + scale.dims().size())); + PADDLE_ENFORCE_EQ(dim_scale[0], + x.dims()[0], + phi::errors::InvalidArgument( + "The scale tensor's shape must be equal to the x " + "tensor's shape, but got [%d] not equal to [%d]", + scale.dims()[0], + x.dims()[0])); + } else /* groupwise dequantization */ { + PADDLE_ENFORCE_EQ( + dim_scale.size(), + 2UL, + phi::errors::InvalidArgument("The scale tensor of dequantize op must " + "be 2D in group-wise mode, but got[%d]", + scale.dims().size())); + PADDLE_ENFORCE_EQ( + dim_scale[0], + (x.dims()[1] + (group_size - 1)) / group_size, + errors::InvalidArgument("The input(weight_scale) dim[0] must be equal " + "to (Input(weight).dim[1] + (group_size -1))" + " / group_size" + "But receive %d and %d", + dim_scale[0], + (x.dims()[1] + (group_size - 1)) / group_size)); + } int n = x.dims()[1]; int k = x.dims()[0]; out->set_dims(common::make_ddim({n, k})); diff --git a/paddle/phi/infermeta/binary.h b/paddle/phi/infermeta/binary.h index c081c1690c28d..82f5fc64d57a5 100644 --- a/paddle/phi/infermeta/binary.h +++ b/paddle/phi/infermeta/binary.h @@ -539,6 +539,7 @@ void WeightDequantizeInferMeta(const MetaTensor& x, const MetaTensor& scale, const std::string& algo, DataType out_dtype, + const int32_t group_size, MetaTensor* out); } // namespace phi diff --git a/paddle/phi/infermeta/multiary.cc b/paddle/phi/infermeta/multiary.cc index 0b2ef29389137..6250b3a3b23c8 100644 --- a/paddle/phi/infermeta/multiary.cc +++ b/paddle/phi/infermeta/multiary.cc @@ -3943,10 +3943,16 @@ void WeightOnlyLinearInferMeta(const MetaTensor& x, const MetaTensor& weight_scale, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, MetaTensor* out) { + PADDLE_ENFORCE((group_size == -1 || group_size == 64 || group_size == 128), + errors::InvalidArgument("group_size must be -1, 64 or 128.")); + + auto weight_scale_dims = weight_scale.dims(); + auto x_dims = x.dims(); auto w_dims = weight.dims(); - auto n = weight_scale.dims()[0]; + auto n = group_size == -1 ? weight_scale_dims[0] : weight_scale_dims[1]; PADDLE_ENFORCE( weight_dtype == "int8" || weight_dtype == "int4", errors::InvalidArgument("quant_method must be 'int8' or 'int4'.")); @@ -3954,10 +3960,6 @@ void WeightOnlyLinearInferMeta(const MetaTensor& x, w_dims.size(), 2UL, errors::InvalidArgument("The input(weight) must be a 2D Tensor.")); - PADDLE_ENFORCE_EQ( - weight_scale.dims().size(), - 1UL, - errors::InvalidArgument("The input(weight_scale) must be a 1D Tensor.")); PADDLE_ENFORCE_EQ( w_dims[0] % 16, 0, @@ -3978,6 +3980,30 @@ void WeightOnlyLinearInferMeta(const MetaTensor& x, "But received Input(X) dim[-1](%s) != Input(Weight) dim[1](%s)", x_dims[x_dims.size() - 1], w_dims[1])); + + // per-channel dequantization + if (group_size == -1) { + PADDLE_ENFORCE_EQ( + weight_scale_dims.size(), + 1UL, + errors::InvalidArgument("The input(weight_scale) must be a 1D Tensor." + "in per-channel mode.")); + } else /* groupwise dequantization */ { + PADDLE_ENFORCE_EQ( + weight_scale_dims.size(), + 2UL, + errors::InvalidArgument("The input(weight_scale) must be a 2D Tensor" + " in groupwise mode.")); + PADDLE_ENFORCE_EQ( + weight_scale_dims[0], + (w_dims[1] + (group_size - 1)) / group_size, + errors::InvalidArgument("The input(weight_scale) dim[0] must be equal " + "to Input(weight) dim[1] / group_size" + "But receive %d and %d", + weight_scale_dims[0], + (w_dims[1] + (group_size - 1)) / group_size)); + } + auto out_dims = x_dims; out_dims[out_dims.size() - 1] = n; out->set_dims(out_dims); diff --git a/paddle/phi/infermeta/multiary.h b/paddle/phi/infermeta/multiary.h index be3f1fba94a80..f51c3dacb1909 100644 --- a/paddle/phi/infermeta/multiary.h +++ b/paddle/phi/infermeta/multiary.h @@ -720,6 +720,7 @@ void WeightOnlyLinearInferMeta(const MetaTensor& x, const MetaTensor& weight_scale, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, MetaTensor* out); void WeightedSampleNeighborsInferMeta(const MetaTensor& row, diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 16da7fbc02128..af60d6ae8da5c 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -5203,6 +5203,7 @@ void UnStackInferMeta(const MetaTensor& x, void WeightQuantizeInferMeta(const MetaTensor& x, const std::string& algo, const int32_t arch, + const int32_t group_size, MetaTensor* out, MetaTensor* scale) { PADDLE_ENFORCE_EQ( @@ -5229,7 +5230,21 @@ void WeightQuantizeInferMeta(const MetaTensor& x, phi::errors::InvalidArgument( "The second dimension of input must be divisible by 16, but got[%d]", x_dims[1])); - std::vector dim_scale({x_dims[1]}); + PADDLE_ENFORCE_EQ( + ((group_size == -1) || (group_size == 64) || (group_size == 128)), + true, + phi::errors::InvalidArgument( + "Currently, group_size only support -1, 64 or 128.")); + + std::vector dim_scale; + if (group_size != -1) { + int64_t scale_dim0 = (x_dims[0] + (group_size - 1)) / group_size; + int64_t scale_dim1 = x_dims[1]; + dim_scale = std::vector({scale_dim0, scale_dim1}); + } else { + dim_scale = std::vector({x_dims[1]}); + } + std::vector dim_out; if (algo == "weight_only_int8" || algo == "llm.int8") { dim_out = std::vector({x_dims[1], x_dims[0]}); diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index f4fca6cd7770d..eae4614a8eb5c 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -477,6 +477,7 @@ void QuantizeXPUInferMeta(const MetaTensor& x, void WeightQuantizeInferMeta(const MetaTensor& x, const std::string& algo, const int32_t arch, + const int32_t group_size, MetaTensor* out, MetaTensor* scale); diff --git a/paddle/phi/kernels/cpu/weight_quantize_kernel.cc b/paddle/phi/kernels/cpu/weight_quantize_kernel.cc index e85b83700b173..313c59e2e6676 100644 --- a/paddle/phi/kernels/cpu/weight_quantize_kernel.cc +++ b/paddle/phi/kernels/cpu/weight_quantize_kernel.cc @@ -28,7 +28,8 @@ void quant_compute(const DeviceContext& dev_ctx, DenseTensor* out, DenseTensor* scale, const std::string& algo, - const int32_t arch) { + const int32_t arch, + const int32_t group_size) { PADDLE_ENFORCE_EQ( ((arch == 80) || (arch == 86) || (arch == 75) || (arch == 70)), true, @@ -51,7 +52,8 @@ void quant_compute(const DeviceContext& dev_ctx, DenseTensor x_int(out->type()); - if ((arch == 80) || (arch == 75) || (arch == 86)) { + if ((arch == 80) || (arch == 75) || (arch == 86) || (arch == 89) || + (arch == 90)) { x_int.Resize({static_cast(m), static_cast(n)}); } else { // phi::Copy may change tensor meta info, here we transpose the quanted @@ -71,9 +73,19 @@ void quant_compute(const DeviceContext& dev_ctx, int_processed_2.Resize(out->dims()); dev_ctx.template Alloc(&int_processed_2); D* int_processed_2_data = int_processed_2.data(); - per_channel_scale(scale_data, x_data, m, n, bits == 8 ? 127.0f : 7.0f); - - per_channel_quant(x_int_data, x_data, scale_data, m, n); + if (group_size == -1) { + per_channel_scale(scale_data, x_data, m, n, bits == 8 ? 127.0f : 7.0f); + per_channel_quant(x_int_data, x_data, scale_data, m, n); + } else { + group_wise_scale(scale_data, + x_data, + m, + n, + bits == 8 ? 127.0f : 7.0f, + static_cast(group_size)); + + group_wise_quant(x_int_data, x_data, scale_data, m, n, group_size); + } if (algo == "llm.int8") { std::vector axis = {1, 0}; funcs::Transpose trans; @@ -105,14 +117,17 @@ void WeightQuantizeKernel(const Context& dev_ctx, const DenseTensor& x, const std::string& algo, const int32_t arch, + const int32_t group_size, DenseTensor* out, DenseTensor* scale) { dev_ctx.template Alloc(out); dev_ctx.template Alloc(scale); if (algo == "weight_only_int8" || algo == "llm.int8") { - quant_compute(dev_ctx, x, out, scale, algo, arch); + quant_compute( + dev_ctx, x, out, scale, algo, arch, group_size); } else if (algo == "weight_only_int4") { - quant_compute(dev_ctx, x, out, scale, algo, arch); + quant_compute( + dev_ctx, x, out, scale, algo, arch, group_size); } else { phi::errors::Unimplemented( "The algo must be in ['weight_only_int8', 'weight_only_int4', " diff --git a/paddle/phi/kernels/funcs/weight_dequant_functor.h b/paddle/phi/kernels/funcs/weight_dequant_functor.h index 1728fa0577ab4..4eed94de7bf4d 100644 --- a/paddle/phi/kernels/funcs/weight_dequant_functor.h +++ b/paddle/phi/kernels/funcs/weight_dequant_functor.h @@ -231,12 +231,133 @@ __global__ void int4_weight_only_dequant(const uint8_t* weight, } } +template +__global__ void int8_weight_only_dequant(const uint8_t* weight, + const T* scales, + T* output, + const int n, + const int k, + const int group_size) { + using Converter = FastWeightOnlyHalfConverter; + AlignedVector vec_weight; + T vec_weight_f16[16]; + AlignedVector vec_out; + + int warp_id = threadIdx.x / 32, lane_id = threadIdx.x % 32; + int tile_id = blockIdx.x * blockDim.x / 32 + warp_id; + // Every two rows of the original weights are interleaved into a row with + // stride of 64, so if each thread processes 16 elements(for int8, we can use + // ldg.128 to load weights), then every group of four adjacent threads will + // alternately process two different row weights for example every 128 + // consecutive int8 elements [128*i, 128*(i+1)-1] of row N under interleave + // layout, the first 64 are from [64*i, 64*(i+1)-1] of row 2N before + // interleaving, and the last 64 are from [64*i, 64*(i+1)-1] of row 2N+1 + // before interleaving. So if each thread loads 16 int8 elements, then the + // elements of the first four and last four threads of each 8 consecutive + // threads will come from row 2N and row 2N+1 respectively before + // interleaving. + int row_id = tile_id * 2 + ((lane_id % 8) > 3 ? 1 : 0); + weight += tile_id * k * 2; + output += row_id * k; + + scales += row_id; +#pragma unroll + for (int i = lane_id * 16; i < k * 2; i += 16 * 32) { + int scale_offset = i / 2 / group_size; + float scale = static_cast(scales[scale_offset * n]); + Load(&weight[i], &vec_weight); +#pragma unroll + for (int p = 0; p < 16; p += Converter::kHalfLength) { + // The rearrangement here counteracts the effect of + // cutlass::add_bias_and_interleave_int8s_inplace Input int8 data layout + // [elt_3 elt_1 elt_2 elt_0] (each elt occupies 8 bits) + // + // Converted fp16 data layout + // [elt_3 elt_2 elt_1 elt_0] (each elt occupies 16 bits) + // vec_weight_f16[p] = static_cast(static_cast(vec_weight[p]) * + // scale); + // fast_cvt_4_packed_signed_i8s_to_2_half2s() + Converter::convert(vec_weight_f16 + p, &vec_weight[p], scale); + } +#pragma unroll + for (int p = 0; p < 16; ++p) { + // The index remapping here is to counteracts the effect of + // cutlass::permute_B_rows_for_mixed_gemm input 0 1 2 3 4 5 6 7 8 9 10 11 + // 12 13 14 15 weight 0 1 8 9 2 3 10 11 4 5 12 13 6 7 14 15 + vec_out[p] = vec_weight_f16[4 * ((p % 8) / 2) + p % 2 + 2 * (p / 8)]; + } + Store(vec_out, &output[i / 128 * 64 + (i % 64)]); + } +} + +template +__global__ void int4_weight_only_dequant(const uint8_t* weight, + const T* scales, + T* output, + const int n, + const int k, + const int group_size) { + using Converter = FastWeightOnlyHalfConverter; + + AlignedVector vec_weight; + T vec_weight_f16[32]; + AlignedVector vec_out; + + int warp_id = threadIdx.x / 32, lane_id = threadIdx.x % 32; + int tile_id = blockIdx.x * blockDim.x / 32 + warp_id; + // Every two rows of the original weights are interleaved into a row with + // stride of 64, so if each thread processes 16 elements(for int8, we can use + // ldg.128 to load weights), then every group of four adjacent threads will + // alternately process two different row weights for example every 128 + // consecutive int8 elements [128*i, 128*(i+1)-1] of row N under interleave + // layout, the first 64 are from [64*i, 64*(i+1)-1] of row 2N before + // interleaving, and the last 64 are from [64*i, 64*(i+1)-1] of row 2N+1 + // before interleaving. So if each thread loads 16 int8 elements, then the + // elements of the first four and last four threads of each 8 consecutive + // threads will come from row 2N and row 2N+1 respectively before + // interleaving. + int row_id = tile_id * 4 + ((lane_id % 8) / 2); + weight += tile_id * k / 2 * 4; + output += row_id * k; + scales += row_id; +#pragma unroll + for (int i = lane_id * 32; i < k * 4; i += 32 * 32) { + Load(&weight[i / 2], &vec_weight); + int scale_offset = i / 4 / group_size; + float scale = static_cast(scales[scale_offset * n]); +#pragma unroll + for (int p = 0; p < 32; p += Converter::kHalfLength) { + // The rearrangement here counteracts the effect of + // cutlass::add_bias_and_interleave_int4s_inplace Input int8 data layout + // [elt_7 elt_5 elt_3 elt_1 elt_6 elt_4 elt_2 elt_0] (each elt + // occupies 4 bits) + // + // Converted fp16 data layout + // [elt_7 elt_6 elt_5 elt_4 elt_3 elt_2 elt_1 elt_0] (each elt + // occupies 16 bits) + // vec_weight_f16[p] = + // static_cast(static_cast(vec_weight[p]) * scale); + Converter::convert(vec_weight_f16 + p, &vec_weight[p / 2], scale); + } +#pragma unroll + for (int p = 0; p < 32; ++p) { + // The index remapping here is to counteracts the effect of + // cutlass::permute_B_rows_for_mixed_gemm input 0 1 2 3 4 5 6 7 8 9 10 11 + // 12 13 14 15 ... 31 weight 0 1 8 9 16 17 24 25 2 3 10 11 18 19 26 27 4 5 + // 12 13 20 21 28 29 6 7 14 15 22 23 30 31 + vec_out[p] = vec_weight_f16[8 * ((p % 8) / 2) + p % 2 + 2 * (p / 8)]; + } + Store(vec_out, &output[i / 256 * 64 + (i % 64)]); + } +} + template void WeightDequantize(const Context& dev_ctx, const DenseTensor& x, const DenseTensor& scale, const std::string& algo, const bool transpose, + const int32_t group_size, DenseTensor* out) { using DataType = typename PDDataTypeTraits::DataType; @@ -246,14 +367,22 @@ void WeightDequantize(const Context& dev_ctx, dim3 grid(n / 32); auto stream = dev_ctx.stream(); - if (algo == "weight_only_int8") { + if (algo == "weight_only_int8" && group_size == -1) { int8_weight_only_dequant<<>>( reinterpret_cast(x.data()), reinterpret_cast(scale.data()), reinterpret_cast(out->data()), n, k); - } else if (algo == "weight_only_int4") { + } else if (algo == "weight_only_int8" && group_size > 0) { + int8_weight_only_dequant<<>>( + reinterpret_cast(x.data()), + reinterpret_cast(scale.data()), + reinterpret_cast(out->data()), + n, + k, + group_size); + } else if (algo == "weight_only_int4" && group_size == -1) { grid.x /= 2; int4_weight_only_dequant<<>>( reinterpret_cast(x.data()), @@ -261,6 +390,15 @@ void WeightDequantize(const Context& dev_ctx, reinterpret_cast(out->data()), n, k); + } else if (algo == "weight_only_int4" && group_size > 0) { + grid.x /= 2; + int4_weight_only_dequant<<>>( + reinterpret_cast(x.data()), + reinterpret_cast(scale.data()), + reinterpret_cast(out->data()), + n, + k, + group_size); } } diff --git a/paddle/phi/kernels/funcs/weight_only_gemv.cu b/paddle/phi/kernels/funcs/weight_only_gemv.cu index aeccf6f2370cd..ff9285693b55f 100644 --- a/paddle/phi/kernels/funcs/weight_only_gemv.cu +++ b/paddle/phi/kernels/funcs/weight_only_gemv.cu @@ -367,6 +367,8 @@ __global__ void int8_weight_only_gemv(const T* input, enum class WeightOnlyQuantType { Int4b, Int8b }; +enum class WeightOnlyType { PerChannel, GroupWise }; + template struct WeightLayoutDetails; @@ -530,8 +532,6 @@ struct WeightOnlyKernelDetails { kElemsPerThread / kActivationElemNumPerAccess; }; -enum class WeightOnlyType { PerChannel, GroupWise }; - struct WeightOnlyPerChannel; template struct WeightOnlyGroupWise; @@ -551,13 +551,12 @@ struct WeightOnlyProperties> { static constexpr int kGroupSize = GS; }; -template + int BlockSize> struct WeightOnlyScaleLoader { - using ElemType = T; using Details = WeightOnlyKernelDetails; static constexpr bool kIsFineGrained = WeightOnlyProperties::kIsFineGrained; @@ -565,25 +564,19 @@ struct WeightOnlyScaleLoader { WeightOnlyProperties::kGroupSize; private: - const ElemType* _scales; - const ElemType* _zeros; + const T* _scales; + const T* _zeros; int _stride; int _offset; public: - __device__ __forceinline__ WeightOnlyScaleLoader(const ElemType* scales, - const ElemType* zeros, + __device__ __forceinline__ WeightOnlyScaleLoader(const T* scales, + const T* zeros, int initial_offset, int stride) : _scales(scales), _zeros(zeros), _stride(stride) { _scales += initial_offset; -#ifndef WIN32 - // linux - if constexpr (Zero) { -#else - // windows if (Zero) { -#endif _zeros += initial_offset; } // Calculate the k dimension index of the element processed by the current @@ -594,10 +587,10 @@ struct WeightOnlyScaleLoader { (threadIdx.x % Details::kThreadsNumPerTile) * Details::kElemsPerThread; } - __device__ __forceinline__ void load(ElemType& scale, // NOLINT - ElemType& zero, // NOLINT - int nid) { + __device__ __forceinline__ void load(T* scale, T* zero, int nid) { int offset = nid * Details::kInterleave; + +// TODO(freeliuzc): cpplint has bug here #ifndef WIN32 if constexpr (kIsFineGrained) { #else @@ -605,15 +598,17 @@ struct WeightOnlyScaleLoader { #endif offset += _offset / kGroupSize * _stride; } - scale = _scales[offset]; + *scale = _scales[offset]; + +// TODO(freeliuzc): cpplint has bug here #ifndef WIN32 if constexpr (Zero) { #else if (Zero) { #endif - zero = _zeros[offset]; + *zero = _zeros[offset]; } else { - zero = static_cast(0.f); + *zero = static_cast(0.f); } } @@ -624,6 +619,272 @@ struct WeightOnlyScaleLoader { __device__ __forceinline__ int offset() { return _offset; } }; // NOLINT +template +struct WeightOnlyConverter {}; + +template <> +struct WeightOnlyConverter { + static __device__ inline void convert(half halves[4], + int8_t signed_chars[4]) { + uint32_t* h = reinterpret_cast(halves); + uint32_t i8s = *reinterpret_cast(signed_chars); + + static constexpr uint32_t mask_for_elt_01 = 0x5150; + static constexpr uint32_t mask_for_elt_23 = 0x5352; + static constexpr uint32_t start_byte_for_fp16 = 0x64646464; + asm volatile("prmt.b32 %0,%1,%2,%3;\n" + : "=r"(h[0]) + : "r"(i8s), "n"(start_byte_for_fp16), "n"(mask_for_elt_01)); + asm volatile("prmt.b32 %0,%1,%2,%3;\n" + : "=r"(h[1]) + : "r"(i8s), "n"(start_byte_for_fp16), "n"(mask_for_elt_23)); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480; + asm volatile("sub.f16x2 %0, %1, %2;\n" + : "=r"(h[0]) + : "r"(h[0]), "r"(I8s_TO_F16s_MAGIC_NUM)); + asm volatile("sub.f16x2 %0, %1, %2;\n" + : "=r"(h[1]) + : "r"(h[1]), "r"(I8s_TO_F16s_MAGIC_NUM)); + } +}; + +template <> +struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int8b> { + static __device__ inline void convert(__nv_bfloat16 halves[4], + int8_t signed_chars[4]) { +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800)) + uint32_t* bf16_result_ptr = reinterpret_cast(halves); + uint32_t i8s = *reinterpret_cast(signed_chars); + + static constexpr uint32_t fp32_base = 0x4B000000; + float fp32_intermediates[4]; + + // Construct FP32s, bfloat does not have enough mantissa for IADD trick + uint32_t* fp32_intermediates_casted = + reinterpret_cast(fp32_intermediates); + fp32_intermediates_casted[0] = __byte_perm(i8s, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(i8s, fp32_base, 0x7651); + fp32_intermediates_casted[2] = __byte_perm(i8s, fp32_base, 0x7652); + fp32_intermediates_casted[3] = __byte_perm(i8s, fp32_base, 0x7653); + + // Subtract out fp32_base + 128 to make the unsigned integer signed. +#pragma unroll + for (int ii = 0; ii < 4; ++ii) { + fp32_intermediates[ii] -= 8388736.f; + } + +// Truncate the fp32 representation and pack up as bfloat16s. +#pragma unroll + for (int ii = 0; ii < 2; ++ii) { + bf16_result_ptr[ii] = __byte_perm(fp32_intermediates_casted[2 * ii + 0], + fp32_intermediates_casted[2 * ii + 1], + 0x7632); + } +#else + // Disable this on architectures older than Ampere since they lack hardware + // for bf16 mma. If one wishes to use HMMA on older hardware, they should + // Convert directly to FP16 using FP16 converters. + assert(false); +#endif + } +}; + +template <> +struct WeightOnlyConverter { + static __device__ inline void convert(half halves[8], + int8_t signed_chars[4]) { + uint32_t* h = reinterpret_cast(halves); + uint32_t i4s = *reinterpret_cast(signed_chars); + + // First, we extract the i4s and construct an intermediate fp16 number. + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + static constexpr uint32_t BOTTOM_MASK = 0x000f000f; + static constexpr uint32_t TOP_MASK = 0x00f000f0; + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400; + + // Note that the entire sequence only requires 1 shift instruction. This is + // thanks to the register packing format and the fact that we force our + // integers to be unsigned, and account for this in the fp16 subtractions. + // In addition, I exploit the fact that sub and fma have the same throughput + // in order to convert elt_23 and elt_67 to fp16 without having to shift + // them to the bottom bits before hand. + + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide + // RAW dependency if we issue immediately before required. + const uint32_t top_i4s = i4s >> 8; + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400 + asm volatile( + "lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[0]) + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400 + asm volatile( + "lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[1]) + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[2]) + : "r"(top_i4s), + "n"(BOTTOM_MASK), + "n"(I4s_TO_F16s_MAGIC_NUM), + "n"(immLut)); + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400 + asm volatile( + "lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[3]) + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + + // I use inline PTX below because I am not sure if the compiler will emit + // float2half instructions if I use the half2 ctor. In this case, I chose + // performance reliability over code readability. + + // This is the half2 {1032, 1032} represented as an integer. + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64086408; + // This is the half2 {1 / 16, 1 / 16} represented as an integer. + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00; + // This is the half2 {-72, -72} represented as an integer. + static constexpr uint32_t NEG_72 = 0xd480d480; + // Finally, we construct the output numbers. + // Convert elt_01 + asm volatile("sub.f16x2 %0, %1, %2;\n" + : "=r"(h[0]) + : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_23 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" + : "=r"(h[1]) + : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_72)); + // Convert elt_45 + asm volatile("sub.f16x2 %0, %1, %2;\n" + : "=r"(h[2]) + : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_67 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" + : "=r"(h[3]) + : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_72)); + } +}; + +template <> +struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int4b> { + static __device__ inline void convert(__nv_bfloat16 halves[8], + int8_t signed_chars[4]) { +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800)) + uint32_t* h = reinterpret_cast(halves); + uint32_t const source_i4s = *reinterpret_cast(signed_chars); + + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t I4s_TO_BF16s_MAGIC_NUM = 0x43004300; + + // We don't have enough mantissa to remove as much shift overhead as FP16, + // so we must loop. No shift needed for first item. + uint32_t i4s = source_i4s; + asm volatile( + "lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[0]) + : "r"(i4s), "n"(MASK), "n"(I4s_TO_BF16s_MAGIC_NUM), "n"(immLut)); +#pragma unroll + for (int ii = 1; ii < 4; ++ii) { + i4s >>= 4; + // (i4s & 0x000f000f) | 0x43004300 + asm volatile( + "lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[ii]) + : "r"(i4s), "n"(MASK), "n"(I4s_TO_BF16s_MAGIC_NUM), "n"(immLut)); + } + + // This is the BF16 {-136, -136} represented as an integer. + static constexpr uint32_t BF16_BIAS = 0xC308C308; + static constexpr uint32_t BF16_ONE = 0x3F803F80; + +// Finally, we construct the output numbers. +#pragma unroll + for (int ii = 0; ii < 4; ++ii) { + // Since this section is for Ampere+, we use bf16 fma to do the bias + // subtraction + asm("fma.rn.bf16x2 %0, %1, %2, %3;\n" + : "=r"(h[ii]) + : "r"(h[ii]), "r"(BF16_ONE), "r"(BF16_BIAS)); + } + +#else + // Disable this on architectures older than Ampere since they lack hardware + // for bf16 mma. If one wishes to use HMMA on older hardware, they should + // Convert directly to FP16 using FP16 converters. + assert(false); +#endif + } +}; + +template +__device__ __forceinline__ void load(T0* dst, T1* src, size_t offset = 0) { + *reinterpret_cast(dst) = + *(reinterpret_cast(src) + offset); +} + +template +struct WeightPostProcessor { + static __device__ __forceinline__ void run(T* weights_vec, + T* weights_f16, + T* scale, + T* zero, + int NPerBlock, + int idx) {} +}; + +template +struct WeightPostProcessor { + static __device__ __forceinline__ void run(T* weights_vec, + T* weights_f16, + T* scale, + T* zero, + int NPerBlock, + int idx) { + using HALF_2_TYPE = typename CUDA_HALF_2_TYPE_TARIS::type; +#pragma unroll + for (int i = 0; i < Details::kShuffleContinous; ++i) { +#pragma unroll + for (int j = 0; j < Details::kShuffleStrided; ++j) { + // Dequantize the weights and arrange the shuffled elements back to + // the correct order in the register array + HALF_2_TYPE v = *reinterpret_cast( + weights_vec + i * Details::kShuffleBasicTile + + j * Details::kShuffleContinous * Details::kShuffleBasicTile); + v = HalfMulAdd::apply( + v, + ConvertDstFunc_2::apply(scale[idx]), + ConvertDstFunc_2::apply(zero[idx])); + weights_f16[(i * Details::kShuffleStrided * Details::kShuffleBasicTile + + j * Details::kShuffleBasicTile + 0) * + NPerBlock + + idx] = v.x; + weights_f16[(i * Details::kShuffleStrided * Details::kShuffleBasicTile + + j * Details::kShuffleBasicTile + 1) * + NPerBlock + + idx] = v.y; + } + } + } +}; + +template +struct WeightPostProcessor { + static __device__ __forceinline__ void run(T* weights_vec, + T* weights_f16, + T* scale, + T* zero, + int NPerBlock, + int idx) { +#pragma unroll + for (int p = 0; p < 16; ++p) { + weights_f16[p * NPerBlock + idx] = + weights_vec[p / 8 + (p % 8) * 2] * scale[idx]; + } + } +}; + template -__global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, +__global__ void weight_only_batched_gemv_multi_warp(const T* in, + const int8_t* qweight, + const T* bias, const T* scales, const T* zeros, - const T* in, - const T* bias, T* out, const int n, const int k) { @@ -650,8 +911,10 @@ __global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, using CvtSrcType = int8_t; using CvtResType = T; using ScaleLoader = - WeightOnlyScaleLoader; - extern __shared__ int8_t shmem[]; + WeightOnlyScaleLoader; + using WeightProcessor = WeightPostProcessor; + + extern __shared__ uint8_t shmem[]; constexpr int Interleave = Details::kInterleave; constexpr int WarpSize = 32; constexpr int Num = Batch * NPerBlock; @@ -673,48 +936,47 @@ __global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, // threads and fp32 for accumulation between threads. T accumulator[Num]; for (int i = 0; i < Num; ++i) { - accumulator[i] = ConvertFloatFunc::apply(0.f); + accumulator[i] = ConvertDstFunc::apply(0.f); } // Iteration in k dimensions for (int local_k = tid * Details::kElemsPerThread; local_k < k * Interleave; local_k += BlockSize * Details::kElemsPerThread) { - T weights_f16[Details::kElemsPerThread * NPerBlock]; // 16 * 2 = 32 + T weights_f16[Details::kElemsPerThread * NPerBlock]; T scale[NPerBlock], zero[NPerBlock]; #pragma unroll for (int idx = 0; idx < NPerBlock; ++idx) { // Load quantized weight and scales/zeros int8_t weights_quantized[Details::kBytePerThread]; - *reinterpret_cast(weights_quantized) = - *reinterpret_cast( - qweight + idx * Interleave * k / Details::kElemsPerByte + - local_k / Details::kElemsPerByte); - scale_loader.load(scale[idx], zero[idx], idx); + load(weights_quantized, + qweight + idx * Interleave * k / Details::kElemsPerByte + + local_k / Details::kElemsPerByte); + scale_loader.load(scale + idx, zero + idx, idx); T weights_vec[Details::kElemsPerThread]; + #pragma unroll for (int i = 0; i < Details::kConvertIters; ++i) { // Use cutlass::FastInterleavedAndBiasedNumericArrayConverter for I2F // type conversion - fast_cvt_4_packed_signed_i8s_to_2_half2s( + WeightOnlyConverter::convert( weights_vec + i * Details::kConvertCount, weights_quantized + i * Details::kConvertCount / Details::kElemsPerByte); } - // TODO(wangbojun) no zero support here -#pragma unroll - for (int p = 0; p < 16; ++p) { - weights_f16[p * NPerBlock + idx] = - weights_vec[p / 8 + (p % 8) * 2] * scale[idx]; - } + // Assign weight and apply scales. + // Currently not support zero. + WeightProcessor::run( + weights_vec, weights_f16, scale, zero, NPerBlock, idx); } #pragma unroll for (int b = 0; b < Batch; ++b) { T in_v[Details::kElemsPerThread]; - // load activation elements - *(float4*)in_v = // NOLINT - *(float4*)(in + b * k + scale_loader.offset()); // NOLINT - *(float4*)(in_v + 8) = // NOLINT - *(float4*)(in + b * k + scale_loader.offset() + 8); // NOLINT +#pragma unroll + for (int idx = 0; idx < Details::kActivationAccessNum; ++idx) { + load(in_v + idx * Details::kActivationElemNumPerAccess, + in + b * k + scale_loader.offset() + + idx * Details::kActivationElemNumPerAccess); + } // Perform vector inner product and accumulate #ifndef WIN32 if constexpr (NPerBlock == 1) { @@ -729,7 +991,7 @@ __global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, *reinterpret_cast(in_v + y), v); } - accumulator[b] = accumulator[b] + static_cast(v.x + v.y); + accumulator[b] = accumulator[b] + ConvertDstFunc::apply(v.x + v.y); } else { #pragma unroll for (int x = 0; x < NPerBlock / 2; ++x) { @@ -752,7 +1014,7 @@ __global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, float reses[Num]; #pragma unroll for (int i = 0; i < Num; ++i) { - reses[i] = static_cast(accumulator[i]); + reses[i] = ConvertFloatFunc::apply(accumulator[i]); } // Each warp completes the internal reduce and writes the [Batch * NPerBlock * @@ -773,343 +1035,384 @@ __global__ void weight_only_batched_gemv_multi_warp(const int8_t* qweight, #else if (Bias) { #endif - bias_v = static_cast(bias[n_start_id + nid]); + bias_v = ConvertFloatFunc::apply(bias[n_start_id + nid]); } int b = i / NPerBlock / Interleave; - out[b * n + n_start_id + nid] = ConvertDstFunc::apply( GeluActivation::apply(v + bias_v)); } } - -#endif - -template -void int8_weight_only_gemv_launcher(const T* input, - const int8_t* weight, - const T* scale_list, - const T* bias, - T* output, - const int k, - const int n, - const bool gelu, - gpuStream_t stream) { -#ifdef PADDLE_WITH_CUDA - dim3 block(kWarpSize * kPerBlockWarpNum); // equal to 512; - dim3 grid(n / kPerBlockWarpNum / - 2); // Note(zhengzekang): Since each warp process 2 rows of matrix. - if (bias) { - if (gelu) { - int8_weight_only_gemv<<>>( - input, weight, scale_list, bias, output, k, n); - } else { - int8_weight_only_gemv<<>>( - input, weight, scale_list, bias, output, k, n); - } - } else { - if (gelu) { - int8_weight_only_gemv<<>>( - input, weight, scale_list, bias, output, k, n); - } else { - int8_weight_only_gemv<<>>( - input, weight, scale_list, bias, output, k, n); - } - } #endif -} - -template <> -void int8_weight_only_gemv_launcher(const float* input, - const int8_t* weight, - const float* scale_list, - const float* bias, - float* output, - const int k, - const int n, - const bool gelu, - gpuStream_t stream) { - // Weightonly GEMV do not support float. - assert(false); -} - -template <> -void int8_weight_only_gemv_launcher(const phi::dtype::bfloat16* input, - const int8_t* weight, - const phi::dtype::bfloat16* scale_list, - const phi::dtype::bfloat16* bias, - phi::dtype::bfloat16* output, - const int k, - const int n, - const bool gelu, - gpuStream_t stream) { - // Environment do not support bf16. - assert(false); -} template -void select_batch_gemv_multi_warp_by_batch(const T* input, - const int8_t* weight, - const T* scale_list, - const T* bias, - T* output, - const int m, - const int k, - const int n, - gpuStream_t stream) { +void select_activation_and_bias(const T* input, + const int8_t* weight, + const T* bias, + const T* scales, + const int m, + const int n, + const int k, + const std::string& act_method, + T* output, + cudaStream_t stream) { #ifdef PADDLE_WITH_CUDA - VLOG(3) << "launch batched gemv multi_block mnk:" << m << " " - << " " << n << " " << k; + static constexpr int kInterleave = WeightLayoutDetails::kInterleave; dim3 grid(n / NPerBlock / kInterleave); dim3 block(BlockSize); - int smem_size = sizeof(float) * BlockSize / 32 * m * NPerBlock * kInterleave; - switch (m) { - case 1: { + int size = sizeof(float) * BlockSize / 32 * Batch * NPerBlock * kInterleave; + if (bias) { + if (act_method == "gelu") { weight_only_batched_gemv_multi_warp - <<>>( - weight, scale_list, /*zeros*/ nullptr, input, bias, output, n, k); - break; - } - case 2: { + <<>>( + input, weight, bias, scales, /*zeros*/ nullptr, output, n, k); + } else if (act_method == "None") { weight_only_batched_gemv_multi_warp - <<>>( - weight, scale_list, /*zeros*/ nullptr, input, bias, output, n, k); - break; - } - case 3: { - weight_only_batched_gemv_multi_warp - <<>>( - weight, scale_list, /*zeros*/ nullptr, input, bias, output, n, k); - break; + <<>>( + input, weight, bias, scales, /*zeros*/ nullptr, output, n, k); + } else { + PADDLE_THROW( + errors::InvalidArgument("Currently, weightonly GEMV act_method " + "only support `gelu`, `None`. ")); } - case 4: { + } else { + if (act_method == "gelu") { weight_only_batched_gemv_multi_warp - <<>>( - weight, scale_list, /*zeros*/ nullptr, input, bias, output, n, k); - break; - } - case 5: { + <<>>( + input, weight, bias, scales, /*zeros*/ nullptr, output, n, k); + } else if (act_method == "None") { weight_only_batched_gemv_multi_warp - <<>>( - weight, scale_list, /*zeros*/ nullptr, input, bias, output, n, k); - break; - } - default: { - throw std::runtime_error("Use unsupported batch for gemv"); - break; + <<>>( + input, weight, bias, scales, /*zeros*/ nullptr, output, n, k); + } else { + PADDLE_THROW( + errors::InvalidArgument("Currently, weightonly GEMV act_method " + "only support `gelu`, `None`. ")); } } #endif } -template -void batched_int8_weight_only_gemv_multi_warp_launcher(const T* input, - const int8_t* weight, - const T* scale_list, - const T* bias, - T* output, - const int m, - const int k, - const int n, - const bool gelu, - gpuStream_t stream) { +template +void weight_only_batched_gemv_launcher( + const T* input, + const int8_t* weight, + const T* bias, + const T* scales, + int m, + int n, + int k, + const std::string& weight_only_quant_type, + const std::string& act_method, + T* output, + cudaStream_t stream) { #ifdef PADDLE_WITH_CUDA - if (bias) { - if (gelu) { - select_batch_gemv_multi_warp_by_batch( - input, weight, scale_list, bias, output, m, k, n, stream); - } else { - select_batch_gemv_multi_warp_by_batch( - input, weight, scale_list, bias, output, m, k, n, stream); + if (weight_only_quant_type == "int4") { + switch (m) { + case 1: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 2: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 3: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 4: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + default: { + throw std::runtime_error( + "Weight only cuda kernel only supported bs <= 4"); + break; + } } - } else { - if (gelu) { - select_batch_gemv_multi_warp_by_batch( - input, weight, scale_list, bias, output, m, k, n, stream); - } else { - select_batch_gemv_multi_warp_by_batch( - input, weight, scale_list, bias, output, m, k, n, stream); + } else if (weight_only_quant_type == "int8") { + switch (m) { + case 1: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 2: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 3: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + case 4: { + select_activation_and_bias( + input, weight, bias, scales, m, n, k, act_method, output, stream); + break; + } + default: { + throw std::runtime_error( + "Weight only cuda kernel only supported bs <= 4"); + break; + } } + } else { + PADDLE_THROW(phi::errors::InvalidArgument( + "WeightOnlyGemvKernel quant_type only support 'int4' or 'int8'.")); } #endif } -template <> -void batched_int8_weight_only_gemv_multi_warp_launcher( - const phi::dtype::bfloat16* input, - const int8_t* weight, - const phi::dtype::bfloat16* scale_list, - const phi::dtype::bfloat16* bias, - phi::dtype::bfloat16* output, - const int m, - const int k, - const int n, - const bool gelu, - gpuStream_t stream) { - // Environment do not support bf16. - assert(false); -} - } // namespace template -void GemvWeightonlyInt8Wrapper(const Context& ctx, - const T* x, - const int8_t* weight, - const T* bias, - const T* weight_scale, - const int m, - const int n, - const int k, - const std::string& act_method, - T* output) { +void WeightOnlyGemvWrapper(const Context& dev_ctx, + const T* input, + const int8_t* weight, + const T* bias, + const T* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + T* output) { using DataType = typename PDDataTypeTraits::DataType; - - bool gelu = false; - if (act_method == "gelu") { - gelu = true; - } else if (act_method == "None") { - gelu = false; - } else { - PADDLE_THROW( - errors::InvalidArgument("Currently, Int8 weightonly GEMV act_method " - "only support `gelu`, `None`. ")); - } - if (m < 1) { - // should no go here since m >=1 - // multi_warp is slightly faster even in m == 1. we don't dispatch to this - // kernel but keep it for future use. - int8_weight_only_gemv_launcher( - reinterpret_cast(x), - weight, - reinterpret_cast(weight_scale), - reinterpret_cast(bias), - reinterpret_cast(output), - k, - n, - gelu, - ctx.stream()); - } else { - batched_int8_weight_only_gemv_multi_warp_launcher( - reinterpret_cast(x), - weight, - reinterpret_cast(weight_scale), + if (weight_only_type == "per_channel") { + PADDLE_ENFORCE_EQ(group_size, + -1, + phi::errors::InvalidArgument( + "group size must be -1 in per-channel mode.")); + + weight_only_batched_gemv_launcher( + reinterpret_cast(input), + reinterpret_cast(weight), reinterpret_cast(bias), - reinterpret_cast(output), + reinterpret_cast(scales), m, - k, n, - gelu, - ctx.stream()); + k, + weight_only_quant_type, + act_method, + reinterpret_cast(output), + dev_ctx.stream()); + } else if (weight_only_type == "group_wise") { + if (group_size == 64) { + weight_only_batched_gemv_launcher>( + reinterpret_cast(input), + reinterpret_cast(weight), + reinterpret_cast(bias), + reinterpret_cast(scales), + m, + n, + k, + weight_only_quant_type, + act_method, + reinterpret_cast(output), + dev_ctx.stream()); + } else if (group_size == 128) { + weight_only_batched_gemv_launcher>( + reinterpret_cast(input), + reinterpret_cast(weight), + reinterpret_cast(bias), + reinterpret_cast(scales), + m, + n, + k, + weight_only_quant_type, + act_method, + reinterpret_cast(output), + dev_ctx.stream()); + } else { + PADDLE_THROW(phi::errors::InvalidArgument( + "WeightOnlyGemvKernel group_size only support 64 or 128.")); + } + } else { + PADDLE_THROW( + phi::errors::InvalidArgument("WeightOnlyGemvKernel type only support " + "'per_channel' or 'group_wise'.")); } } +template <> +void WeightOnlyGemvWrapper(const phi::GPUContext& dev_ctx, + const float* input, + const int8_t* weight, + const float* bias, + const float* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + float* output) { + PADDLE_THROW(phi::errors::Unimplemented( + "WeightOnlyGemvKernel type only support 'float16' and 'bfloa16." + "Not support float32.")); +} + template -void GemvWeightonlyInt8Kernel(const Context& dev_ctx, - const DenseTensor& x, - const DenseTensor& weight, - const paddle::optional& bias, - const DenseTensor& weight_scale, - const std::string& act_method, - DenseTensor* out) { +void WeightOnlyGemvKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& weight, + const paddle::optional& bias, + const DenseTensor& weight_scale, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + DenseTensor* out) { const T* x_data = x.data(); - const int8_t* weight_data = - weight.data(); // Actually, we pass the weight datatype is - // uint8_t type. + const int8_t* weight_data = weight.data(); + // Actually, we pass the weight datatype is uint8_t type. const T* bias_data = bias ? bias.get().data() : nullptr; const T* weight_scale_data = weight_scale.data(); T* out_data = dev_ctx.template Alloc(out); int m = x.dims()[0]; int k = x.dims()[1]; int n = weight.dims()[0]; - GemvWeightonlyInt8Wrapper(dev_ctx, - x_data, - weight_data, - bias_data, - weight_scale_data, - m, - n, - k, - act_method, - out_data); -} -template void GemvWeightonlyInt8Wrapper(const phi::GPUContext& ctx, - const phi::dtype::float16* x, - const int8_t* weight, - const phi::dtype::float16* bias, - const phi::dtype::float16* weight_scale, - const int m, - const int n, - const int k, - const std::string& act_method, - phi::dtype::float16* output); - -template void GemvWeightonlyInt8Wrapper( - const phi::GPUContext& ctx, - const phi::dtype::bfloat16* x, - const int8_t* weight, - const phi::dtype::bfloat16* bias, - const phi::dtype::bfloat16* weight_scale, - const int m, - const int n, - const int k, - const std::string& act_method, - phi::dtype::bfloat16* output); - -// template void GemvWeightonlyInt8Wrapper(const phi::GPUContext& ctx, -// const float* x, -// const int8_t* weight, -// const float* bias, -// const float* weight_scale, -// const int m, -// const int n, -// const int k, -// const std::string& act_method, -// float* output); + WeightOnlyGemvWrapper(dev_ctx, + x_data, + weight_data, + bias_data, + weight_scale_data, + m, + n, + k, + group_size, + weight_only_quant_type, + weight_only_type, + act_method, + out_data); +} +template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, + const float* input, + const int8_t* weight, + const float* bias, + const float* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + float* output); + +template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, + const phi::dtype::float16* input, + const int8_t* weight, + const phi::dtype::float16* bias, + const phi::dtype::float16* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + phi::dtype::float16* output); + +template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, + const phi::dtype::bfloat16* input, + const int8_t* weight, + const phi::dtype::bfloat16* bias, + const phi::dtype::bfloat16* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + phi::dtype::bfloat16* output); } // namespace phi diff --git a/paddle/phi/kernels/funcs/weight_only_gemv.h b/paddle/phi/kernels/funcs/weight_only_gemv.h index 8a2cb1d5b4f34..7f0b4aa7fbc2c 100644 --- a/paddle/phi/kernels/funcs/weight_only_gemv.h +++ b/paddle/phi/kernels/funcs/weight_only_gemv.h @@ -19,15 +19,18 @@ limitations under the License. */ namespace phi { template -void GemvWeightonlyInt8Wrapper(const Context& ctx, - const T* x, - const int8_t* weight, - const T* bias, - const T* weight_scale, - const int m, - const int n, - const int k, - const std::string& act_method, - T* output); +void WeightOnlyGemvWrapper(const Context& dev_ctx, + const T* input, + const int8_t* weight, + const T* bias, + const T* scales, + int m, + int n, + int k, + int group_size, + const std::string& weight_only_quant_type, + const std::string& weight_only_type, + const std::string& act_method, + T* output); } // namespace phi diff --git a/paddle/phi/kernels/gpu/weight_dequantize_kernel.cu b/paddle/phi/kernels/gpu/weight_dequantize_kernel.cu index fce785804c344..77e71b950ddfa 100644 --- a/paddle/phi/kernels/gpu/weight_dequantize_kernel.cu +++ b/paddle/phi/kernels/gpu/weight_dequantize_kernel.cu @@ -29,11 +29,12 @@ void WeightDequantizeKernel(const Context& dev_ctx, const DenseTensor& scale, const std::string& algo, DataType out_dtype, + int32_t group_size, DenseTensor* out) { #if defined(PADDLE_WITH_CUTLASS) auto out_dims = out->dims(); dev_ctx.template Alloc(out); - WeightDequantize(dev_ctx, x, scale, algo, true, out); + WeightDequantize(dev_ctx, x, scale, algo, true, group_size, out); out->Resize({{out_dims[1], out_dims[0]}}); auto out_tmp = Transpose(dev_ctx, *out, {1, 0}); out->ShareDataWith(out_tmp); diff --git a/paddle/phi/kernels/gpu/weight_only_linear_grad_kernel.cu b/paddle/phi/kernels/gpu/weight_only_linear_grad_kernel.cu index c5dc7a15db6e4..de6c2742590b3 100644 --- a/paddle/phi/kernels/gpu/weight_only_linear_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/weight_only_linear_grad_kernel.cu @@ -33,6 +33,7 @@ void WeightOnlyLinearGradKernel(const Context& dev_ctx, const DenseTensor& out_grad, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, DenseTensor* x_grad) { #if defined(PADDLE_WITH_CUTLASS) PADDLE_ENFORCE_EQ( @@ -41,6 +42,12 @@ void WeightOnlyLinearGradKernel(const Context& dev_ctx, phi::errors::InvalidArgument( "Currently weightonly linear grad only support arch = 80 or 86. ")); + PADDLE_ENFORCE_EQ( + group_size, + -1, + phi::errors::InvalidArgument( + "Currently weightonly linear grad only support per-channel mode. ")); + int n = weight_scale.dims()[0]; int k = weight.dims()[1]; dev_ctx.template Alloc(x_grad); @@ -49,8 +56,13 @@ void WeightOnlyLinearGradKernel(const Context& dev_ctx, dev_ctx.template Alloc(&weight_dequantized); std::string algo = weight_dtype == "int8" ? "weight_only_int8" : "weight_only_int4"; - WeightDequantize( - dev_ctx, weight, weight_scale, algo, true, &weight_dequantized); + WeightDequantize(dev_ctx, + weight, + weight_scale, + algo, + true, + group_size, + &weight_dequantized); MatmulKernel( dev_ctx, out_grad, weight_dequantized, false, false, x_grad); #else diff --git a/paddle/phi/kernels/gpu/weight_only_linear_kernel.cu b/paddle/phi/kernels/gpu/weight_only_linear_kernel.cu index 32fb9951bfa47..c41b86148291d 100644 --- a/paddle/phi/kernels/gpu/weight_only_linear_kernel.cu +++ b/paddle/phi/kernels/gpu/weight_only_linear_kernel.cu @@ -31,6 +31,7 @@ void WeightOnlyLinearKernel(const Context& dev_ctx, const DenseTensor& weight_scale, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, DenseTensor* out) { #if defined(PADDLE_WITH_CUTLASS) PADDLE_ENFORCE_EQ( @@ -50,12 +51,12 @@ void WeightOnlyLinearKernel(const Context& dev_ctx, T* out_data = out->data(); const auto x_dims = x.dims(); const auto w_dims = weight.dims(); - int n = weight_scale.dims()[0]; + int n = group_size > 0 ? weight_scale.dims()[1] : weight_scale.dims()[0]; int k = w_dims[1]; int m = x.numel() / k; // m > 3: run gemm. - if (m > 3 || weight_dtype == "int4" || (arch == 70)) { + if (m > 3 || (arch == 70)) { /* Note(Zhengzekang): If using arch = 70, we always dispatch to weightonly Gemm, @@ -157,19 +158,39 @@ we havenot support sm70 weightonly gemv, because sm70 weight layout is RowMajor. PADDLE_THROW(phi::errors::Unimplemented( "Please compile with cutlass to make cutlass available")); #endif - } else { // m == 1: gemv + } else { // m <= 3: gemv if (weight_dtype == "int8") { - GemvWeightonlyInt8Wrapper(dev_ctx, - x_data, - weight_data, - bias_data, - weight_scale_data, - m, - n, - k, - "None", - out->data()); - } // TODO(lizhenyun) support weight_only_gemv for int4. + WeightOnlyGemvWrapper( + dev_ctx, + x_data, + weight_data, + bias_data, + weight_scale_data, + m, + n, + k, + group_size, + "int8", + group_size > 0 ? "group_wise" : "per_channel", + "None", + out->data()); + + } else if (weight_dtype == "int4") { + WeightOnlyGemvWrapper( + dev_ctx, + x_data, + weight_data, + bias_data, + weight_scale_data, + m, + n, + k, + group_size, + "int4", + group_size > 0 ? "group_wise" : "per_channel", + "None", + out->data()); + } } } } // namespace phi diff --git a/paddle/phi/kernels/gpu/weight_quantize_kernel.cu b/paddle/phi/kernels/gpu/weight_quantize_kernel.cu index 0c0024fc9ece3..8cd5598e2e92a 100644 --- a/paddle/phi/kernels/gpu/weight_quantize_kernel.cu +++ b/paddle/phi/kernels/gpu/weight_quantize_kernel.cu @@ -26,8 +26,15 @@ void WeightQuantizeKernel(const Context& dev_ctx, const DenseTensor& x, const std::string& algo, const int32_t arch, + const int32_t group_size, DenseTensor* out, DenseTensor* scale) { + PADDLE_ENFORCE_EQ( + ((group_size == -1) || (group_size == 64) || (group_size == 128)), + true, + phi::errors::InvalidArgument( + "Currently, group_size only support -1(per-channel), 64 or 128.")); + DenseTensor quanted_x; dev_ctx.template Alloc(out); dev_ctx.template Alloc(scale); diff --git a/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h b/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h index d521090816108..2905fd14e6b33 100644 --- a/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h +++ b/paddle/phi/kernels/impl/weight_quantize_kernel_impl.h @@ -56,6 +56,27 @@ void per_channel_scale( } } +template +void group_wise_scale(T* scale, + const T* input, + size_t m, + size_t n, + float bound, + size_t group_size) { + for (size_t i = 0; i < n; ++i) { + for (size_t j = 0; j < m; j += group_size) { + float max = static_cast(0.f); + for (size_t k = 0; k < group_size && j + k < m; ++k) { + max = static_cast(xabs(input[(j + k) * n + i])) > max + ? static_cast(xabs(input[(j + k) * n + i])) + : max; + } + scale[static_cast(j / group_size) * n + i] = + static_cast(max / bound); + } + } +} + template void per_channel_quant(int8_t* output, const T* input, @@ -102,6 +123,55 @@ void per_channel_quant(int8_t* output, } } +template +void group_wise_quant(int8_t* output, + const T* input, + const T* scale, + size_t num_rows, + size_t num_cols, + const int group_size) { + size_t bytes_per_out_col = num_cols * quant_bit / 8; + for (size_t ii = 0; ii < num_rows; ++ii) { + int8_t* current_quantized_weight_row = output + ii * bytes_per_out_col; + const T* current_weight_row = input + ii * num_cols; + for (size_t jj = 0; jj < bytes_per_out_col; ++jj) { + if (quant_bit == 8) { + size_t scale_cur_offset = jj + (ii / group_size) * num_cols; + const float col_scale = static_cast(scale[scale_cur_offset]); + const float weight_elt = static_cast(current_weight_row[jj]); + const float scaled_weight = round(weight_elt / col_scale); + const int8_t clipped_weight = static_cast( + std::max(-127.f, std::min(127.f, scaled_weight))); + current_quantized_weight_row[jj] = clipped_weight; + } else if (quant_bit == 4) { + // We will pack two int4 elements per iteration of the inner loop. + int8_t packed_int4s = 0; + for (int packed_idx = 0; packed_idx < 2; ++packed_idx) { + const size_t input_idx = 2 * jj + packed_idx; + if (input_idx < num_cols) { + size_t scale_cur_offset = input_idx + (ii / group_size) * num_cols; + const float col_scale = static_cast(scale[scale_cur_offset]); + const float weight_elt = + static_cast(current_weight_row[input_idx]); + const float scaled_weight = round(weight_elt / col_scale); + int int_weight = static_cast(scaled_weight); + const int8_t clipped_weight = std::max(-7, std::min(7, int_weight)); + + // Kill the sign extension bits (hence 0x0F mask) then shift to + // upper bits if packing the second int4 and or the bits into the + // final result. + packed_int4s |= ((clipped_weight & 0x0F) << (4 * packed_idx)); + } + } + current_quantized_weight_row[jj] = packed_int4s; + } else { + phi::errors::Unimplemented("Unsupported quantization bits: %d", + quant_bit); + } + } + } +} + template void add_bias_and_interleave_inplace(int8_t* tensor_ptr, size_t num_elts) { const size_t num_bytes = num_elts * quant_bit / 8; diff --git a/paddle/phi/kernels/weight_dequantize_kernel.h b/paddle/phi/kernels/weight_dequantize_kernel.h index 3a0a10924b57e..59bc406d3b689 100644 --- a/paddle/phi/kernels/weight_dequantize_kernel.h +++ b/paddle/phi/kernels/weight_dequantize_kernel.h @@ -24,6 +24,7 @@ void WeightDequantizeKernel(const Context& dev_ctx, const DenseTensor& scale, const std::string& algo, DataType out_dtype, + int32_t group_size, DenseTensor* out); } // namespace phi diff --git a/paddle/phi/kernels/weight_only_linear_grad_kernel.h b/paddle/phi/kernels/weight_only_linear_grad_kernel.h index af05059c488f3..5ac26f03b9e65 100644 --- a/paddle/phi/kernels/weight_only_linear_grad_kernel.h +++ b/paddle/phi/kernels/weight_only_linear_grad_kernel.h @@ -27,6 +27,7 @@ void WeightOnlyLinearGradKernel(const Context& dev_ctx, const DenseTensor& out_grad, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, DenseTensor* x_grad); } // namespace phi diff --git a/paddle/phi/kernels/weight_only_linear_kernel.h b/paddle/phi/kernels/weight_only_linear_kernel.h index 17037fb531f06..4ec3bbcd82ead 100644 --- a/paddle/phi/kernels/weight_only_linear_kernel.h +++ b/paddle/phi/kernels/weight_only_linear_kernel.h @@ -26,5 +26,6 @@ void WeightOnlyLinearKernel(const Context& dev_ctx, const DenseTensor& weight_scale, const std::string& weight_dtype, const int32_t arch, + const int32_t group_size, DenseTensor* out); } // namespace phi diff --git a/paddle/phi/kernels/weight_quantize_kernel.h b/paddle/phi/kernels/weight_quantize_kernel.h index b906e68a40338..17adb5e21d59c 100644 --- a/paddle/phi/kernels/weight_quantize_kernel.h +++ b/paddle/phi/kernels/weight_quantize_kernel.h @@ -23,6 +23,7 @@ void WeightQuantizeKernel(const Context& dev_ctx, const DenseTensor& x, const std::string& algo, const int32_t arch, + const int32_t group_size, DenseTensor* out, DenseTensor* scale); diff --git a/python/paddle/nn/quant/quantized_linear.py b/python/paddle/nn/quant/quantized_linear.py index 155ea48c063aa..059ecc463605f 100644 --- a/python/paddle/nn/quant/quantized_linear.py +++ b/python/paddle/nn/quant/quantized_linear.py @@ -36,7 +36,7 @@ def _get_arch_info(): ) -def weight_quantize(x, algo="weight_only_int8", arch=None): +def weight_quantize(x, algo="weight_only_int8", arch=None, group_size=-1): """ Quantization function for weight_only and llm.int8's weight. @@ -45,6 +45,7 @@ def weight_quantize(x, algo="weight_only_int8", arch=None): algo (str): The algo that is x will be apply, must be one of 'weight_only_int8', 'weight_only_int4' and 'llm.int8', default: 'weight_only_int8'. arch (int): The compute arch for target device. For example, A100 is 80, v100 is 70, if you do not assign arch, we will get arch from your device, default: None. + group_size (int): The group size for weight quantization. -1 stands for default per-channel mode. Currently only support 64 or 128. Returns: out (Tensor): The Tensor which is the quantitative results, the data type is int8, the shape is transposition of x. @@ -71,8 +72,12 @@ def weight_quantize(x, algo="weight_only_int8", arch=None): arch == 70 or arch == 80 or arch == 86 or arch == 75 ), f"Currently weight_quantize only support SM70/75/80/86. but got {arch} " + assert ( + group_size == -1 or group_size == 64 or group_size == 128 + ), f"Currently group_size only support -1/64/128. but got {group_size} " + if in_dynamic_mode(): - return _C_ops.weight_quantize(x, algo, arch) + return _C_ops.weight_quantize(x, algo, arch, group_size) else: type = "weight_quantize" helper = LayerHelper(type, **locals()) @@ -83,12 +88,14 @@ def weight_quantize(x, algo="weight_only_int8", arch=None): type=type, inputs={"x": x}, outputs={'out': out, "scale": scale}, - attrs={"algo": algo, "arch": arch}, + attrs={"algo": algo, "arch": arch, "group_size": group_size}, ) return (out, scale) -def weight_dequantize(x, scale, algo="weight_only_int8", out_dtype='float16'): +def weight_dequantize( + x, scale, algo="weight_only_int8", out_dtype='float16', group_size=-1 +): """ Dequantization function for weight_only and llm.int8's weight. @@ -114,12 +121,16 @@ def weight_dequantize(x, scale, algo="weight_only_int8", out_dtype='float16'): >>> out, scale = weight_quantize(x, algo='weight_only_int8') >>> x_dequant = weight_dequantize(out, scale) """ + assert ( + group_size == -1 or group_size == 64 or group_size == 128 + ), f"Currently group_size only support -1/64/128. but got {group_size} " + check_dtype( out_dtype, 'out_dtype', ['float16', 'bfloat16'], 'weight_dequantize' ) out_dtype = convert_np_dtype_to_dtype_(out_dtype) if in_dynamic_mode(): - return _C_ops.weight_dequantize(x, scale, algo, out_dtype) + return _C_ops.weight_dequantize(x, scale, algo, out_dtype, group_size) else: type = "weight_dequantize" helper = LayerHelper(type, **locals()) @@ -129,13 +140,23 @@ def weight_dequantize(x, scale, algo="weight_only_int8", out_dtype='float16'): type=type, inputs={"x": x, "scale": scale}, outputs={'out': out}, - attrs={"algo": algo, "out_dtype": out_dtype}, + attrs={ + "algo": algo, + "out_dtype": out_dtype, + "group_size": group_size, + }, ) return out def weight_only_linear( - x, weight, bias=None, weight_scale=None, weight_dtype="int8", arch=None + x, + weight, + bias=None, + weight_scale=None, + weight_dtype="int8", + arch=None, + group_size=-1, ): """ Applies matrix multiplication of two tensors and then bias addition if provided. @@ -149,6 +170,7 @@ def weight_only_linear( weight_scale (Tensor|None): The input scale Tensor Provided to weight for dequantization. Its rank must be 1. weight_dtype(str): The dtype of weight Tensor, must be one of 'int8', 'int4', Defaulted to 'int8'. arch (int): The compute arch for target device. For example, A100 is 80, v100 is 70, if you do not assign arch, we will get arch from your device, default: None. + group_size (int): The group size for weight quantization. -1 stands for default per-channel mode. Currently only support 64 or 128. Returns: Tensor: the output Tensor, the data type is the same as that of x. @@ -174,10 +196,13 @@ def weight_only_linear( assert ( arch == 70 or arch == 80 or arch == 86 or arch == 75 ), f"Currently weight_quantize only support SM70/75/80/86. but got {arch} " + assert ( + group_size == -1 or group_size == 64 or group_size == 128 + ), f"Currently weight_quantize only support group size of -1, 64 or 128. but got {group_size} " if in_dynamic_mode(): out = _C_ops.weight_only_linear( - x, weight, bias, weight_scale, weight_dtype, arch + x, weight, bias, weight_scale, weight_dtype, arch, group_size ) return out else: @@ -195,7 +220,11 @@ def weight_only_linear( } if bias is not None: inputs["bias"] = [bias] - attrs = {'weight_dtype': weight_dtype, 'arch': arch} + attrs = { + 'weight_dtype': weight_dtype, + 'arch': arch, + 'group_size': group_size, + } out = helper.create_variable_for_type_inference(dtype) diff --git a/test/quantization/test_weight_only_linear.py b/test/quantization/test_weight_only_linear.py index c7bbc1c658267..f3749d0b4fb15 100644 --- a/test/quantization/test_weight_only_linear.py +++ b/test/quantization/test_weight_only_linear.py @@ -72,6 +72,7 @@ def config(self): self.out_features = 256 self.weight_dtype = "int8" self.static = False + self.group_size = -1 def weightQuantizeCPUGPUConsistenceCheck(self, weight_float): for arch in [70, 75, 80, 86]: @@ -83,6 +84,7 @@ def weightQuantizeCPUGPUConsistenceCheck(self, weight_float): if self.weight_dtype == "int8" else "weight_only_int4", arch=arch, + group_size=self.group_size, ) weight_cpu, weight_scale_cpu = Q.weight_quantize( weight_float.cpu(), @@ -90,6 +92,7 @@ def weightQuantizeCPUGPUConsistenceCheck(self, weight_float): if self.weight_dtype == "int8" else "weight_only_int4", arch=arch, + group_size=self.group_size, ) np.testing.assert_allclose( weight_gpu.numpy(), weight_cpu.numpy(), atol=1.5 @@ -106,7 +109,7 @@ def weightQuantizeCPUGPUConsistenceCheck(self, weight_float): def setUp(self): self.config() if self.dtype == "bfloat16" or self.weight_dtype == "int4": - self.atol = 1e-1 + self.atol = 1.5e-1 x = np.random.random((self.batch, self.token, self.in_features)) self.x = paddle.to_tensor(x, dtype=self.dtype) if self.bias: @@ -136,6 +139,7 @@ def setUp(self): algo="weight_only_int8" if self.weight_dtype == "int8" else "weight_only_int4", + group_size=self.group_size, ) def get_linear_out(self): @@ -149,6 +153,7 @@ def get_weight_only_linear_out(self): bias=self.bias, weight_scale=self.weight_scale, weight_dtype=self.weight_dtype, + group_size=self.group_size, ) return out.numpy() @@ -185,6 +190,7 @@ def get_weight_only_linear_out_static(self): bias, weight_scale, self.weight_dtype, + group_size=self.group_size, ) feed_dict = { 'x': x_np, @@ -351,59 +357,188 @@ def config(self): not core.is_compiled_with_cuda() or get_cuda_version() < 11020, "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", ) -class WeightOnlyLinearTestCaseStatic(WeightOnlyLinearTestCase): +class WeightOnlyLinearTestCase11(WeightOnlyLinearTestCase): def config(self): super().config() - self.static = True + self.dtype = 'float16' + self.weight_dtype = "int4" + self.batch = 1 + self.token = 1 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() or get_cuda_version() < 11020, + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", +) +class WeightOnlyLinearTestCase12(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'float16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11020 - or paddle.device.cuda.get_device_capability()[0] < 8, + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase13(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase14(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase15(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 64 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase16(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 128 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() or get_cuda_version() < 11020, "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", ) -class WeightOnlyLinearBackwardAndWeightDequantizeTestCase(unittest.TestCase): - def test_weightonly_linear_backward(self): - x = ( - paddle.rand(shape=(128, 4096), dtype='float16') - * 1 - / math.sqrt(4096) - ) - x.stop_gradient = False - quant_x = copy.deepcopy(x) - quant_x.stop_gradient = False - weight = ( - paddle.rand(shape=(4096, 12288), dtype='float16') - * 1 - / math.sqrt(4096) - ) +class WeightOnlyLinearTestCase17(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'float16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 64 - quant_weight, quant_scale = Q.weight_quantize( - x=weight.cuda(), algo='weight_only_int8' - ) - dequant_weight = Q.weight_dequantize(quant_weight.cuda(), quant_scale) - np.testing.assert_allclose(weight, dequant_weight, rtol=1e-2, atol=1e-2) - quant_out = Q.weight_only_linear( - x=quant_x, - weight=quant_weight, - weight_scale=quant_scale, - weight_dtype="int8", - ) - out = paddle.matmul(x=x, y=weight) - np.testing.assert_allclose(quant_out, out, rtol=1e-3, atol=1e-3) +@unittest.skipIf( + not core.is_compiled_with_cuda() or get_cuda_version() < 11020, + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", +) +class WeightOnlyLinearTestCase18(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'float16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 128 - quant_out.backward() - out.backward() - np.testing.assert_allclose(quant_x.grad, x.grad, rtol=1e-3, atol=1e-3) + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase19(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int4" + self.bias = False + self.batch = 1 + self.token = 2 + self.group_size = 128 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase20(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int8" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 64 + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8 + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8 or core is not support bfloat16", +) +class WeightOnlyLinearTestCase21(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.dtype = 'bfloat16' + self.weight_dtype = "int8" + self.bias = False + self.batch = 1 + self.token = 1 + self.group_size = 128 @unittest.skipIf( not core.is_compiled_with_cuda() or get_cuda_version() < 11020, "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", ) -class WeightOnlyLinearTestCase11(WeightOnlyLinearTestCase): +class WeightOnlyLinearTestCase22(WeightOnlyLinearTestCase): def config(self): super().config() self.dtype = 'float16' @@ -416,7 +551,7 @@ def config(self): not core.is_compiled_with_cuda() or get_cuda_version() < 11020, "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", ) -class WeightOnlyLinearTestCase12(WeightOnlyLinearTestCase): +class WeightOnlyLinearTestCase23(WeightOnlyLinearTestCase): def config(self): super().config() self.dtype = 'float16' @@ -432,7 +567,7 @@ def config(self): or paddle.device.cuda.get_device_capability()[0] < 8, "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", ) -class WeightOnlyLinearTestCase13(WeightOnlyLinearTestCase): +class WeightOnlyLinearTestCase24(WeightOnlyLinearTestCase): def config(self): super().config() self.dtype = 'bfloat16' @@ -441,5 +576,57 @@ def config(self): self.out_features = 288 +@unittest.skipIf( + not core.is_compiled_with_cuda() or get_cuda_version() < 11020, + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", +) +class WeightOnlyLinearTestCaseStatic(WeightOnlyLinearTestCase): + def config(self): + super().config() + self.static = True + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or get_cuda_version() < 11020 + or paddle.device.cuda.get_device_capability()[0] < 8, + "quantized_matmul requires CUDA >= 11.2 and CUDA_ARCH >= 8", +) +class WeightOnlyLinearBackwardAndWeightDequantizeTestCase(unittest.TestCase): + def test_weightonly_linear_backward(self): + x = ( + paddle.rand(shape=(128, 4096), dtype='float16') + * 1 + / math.sqrt(4096) + ) + x.stop_gradient = False + quant_x = copy.deepcopy(x) + quant_x.stop_gradient = False + weight = ( + paddle.rand(shape=(4096, 12288), dtype='float16') + * 1 + / math.sqrt(4096) + ) + + quant_weight, quant_scale = Q.weight_quantize( + x=weight.cuda(), algo='weight_only_int8' + ) + dequant_weight = Q.weight_dequantize(quant_weight.cuda(), quant_scale) + np.testing.assert_allclose(weight, dequant_weight, rtol=1e-2, atol=1e-2) + + quant_out = Q.weight_only_linear( + x=quant_x, + weight=quant_weight, + weight_scale=quant_scale, + weight_dtype="int8", + ) + out = paddle.matmul(x=x, y=weight) + np.testing.assert_allclose(quant_out, out, rtol=1e-3, atol=1e-3) + + quant_out.backward() + out.backward() + np.testing.assert_allclose(quant_x.grad, x.grad, rtol=1e-3, atol=1e-3) + + if __name__ == '__main__': unittest.main() From f2ee1769f63c43cd99d3842ccf3155a8235617d9 Mon Sep 17 00:00:00 2001 From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com> Date: Wed, 27 Dec 2023 14:03:54 +0800 Subject: [PATCH 080/146] change_cc_test_old (#60356) * change_cc_test_old * update * update --- .../ir_adaptor/translator/type_translator.h | 2 +- .../fluid/pir/dialect/op_generator/op_gen.py | 2 +- .../operator/utils/op_yaml_info_parser.h | 10 ++++----- test/cpp/pir/core/CMakeLists.txt | 22 +++---------------- 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/ir_adaptor/translator/type_translator.h b/paddle/fluid/ir_adaptor/translator/type_translator.h index 255795c92d807..00b369259718e 100644 --- a/paddle/fluid/ir_adaptor/translator/type_translator.h +++ b/paddle/fluid/ir_adaptor/translator/type_translator.h @@ -35,7 +35,7 @@ class TypeTranslator { using VarType = paddle::framework::proto::VarType; private: - TypeTranslator(); // Disallow instantiation outside of the class. + TEST_API TypeTranslator(); // Disallow instantiation outside of the class. std::unordered_map handlers; public: diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index e3fbba6ed7bf7..7dd754e868f86 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -41,7 +41,7 @@ # Note(Galaxy1458) The need_export_symbol_op_list is used # for some unittests these need to export symbol op compiled with dynamic lib. -need_export_symbol_op_list = ['AbsOp', 'FullOp'] +need_export_symbol_op_list = ['AbsOp', 'FullOp', 'UniformOp'] # ===================================== # String Template for h file code gen diff --git a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h index fd4004730c906..4ff03f336dae2 100644 --- a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h +++ b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h @@ -23,13 +23,13 @@ class OpYamlInfoParser { public: OpYamlInfoParser() = delete; - explicit OpYamlInfoParser(const OpInfoTuple& op_info_tuple, - bool is_legacy_op = false); + TEST_API explicit OpYamlInfoParser(const OpInfoTuple& op_info_tuple, + bool is_legacy_op = false); - bool IsTensorAttribute(size_t index) const; - size_t InputTensorNumber() const; + TEST_API bool IsTensorAttribute(size_t index) const; + TEST_API size_t InputTensorNumber() const; - const std::string& AttrTypeName(const std::string& name) const; + TEST_API const std::string& AttrTypeName(const std::string& name) const; const std::string& TensorAttrTypeName(const std::string& name) const; const std::vector& TensorParams(bool is_kernel = false) const; diff --git a/test/cpp/pir/core/CMakeLists.txt b/test/cpp/pir/core/CMakeLists.txt index c53dbb3a71666..9663acf10f4aa 100644 --- a/test/cpp/pir/core/CMakeLists.txt +++ b/test/cpp/pir/core/CMakeLists.txt @@ -42,26 +42,10 @@ cc_test( SRCS ir_parser_test.cc DEPS gtest op_dialect_vjp pir) -cc_test_old(ir_op_info_test SRCS op_info_test.cc DEPS gtest pir) -cc_test_old( - ir_op_yaml_info_parser_test - SRCS - op_yaml_info_parser_test.cc - DEPS - gtest - op_dialect - op_dialect_vjp - pir) +paddle_test(ir_op_info_test SRCS op_info_test.cc) +paddle_test(ir_op_yaml_info_parser_test SRCS op_yaml_info_parser_test.cc) -cc_test_old( - ir_type_converter_test - SRCS - ir_type_converter_test.cc - DEPS - gtest - program_translator - op_dialect_vjp - pir) +paddle_test(ir_type_converter_test SRCS ir_type_converter_test.cc) paddle_test(type_interface_test SRCS type_interface_test.cc DEPS test_dialect gtest) From 1862518f3076bc2e2a742e63f25d856059c3eb7f Mon Sep 17 00:00:00 2001 From: Ghost Screaming Date: Wed, 27 Dec 2023 14:06:30 +0800 Subject: [PATCH 081/146] [AutoParallel] Fix pipeline parallel get none grad in non-computatio rank. (#60214) * [AutoParallel] Fix pipeline parallel get none grad in non-computation rank. * fix optimizer update parameter is uninitialized * fix gradient clip --------- Co-authored-by: LiYuRio --- .../eager/accumulation/accumulation_node.cc | 3 +- paddle/fluid/eager/utils.cc | 7 ++ paddle/fluid/pybind/eager_method.cc | 4 ++ paddle/fluid/pybind/eager_properties.cc | 5 +- paddle/fluid/pybind/eager_utils.cc | 3 +- .../phi/api/yaml/generator/dist_bw_api_gen.py | 4 +- paddle/phi/infermeta/unary.cc | 1 + .../paddle/distributed/auto_parallel/api.py | 50 --------------- python/paddle/nn/clip.py | 64 +++++-------------- python/paddle/optimizer/optimizer.py | 13 +++- .../semi_auto_parallel_simple_net_sp.py | 2 +- 11 files changed, 51 insertions(+), 105 deletions(-) diff --git a/paddle/fluid/eager/accumulation/accumulation_node.cc b/paddle/fluid/eager/accumulation/accumulation_node.cc index c15739385dd43..24ec12a06a60f 100644 --- a/paddle/fluid/eager/accumulation/accumulation_node.cc +++ b/paddle/fluid/eager/accumulation/accumulation_node.cc @@ -178,7 +178,8 @@ GradNodeAccumulation::operator()( if (!weak_grad_.expired() && !is_new_grad) { auto grad = weak_grad_.lock(); - if (grad_out.defined() && grad_out.initialized()) { + if (grad_out.defined() && + (grad_out.is_dist_tensor() || grad_out.initialized())) { CopyOrAddTensor(grad.get(), grad_out, is_fake_empty_); } // else { do nothing since there is no valid value in grad out tensor } diff --git a/paddle/fluid/eager/utils.cc b/paddle/fluid/eager/utils.cc index 1bc28549cb0c4..16fd52e8512ee 100644 --- a/paddle/fluid/eager/utils.cc +++ b/paddle/fluid/eager/utils.cc @@ -617,6 +617,13 @@ void EagerUtils::FillZeroForEmptyGradInput(paddle::Tensor* in_grad, *(static_cast(in_grad->impl().get()) ->unsafe_mutable_value()) = *(static_cast(tensor_with_zero.impl().get())); + } else { + *(static_cast(in_grad->impl().get()) + ->unsafe_mutable_value()) = + phi::DenseTensor( + std::make_shared( + nullptr, 0, phi::distributed::GetDefaultPlace()), + phi::DenseTensorMeta()); } } else { auto tensor_with_zero = diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 5c35e41eab0c9..5a102f3c75cc5 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -2900,6 +2900,10 @@ static PyObject* tensor__grad_ivar(TensorObject* self, if (meta && meta->Grad().initialized()) { return ToPyObject(meta->Grad()); } else { + if (meta && !meta->Grad().initialized() && meta->Grad().impl() && + meta->Grad().is_dist_tensor()) { + return ToPyObject(meta->Grad(), false); + } RETURN_PY_NONE } EAGER_CATCH_AND_THROW_RETURN_NULL diff --git a/paddle/fluid/pybind/eager_properties.cc b/paddle/fluid/pybind/eager_properties.cc index 64bba2f70eeee..2a2b94b715abd 100644 --- a/paddle/fluid/pybind/eager_properties.cc +++ b/paddle/fluid/pybind/eager_properties.cc @@ -287,10 +287,13 @@ PyObject* tensor_properties_get_grad(TensorObject* self, void* closure) { EAGER_TRY VLOG(6) << "Get grad for tensor: " << self->tensor.name(); auto meta = egr::EagerUtils::nullable_autograd_meta(self->tensor); - VLOG(6) << meta << " initialized: " << meta->Grad().initialized(); if (meta && meta->Grad().initialized()) { return ToPyObject(meta->Grad()); } else { + if (meta && !meta->Grad().initialized() && meta->Grad().impl() && + meta->Grad().is_dist_tensor()) { + return ToPyObject(meta->Grad(), false); + } RETURN_PY_NONE } EAGER_CATCH_AND_THROW_RETURN_NULL diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 9889d9511b4d0..e23217feacb65 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -2475,7 +2475,8 @@ paddle::Tensor PyTensorHook::operator()(const paddle::Tensor& var) { PyObject* res = nullptr; try { - PyObject* p_tmp_var = ToPyObject(var); + bool return_py_none_if_not_initialize = var.is_dist_tensor() ? false : true; + PyObject* p_tmp_var = ToPyObject(var, return_py_none_if_not_initialize); res = PyObject_CallFunctionObjArgs(py_func_, p_tmp_var, nullptr); Py_DECREF(p_tmp_var); } catch (platform::EnforceNotMet& e) { diff --git a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py index c7ec9ace290ac..3fd8d8a383f3e 100644 --- a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py @@ -126,7 +126,7 @@ MULTI_SINGLE_OUT_CREATION_TEMPLATE_NO_SPMD = """ auto dist_out_{idx} = SetKernelDistOutput({name}); auto dense_out_{idx} = dist_out_{idx} ? dist_out_{idx}->unsafe_mutable_value() : nullptr; - if (dense_out_{idx} && !rank_is_in_current_mesh && dist_out_{idx}->defined()) {{ + if (dense_out_{idx} && !rank_is_in_current_mesh && !dist_out_{idx}->defined()) {{ *dense_out_{idx} = phi::DenseTensor( std::make_shared(nullptr, 0, phi::distributed::GetDefaultPlace()), phi::DenseTensorMeta()); @@ -137,7 +137,7 @@ CreateKernelDistOutput({name}, !rank_is_in_current_mesh, spmd_info.second[{idx}]); phi::distributed::DistTensor* dist_out_{idx} = shared_dist_out_{idx}.get(); phi::DenseTensor* dense_out_{idx} = dist_out_{idx} ? dist_out_{idx}->unsafe_mutable_value() : nullptr; - if (dense_out_{idx} && !rank_is_in_current_mesh && dist_out_{idx}->defined()) {{ + if (dense_out_{idx} && !rank_is_in_current_mesh && !dist_out_{idx}->defined()) {{ *dense_out_{idx} = phi::DenseTensor( std::make_shared(nullptr, 0, phi::distributed::GetDefaultPlace()), phi::DenseTensorMeta()); diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index af60d6ae8da5c..d221c13968910 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -4023,6 +4023,7 @@ void SequenceMaskScalarInferMeta(const MetaTensor& x, void SquaredL2NormInferMeta(const MetaTensor& x, MetaTensor* out) { out->set_dims({1}); + out->set_dtype(x.dtype()); } void SqueezeInferMeta(const MetaTensor& x, diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index bef09fa95fc8d..67ef633485add 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -515,9 +515,7 @@ def __init__(self, optimizer, shard_fn=None): optimizer.helper = paddle.base.layer_helper.LayerHelper( optimizer.__class__.__name__ ) - # solve global_clip for auto_parallel self._shard_clip = False - self._generate_flag = False if ( hasattr(optimizer, "_grad_clip") and optimizer._grad_clip is not None @@ -564,40 +562,15 @@ def _shard_accumulator(self, param): placements=placements, ) - def generate_pp_mesh(self, all_process_ids=[]): - pp_mesh = None - if len(all_process_ids) <= 1: - return pp_mesh - else: - mesh = np.array(all_process_ids) - for i in range(mesh.shape[-1]): - ranks = mesh[:, i].tolist() - if dist.get_rank() in ranks: - pp_mesh = dist.ProcessMesh(ranks) - return pp_mesh - def step(self): if not isinstance(self._inner_opt._parameter_list[0], dict): params_grads = [] - all_process_ids = [] for param in self._inner_opt._parameter_list: if param.stop_gradient: continue if param._grad_ivar() is not None: grad_var = param._grad_ivar() params_grads.append((param, grad_var)) - if ( - not self._generate_flag - and self._shard_clip - and param.is_dist() - ): - if param.process_mesh.process_ids not in all_process_ids: - all_process_ids.append(param.process_mesh.process_ids) - if not self._generate_flag and self._shard_clip: - self._inner_opt._grad_clip._pp_mesh = self.generate_pp_mesh( - all_process_ids - ) - self._generate_flag = True for p, g in params_grads: self._shard_accumulator(p) self._inner_opt._apply_optimize( @@ -606,36 +579,13 @@ def step(self): else: for param_group in self._inner_opt._param_groups: params_grads = defaultdict(lambda: []) - all_process_ids = [] - shard_clip_flag = False for param in param_group['params']: if param.stop_gradient: continue if param._grad_ivar() is not None: grad_var = param._grad_ivar() params_grads['params'].append((param, grad_var)) - if ( - not self._generate_flag - and "grad_clip" in param_group.keys() - and isinstance( - param_group["grad_clip"], - paddle.nn.ClipGradByGlobalNorm, - ) - and param.is_dist() - ): - if ( - param.process_mesh.process_ids - not in all_process_ids - ): - all_process_ids.append( - param.process_mesh.process_ids - ) - shard_clip_flag = True - if shard_clip_flag: - param_group["grad_clip"]._pp_mesh = self.generate_pp_mesh( - all_process_ids - ) params_grads.update( {k: v for k, v in param_group.items() if k != 'params'} ) diff --git a/python/paddle/nn/clip.py b/python/paddle/nn/clip.py index 6cd80a4f6a3da..b614a6c407077 100644 --- a/python/paddle/nn/clip.py +++ b/python/paddle/nn/clip.py @@ -18,6 +18,7 @@ import paddle import paddle.autograd as imperative_base +import paddle.distributed as dist from paddle import _C_ops from paddle.base import core, framework, unique_name from paddle.base.data_feeder import check_variable_and_dtype @@ -661,8 +662,6 @@ def __init__( # are so many hard code depends on `add_n` in the legacy static # manual hybrid-parallel. self._async_add_n = None - # just for auto parallel. - self._pp_mesh = None def __str__(self): return "Gradient Clip By GlobalNorm, global_norm=%f" % (self.clip_norm) @@ -673,6 +672,8 @@ def _dygraph_clip(self, params_grads): sum_square_list = [] sum_square_list_fp16 = [] sum_square_list_fp32 = [] + src_mesh = params_grads[0][0].process_mesh + for p, g in params_grads: if g is None: continue @@ -689,6 +690,14 @@ def _dygraph_clip(self, params_grads): merge_grad = get_tensor_from_selected_rows(merge_grad) sum_square = _squared_l2_norm(merge_grad) + + # if the gradient mesh is not equal to src mesh + # do reshard to get the result of squared_l2 from other pp stage mesh + if src_mesh is not None and g.process_mesh != src_mesh: + sum_square = dist.reshard( + sum_square, src_mesh, sum_square.placements + ) + if ( sum_square.dtype == core.VarDesc.VarType.FP16 or sum_square.dtype == core.VarDesc.VarType.BF16 @@ -715,64 +724,21 @@ def async_add_n(var_list): global_norm_var = [] if len(sum_square_list_fp16) > 0: global_norm_var_fp16 = async_add_n(sum_square_list_fp16) - if self._pp_mesh is not None: - # sync pp - global_norm_var_fp16 = ( - paddle.distributed.auto_parallel.api.dtensor_from_local( - global_norm_var_fp16._local_value().reshape([-1]), - self._pp_mesh, - [paddle.distributed.Partial()], - ) - ) - global_norm_var_fp16 = paddle.distributed.reshard( - global_norm_var_fp16, - self._pp_mesh, - [paddle.distributed.Replicate()], - ) global_norm_var.append(global_norm_var_fp16.astype(sum_dtype)) if len(sum_square_list_fp32) > 0: global_norm_var_fp32 = async_add_n(sum_square_list_fp32) - if self._pp_mesh is not None: - # sync pp - global_norm_var_fp32 = ( - paddle.distributed.auto_parallel.api.dtensor_from_local( - global_norm_var_fp32._local_value().reshape([-1]), - self._pp_mesh, - [paddle.distributed.Partial()], - ) - ) - global_norm_var_fp32 = paddle.distributed.reshard( - global_norm_var_fp32, - self._pp_mesh, - [paddle.distributed.Replicate()], - ) if sum_dtype == 'float32': global_norm_var.append(global_norm_var_fp32) else: global_norm_var.append(global_norm_var_fp32.astype(sum_dtype)) if len(sum_square_list) > 0: global_norm_var_fp64 = async_add_n(sum_square_list) - if self._pp_mesh is not None: - # sync pp - global_norm_var_fp64 = ( - paddle.distributed.auto_parallel.api.dtensor_from_local( - global_norm_var_fp64._local_value().reshape([-1]), - self._pp_mesh, - [paddle.distributed.Partial()], - ) - ) - global_norm_var_fp64 = paddle.distributed.reshard( - global_norm_var_fp64, - self._pp_mesh, - [paddle.distributed.Replicate()], - ) global_norm_var.append(global_norm_var_fp64) - if self._pp_mesh is not None: - global_norm_var = [t._local_value() for t in global_norm_var] + global_norm_var = async_add_n(global_norm_var) global_norm_var = paddle.sqrt(global_norm_var) max_global_norm = paddle.full( - shape=[], dtype=global_norm_var.dtype, fill_value=self.clip_norm + shape=[], dtype=sum_dtype, fill_value=self.clip_norm ) need_clip = False @@ -800,6 +766,10 @@ def async_add_n(var_list): if clip_var.dtype != g.dtype else clip_var ) + if clip_input.process_mesh != g.process_mesh: + clip_input = paddle.distributed.reshard( + clip_input, g.process_mesh, clip_input.placements + ) new_grad = paddle.multiply(g, clip_input) params_and_grads.append((p, new_grad)) else: diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 83e2d8787ce55..3307a88ca48f4 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -1193,7 +1193,13 @@ def _create_optimization_pass( self._set_auxiliary_var('found_inf', False) if isinstance(parameters_and_grads, list): for param_and_grad in parameters_and_grads: - if param_and_grad[1] is None: + # Parameters can be uninitialized in pipeline parallel of semi-auto parallel. + # Since gradient clip and parameters update mixed up in one interface, so we + # need to filter again here. + if ( + param_and_grad[1] is None + or not param_and_grad[1]._is_initialized() + ): continue if param_and_grad[0].stop_gradient is False: self._append_optimize_op( @@ -1201,7 +1207,10 @@ def _create_optimization_pass( ) else: for param_and_grad in parameters_and_grads['params']: - if param_and_grad[1] is None: + if ( + param_and_grad[1] is None + or not param_and_grad[1]._is_initialized() + ): continue if param_and_grad[0].stop_gradient is False: param_grad_dict = {} diff --git a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_simple_net_sp.py b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_simple_net_sp.py index 43f03ec82ee4b..00d3c707a90c5 100644 --- a/test/auto_parallel/hybrid_strategy/semi_auto_parallel_simple_net_sp.py +++ b/test/auto_parallel/hybrid_strategy/semi_auto_parallel_simple_net_sp.py @@ -209,7 +209,7 @@ def test_dp_mp_sp_demo_net(self): for param, param_base in zip( self.dp_mp_sp_parameters, self.base_parameters ): - if param.grad is not None: + if param.grad._is_initialized(): self.check_tensor_eq(param, param_base) self.check_tensor_eq(param.grad, param_base.grad) From c8ef957e814aa5c138ad3143528519f23e81d3fd Mon Sep 17 00:00:00 2001 From: chen2016013 <111894720+chen2016013@users.noreply.github.com> Date: Wed, 27 Dec 2023 14:31:38 +0800 Subject: [PATCH 082/146] [PIR] Fix and open PIR test in test_cond (#60270) * fix python bugs * update * fix * update * fix bug * fix bug * add todo --- .../instruction/tuple_pop_instruction.cc | 43 +++- .../instruction/tuple_push_instruction.cc | 9 +- .../pir_adaptor/pir_adaptor_util.cc | 58 ++++- .../pir_adaptor/pir_adaptor_util.h | 3 +- .../phi/kernels/gpu/batch_norm_grad_kernel.cu | 10 +- paddle/phi/kernels/gpu/batch_norm_kernel.cu | 13 +- python/paddle/base/backward.py | 21 ++ python/paddle/optimizer/optimizer.py | 35 ++- python/paddle/pir/core.py | 2 + test/ir/pir/cinn/test_cinn_sub_graph.py | 7 +- test/legacy_test/test_cond.py | 227 ++++++++++-------- 11 files changed, 292 insertions(+), 136 deletions(-) diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc b/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc index d86ee66a9d1e9..a3a8f4461865e 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc @@ -73,6 +73,45 @@ static std::stack PopElements(VariableRefArray* var_array, } return rtn; } +void ShareVarData(const Variable* src_var, Variable* dst_var) { + if (src_var->IsType()) { + auto& src_tensor = src_var->Get(); + auto* tmp_dst_tensor = dst_var->GetMutable(); + if (src_tensor.numel() == 0) { + tmp_dst_tensor->set_meta(src_tensor.meta()); + return; + } + tmp_dst_tensor->ShareDataWith(src_tensor); + } else if (src_var->IsType()) { + auto* tmp_dst_slr = dst_var->GetMutable(); + auto* dst_t = tmp_dst_slr->mutable_value(); + auto& src_slr = src_var->Get(); + auto& src_t = src_slr.value(); + if (src_t.numel() == 0) { + dst_t->set_meta(src_t.meta()); + return; + } + dst_t->ShareDataWith(src_t); + } else if (src_var->IsType()) { + auto src_tensor_array = src_var->Get(); + auto* dst_tensor_array = dst_var->GetMutable(); + if (src_tensor_array.numel() == 0) return; + dst_tensor_array->clear(); + for (auto src_tensor : src_tensor_array) { + phi::DenseTensor* tmp_dst_tensor = new phi::DenseTensor(); + if (src_tensor.numel() == 0) { + tmp_dst_tensor->set_meta(src_tensor.meta()); + } else { + tmp_dst_tensor->ShareDataWith(src_tensor); + } + dst_tensor_array->push_back(*tmp_dst_tensor); + } + } else { + PADDLE_THROW(phi::errors::PreconditionNotMet( + "Output only support DenseTensorType " + "or SelectedRowsType or TensorArrayType")); + } +} void TuplePopInstruction::Run() { VLOG(6) << "run tuple_pop instruction"; @@ -89,9 +128,7 @@ void TuplePopInstruction::Run() { VLOG(6) << "pop back var: " << front_var; auto outlet_element_value = tuple_pop_op_.outlet_element(i); auto grad_var = value_exe_info_->GetVarByValue(outlet_element_value); - grad_var->GetMutable()->ShareDataWith( - front_var->Get()); - + ShareVarData(front_var, grad_var); Variable* gc_front_var = const_cast(front_var); AddEagerGCVar(gc_front_var); } diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc b/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc index 78a174ba1c977..bb01125bf3eca 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc @@ -68,6 +68,7 @@ TuplePushInstruction::TuplePushInstruction(size_t id, } void TuplePushInstruction::Run() { + VLOG(4) << "run tuple_push instruction"; if (tuple_push_op_.tuple_size() == 0) { stack_element_var_array_->emplace_back(nullptr); } else { @@ -80,10 +81,12 @@ void TuplePushInstruction::Run() { int stack_size = tuple_push_op_.tuple_size(); auto var_name = value_2_var_name.at(inlet_element_value); - std::string new_name = var_name + "copied_" + - std::to_string(stack_element_var_array_->size()); + auto num_str = std::to_string(stack_element_var_array_->size()); + std::string new_name = var_name + "_copied_" + num_str; auto* copy_var = value_exe_info_->GetScope()->Var(new_name); - DeepCopyVariable(var, copy_var, value_exe_info_, stack_size); + bool is_optional = (inlet_element_value.impl() == nullptr || + !inlet_element_value.type()); + DeepCopyVariable(var, copy_var, value_exe_info_, stack_size, is_optional); VLOG(10) << "done DeepCopyVariable " << new_name; stack_element_var_array_->emplace_back(copy_var); VLOG(6) << "push back var: " << new_name << "[" << copy_var << "]" diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index eba12327d10a0..7f110b49b218f 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -291,27 +291,70 @@ void CheckInputVars(pir::Operation* op, void DeepCopyVariable(const Variable* src_var, Variable* dst_var, ValueExecutionInfo* value_exe_info, - uint32_t stack_size) { + uint32_t stack_size, + bool is_optional) { if (src_var->IsType()) { - auto* tmp_dst_tensor = dst_var->GetMutable(); auto& src_tensor = src_var->Get(); + auto* tmp_dst_tensor = dst_var->GetMutable(); tmp_dst_tensor->set_lod(src_tensor.lod()); + // NOTE(chenxi67): why add ? In some case(e.g. + // Opresult reserve_space generated by BatchNorm Op), Variable pointer is + // initialized but the content it hold (DenseTensor for most cases) does not + // have holder. In this case we only do set_meta but not copy Tensor. + if (src_tensor.numel() == 0) { + tmp_dst_tensor->set_meta(src_tensor.meta()); + return; + } + if (!src_tensor.initialized()) { + if (is_optional) { + dst_var = nullptr; + return; + } else { + PADDLE_THROW(platform::errors::PermissionDenied( + "DenseTensor shouldn't be null")); + } + } framework::TensorCopy(src_tensor, src_tensor.place(), tmp_dst_tensor); } else if (src_var->IsType()) { - auto* tmp_dst_slr = dst_var->GetMutable(); auto& src_slr = src_var->Get(); + auto* tmp_dst_slr = dst_var->GetMutable(); tmp_dst_slr->set_rows(src_slr.rows()); tmp_dst_slr->set_height(src_slr.height()); - auto& src_t = src_slr.value(); auto* dst_t = tmp_dst_slr->mutable_value(); + if (src_t.numel() == 0) { + dst_t->set_meta(src_t.meta()); + return; + } + if (!src_slr.initialized()) { + if (is_optional) { + dst_var = nullptr; + return; + } else { + PADDLE_THROW(platform::errors::PermissionDenied( + "SelectedRows shouldn't be null")); + } + } framework::TensorCopy(src_t, src_t.place(), dst_t); } else if (src_var->IsType()) { auto src_tensor_array = src_var->Get(); auto* dst_tensor_array = dst_var->GetMutable(); + if (!src_tensor_array.initialized()) { + if (is_optional) { + dst_var = nullptr; + return; + } else { + PADDLE_THROW(platform::errors::PermissionDenied( + "TensorArray shouldn't be null")); + } + } dst_tensor_array->clear(); for (auto src_tensor : src_tensor_array) { phi::DenseTensor* tmp_dst_tensor = new phi::DenseTensor(); + if (src_tensor.numel() == 0) { + tmp_dst_tensor->set_meta(src_tensor.meta()); + continue; + } framework::TensorCopy(src_tensor, src_tensor.place(), tmp_dst_tensor); dst_tensor_array->push_back(*tmp_dst_tensor); } @@ -323,7 +366,8 @@ void DeepCopyVariable(const Variable* src_var, std::string new_name = "copied_" + std::to_string(stack_size) + '_' + value_exe_info->GetVarName(src_ref_var); auto tmp_dst_var = value_exe_info->GetScope()->Var(new_name); - DeepCopyVariable(src_ref_var, tmp_dst_var, value_exe_info, stack_size); + DeepCopyVariable( + src_ref_var, tmp_dst_var, value_exe_info, stack_size, is_optional); dst_ref_array->emplace_back(tmp_dst_var); } @@ -338,10 +382,10 @@ void BuildValue(pir::Value value, const std::string& var_name_prefix, ValueExecutionInfo* value_exe_info) { if (!IsInvalid(value)) { - VLOG(8) << "Value is not invalid, so skip build a variable."; + VLOG(8) << "Value " << value.impl() + << " is not invalid, so skip build a variable."; return; } - Variable* var = nullptr; auto& value_2_var_name = value_exe_info->GetValue2VarName(); if (value_2_var_name.find(value) != value_2_var_name.end()) { diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h index 2fe518931b419..cd1ca07bbe23d 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h @@ -136,7 +136,8 @@ void BuildScope(const pir::Block& block, void DeepCopyVariable(const Variable* src_var, Variable* dst_var, ValueExecutionInfo* value_exe_info, - uint32_t stack_size); + uint32_t stack_size, + bool is_optional); void BuildRuntimeContext(pir::Operation* op, const ValueExecutionInfo& value_exec_info, diff --git a/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu b/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu index c275f58ff734b..ea189d4a328b8 100644 --- a/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu @@ -967,7 +967,11 @@ void BatchNormGradFunctor(const Context &ctx, workspace_tensor.Resize({static_cast(workspace_size)}); workspace_ptr = static_cast(ctx.template Alloc(&workspace_tensor)); - + uint8_t *reserve_space_ptr = nullptr; + if (reserve_space_size != 0) { + reserve_space_ptr = + const_cast(reserve_space->template data()); + } PADDLE_ENFORCE_GPU_SUCCESS( phi::dynload::cudnnBatchNormalizationBackwardEx( /*handle=*/ctx.cudnn_handle(), @@ -1002,7 +1006,9 @@ void BatchNormGradFunctor(const Context &ctx, /*workspace=*/workspace_ptr, /*workSpaceSizeInBytes=*/workspace_size, /*reserveSpace=*/ - const_cast(reserve_space->template data()), + // const_cast(reserve_space->template + // data()), + reserve_space_ptr, /*reserveSpaceSizeInBytes=*/reserve_space_size)); #else PADDLE_ENFORCE_GPU_SUCCESS( diff --git a/paddle/phi/kernels/gpu/batch_norm_kernel.cu b/paddle/phi/kernels/gpu/batch_norm_kernel.cu index f01a4ee860d81..8f010319370a4 100644 --- a/paddle/phi/kernels/gpu/batch_norm_kernel.cu +++ b/paddle/phi/kernels/gpu/batch_norm_kernel.cu @@ -846,10 +846,8 @@ void BatchNormKernel(const Context &ctx, } } else { int64_t reserve_space_size = 0; - void *reserve_space_ptr = nullptr; - DenseTensor reserve_space_tensor; if (reserve_space == nullptr) { - reserve_space = &reserve_space_tensor; + reserve_space = new DenseTensor(); } reserve_space->Resize({reserve_space_size}); ctx.template Alloc(reserve_space); @@ -896,6 +894,12 @@ void BatchNormKernel(const Context &ctx, ctx.template Alloc>(saved_variance); if ((N * H * W * D) == 1) { + int64_t reserve_space_size = 0; + if (reserve_space == nullptr) { + reserve_space = new DenseTensor(); + } + reserve_space->Resize({reserve_space_size}); + ctx.template Alloc(reserve_space); // Only 1 element in normalization dimension, // skip the batch norm calculation, let y = x. phi::Copy(ctx, x, ctx.GetPlace(), false, y); @@ -1127,13 +1131,12 @@ void BatchNormKernel(const Context &ctx, void *reserve_space_ptr = nullptr; void *workspace_ptr = nullptr; DenseTensor workspace_tensor; - DenseTensor reserve_space_tensor; // Create reserve space and workspace for batch norm. // Create tensor for each batchnorm op, it will be used in the // backward. Thus this tensor shouldn't be temp. // auto *reserve_space = ctx.Output("ReserveSpace"); if (reserve_space == nullptr) { - reserve_space = &reserve_space_tensor; + reserve_space = new DenseTensor(); } PADDLE_ENFORCE_NOT_NULL( reserve_space, diff --git a/python/paddle/base/backward.py b/python/paddle/base/backward.py index fab33e5fcdabd..4bdb55ce05ce5 100755 --- a/python/paddle/base/backward.py +++ b/python/paddle/base/backward.py @@ -1949,6 +1949,27 @@ def _get_no_grad_set_name(no_grad_set): return no_grad_set_name +def _get_no_grad_set_value(no_grad_set): + no_grad_set_value = paddle.autograd.backward_utils.ValueSet() + if no_grad_set is not None: + if isinstance(no_grad_set, (set, list, tuple)): + for i, no_grad_value in enumerate(no_grad_set): + if isinstance(no_grad_value, paddle.pir.Value): + no_grad_set_value.add(no_grad_value) + else: + raise TypeError( + "The type of no_grad_set's member must be paddle.pir.Value, but received %s." + % (type(no_grad_value)) + ) + else: + raise TypeError( + "The type of no_grad_set should be set or list or tuple, but received {}".format( + type(no_grad_set) + ) + ) + return no_grad_set_value + + @framework.static_only def append_backward( loss, diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 3307a88ca48f4..3a64f2095f30a 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -36,7 +36,11 @@ from paddle.regularizer import L2Decay from ..base import framework, unique_name -from ..base.backward import _get_no_grad_set_name, append_backward +from ..base.backward import ( + _get_no_grad_set_name, + _get_no_grad_set_value, + append_backward, +) from ..base.framework import Parameter from ..base.layer_helper import LayerHelper from .lr import LRScheduler @@ -1644,15 +1648,26 @@ def append_regularization_ops( return params_and_grads def _get_no_grad_set(self, loss, no_grad_set=None): - no_grad_set = _get_no_grad_set_name(no_grad_set) - parameters = loss.block.program.global_block().all_parameters() - param_no_trainable = { - param.name for param in parameters if param.stop_gradient is True - } - # If the parameter is no trainable, it should not have a gradient. - no_grad_set.update(param_no_trainable) - - return no_grad_set + if in_pir_mode(): + no_grad_set = _get_no_grad_set_value(no_grad_set) + parameters = loss.block.program.global_block().all_parameters() + param_no_trainable = [ + param for param in parameters if param.stop_gradient is True + ] + # If the parameter is no trainable, it should not have a gradient. + no_grad_set.update(param_no_trainable) + return no_grad_set + else: + no_grad_set = _get_no_grad_set_name(no_grad_set) + parameters = loss.block.program.global_block().all_parameters() + param_no_trainable = { + param.name + for param in parameters + if param.stop_gradient is True + } + # If the parameter is no trainable, it should not have a gradient. + no_grad_set.update(param_no_trainable) + return no_grad_set @framework.non_static_only def clear_grad(self, set_to_zero=True): diff --git a/python/paddle/pir/core.py b/python/paddle/pir/core.py index 1933ef7fabf84..afb9ff1ac75ab 100644 --- a/python/paddle/pir/core.py +++ b/python/paddle/pir/core.py @@ -20,6 +20,7 @@ from paddle.base.libpaddle.pir import ( Program, get_current_insertion_point, + reset_insertion_point_to_start, set_insertion_point, set_insertion_point_to_block_end, ) @@ -300,6 +301,7 @@ def create_parameter( main_program.move_parameters_from(startup_program) with program_guard(default_main_program()): + reset_insertion_point_to_start() param = parameter(op_result_name, dtype, shape) trainable = kwargs.get('trainable', True) param.stop_gradient = not trainable diff --git a/test/ir/pir/cinn/test_cinn_sub_graph.py b/test/ir/pir/cinn/test_cinn_sub_graph.py index ad4c65d3d3541..32b0bd5779dd9 100644 --- a/test/ir/pir/cinn/test_cinn_sub_graph.py +++ b/test/ir/pir/cinn/test_cinn_sub_graph.py @@ -203,9 +203,10 @@ def test_forward(self): cinn_out = self.train(use_cinn=True) dy_out = self.train(use_cinn=False) - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 - ) + # TODO(zhangliujie) fix precision error + # np.testing.assert_allclose( + # cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 + # ) class TestCinnDropout(TestCinnSubGraphBase): diff --git a/test/legacy_test/test_cond.py b/test/legacy_test/test_cond.py index 393d9f574ba9a..1323d7caa6eae 100644 --- a/test/legacy_test/test_cond.py +++ b/test/legacy_test/test_cond.py @@ -696,78 +696,93 @@ def backward_value_helper(self, cond_func, use_cuda): Helper function that compares calculated backward value is close to dy/dx """ paddle.enable_static() - if not paddle.framework.in_pir_mode(): - pass main_program = paddle.static.Program() main_program.random_seed = 123 startup_program = paddle.static.Program() startup_program.random_seed = 123 - with paddle.static.program_guard(main_program, startup_program): - img = paddle.static.data( - name='image', shape=[-1, 9], dtype='float32' - ) - img.stop_gradient = False - img.persistable = True - label = paddle.static.data( - name='label', shape=[-1, 1], dtype='int64' - ) - i = paddle.static.data(name="i", shape=[1], dtype='int32') - loss = cond_func(i, img, label) - grad_list = append_backward(loss) - place = base.CUDAPlace(0) if use_cuda else base.CPUPlace() - exe = base.Executor(place) - exe.run(startup_program) - - num_devices = 1 - - delta = 0.005 - for feed_i in range(0, 10): - feed_img = np.random.random(size=[1, 9]).astype(np.float32) - feed_label = np.random.randint( - low=0, high=10, size=[1, 1], dtype=np.int64 - ) - if paddle.framework.in_pir_mode(): - for p, g in grad_list: - if p.is_same(img): - dimg = g - img_grad, loss_value = exe.run( - main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img, - 'label': feed_label, - }, - fetch_list=[dimg, loss], + with paddle.static.scope_guard(paddle.static.Scope()): + with paddle.static.program_guard(main_program, startup_program): + img = paddle.static.data( + name='image', shape=[-1, 9], dtype='float32' ) - else: - img_grad, loss_value = exe.run( - main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img, - 'label': feed_label, - }, - fetch_list=[img.grad_name, loss.name], + img.stop_gradient = False + img.persistable = True + label = paddle.static.data( + name='label', shape=[-1, 1], dtype='int64' ) - - numerical_grad = np.zeros(shape=[num_devices, 9], dtype=np.float32) - feed_img_delta = np.copy(feed_img) - for j in range(9): - feed_img_delta[0][j] = feed_img[0][j] + delta - loss_delta = exe.run( - main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img_delta, - 'label': feed_label, - }, - fetch_list=[loss], + i = paddle.static.data(name="i", shape=[1], dtype='int32') + loss = cond_func(i, img, label) + grad_list = append_backward(loss) + place = base.CUDAPlace(0) if use_cuda else base.CPUPlace() + exe = base.Executor(place) + exe.run(startup_program) + + num_devices = 1 + + delta = 0.005 + for feed_i in range(0, 10): + feed_img = np.random.random(size=[1, 9]).astype(np.float32) + feed_label = np.random.randint( + low=0, high=10, size=[1, 1], dtype=np.int64 + ) + if paddle.framework.in_pir_mode(): + for p, g in grad_list: + if p.is_same(img): + dimg = g + img_grad, loss_value = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img, + 'label': feed_label, + }, + fetch_list=[dimg, loss], + ) + else: + img_grad, loss_value = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img, + 'label': feed_label, + }, + fetch_list=[img.grad_name, loss.name], + ) + + numerical_grad = np.zeros( + shape=[num_devices, 9], dtype=np.float32 + ) + feed_img_delta = np.copy(feed_img) + for j in range(9): + feed_img_delta[0][j] = feed_img[0][j] + delta + if paddle.framework.in_pir_mode(): + for p, g in grad_list: + if p.is_same(img): + dimg = g + _, loss_delta = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img_delta, + 'label': feed_label, + }, + fetch_list=[dimg, loss], + ) + else: + loss_delta = exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img_delta, + 'label': feed_label, + }, + fetch_list=[loss], + ) + numerical_grad[0][j] = (loss_delta - loss_value) / delta + feed_img_delta[0][j] = feed_img[0][j] + np.testing.assert_allclose( + img_grad, numerical_grad, rtol=0.05, atol=0.05 ) - numerical_grad[0][j] = (loss_delta - loss_value) / delta - feed_img_delta[0][j] = feed_img[0][j] - np.testing.assert_allclose( - img_grad, numerical_grad, rtol=0.05, atol=0.05 - ) def add_optimizer_helper(self, cond_func, use_cuda): """ @@ -775,38 +790,41 @@ def add_optimizer_helper(self, cond_func, use_cuda): """ main_program = paddle.static.Program() startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program, startup_program): - img = paddle.static.data( - name='image', shape=[-1, 784], dtype='float32' - ) - label = paddle.static.data( - name='label', shape=[-1, 1], dtype='int64' - ) - i = paddle.static.data(name="i", shape=[1], dtype='int32') - loss = cond_func(i, img, label) - optimizer = paddle.optimizer.SGD(learning_rate=0.1) - optimizer.minimize(loss) - - place = base.CUDAPlace(0) if use_cuda else base.CPUPlace() - exe = base.Executor(place) - exe.run(startup_program) - - for feed_i in range(0, 10): - feed_img = np.random.random(size=[16, 784]).astype(np.float32) - feed_label = np.random.randint( - low=0, high=10, size=[16, 1], dtype=np.int64 - ) - exe.run( - main_program, - feed={ - 'i': np.full((1), feed_i, np.int32), - 'image': feed_img, - 'label': feed_label, - }, - fetch_list=[loss], - ) + with paddle.static.scope_guard(paddle.static.Scope()): + with paddle.static.program_guard(main_program, startup_program): + img = paddle.static.data( + name='image', shape=[-1, 784], dtype='float32' + ) + img.stop_gradient = False + img.persistable = True + label = paddle.static.data( + name='label', shape=[-1, 1], dtype='int64' + ) + i = paddle.static.data(name="i", shape=[1], dtype='int32') + loss = cond_func(i, img, label) + optimizer = paddle.optimizer.SGD(learning_rate=0.1) + optimizer.minimize(loss) + + place = base.CUDAPlace(0) if use_cuda else base.CPUPlace() + exe = base.Executor(place) + exe.run(startup_program) + + for feed_i in range(0, 10): + feed_img = np.random.random(size=[16, 784]).astype(np.float32) + feed_label = np.random.randint( + low=0, high=10, size=[16, 1], dtype=np.int64 + ) + exe.run( + main_program, + feed={ + 'i': np.full((1), feed_i, np.int32), + 'image': feed_img, + 'label': feed_label, + }, + fetch_list=[loss], + ) - # @test_with_pir_api + @test_with_pir_api def test_cond_backward(self): paddle.enable_static() @@ -821,9 +839,11 @@ def cond_func(i, img, label): self.backward_value_helper(cond_func, core.is_compiled_with_cuda()) self.add_optimizer_helper(cond_func, core.is_compiled_with_cuda()) - # @test_with_pir_api + @test_with_pir_api def test_half_nested_cond_backward(self): paddle.enable_static() + np.random.seed(2023) + paddle.seed(2023) def branch(i, img, label): return paddle.static.nn.cond( @@ -846,22 +866,25 @@ def cond_func_simple_net_at_false(i, img, label): cond_func_simple_net_at_true, core.is_compiled_with_cuda(), ) - self.add_optimizer_helper( - cond_func_simple_net_at_true, - core.is_compiled_with_cuda(), - ) + self.backward_value_helper( cond_func_simple_net_at_false, core.is_compiled_with_cuda(), ) + self.add_optimizer_helper( + cond_func_simple_net_at_true, + core.is_compiled_with_cuda(), + ) self.add_optimizer_helper( cond_func_simple_net_at_false, core.is_compiled_with_cuda(), ) - # @test_with_pir_api + @test_with_pir_api def test_nested_cond_backward(self): paddle.enable_static() + np.random.seed(2023) + paddle.seed(2023) def branch(i, img, label, mod_two): if mod_two: From ac2de38a80dd6f2c66b4ef4ad515e209a0470fef Mon Sep 17 00:00:00 2001 From: Yichen Zhang <32740647+pkuzyc@users.noreply.github.com> Date: Wed, 27 Dec 2023 14:33:43 +0800 Subject: [PATCH 083/146] [Auto Parallel] Add unshard_dtensor api (#60272) * add dynamic part of unshard * add unshard_dtensor api * handle Parameter type in unshard_dtensor --- python/paddle/distributed/__init__.py | 2 + .../paddle/distributed/auto_parallel/api.py | 70 +++++++++++++ test/auto_parallel/CMakeLists.txt | 4 + .../semi_auto_parallel_unshard_dtensor_api.py | 99 +++++++++++++++++++ ...test_semi_auto_parallel_unshard_dtensor.py | 40 ++++++++ 5 files changed, 215 insertions(+) create mode 100644 test/auto_parallel/semi_auto_parallel_unshard_dtensor_api.py create mode 100644 test/auto_parallel/test_semi_auto_parallel_unshard_dtensor.py diff --git a/python/paddle/distributed/__init__.py b/python/paddle/distributed/__init__.py index d6bc43dbce287..b7fa016e78b5a 100644 --- a/python/paddle/distributed/__init__.py +++ b/python/paddle/distributed/__init__.py @@ -87,6 +87,7 @@ to_static, Strategy, DistModel, + unshard_dtensor, ) from .fleet import BoxPSDataset # noqa: F401 @@ -169,4 +170,5 @@ "to_static", "Strategy", "DistModel", + "unshard_dtensor", ] diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index 67ef633485add..f8eb3f71f89b9 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -1433,3 +1433,73 @@ def to_static( dist_loader = dist_model.dist_loader return dist_model, dist_loader + + +def unshard_dtensor(dist_tensor): + """ + Converts a distributed tensor to a dense tensor. ``unshard_dtensor`` + first make the ``dist_tensor`` be ``Replicate`` state on all processes and + then converts it to a dense ``paddle.Tensor``. It can be treated as a + reverse operation of ``shard_tensor``. + + Args: + dist_tensor (paddle.Tensor): The distributed tensor which is constructed + from a dense tensor with ``shard_tensor``. + + Returns: + paddle.Tensor: The original dense tensor of the input ``dist_tensor``. + + Examples: + .. code-block:: python + + >>> import paddle + >>> import paddle.distributed as dist + >>> from paddle.distributed import Replicate, Shard + + >>> # doctest: +REQUIRES(env:DISTRIBUTED) + >>> mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + >>> original_tensor = paddle.rand([4, 1024, 512]) + >>> dist_tensor = dist.shard_tensor(original_tensor, mesh, [Shard(0)]) + >>> # dense_tensor's shape is the same as original_tensor + >>> dense_tensor = dist.unshard_dtensor(dist_tensor) + """ + if paddle.in_dynamic_mode(): + # if the input is not a distributed + # tensor, return it directly + if dist_tensor.is_dist() is False: + raise ValueError("The input should be a distributed tensor.") + + mesh = dist_tensor.process_mesh + placements = dist_tensor.placements + replicate_placements = [dist.Replicate()] * len(placements) + r_dist_tensor = reshard(dist_tensor, mesh, replicate_placements) + + if isinstance(dist_tensor, EagerParamBase): + return EagerParamBase.from_tensor( + r_dist_tensor._local_value(), + **dist_tensor.__dict__, + ) + else: + return paddle.Tensor(r_dist_tensor._local_value()) + + else: + assert isinstance( + dist_tensor, Variable + ), "the input type of 'unshard_dtensor' should be Variable, but got [{}]".format( + dist_tensor + ) + # in static mode, 'distributed tensor' and 'dense tensor' are all + # Varialble type, the distributed attribute is a property of the Varibale. + # So, it's no need to convert the distributed tensor to a dense tensor. + # We only need to modify its distributed attribute. + empty_dist_attr = ( + dist.auto_parallel.static.dist_attribute.TensorDistAttr() + ) + dist_tensor.dist_attr = empty_dist_attr + + # remove the distributed tensor from dist_context + default_dist_ctx = get_default_distributed_context() + serial_tensor_id = dist_tensor.desc.original_id() + default_dist_ctx._dist_tensors_for_program.pop(serial_tensor_id, None) + + return dist_tensor diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index 6a90434d3b198..04d6219c5946e 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -190,6 +190,10 @@ if(WITH_DISTRIBUTE AND WITH_GPU) py_test_modules(test_gpt_with_prim MODULES test_gpt_with_prim) set_tests_properties(test_gpt_with_prim PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 200) + py_test_modules(test_semi_auto_parallel_unshard_dtensor MODULES + test_semi_auto_parallel_unshard_dtensor) + set_tests_properties(test_semi_auto_parallel_unshard_dtensor + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 100) # End of unittests WITH multi cards and timeout # NOTE(zyl): unittests WITH multi cards and WITHOUT timeout diff --git a/test/auto_parallel/semi_auto_parallel_unshard_dtensor_api.py b/test/auto_parallel/semi_auto_parallel_unshard_dtensor_api.py new file mode 100644 index 0000000000000..f67381405a639 --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_unshard_dtensor_api.py @@ -0,0 +1,99 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import paddle +import paddle.distributed as dist +from paddle.base.dygraph.base import switch_to_static_graph +from paddle.distributed import Replicate, Shard +from paddle.distributed.auto_parallel.static.dist_context import ( + get_default_distributed_context, +) + + +class TestUnshardDTensor(unittest.TestCase): + def __init__(self): + self.mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + def run_dynamic(self): + ori_tensor = paddle.rand([4, 1024, 512]) + d_tensor = dist.shard_tensor(ori_tensor, self.mesh, [Shard(0)]) + dense_tensor = dist.unshard_dtensor(d_tensor) + self.assertListEqual(dense_tensor.shape, ori_tensor.shape) + self.assertFalse(dense_tensor.is_dist()) + + ori_parameter = paddle.create_parameter([1024, 512], dtype='float32') + d_tensor = dist.shard_tensor(ori_parameter, self.mesh, [Shard(0)]) + dense_parameter = dist.unshard_dtensor(d_tensor) + self.assertListEqual(dense_parameter.shape, ori_parameter.shape) + self.assertFalse(dense_parameter.is_dist()) + self.assertTrue( + isinstance(dense_parameter, paddle.base.framework.EagerParamBase) + ) + + @switch_to_static_graph + def run_static(self): + ori_tensor = paddle.static.data( + name="input", + shape=[4, 1024, 512], + dtype='float32', + ) + self.assertIsNone(ori_tensor.dist_attr.process_mesh) + d_tensor = dist.shard_tensor(ori_tensor, self.mesh, [Shard(0)]) + + default_dist_context = get_default_distributed_context() + dist_input = default_dist_context.get_dist_tensor_for_program( + ori_tensor + ) + self.assertEqual(dist_input.dist_attr.process_mesh, self.mesh) + + dense_tensor = dist.unshard_dtensor(d_tensor) + dist_input = default_dist_context.get_dist_tensor_for_program( + ori_tensor + ) + self.assertTupleEqual(dense_tensor.shape, ori_tensor.shape) + self.assertIsNone(dense_tensor.dist_attr.process_mesh) + self.assertIsNone(dist_input) + + def run_dy2static(self): + @paddle.jit.to_static(full_graph=True) + def unshard_func(): + mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + input = paddle.rand([4, 1024, 512]) + d_tensor = dist.shard_tensor(input, mesh, [Replicate()]) + dense_tensor = dist.unshard_dtensor(d_tensor) + return input, dense_tensor + + dy_ori_tensor, dy_dense_tensor = unshard_func() + st_ori_tensor = unshard_func.outputs[0] + st_dense_tensor = unshard_func.outputs[1] + self.assertListEqual(dy_dense_tensor.shape, dy_ori_tensor.shape) + self.assertFalse(dy_dense_tensor.is_dist()) + + default_dist_context = get_default_distributed_context() + dist_input = default_dist_context.get_dist_tensor_for_program( + st_ori_tensor + ) + self.assertIsNone(st_dense_tensor.dist_attr.process_mesh) + self.assertIsNone(dist_input) + + def run_test_cases(self): + self.run_dynamic() + self.run_static() + self.run_dy2static() + + +if __name__ == "__main__": + TestUnshardDTensor().run_test_cases() diff --git a/test/auto_parallel/test_semi_auto_parallel_unshard_dtensor.py b/test/auto_parallel/test_semi_auto_parallel_unshard_dtensor.py new file mode 100644 index 0000000000000..a62527eb1647b --- /dev/null +++ b/test/auto_parallel/test_semi_auto_parallel_unshard_dtensor.py @@ -0,0 +1,40 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelUnshardDTensor(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp( + num_of_devices=2, + timeout=100, + ) + self._default_envs = {"dtype": "float32", "seed": "2023"} + self._changeable_envs = {"backend": ["gpu"]} + + def test_api_function(self): + envs_list = test_base.gen_product_envs_list( + {"dtype": "float32", "seed": "2023"}, {"backend": ["gpu"]} + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_unshard_dtensor_api.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() From caa171552152424a2adcfe6bef9babb2039434a2 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Wed, 27 Dec 2023 14:54:07 +0800 Subject: [PATCH 084/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.272?= =?UTF-8?q?=E3=80=91Migrate=20LookAhead=20to=20pir=20(#60346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/incubate/optimizer/lookahead.py | 78 +++++++++++++------ python/paddle/optimizer/sgd.py | 4 +- test/legacy_test/test_lookahead.py | 28 ++++--- 3 files changed, 75 insertions(+), 35 deletions(-) diff --git a/python/paddle/incubate/optimizer/lookahead.py b/python/paddle/incubate/optimizer/lookahead.py index 821b5c3ce036c..12cb00ba7a3ff 100644 --- a/python/paddle/incubate/optimizer/lookahead.py +++ b/python/paddle/incubate/optimizer/lookahead.py @@ -17,7 +17,9 @@ from paddle.base.dygraph import base as imperative_base from paddle.base.framework import Variable from paddle.base.layer_helper import LayerHelper +from paddle.framework import in_pir_mode from paddle.optimizer import Optimizer +from paddle.pir.core import create_parameter __all__ = [] @@ -121,7 +123,9 @@ def __init__(self, inner_optimizer, alpha=0.5, k=5, name=None): self.inner_optimizer = inner_optimizer if self.inner_optimizer._parameter_list is None: parameters = ( - framework.default_main_program().global_block().all_parameters() + paddle.static.default_main_program() + .global_block() + .all_parameters() ) else: parameters = self.inner_optimizer._parameter_list @@ -186,40 +190,64 @@ def step(self): ) def _create_accumulators(self, block, parameters): - assert isinstance(block, framework.Block) + assert isinstance(block, (framework.Block, paddle.pir.Block)) for p in parameters: self._add_accumulator(self._slow_str, p) def _increment_global_var(self): - if self._global_step_var is None: - self._global_step_var = paddle.static.create_global_var( - name=unique_name.generate("lookahead_step"), - shape=[1], - value=0, - dtype='int32', - persistable=True, + if in_pir_mode(): + if self._global_step_var is None: + self._global_step_var = create_parameter( + dtype='int32', + shape=[1], + name=unique_name.generate("lookahead_step"), + trainable=False, + initializer=paddle.nn.initializer.ConstantInitializer( + value=0.0, force_cpu=False + ), + ) + self._global_step_var = paddle.increment(self._global_step_var, 1.0) + else: + if self._global_step_var is None: + self._global_step_var = paddle.static.create_global_var( + name=unique_name.generate("lookahead_step"), + shape=[1], + value=0, + dtype='int32', + persistable=True, + ) + + self.helper.append_op( + type='increment', + inputs={'X': [self._global_step_var]}, + outputs={'Out': [self._global_step_var]}, + attrs={'step': 1.0}, ) - self.helper.append_op( - type='increment', - inputs={'X': [self._global_step_var]}, - outputs={'Out': [self._global_step_var]}, - attrs={'step': 1.0}, - ) - def _append_optimize_op(self, block, param_and_grad): one_var = paddle.ones(shape=[1], dtype='int32', name='lookahead_ones') zero_var = paddle.zeros( shape=[1], dtype='int32', name='lookahead_zeros' ) - k_var = paddle.static.create_global_var( - name=unique_name.generate("lookahead_k"), - shape=[1], - value=self.k, - dtype='int32', - persistable=True, - ) + if in_pir_mode(): + k_var = create_parameter( + dtype='int32', + shape=[1], + name=unique_name.generate("lookahead_k"), + trainable=False, + initializer=paddle.nn.initializer.ConstantInitializer( + value=float(self.k), force_cpu=False + ), + ) + else: + k_var = paddle.static.create_global_var( + name=unique_name.generate("lookahead_k"), + shape=[1], + value=self.k, + dtype='int32', + persistable=True, + ) mod = paddle.remainder(self._global_step_var, k_var) @@ -284,7 +312,9 @@ def minimize( >>> lookahead.clear_grad() """ - assert isinstance(loss, Variable), "The loss should be an Tensor." + assert isinstance( + loss, (Variable, paddle.pir.Value) + ), "The loss should be an Tensor." # Apply inner optimizer to the main_program optimize_ops, params_grads = self.inner_optimizer.minimize( diff --git a/python/paddle/optimizer/sgd.py b/python/paddle/optimizer/sgd.py index dcd8e17d51cb6..233261f7f769b 100644 --- a/python/paddle/optimizer/sgd.py +++ b/python/paddle/optimizer/sgd.py @@ -14,7 +14,7 @@ import warnings -from paddle import _C_ops +from paddle import _C_ops, pir from ..base import framework from ..base.dygraph import no_grad @@ -93,7 +93,7 @@ def __init__( self._master_weights = {} def _create_accumulators(self, block, parameters): - assert isinstance(block, framework.Block) + assert isinstance(block, (framework.Block, pir.Block)) if isinstance(parameters, dict): parameters = self._update_param_group(parameters) diff --git a/test/legacy_test/test_lookahead.py b/test/legacy_test/test_lookahead.py index f9eeee2f651a4..911db37cd8977 100644 --- a/test/legacy_test/test_lookahead.py +++ b/test/legacy_test/test_lookahead.py @@ -18,6 +18,8 @@ import paddle from paddle import base, nn +from paddle.base.framework import in_pir_mode +from paddle.pir_utils import test_with_pir_api LOOKAHEAD_K = 5 LOOKAHEAD_ALPHA = 0.2 @@ -25,20 +27,21 @@ class TestLookAhead(unittest.TestCase): + @test_with_pir_api def test_lookahead_static(self): paddle.enable_static() place = base.CPUPlace() shape = [2, 3, 8, 8] exe = base.Executor(place) - train_program = base.Program() - startup = base.Program() - with base.program_guard(train_program, startup): + train_program = paddle.static.Program() + startup = paddle.static.Program() + with paddle.static.program_guard(train_program, startup): with base.unique_name.guard(): data = paddle.static.data( name='X', shape=[None, 1], dtype='float32' ) - hidden = paddle.static.nn.fc(x=data, size=10) - loss = paddle.mean(hidden) + hidden = paddle.nn.Linear(1, 10) + loss = paddle.mean(hidden(data)) optimizer = paddle.optimizer.SGD(learning_rate=SGD_LR) lookahead = paddle.incubate.optimizer.LookAhead( @@ -55,13 +58,20 @@ def test_lookahead_static(self): fast_param - slow_param ) x = np.random.random(size=(10, 1)).astype('float32') + if in_pir_mode(): + for op in train_program.global_block().ops: + if op.name() == 'pd_op.add_grad': + bias_grad = op.result(1) + fetch_list = [hidden.bias, bias_grad] + else: + fetch_list = [ + 'linear_0.b_0', + 'linear_0.b_0@GRAD', + ] latest_b, b_grad = exe.run( program=train_program, feed={'X': x}, - fetch_list=[ - 'fc_0.b_0', - 'fc_0.b_0@GRAD', - ], + fetch_list=fetch_list, ) if i == 0: slow_param = latest_b From 99528466f8f3399f1d84ec3d37aacbc396c445c9 Mon Sep 17 00:00:00 2001 From: Leo Chen <39020268+leo0519@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:16:54 +0800 Subject: [PATCH 085/146] Fix and refactor trt_skip_layernorm ut (#59980) --- .../test_trt_skip_layernorm_fuse_pass.py | 213 +++++------------- 1 file changed, 54 insertions(+), 159 deletions(-) diff --git a/test/ir/inference/test_trt_skip_layernorm_fuse_pass.py b/test/ir/inference/test_trt_skip_layernorm_fuse_pass.py index 4d671c33cdfb8..7fe31ae5b4034 100644 --- a/test/ir/inference/test_trt_skip_layernorm_fuse_pass.py +++ b/test/ir/inference/test_trt_skip_layernorm_fuse_pass.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,39 +25,46 @@ from paddle.base.core import AnalysisConfig, PassVersionChecker -class SkipLayernormFusePassTest0(InferencePassTest): +class SkipLayernormFusePassTest(InferencePassTest): def setUp(self): + self.set_args() + input_shape_with_batch = [self.batch_size] + self.input_shape + min_input_shape_with_batch = [1] + self.min_input_shape with base.program_guard(self.main_program, self.startup_program): data1 = paddle.static.data( - name="data1", shape=[-1, 3, 128, 128], dtype="float32" + name='data1', shape=[-1] + self.input_shape, dtype='float32' ) data2 = paddle.static.data( - name="data2", shape=[-1, 3, 128, 128], dtype="float32" - ) - eltwise_out = self.append_eltwise(data1, data2) - out = paddle.nn.functional.layer_norm( - eltwise_out, eltwise_out.shape[1:] + name='data2', shape=[-1] + self.input_shape, dtype='float32' ) + eltwise_out = paddle.add(data1, data2) + out = paddle.nn.LayerNorm(eltwise_out.shape[-1:])(eltwise_out) self.feeds = { - "data1": np.random.random([1, 3, 128, 128]).astype("float32"), - "data2": np.random.random([1, 3, 128, 128]).astype("float32"), + 'data1': np.random.random(input_shape_with_batch).astype('float32'), + 'data2': np.random.random(input_shape_with_batch).astype('float32'), } self.enable_trt = True - self.trt_parameters = SkipLayernormFusePassTest0.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False + self.trt_parameters = SkipLayernormFusePassTest.TensorRTParam( + 1 << 30, 32, 0, self.trt_precision, True, False ) - self.dynamic_shape_params = ( - SkipLayernormFusePassTest0.DynamicShapeParam( - {'data1': [1, 1, 1, 128], 'data2': [1, 1, 1, 128]}, - {'data1': [1, 3, 128, 128], 'data2': [1, 3, 128, 128]}, - {'data1': [1, 3, 128, 128], 'data2': [1, 3, 128, 128]}, - False, - ) + self.dynamic_shape_params = SkipLayernormFusePassTest.DynamicShapeParam( + { + 'data1': min_input_shape_with_batch, + 'data2': min_input_shape_with_batch, + }, + {'data1': input_shape_with_batch, 'data2': input_shape_with_batch}, + {'data1': input_shape_with_batch, 'data2': input_shape_with_batch}, + False, ) self.fetch_list = [out] - def append_eltwise(self, data1, data2): - return paddle.add(data1, data2) + def set_args(self): + self.input_shape = [3, 128, 256] + self.batch_size = 1 + self.trt_precision = AnalysisConfig.Precision.Float32 + self.min_input_shape = [1, 1, 256] + self.atol = 1e-2 + self.rtol = 1e-5 def test_check_output(self): opt_path = os.path.join(self.path, '_opt_cache') @@ -65,154 +72,42 @@ def test_check_output(self): shutil.rmtree(opt_path) if core.is_compiled_with_cuda(): use_gpu = True - self.check_output_with_option(use_gpu, atol=0.01, rtol=0.00001) - self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') - ) - - -class SkipLayernormFusePassTest1(InferencePassTest): - def setUp(self): - with base.program_guard(self.main_program, self.startup_program): - data1 = paddle.static.data( - name="data1", shape=[-1, 256, 1536], dtype="float32" - ) - data2 = paddle.static.data( - name="data2", shape=[-1, 256, 1536], dtype="float32" - ) - eltwise_out = self.append_eltwise(data1, data2) - - out = paddle.nn.functional.layer_norm( - eltwise_out, eltwise_out.shape[1:] - ) - - self.feeds = { - "data1": np.random.random([1, 256, 1536]).astype("float32"), - "data2": np.random.random([1, 256, 1536]).astype("float32"), - } - self.enable_trt = True - self.trt_parameters = SkipLayernormFusePassTest1.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Float32, True, False - ) - self.dynamic_shape_params = ( - SkipLayernormFusePassTest1.DynamicShapeParam( - {'data1': [1, 1, 1], 'data2': [1, 1, 1]}, - {'data1': [1, 384, 1536], 'data2': [1, 384, 1536]}, - {'data1': [1, 384, 1536], 'data2': [1, 384, 1536]}, - False, - ) - ) - self.fetch_list = [out] - - def append_eltwise(self, data1, data2): - return paddle.add(data1, data2) - - def test_check_output(self): - opt_path = os.path.join(self.path, '_opt_cache') - if os.path.exists(opt_path): - shutil.rmtree(opt_path) - if core.is_compiled_with_cuda(): - use_gpu = True - self.check_output_with_option(use_gpu, atol=0.01, rtol=0.00001) - self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') - ) - - -class SkipLayernormFusePassTest2(InferencePassTest): - def setUp(self): - with base.program_guard(self.main_program, self.startup_program): - data1 = paddle.static.data( - name="data1", shape=[-1, 128, 64, 768], dtype="float32" - ) - data2 = paddle.static.data( - name="data2", shape=[-1, 128, 64, 768], dtype="float32" + self.check_output_with_option( + use_gpu, atol=self.atol, rtol=self.rtol ) - eltwise_out = self.append_eltwise(data1, data2) - - out = paddle.nn.functional.layer_norm( - eltwise_out, eltwise_out.shape[1:] - ) - - self.feeds = { - "data1": np.random.random([1, 128, 64, 768]).astype("float32"), - "data2": np.random.random([1, 128, 64, 768]).astype("float32"), - } - self.enable_trt = True - self.trt_parameters = SkipLayernormFusePassTest2.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False - ) - self.dynamic_shape_params = ( - SkipLayernormFusePassTest2.DynamicShapeParam( - {'data1': [1, 1, 1, 1], 'data2': [1, 1, 1, 1]}, - {'data1': [1, 128, 64, 768], 'data2': [1, 128, 64, 768]}, - {'data1': [1, 128, 64, 768], 'data2': [1, 128, 64, 768]}, - False, - ) - ) - self.fetch_list = [out] - - def append_eltwise(self, data1, data2): - return paddle.add(data1, data2) - - def test_check_output(self): - opt_path = os.path.join(self.path, '_opt_cache') - if os.path.exists(opt_path): - shutil.rmtree(opt_path) - if core.is_compiled_with_cuda(): - use_gpu = True - self.check_output_with_option(use_gpu, atol=0.1, rtol=0.00001) self.assertTrue( PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') ) -class SkipLayernormFusePassTest3(InferencePassTest): - def setUp(self): - with base.program_guard(self.main_program, self.startup_program): - data1 = paddle.static.data( - name="data1", shape=[-1, 128, 128], dtype="float32" - ) - data2 = paddle.static.data( - name="data2", shape=[-1, 128, 128], dtype="float32" - ) - eltwise_out = self.append_eltwise(data1, data2) +class SkipLayernormFusePassTest1(SkipLayernormFusePassTest): + def set_args(self): + self.input_shape = [256, 1536] + self.batch_size = 1 + self.trt_precision = AnalysisConfig.Precision.Float32 + self.min_input_shape = [1, 1] + self.atol = 1e-2 + self.rtol = 1e-5 - out = paddle.nn.functional.layer_norm( - eltwise_out, eltwise_out.shape[1:] - ) - self.feeds = { - "data1": np.random.random([1, 128, 128]).astype("float32"), - "data2": np.random.random([1, 128, 128]).astype("float32"), - } - self.enable_trt = True - self.trt_parameters = SkipLayernormFusePassTest3.TensorRTParam( - 1 << 30, 32, 0, AnalysisConfig.Precision.Half, True, False - ) - self.dynamic_shape_params = ( - SkipLayernormFusePassTest3.DynamicShapeParam( - {'data1': [1, 1, 1], 'data2': [1, 1, 1]}, - {'data1': [1, 128, 128], 'data2': [1, 128, 128]}, - {'data1': [1, 128, 128], 'data2': [1, 128, 128]}, - False, - ) - ) - self.fetch_list = [out] +class SkipLayernormFusePassTest2(SkipLayernormFusePassTest): + def set_args(self): + self.input_shape = [128, 64, 768] + self.batch_size = 1 + self.trt_precision = AnalysisConfig.Precision.Half + self.min_input_shape = [1, 1, 1] + self.atol = 1e-1 + self.rtol = 1e-5 - def append_eltwise(self, data1, data2): - return paddle.add(data1, data2) - def test_check_output(self): - opt_path = os.path.join(self.path, '_opt_cache') - if os.path.exists(opt_path): - shutil.rmtree(opt_path) - if core.is_compiled_with_cuda(): - use_gpu = True - self.check_output_with_option(use_gpu, atol=0.1, rtol=0.00001) - self.assertTrue( - PassVersionChecker.IsCompatible('tensorrt_subgraph_pass') - ) +class SkipLayernormFusePassTest3(SkipLayernormFusePassTest): + def set_args(self): + self.input_shape = [128, 256] + self.batch_size = 1 + self.trt_precision = AnalysisConfig.Precision.Half + self.min_input_shape = [1, 1] + self.atol = 1e-1 + self.rtol = 1e-5 if __name__ == "__main__": From 430894e389dcca5108b296538fa40f305c67082b Mon Sep 17 00:00:00 2001 From: feifei-111 <2364819892@qq.com> Date: Wed, 27 Dec 2023 15:27:37 +0800 Subject: [PATCH 086/146] fix data_op device for gpu pinned tensor (#60357) --- .../pir/transforms/pd_op_to_kernel_pass.cc | 7 ++++++- .../executor/function_graph.py | 4 +--- .../executor/opcode_executor.py | 2 +- python/paddle/jit/sot/profiler.py | 17 ++++------------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 4731b61541e21..91ca8a0d4b3f6 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -689,7 +689,12 @@ phi::KernelKey GetKernelKey( auto data_place = op->attributes().at("place").dyn_cast().data(); - auto backend = paddle::experimental::ParseBackend(data_place); + phi::Backend backend; + if (data_place.GetType() == AllocationType::GPUPINNED) { + backend = phi::Backend::CPU; + } else { + backend = paddle::experimental::ParseBackend(data_place); + } return {backend, phi::DataLayout::ANY, diff --git a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py index ab525a71e360d..8d86c0565baa4 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/function_graph.py +++ b/python/paddle/jit/sot/opcode_translator/executor/function_graph.py @@ -242,9 +242,7 @@ def collect(inp): def guard_fn(self) -> Guard: with tmp_name_guard(): guards = [] - with EventGuard( - "guard_fn: find vars and make stringify guard", event_level=1 - ): + with EventGuard("guard_fn: find vars and make stringify guard"): for variable in find_traceable_vars( self.input_variables + list(self._global_guarded_variables) ): diff --git a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py index 66efe59674234..655eb7317dccc 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py +++ b/python/paddle/jit/sot/opcode_translator/executor/opcode_executor.py @@ -562,7 +562,7 @@ def step(self, instr: Instruction): opname = instr.opname if instr.opname != "PRECALL" else "PRECALL__CALL" assert opname != "CALL", "CALL should fused with PRECALL" - with EventGuard(f"{opname}", event_level=1): + with EventGuard(f"{opname}", event_level=2): return getattr(self, opname)(instr) # run single step. def indexof(self, instr: Instruction): diff --git a/python/paddle/jit/sot/profiler.py b/python/paddle/jit/sot/profiler.py index 8315e03dd37f5..9312552f4d8cf 100644 --- a/python/paddle/jit/sot/profiler.py +++ b/python/paddle/jit/sot/profiler.py @@ -18,7 +18,7 @@ from paddle.framework import core -_event_level = int(os.environ.get("EVENT_LEVEL", "-1")) +_event_level = int(os.environ.get("EVENT_LEVEL", "0")) class SotProfiler: @@ -37,7 +37,7 @@ def disable(self): @contextmanager -def EventGuard(event_name, event_level=0): +def EventGuard(event_name, event_level=1): try: global _event_level need_pop = False @@ -50,20 +50,11 @@ def EventGuard(event_name, event_level=0): core.nvprof_nvtx_pop() -if _event_level == -1: - - @contextmanager - def _EmptyEventGuard(event_name, event_level=0): - yield - - EventGuard = _EmptyEventGuard # noqa: F811 - - -def event_register(event_name, event_level=0): +def event_register(event_name, event_level=1): def event_wrapper(func): @wraps(func) def call_with_event(*args, **kwargs): - with EventGuard(event_name, event_level=0): + with EventGuard(event_name, event_level=event_level): return func(*args, **kwargs) return call_with_event From 50aebcf50439ef9b6346d8ec995cc3edb8d45d7e Mon Sep 17 00:00:00 2001 From: JYChen Date: Wed, 27 Dec 2023 15:28:21 +0800 Subject: [PATCH 087/146] optimize slice over limit case in static (#60276) * optimize slice over limit case in static * fix code --- python/paddle/base/variable_index.py | 37 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index 0df9ebc5513da..ca3a107765dcb 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -219,6 +219,26 @@ def deal_advanced_index(ori_tensor, indices, is_for_setitem): ) +def slice_is_same_to_original(start, end, step): + if start is None and end is None and step is None: + return True + + # If there is Variable, we cannot determine whether it is the same to original. + if isinstance( + start, (paddle.base.Variable, paddle.pir.Value, paddle.pir.OpResult) + ): + return False + if isinstance( + end, (paddle.base.Variable, paddle.pir.Value, paddle.pir.OpResult) + ): + return False + if isinstance( + step, (paddle.base.Variable, paddle.pir.Value, paddle.pir.OpResult) + ): + return False + return start == 0 and end == MAX_INTEGER and step == 1 + + def parse_index(x, indices): advanced_index = [None] * 2 * len(x.shape) # content is (dim, index) # for set_value / slice / strided_slice OP @@ -283,9 +303,10 @@ def parse_index(x, indices): start = slice_item.start end = slice_item.stop step = slice_item.step - estimated_dim += 1 - dim += 1 + if start is None and end is None and step is None: + estimated_dim += 1 + dim += 1 continue step = 1 if step is None else step @@ -294,6 +315,16 @@ def parse_index(x, indices): if end is None: end = MAX_INTEGER if step > 0 else -1 + if not ( + is_tensor_array + or isinstance(end, (paddle.base.Variable, paddle.pir.Value)) + or isinstance(step, (paddle.base.Variable, paddle.pir.Value)) + ): + if x.shape[dim] != -1 and end >= x.shape[dim]: + end = MAX_INTEGER if step > 0 else -1 + estimated_dim += 1 + dim += 1 + elif isinstance(slice_item, (list, tuple)): advanced_index[estimated_dim] = ( estimated_dim, @@ -356,7 +387,7 @@ def parse_index(x, indices): slice_item ) ) - if not (start is None or end is None or step is None): + if not slice_is_same_to_original(start, end, step): starts.append(start) ends.append(end) steps.append(step) From 8f598d9fba34e39169ba96fd4828ce874dc133ce Mon Sep 17 00:00:00 2001 From: LoneRanger <836253168@qq.com> Date: Wed, 27 Dec 2023 16:27:57 +0800 Subject: [PATCH 088/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.75-76?= =?UTF-8?q?=E3=80=91Migrate=20some=20ops=20into=20pir=20(#59627)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/nn/functional/common.py | 2 +- python/paddle/nn/functional/loss.py | 10 +++++++--- test/legacy_test/test_fold_op.py | 4 ++-- test/legacy_test/test_sigmoid_focal_loss.py | 2 ++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index 8988e89111c09..1fb678efd0b13 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -2401,7 +2401,7 @@ def _is_list_or_turple_(data): "of 2 or 4 integers" ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): out = _C_ops.fold( x, output_sizes, kernel_sizes, strides, paddings, dilations ) diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index d1611106b7c52..809056cf39aaf 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -3176,7 +3176,7 @@ def sigmoid_focal_loss( ) ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): place = _current_expected_place() one = _C_ops.full(logit.shape, 1.0, logit.dtype, place) @@ -3193,7 +3193,10 @@ def sigmoid_focal_loss( ), ) - alpha = base.dygraph.base.to_variable([alpha], dtype=loss.dtype) + if in_dynamic_mode(): + alpha = base.dygraph.base.to_variable([alpha], dtype=loss.dtype) + else: + alpha = paddle.to_tensor(alpha, dtype=loss.dtype) alpha_t = _C_ops.add( _C_ops.multiply(alpha, label), _C_ops.multiply( @@ -3202,7 +3205,8 @@ def sigmoid_focal_loss( ) loss = _C_ops.multiply(alpha_t, loss) - gamma = base.dygraph.base.to_variable([gamma], dtype=loss.dtype) + if in_dynamic_mode(): + gamma = base.dygraph.base.to_variable([gamma], dtype=loss.dtype) gamma_t = _C_ops.pow(_C_ops.subtract(one, p_t), gamma) loss = _C_ops.multiply(gamma_t, loss) diff --git a/test/legacy_test/test_fold_op.py b/test/legacy_test/test_fold_op.py index 8e4ab1971b7ae..18aa7886bff7b 100644 --- a/test/legacy_test/test_fold_op.py +++ b/test/legacy_test/test_fold_op.py @@ -133,10 +133,10 @@ def setUp(self): self.set_data() def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X'], 'Y') + self.check_grad(['X'], 'Y', check_pir=True) class TestFold_Complex64(TestFoldOp): diff --git a/test/legacy_test/test_sigmoid_focal_loss.py b/test/legacy_test/test_sigmoid_focal_loss.py index b151d4c56a21e..9142375f37694 100644 --- a/test/legacy_test/test_sigmoid_focal_loss.py +++ b/test/legacy_test/test_sigmoid_focal_loss.py @@ -18,6 +18,7 @@ import paddle from paddle import base +from paddle.pir_utils import test_with_pir_api def call_sfl_functional( @@ -119,6 +120,7 @@ def calc_sigmoid_focal_loss( class TestSigmoidFocalLoss(unittest.TestCase): + @test_with_pir_api def test_SigmoidFocalLoss(self): logit_np = np.random.uniform(0.1, 0.8, size=(2, 3, 4, 10)).astype( np.float64 From c79c631b7d728baac87e312dfa5a7a9694790229 Mon Sep 17 00:00:00 2001 From: enzodechine Date: Wed, 27 Dec 2023 17:37:25 +0800 Subject: [PATCH 089/146] bind bf16 strided_slice&grad (#60382) --- paddle/phi/kernels/strided_slice_grad_kernel.cc | 3 ++- paddle/phi/kernels/strided_slice_kernel.cc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/phi/kernels/strided_slice_grad_kernel.cc b/paddle/phi/kernels/strided_slice_grad_kernel.cc index 7582f751bf16a..8c5c90783133c 100644 --- a/paddle/phi/kernels/strided_slice_grad_kernel.cc +++ b/paddle/phi/kernels/strided_slice_grad_kernel.cc @@ -78,5 +78,6 @@ PD_REGISTER_KERNEL(strided_slice_grad, int, int16_t, float, - phi::dtype::float16) {} + phi::dtype::float16, + phi::dtype::bfloat16) {} #endif diff --git a/paddle/phi/kernels/strided_slice_kernel.cc b/paddle/phi/kernels/strided_slice_kernel.cc index 0852cc8830e2c..2bc9325de1ee7 100644 --- a/paddle/phi/kernels/strided_slice_kernel.cc +++ b/paddle/phi/kernels/strided_slice_kernel.cc @@ -76,5 +76,6 @@ PD_REGISTER_KERNEL(strided_slice, int, int16_t, float, - phi::dtype::float16) {} + phi::dtype::float16, + phi::dtype::bfloat16) {} #endif From fd9e67c14b6f45d5bb9c2e754d89ae0b3b80b88e Mon Sep 17 00:00:00 2001 From: enzodechine Date: Wed, 27 Dec 2023 17:37:44 +0800 Subject: [PATCH 090/146] [XPU]support bf16 elementwise_sub and div (#60386) * support bf16 elementwise_sub and div * support bf16 elementwise_sub and div --- paddle/phi/backends/xpu/xpu3_op_list.cc | 10 ++++++++-- .../phi/kernels/xpu/elementwise_divide_grad_kernel.cc | 1 + paddle/phi/kernels/xpu/elementwise_divide_kernel.cc | 1 + .../kernels/xpu/elementwise_subtract_grad_kernel.cc | 1 + paddle/phi/kernels/xpu/elementwise_subtract_kernel.cc | 1 + 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/paddle/phi/backends/xpu/xpu3_op_list.cc b/paddle/phi/backends/xpu/xpu3_op_list.cc index 623f63444c308..016e5ef917af5 100644 --- a/paddle/phi/backends/xpu/xpu3_op_list.cc +++ b/paddle/phi/backends/xpu/xpu3_op_list.cc @@ -266,10 +266,13 @@ XPUOpMap& get_kl3_ops() { phi::DataType::INT64, phi::DataType::INT32})}, {"elementwise_div_grad", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"elementwise_div", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, + phi::DataType::BFLOAT16, phi::DataType::INT64, phi::DataType::INT32})}, {"elementwise_floordiv", @@ -295,10 +298,13 @@ XPUOpMap& get_kl3_ops() { {"elementwise_pow", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"elementwise_sub_grad", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"elementwise_sub", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, + phi::DataType::BFLOAT16, phi::DataType::INT32, phi::DataType::INT64})}, {"elementwise_mod", diff --git a/paddle/phi/kernels/xpu/elementwise_divide_grad_kernel.cc b/paddle/phi/kernels/xpu/elementwise_divide_grad_kernel.cc index 3b20874b5f312..eeba11974c304 100644 --- a/paddle/phi/kernels/xpu/elementwise_divide_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/elementwise_divide_grad_kernel.cc @@ -59,4 +59,5 @@ PD_REGISTER_KERNEL(divide_grad, ALL_LAYOUT, phi::DivideGradKernel, phi::dtype::float16, + phi::dtype::bfloat16, float) {} diff --git a/paddle/phi/kernels/xpu/elementwise_divide_kernel.cc b/paddle/phi/kernels/xpu/elementwise_divide_kernel.cc index 2f608879cd7e0..41f20b061fae6 100644 --- a/paddle/phi/kernels/xpu/elementwise_divide_kernel.cc +++ b/paddle/phi/kernels/xpu/elementwise_divide_kernel.cc @@ -50,5 +50,6 @@ PD_REGISTER_KERNEL(divide, phi::DivideKernel, float, phi::dtype::float16, + phi::dtype::bfloat16, int, int64_t) {} diff --git a/paddle/phi/kernels/xpu/elementwise_subtract_grad_kernel.cc b/paddle/phi/kernels/xpu/elementwise_subtract_grad_kernel.cc index d22b369619d40..f61a5f5de9410 100644 --- a/paddle/phi/kernels/xpu/elementwise_subtract_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/elementwise_subtract_grad_kernel.cc @@ -53,4 +53,5 @@ PD_REGISTER_KERNEL(subtract_grad, ALL_LAYOUT, phi::SubtractGradKernel, phi::dtype::float16, + phi::dtype::bfloat16, float) {} diff --git a/paddle/phi/kernels/xpu/elementwise_subtract_kernel.cc b/paddle/phi/kernels/xpu/elementwise_subtract_kernel.cc index a3252b7534dbf..8ba3c47a456e9 100644 --- a/paddle/phi/kernels/xpu/elementwise_subtract_kernel.cc +++ b/paddle/phi/kernels/xpu/elementwise_subtract_kernel.cc @@ -44,5 +44,6 @@ PD_REGISTER_KERNEL(subtract, phi::SubtractKernel, float, phi::dtype::float16, + phi::dtype::bfloat16, int, int64_t) {} From 363a11b3043fc4db9fb9b4e25b91c71218b54b61 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Wed, 27 Dec 2023 18:34:08 +0800 Subject: [PATCH 091/146] [PIR] inplace pass support sub block (#60369) * inplace pass support sub block * update * update * fix typo --- paddle/fluid/pir/transforms/inplace_pass.cc | 125 ++++++++++-------- .../transforms/transform_general_functions.cc | 58 +++++++- .../transforms/transform_general_functions.h | 30 ++++- paddle/fluid/pybind/control_flow_api.cc | 50 +------ 4 files changed, 150 insertions(+), 113 deletions(-) diff --git a/paddle/fluid/pir/transforms/inplace_pass.cc b/paddle/fluid/pir/transforms/inplace_pass.cc index eaaaeba7b28b6..b836617321f8c 100644 --- a/paddle/fluid/pir/transforms/inplace_pass.cc +++ b/paddle/fluid/pir/transforms/inplace_pass.cc @@ -28,6 +28,7 @@ #include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h" #include "paddle/fluid/pir/dialect/operator/utils/utils.h" #include "paddle/fluid/pir/transforms/inplace_pass.h" +#include "paddle/fluid/pir/transforms/transform_general_functions.h" #include "paddle/phi/core/flags.h" #include "paddle/pir/core/builtin_op.h" #include "paddle/pir/core/operation.h" @@ -36,17 +37,17 @@ PHI_DECLARE_string(ir_inplace_kernel_blacklist); -namespace details { +namespace { using TensorType = paddle::dialect::AllocatedDenseTensorType; -static std::unordered_set ignore_shape_check_ops = { +std::unordered_set IgnoreShapeCheckOps = { paddle::dialect::ReshapeOp::name(), paddle::dialect::SqueezeOp::name(), paddle::dialect::UnsqueezeOp::name(), }; -static std::unordered_set relax_shape_check_ops = { +std::unordered_set RelaxShapeCheckOps = { paddle::dialect::ReshapeGradOp::name(), paddle::dialect::AddGradOp::name(), }; @@ -54,7 +55,7 @@ static std::unordered_set relax_shape_check_ops = { // NOTE(zhangbo): Which kind of value can be deleted? // (1) Value's type needs to be AllocatedDenseTensorType or // AllocatedSelectedRowsType; (2) Value's is not persisable. -static bool CanBeDeleted(pir::Value value) { +bool CanBeDeleted(pir::Value value) { if (!value.type()) { return false; } @@ -66,10 +67,10 @@ static bool CanBeDeleted(pir::Value value) { return !(persist_attr && persist_attr.data()); } -static bool CanDoInplace(const std::unordered_set& eager_dels, - pir::Value input, - pir::Value output, - const std::string& op_name) { +bool CanDoInplace(const std::unordered_set& eager_dels, + pir::Value input, + pir::Value output, + const std::string& op_name) { if (!input.type() || !output.type()) { return false; } @@ -83,7 +84,7 @@ static bool CanDoInplace(const std::unordered_set& eager_dels, return false; } - if (details::ignore_shape_check_ops.count(op_name) > 0 && + if (IgnoreShapeCheckOps.count(op_name) > 0 && eager_dels.count(input) != 0) { VLOG(9) << " -- reshape, squeeze, unsqueeze do not need check shape, " "can do inplace"; @@ -141,7 +142,7 @@ static bool CanDoInplace(const std::unordered_set& eager_dels, return in_numel == out_numel; }; bool equal = false; - bool relax = (details::relax_shape_check_ops.count(op_name) > 0); + bool relax = (RelaxShapeCheckOps.count(op_name) > 0); if (relax) { equal = is_numel_euqal_loose_version(input_alloc_tensor_type, output_alloc_tensor_type); @@ -164,7 +165,7 @@ static bool CanDoInplace(const std::unordered_set& eager_dels, return true; } -static bool IsNoNeedBuffer(pir::Operation* op, pir::Value value) { +bool IsNoNeedBuffer(pir::Operation* op, pir::Value value) { if (op->dialect()->name().compare(paddle::dialect::KernelDialect::name()) != 0) { VLOG(8) << op->name() @@ -194,9 +195,9 @@ static bool IsNoNeedBuffer(pir::Operation* op, pir::Value value) { // NOTE(zhangbo): pd_op.feed's output and pd_op.fetch's input can not be eager // deleted. -static std::unordered_set GetSkipDeletionValues(pir::Block* block) { +std::unordered_set GetSkipDeletionValues(const pir::Block& block) { std::unordered_set skip_dels; - for (auto& op : *block) { + for (auto& op : block) { if (op.dialect()->name().compare(paddle::dialect::KernelDialect::name()) != 0) { continue; @@ -223,11 +224,11 @@ static std::unordered_set GetSkipDeletionValues(pir::Block* block) { // NOTE(zhangbo): For inplace Pass, currently only the kernel_dialect operator // is supported. Therefore, this function only returns the values in the // kernel_dialect operator that can be eager deleted. -static void GetEagerDelValueOfOp( - pir::Block* block, +void GetEagerDelValueOfOp( + const pir::Block& block, const std::unordered_set& skip_dels, std::unordered_map* del_value_2_op) { - for (auto& op : *block) { + for (auto& op : block) { std::string upper_op_name = op.name(); if (op.dialect()->name().compare(paddle::dialect::KernelDialect::name()) == 0) { @@ -259,18 +260,19 @@ static void GetEagerDelValueOfOp( } } - if (op.isa()) { - auto if_op = op.dyn_cast(); - GetEagerDelValueOfOp(&if_op.true_block(), skip_dels, del_value_2_op); - VLOG(8) << "GetEagerDelValueOfOp for IfOp true block"; - GetEagerDelValueOfOp(&if_op.false_block(), skip_dels, del_value_2_op); - VLOG(8) << "GetEagerDelValueOfOp for IfOp false block"; + if (op.num_regions() > 0) { + for (size_t i = 0; i < op.num_regions(); ++i) { + for (const auto& inner_block : op.region(i)) { + GetEagerDelValueOfOp(inner_block, skip_dels, del_value_2_op); + } + } + VLOG(8) << "GetEagerDelValueOfOp for sub block"; } } } -static std::unordered_map> -GetEagerDeletionValues(pir::Block* block) { +std::unordered_map> +GetEagerDeletionValues(const pir::Block& block) { std::unordered_set skip_dels = GetSkipDeletionValues(block); std::unordered_map del_value_2_op; @@ -285,8 +287,8 @@ GetEagerDeletionValues(pir::Block* block) { return eager_dels; } -static std::unordered_map GetInplaceOps( - pir::Block* block) { +std::unordered_map GetInplaceOps( + const pir::Block& block) { const auto eager_dels = GetEagerDeletionValues(block); std::unordered_map inplace_ops; @@ -295,7 +297,7 @@ static std::unordered_map GetInplaceOps( std::unordered_set reused_input_values; std::unordered_set reused_output_values; - for (auto& op : *block) { + for (auto& op : block) { for (size_t i = 0; i < op.num_operands(); ++i) { visited_values.insert(op.operand_source(i)); } @@ -391,6 +393,8 @@ static std::unordered_map GetInplaceOps( std::unordered_map inplace_out_2_in = upper_inplace_op_info_parser.GetInplaceIdMap(); + const auto used_external_values = GetUsedExternalValue(block); + bool can_do_inplace = true; for (auto& kv : inplace_out_2_in) { uint32_t out_slot = kv.first; @@ -403,12 +407,19 @@ static std::unordered_map GetInplaceOps( (visited_values.count(op.result(out_slot)) > 0) || (!CanBeDeleted(op.result(out_slot))) || (reused_input_values.count(op.operand_source(in_slot)) > 0) || - (reused_output_values.count(op.result(out_slot)) > 0)) { + (reused_output_values.count(op.result(out_slot)) > 0) || + (std::find(used_external_values.begin(), + used_external_values.end(), + op.operand_source(in_slot)) != + used_external_values.end()) || + (std::find(used_external_values.begin(), + used_external_values.end(), + op.result(out_slot)) != used_external_values.end())) { can_do_inplace = false; VLOG(6) << upper_op_name << "'s value has been visited or reused by other inplace op, " "so that can't do inplace when setting relax to :" - << (details::relax_shape_check_ops.count(upper_op_name) > 0); + << (RelaxShapeCheckOps.count(upper_op_name) > 0); VLOG_IF( 8, ((in_slot < op.num_operands()) && (out_slot < op.num_results()))) << " -- operand " << in_slot << " and result " << out_slot @@ -450,45 +461,43 @@ static std::unordered_map GetInplaceOps( } return inplace_ops; } -} // namespace details +} // namespace class InplacePass : public pir::Pass { public: InplacePass() : pir::Pass("inplace_pass", 3) {} void Run(pir::Operation* op) override { - auto module_op = op->dyn_cast(); - IR_ENFORCE(module_op, "inplace_pass should run on module op."); - auto& block = module_op.block(); - - auto inplace_ops = details::GetInplaceOps(&block); int64_t num_rewrites_{0}; - for (auto kv : inplace_ops) { - VLOG(6) << "Do inplace for: " - << kv.first->attributes() - .at("op_name") - .dyn_cast() - .AsString(); - pir::Block::Iterator insert_pos = - std::find(block.begin(), block.end(), *kv.first); - IR_ENFORCE(insert_pos != block.end(), - "Operator %s not found in block.", - kv.first->name()); - - kv.first->set_attribute( - "op_name", - pir::StrAttribute::get(pir::IrContext::Instance(), kv.second)); - kv.first->set_attribute( - "is_inplace", - pir::BoolAttribute::get(pir::IrContext::Instance(), true)); - num_rewrites_++; + for (size_t i = 0; i < op->num_regions(); ++i) { + auto& region = op->region(i); + for (auto& block : region) { + auto inplace_ops = GetInplaceOps(block); + + for (const auto& kv : inplace_ops) { + VLOG(6) << "Do inplace for: " + << kv.first->attributes() + .at("op_name") + .dyn_cast() + .AsString(); + pir::Block::Iterator insert_pos = + std::find(block.begin(), block.end(), *kv.first); + IR_ENFORCE(insert_pos != block.end(), + "Operator %s not found in block.", + kv.first->name()); + + kv.first->set_attribute( + "op_name", + pir::StrAttribute::get(pir::IrContext::Instance(), kv.second)); + kv.first->set_attribute( + "is_inplace", + pir::BoolAttribute::get(pir::IrContext::Instance(), true)); + num_rewrites_++; + } + } } PrintStatistics(num_rewrites_); } - - bool CanApplyOn(pir::Operation* op) const override { - return op->isa<::pir::ModuleOp>() && op->num_regions() > 0; - } }; namespace pir { diff --git a/paddle/fluid/pir/transforms/transform_general_functions.cc b/paddle/fluid/pir/transforms/transform_general_functions.cc index d0d44b1a720af..7f9f74cb6710a 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.cc +++ b/paddle/fluid/pir/transforms/transform_general_functions.cc @@ -14,12 +14,47 @@ #include "paddle/fluid/pir/transforms/transform_general_functions.h" +#include + #include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" #include "paddle/fluid/pir/dialect/operator/ir/op_type.h" #include "paddle/pir/core/builtin_op.h" #include "paddle/pir/core/op_operand.h" #include "paddle/pir/core/parameter.h" #include "paddle/pir/core/program.h" +#include "paddle/pir/core/value.h" + +namespace { + +void GetUsedExternalValueImpl( + std::unordered_set& defined_values, // NOLINT + std::vector& used_values, // NOLINT + const pir::Operation& op) { + for (size_t index = 0; index < op.num_operands(); ++index) { + pir::Value value = op.operand_source(index); + if (defined_values.find(value) == defined_values.end()) { + used_values.push_back(value); + defined_values.insert(value); + } + } + for (auto& region : op) { + for (auto& block : region) { + for (auto value : block.args()) { + defined_values.insert(value); + } + } + for (auto& block : region) { + for (auto& inner_op : block) { + GetUsedExternalValueImpl(defined_values, used_values, inner_op); + } + } + } + for (size_t index = 0; index < op.num_results(); ++index) { + defined_values.insert(op.result(index)); + } +} + +} // namespace namespace pir { @@ -58,7 +93,7 @@ pir::Type GetDataTypeFromValue(pir::Value value) { return value.type().dyn_cast().dtype(); } -Operation* GetDefiningOpForInput(Operation* op, uint32_t index) { +Operation* GetDefiningOpForInput(const Operation* op, uint32_t index) { PADDLE_ENFORCE_EQ( index < op->num_operands() && op->operand_source(index), true, @@ -66,8 +101,8 @@ Operation* GetDefiningOpForInput(Operation* op, uint32_t index) { return op->operand_source(index).dyn_cast().owner(); } -std::vector> GetUseOpsForOutput(Operation* op, - uint32_t index) { +std::vector> GetUseOpsForOutput( + const Operation* op, uint32_t index) { PADDLE_ENFORCE_EQ( index < op->num_results(), true, @@ -80,4 +115,21 @@ std::vector> GetUseOpsForOutput(Operation* op, return use_ops; } +std::vector GetUsedExternalValue(const pir::Operation& op) { + std::unordered_set defined_values{nullptr}; + std::vector used_values; + GetUsedExternalValueImpl(defined_values, used_values, op); + return used_values; +} + +std::vector GetUsedExternalValue(const pir::Block& block) { + auto& args = block.args(); + std::unordered_set defined_values(args.begin(), args.end()); + std::vector used_values; + for (auto& op : block) { + GetUsedExternalValueImpl(defined_values, used_values, op); + } + return used_values; +} + } // namespace pir diff --git a/paddle/fluid/pir/transforms/transform_general_functions.h b/paddle/fluid/pir/transforms/transform_general_functions.h index e653f5d4713c1..3c909accf1b5f 100644 --- a/paddle/fluid/pir/transforms/transform_general_functions.h +++ b/paddle/fluid/pir/transforms/transform_general_functions.h @@ -57,23 +57,43 @@ pir::Type GetDataTypeFromValue(pir::Value value); /** * @brief Get an operation that defines the specific input of the operation. * - * @param Operation* pointer to an operation + * @param const Operation* const pointer to an operation * @param uint32_t index of operand of the operation * * @return Operation* */ -Operation* GetDefiningOpForInput(Operation* op, uint32_t index); +Operation* GetDefiningOpForInput(const Operation* op, uint32_t index); /** * @brief Get operations and the index of designative op operand (op result) that use the specific output of the operation. * - * @param Operation* pointer to an operation + * @param const Operation* cosnt pointer to an operation * @param uint32_t index of result of the operation * @return std::vector> */ -std::vector> GetUseOpsForOutput(Operation* op, - uint32_t index); +std::vector> GetUseOpsForOutput( + const Operation* op, uint32_t index); + +/** +* @brief Get the value of the input and output of the specified op in the +external block. +* +* @param const Operation& const reference to an operation + +* @return std::vector +*/ +std::vector GetUsedExternalValue(const Operation& op); + +/** + * @brief Get the external value of the input and output of all op which in the + specified block. + * + * @param const Block& const reference to an block + + * @return std::vector + */ +std::vector GetUsedExternalValue(const Block& block); } // namespace pir diff --git a/paddle/fluid/pybind/control_flow_api.cc b/paddle/fluid/pybind/control_flow_api.cc index 2979d944e0bbf..2cf9bcd424ffe 100644 --- a/paddle/fluid/pybind/control_flow_api.cc +++ b/paddle/fluid/pybind/control_flow_api.cc @@ -24,6 +24,7 @@ #include "paddle/fluid/pir/dialect/operator/ir/api_builder.h" #include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" +#include "paddle/fluid/pir/transforms/transform_general_functions.h" #include "paddle/fluid/platform/enforce.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/common/place.h" @@ -111,51 +112,6 @@ void BindAssertOp(py::module* m) { "as_operation", &AssertOp::operation, return_value_policy::reference); } -void GetUsedExternalValueImpl( - std::unordered_set& defined_values, // NOLINT - std::vector& used_values, // NOLINT - const Operation& op) { - for (size_t index = 0; index < op.num_operands(); ++index) { - Value value = op.operand_source(index); - if (defined_values.find(value) == defined_values.end()) { - used_values.push_back(value); - defined_values.insert(value); - } - } - for (auto& region : op) { - for (auto& block : region) { - for (auto value : block.args()) { - defined_values.insert(value); - } - } - for (auto& block : region) { - for (auto& inner_op : block) { - GetUsedExternalValueImpl(defined_values, used_values, inner_op); - } - } - } - for (size_t index = 0; index < op.num_results(); ++index) { - defined_values.insert(op.result(index)); - } -} - -std::vector GetUsedExternalValue(const Operation& op) { - std::unordered_set defined_values{nullptr}; - std::vector used_values; - GetUsedExternalValueImpl(defined_values, used_values, op); - return used_values; -} - -std::vector GetUsedExternalValue(const Block& block) { - auto& args = block.args(); - std::unordered_set defined_values(args.begin(), args.end()); - std::vector used_values; - for (auto& op : block) { - GetUsedExternalValueImpl(defined_values, used_values, op); - } - return used_values; -} - Value BuildHasElementsOp(Operation& fwd_op) { // NOLINT PADDLE_ENFORCE(fwd_op.isa(), phi::errors::PreconditionNotMet( @@ -246,9 +202,9 @@ void PyIfOp::UpdateOutput() { void BindControlFlowApi(py::module* m) { m->def("get_used_external_value", - [](const Operation& op) { return GetUsedExternalValue(op); }); + [](const Operation& op) { return pir::GetUsedExternalValue(op); }); m->def("get_used_external_value", - [](const Block& block) { return GetUsedExternalValue(block); }); + [](const Block& block) { return pir::GetUsedExternalValue(block); }); m->def("build_pipe_for_block", BuildPipeForBlock); m->def("cf_has_elements", BuildHasElementsOp); m->def("cf_yield", [](py::list inputs) { From d1344c9427b9199d195b0a5c13532e2c6b47d552 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Wed, 27 Dec 2023 18:43:05 +0800 Subject: [PATCH 092/146] =?UTF-8?q?=E3=80=90pir=E3=80=91delete=20wrong=20o?= =?UTF-8?q?ld=20ir=20while=5Floop=20test=20add=20pir=20test=20(#60328)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimize backward * modfiy while_loop * delete print * modify append_full_like use copy value * clear * clear --- python/paddle/autograd/ir_backward.py | 17 ++--- test/legacy_test/test_while_loop_op.py | 95 +++++++++++++------------- 2 files changed, 51 insertions(+), 61 deletions(-) diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index a8ac124e6e2b1..eed96992a1d52 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -574,7 +574,6 @@ def make_input_with_input_stopgradient(op): return inputs, input_grad_stopgradients def update_input_grad_map(op, input_grads, all_inputs): - _, fwd_value_to_block_argument_map = argument_to_value(op) i = 0 for input, grad_semantic in zip(all_inputs, get_grad_semantic_info(op)): if not grad_semantic: @@ -631,8 +630,11 @@ def append_yield( if len(state.value_to_valuegrad[value]) > 1: append_add_n(value) else: + new_value = return_map_value( + value, control_flow_value_to_copyvalue_map + ) value_grad = append_full_like( - 0.0, value, value, state, backward_ops + 0.0, new_value, value, state, backward_ops ) input_grad = state.value_to_valuegrad[value][0][0] @@ -762,16 +764,6 @@ def argument_to_value(while_op): for sub_fwd_block, sub_bwd_block in zip( op.blocks(), grad_op.blocks() ): - # update grad_op structure - if grad_op.name() == "pd_op.while": - ( - _, - sub_bwd_block_argument_to_value_map, - ) = argument_to_value(grad_op) - else: - sub_bwd_block_argument_to_value_map = ( - ValueDict() - ) sub_state = state.copy(sub_fwd_block) sub_backward_ops = [] append_backward_ops( @@ -784,7 +776,6 @@ def argument_to_value(while_op): no_grad_set, sub_backward_ops, sub_state, - sub_bwd_block_argument_to_value_map, ) # update input_grad map update_input_grad_map(op, input_grads, origin_inputs) diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index ca874defb6b0d..42582d092fa6f 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -254,66 +254,63 @@ def internal_body(j, init, sums): class TestApiWhileLoop_Backward(unittest.TestCase): - # TODO(zhangbo): Support while grad exe for pir - # @test_with_pir_api def test_while_loop_backward(self): - def cond(i, x): - return paddle.less_than(i, eleven) + with paddle.pir_utils.IrGuard(): + + def cond(i, x): + return paddle.less_than(i, eleven) + + def body(i, x): + x = paddle.multiply(x=i, y=i) + i = paddle.increment(i) + return [i, x] + + main_program = paddle.static.Program() + startup_program = paddle.static.Program() + with paddle.static.program_guard(main_program, startup_program): + i = paddle.static.data(name='i', shape=[1], dtype='float32') + i.stop_gradient = False + i.persistable = True + eleven = paddle.tensor.fill_constant( + shape=[1], dtype='float32', value=11 + ) + one = paddle.tensor.fill_constant( + shape=[1], dtype='float32', value=1 + ) + x = paddle.static.data(name='x', shape=[1], dtype='float32') + x.stop_gradient = False + x.persistable = True - def body(i, x): - x = paddle.multiply(x=i, y=i) - i = paddle.increment(i) - return [i, x] + out = paddle.static.nn.while_loop(cond, body, [i, x]) + mean = paddle.mean(out[1]) + grad_list = append_backward(mean) - main_program = paddle.static.Program() - startup_program = paddle.static.Program() - with paddle.static.program_guard(main_program, startup_program): - i = paddle.static.data(name='i', shape=[1], dtype='float32') - i.stop_gradient = False - i.persistable = True - eleven = paddle.tensor.fill_constant( - shape=[1], dtype='float32', value=11 - ) - one = paddle.tensor.fill_constant( - shape=[1], dtype='float32', value=1 + place = ( + base.CUDAPlace(0) + if core.is_compiled_with_cuda() + else base.CPUPlace() ) - x = paddle.static.data(name='x', shape=[1], dtype='float32') - x.stop_gradient = False - x.persistable = True - - out = paddle.static.nn.while_loop(cond, body, [i, x]) - mean = paddle.mean(out[1]) - grad_list = append_backward(mean) + exe = base.Executor(place) - place = ( - base.CUDAPlace(0) - if core.is_compiled_with_cuda() - else base.CPUPlace() - ) - exe = base.Executor(place) + feed_i = np.ones(1).astype('float32') + feed_x = np.ones(1).astype('float32') + data = np.asarray([100]).astype('float32') + i_grad = np.asarray([0]).astype('float32') + x_grad = np.asarray([0]).astype('float32') - feed_i = np.ones(1).astype('float32') - feed_x = np.ones(1).astype('float32') - data = np.asarray([100]).astype('float32') - i_grad = np.asarray([110]).astype('float32') - - if paddle.framework.in_pir_mode(): for p, g in grad_list: - if p == i: + if p.is_same(i): di = g + elif p.is_same(x): + dx = g res = exe.run( main_program, feed={'i': feed_i, 'x': feed_x}, - fetch_list=[mean, di], + fetch_list=[mean, di, dx], ) - else: - res = exe.run( - main_program, - feed={'i': feed_i, 'x': feed_x}, - fetch_list=[mean.name, i.grad_name, x.grad_name], - ) - np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) - np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) + np.testing.assert_allclose(np.asarray(res[2]), x_grad, rtol=1e-05) @test_with_pir_api def test_while_loop_backward2(self): @@ -356,6 +353,7 @@ def body(i, x): fetch_list = [out[1]] for p, g in grad_list: fetch_list.append(g) + res = exe.run( main_program, feed={'i': feed_i, 'x': feed_x}, @@ -367,6 +365,7 @@ def body(i, x): feed={'i': feed_i, 'x': feed_x}, fetch_list=[out[1].name, i.grad_name, x.grad_name], ) + np.testing.assert_allclose(np.asarray(res[0]), data, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[1]), i_grad, rtol=1e-05) np.testing.assert_allclose(np.asarray(res[2]), x_grad, rtol=1e-05) From cdeb3a632de460323dc2fec5e872898fafaeb7ca Mon Sep 17 00:00:00 2001 From: lijialin03 <124568209+lijialin03@users.noreply.github.com> Date: Wed, 27 Dec 2023 19:07:03 +0800 Subject: [PATCH 093/146] fix bug of lbfgs test=develop (#60219) * fix bug of lbfgs test=develop * update 1 * update 2 * update 3 test file --- python/paddle/optimizer/lbfgs.py | 12 +++++------- test/legacy_test/test_lbfgs_class.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/python/paddle/optimizer/lbfgs.py b/python/paddle/optimizer/lbfgs.py index 215473ff3a740..936b71b232d4d 100644 --- a/python/paddle/optimizer/lbfgs.py +++ b/python/paddle/optimizer/lbfgs.py @@ -155,12 +155,7 @@ def _strong_wolfe( gtd_new = paddle.dot(grad_new, d) # bracket an interval containing a point satisfying the Wolfe criteria - t_prev, f_prev, g_prev, gtd_prev = ( - paddle.to_tensor(0, dtype=grad.dtype), - loss, - grad, - gtd, - ) + t_prev, f_prev, g_prev, gtd_prev = (0, loss, grad, gtd) done = False ls_iter = 0 while ls_iter < max_ls: @@ -227,7 +222,10 @@ def _strong_wolfe( low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[-1] else (1, 0) while not done and ls_iter < max_ls: # line-search bracket is so small - if paddle.abs(bracket[1] - bracket[0]) * d_norm < tolerance_change: + bracket_ls = bracket[1] - bracket[0] + if not isinstance(bracket_ls, paddle.Tensor): + bracket_ls = paddle.to_tensor(bracket_ls, dtype=gtd_new.dtype) + if paddle.abs(bracket_ls) * d_norm < tolerance_change: break # compute new trial value diff --git a/test/legacy_test/test_lbfgs_class.py b/test/legacy_test/test_lbfgs_class.py index 47c0d36b9ecdd..631d21962e398 100644 --- a/test/legacy_test/test_lbfgs_class.py +++ b/test/legacy_test/test_lbfgs_class.py @@ -498,6 +498,16 @@ def func3(x, alpha, d): paddle.to_tensor([1.0]), max_ls=1, ) + lbfgs._strong_wolfe( + func2, + paddle.to_tensor([1.0]), + -0.001, + paddle.to_tensor([1.0]), + paddle.to_tensor([1.0]), + paddle.to_tensor([1.0]), + paddle.to_tensor([1.0]), + max_ls=1, + ) lbfgs._strong_wolfe( func3, From 9faa23f7e835b24d698d014e29e7765f0fd105a5 Mon Sep 17 00:00:00 2001 From: Yichen Zhang <32740647+pkuzyc@users.noreply.github.com> Date: Wed, 27 Dec 2023 21:18:00 +0800 Subject: [PATCH 094/146] fix the randomness in c_softmax_with_cross_entropy (#60370) --- .../collective/c_softmax_with_cross_entropy_op.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu index f8f43d5c9da48..88bd57f55016c 100644 --- a/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu +++ b/paddle/fluid/operators/collective/c_softmax_with_cross_entropy_op.cu @@ -295,10 +295,8 @@ struct CSoftmaxWithCrossEntropyFunctor { sum_exp_logits = ctx.AllocateTmpTensor({N, 1}, dev_ctx); sum_exp_logits.mutable_data(place); - auto eigen_sum_exp_logits = - phi::funcs::EigenMatrix::From(sum_exp_logits); - eigen_sum_exp_logits.device(*dev_ctx.eigen_device()) = - eigen_softmax.sum(along_axis); + phi::SumKernel( + dev_ctx, softmax_2d, {-1}, softmax_2d.dtype(), true, &sum_exp_logits); if (comm_ctx) { comm_ctx->AllReduce(&sum_exp_logits, sum_exp_logits, ncclSum, stream); @@ -333,6 +331,8 @@ struct CSoftmaxWithCrossEntropyFunctor { N); } + auto eigen_sum_exp_logits = + phi::funcs::EigenMatrix::From(sum_exp_logits); eigen_softmax.device(*dev_ctx.eigen_device()) = (eigen_softmax * eigen_sum_exp_logits.inverse().broadcast(one_by_class)); From 04bceca9d67057b3495e4cb75cc15f580bcf711f Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Wed, 27 Dec 2023 21:34:01 +0800 Subject: [PATCH 095/146] [PIR] support mutable loop_vars in while_loop. (#60330) --- .../dialect/operator/ir/control_flow_op.cc | 122 ++++++++++++++++-- .../pir/dialect/operator/ir/control_flow_op.h | 7 +- .../pir/dialect/operator/ir/op_dialect.cc | 3 +- paddle/fluid/pybind/control_flow_api.cc | 85 ++++++++++-- paddle/fluid/pybind/control_flow_api.h | 16 +++ paddle/fluid/pybind/pir.cc | 10 +- paddle/phi/infermeta/unary.cc | 1 + paddle/pir/core/block.cc | 16 ++- paddle/pir/core/block.h | 2 + paddle/pir/core/interface_support.h | 4 +- paddle/pir/core/interface_value.h | 4 +- paddle/pir/core/region.cc | 10 ++ paddle/pir/core/region.h | 7 +- python/paddle/static/nn/control_flow.py | 14 +- test/ir/pir/test_ir_pybind.py | 3 - test/ir/pir/test_while_api.py | 10 +- test/legacy_test/test_while_loop_op.py | 6 +- 17 files changed, 258 insertions(+), 62 deletions(-) diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index a898965f1f702..040fbb2837711 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -287,20 +287,30 @@ std::vector> IfOp::Vjp( void WhileOp::Build(pir::Builder &builder, // NOLINT pir::OperationArgument &argument, // NOLINT pir::Value cond, - const std::vector &inputs) { + const std::vector &inputs, + bool construct_body) { argument.AddInput(cond); argument.AddInputs(inputs); - auto &body = argument.AddRegion().emplace_back(); std::vector outs_stop_gradient; - for (auto val : inputs) { - argument.AddOutput(val.type()); - auto arg = body.AddArgument(val.type()); - - auto bool_attr = val.attribute(kStopGradientAttrName); - arg.set_attribute(kStopGradientAttrName, - bool_attr ? bool_attr : builder.bool_attr(false)); - outs_stop_gradient.push_back(bool_attr ? bool_attr - : builder.bool_attr(false)); + if (construct_body) { + auto &body = argument.AddRegion().emplace_back(); + for (auto val : inputs) { + argument.AddOutput(val.type()); + auto arg = body.AddArgument(val.type()); + auto bool_attr = val.attribute(kStopGradientAttrName); + outs_stop_gradient.push_back(bool_attr ? bool_attr + : builder.bool_attr(false)); + arg.set_attribute(kStopGradientAttrName, + bool_attr ? bool_attr : builder.bool_attr(false)); + } + } else { + argument.AddRegion(nullptr); + for (auto val : inputs) { + argument.AddOutput(val.type()); + auto bool_attr = val.attribute(kStopGradientAttrName); + outs_stop_gradient.push_back(bool_attr ? bool_attr + : builder.bool_attr(false)); + } } argument.AddAttribute( @@ -343,6 +353,96 @@ void WhileOp::Print(pir::IrPrinter &printer) { os << "\n }"; } +void WhileOp::VerifySig() { + VLOG(4) << "Start Verifying inputs, outputs and attributes for: WhileOp."; + auto input_size = num_operands(); + PADDLE_ENFORCE_GE( + input_size, + 1u, + phi::errors::PreconditionNotMet( + "The size %d of inputs must be greater or equal to 1.", input_size)); + + if (auto cond_type = operand_type(0).dyn_cast()) { + PADDLE_ENFORCE_EQ( + cond_type.dtype().isa(), + true, + phi::errors::PreconditionNotMet( + "Type validation failed for the 0th input, it should be a " + "bool DenseTensorType.")); + } else if (auto cond_type = + operand_type(0).dyn_cast()) { + PADDLE_ENFORCE_EQ( + cond_type.dtype().isa(), + true, + phi::errors::PreconditionNotMet( + "Type validation failed for the 0th input, it should be a " + "bool DenseTensorType.")); + } else { + PADDLE_THROW(phi::errors::PreconditionNotMet( + "Currently, the while op cond input only support bool dense_tensor " + "and bool allocated_dense_tensor.")); + } + PADDLE_ENFORCE_EQ((*this)->num_regions(), + 1u, + phi::errors::PreconditionNotMet( + "The size %d of regions must be equal to 1.", + (*this)->num_regions())); + auto output_size = num_results(); + PADDLE_ENFORCE_EQ(output_size + 1, + input_size, + phi::errors::PreconditionNotMet( + "The result size (%d) not equal to input size(%d) + 1.", + num_results(), + input_size)); + for (size_t index = 0; index < output_size; ++index) { + PADDLE_ENFORCE_EQ( + operand_type(index + 1), + result_type(index), + phi::errors::PreconditionNotMet( + "The (%d) result and operand type is not equal.", index)); + } +} + +void WhileOp::VerifyRegion() { + VLOG(4) << "Start verifying sub regions for: WhileOp."; + PADDLE_ENFORCE_EQ( + (*this)->region(0).size(), + 1u, + phi::errors::PreconditionNotMet("The size %d of body_region must be 1.", + (*this)->region(0).size())); + auto &body_block = body(); + auto output_size = num_results(); + PADDLE_ENFORCE_EQ( + body_block.args_size(), + output_size, + phi::errors::PreconditionNotMet( + "The result size (%d) not equal to block args size(%d) + 1.", + output_size, + body_block.args_size())); + + PADDLE_ENFORCE_EQ( + body_block.empty(), + false, + phi::errors::PreconditionNotMet("The body block is empty.")); + + auto yield_op = body_block.back().dyn_cast(); + auto input_size = num_operands(); + PADDLE_ENFORCE_EQ( + yield_op && yield_op.num_operands() == input_size, + true, + phi::errors::PreconditionNotMet( + "The body block yield size not equal to operands size.")); + // Todo: fix other bugs and make the following code work. + // for (size_t index = 0; index < input_size; ++index) { + // PADDLE_ENFORCE_EQ( + // operand_type(index), + // yield_op.operand_type(index), + // phi::errors::PreconditionNotMet( + // "The (%d) operand and block yield type is not equal.", index)); + // } + VLOG(4) << "Successful end verifying sub regions for: WhileOp."; +} + std::vector> WhileOp::Vjp( pir::Operation *op, const std::vector> &inputs, diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h index baffcadc12718..3c86d56d11616 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.h @@ -77,13 +77,14 @@ class WhileOp : public pir::Op { static void Build(pir::Builder &builder, // NOLINT pir::OperationArgument &argument, // NOLINT pir::Value cond, - const std::vector &inputs); + const std::vector &inputs, + bool construct_body = true); TEST_API pir::Block &body(); pir::Value cond(); const pir::Block::ArgListType &block_args() { return body().args(); } void Print(pir::IrPrinter &printer); // NOLINT - void VerifySig() {} - void VerifyRegion() {} + void VerifySig(); + void VerifyRegion(); static std::vector> Vjp( pir::Operation *op, const std::vector> &inputs_, diff --git a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc index 8cd6375dbe7b6..7b5959a542e7a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc @@ -35,8 +35,7 @@ OperatorDialect::OperatorDialect(pir::IrContext *ctx) ctx->GetOrRegisterDialect<::pir::ControlFlowDialect>(); auto info = ctx->GetRegisteredOpInfo(pir::TuplePushOp::name()); info.AttachInterface(std::move( - pir::InterfaceValue:: - Get())); + pir::InterfaceValue::Get())); } void OperatorDialect::initialize() { diff --git a/paddle/fluid/pybind/control_flow_api.cc b/paddle/fluid/pybind/control_flow_api.cc index 2cf9bcd424ffe..42beed478d821 100644 --- a/paddle/fluid/pybind/control_flow_api.cc +++ b/paddle/fluid/pybind/control_flow_api.cc @@ -40,6 +40,8 @@ using paddle::dialect::AssertOp; using paddle::dialect::HasElementsOp; using paddle::dialect::IfOp; using paddle::dialect::WhileOp; +using paddle::pybind::PyIfOp; +using paddle::pybind::PyWhileOp; using pir::Block; using pir::Builder; using pir::Operation; @@ -51,8 +53,6 @@ using pir::Type; using pir::Value; using pir::YieldOp; using pybind11::return_value_policy; - -using paddle::pybind::PyIfOp; namespace { void BindIfOp(py::module* m) { @@ -79,22 +79,24 @@ void BindIfOp(py::module* m) { } void BindWhileOp(py::module* m) { - m->def("build_while_op", [](Value cond, py::list loop_vars) { + m->def("build_while_op", [](Value cond, py::list loop_vars) -> PyWhileOp { std::vector loop_values; for (auto var : loop_vars) { loop_values.push_back(var.cast()); } - return ApiBuilder::Instance().GetBuilder()->Build(cond, - loop_values); + return PyWhileOp( + ApiBuilder::Instance().GetBuilder()->Build(cond, loop_values)); }); - py::class_ while_op(*m, "WhileOp", R"DOC( + py::class_ while_op(*m, "WhileOp", R"DOC( WhileOp in python api. )DOC"); - while_op.def("body", &WhileOp::body, return_value_policy::reference) - .def("as_operation", &WhileOp::operation, return_value_policy::reference) + while_op.def("body", &PyWhileOp::body, return_value_policy::reference) + .def( + "as_operation", &PyWhileOp::operation, return_value_policy::reference) .def("block_arguments", &WhileOp::block_args, - return_value_policy::reference); + return_value_policy::reference) + .def("optimize_update", &PyWhileOp::OptimizeUpdate); } void BindAssertOp(py::module* m) { @@ -183,7 +185,7 @@ PyIfOp::PyIfOp(IfOp if_op) : IfOp(if_op) { void PyIfOp::UpdateOutput() { PADDLE_ENFORCE_NOT_NULL( - *this, + operation_, paddle::platform::errors::InvalidArgument( "The if_op in PyIfOp used to update output can't be nullptr")); auto block = parent(); @@ -197,7 +199,68 @@ void PyIfOp::UpdateOutput() { cond(), true_region().TakeBack(), false_region().TakeBack()); block->Assign(iter, new_if_op); IfOp::operator=(new_if_op); - VerifyRegion(); + operation_->Verify(); +} + +PyWhileOp::PyWhileOp(WhileOp while_op) : WhileOp(while_op) { + PADDLE_ENFORCE_NOT_NULL( + operation_, + paddle::platform::errors::InvalidArgument( + "The while_op used to construct PyWhileOp can't be nullptr")); +} + +std::vector PyWhileOp::OptimizeUpdate() { + PADDLE_ENFORCE_NOT_NULL(operation_, + paddle::platform::errors::InvalidArgument( + "The while_op in PyWhileOp used to remove unused " + "loop vars can't be nullptr")); + auto parent_block = parent(); + PADDLE_ENFORCE_NOT_NULL( + parent_block, + paddle::platform::errors::InvalidArgument( + "The parent block of while_op which used to remove " + "unused loop vars can't be nullptr")); + + operation_->Verify(); + auto& body_block = body(); + auto yield_op = body_block.back().dyn_cast(); + auto operand_num = operation_->num_operands(); + bool no_change = true; + std::vector index_vec; + std::vector res, new_input, new_yield_val{yield_op.operand_source(0)}; + for (uint32_t i = 0; i < num_results(); ++i) { + res.push_back(result(i)); + } + for (size_t operand_index = 1u, arg_index = 0u; operand_index < operand_num; + ++operand_index) { + if (yield_op.operand_source(operand_index) == body_block.arg(arg_index)) { + body_block.arg(arg_index).ReplaceAllUsesWith( + operand_source(operand_index)); + body_block.EraseArgument(arg_index); + no_change = false; + res[operand_index - 1u] = operand_source(operand_index); + } else { + new_input.push_back(operand_source(operand_index)); + index_vec.push_back(operand_index - 1u); + new_yield_val.push_back(yield_op.operand_source(operand_index)); + ++arg_index; + } + } + if (no_change) return res; + Block::Iterator iter = **this; + Builder builder(ir_context(), false); + auto new_while_op = builder.Build(cond(), new_input, false); + new_while_op->region(0).swap(std::move(operation_->region(0))); + parent_block->Assign(iter, new_while_op); + WhileOp::operator=(new_while_op); + body_block.pop_back(); + builder.SetInsertionPointToBlockEnd(&body_block); + builder.Build(new_yield_val); + operation_->Verify(); + for (size_t result_index = 0; result_index < num_results(); ++result_index) { + res[index_vec[result_index]] = result(result_index); + } + return res; } void BindControlFlowApi(py::module* m) { diff --git a/paddle/fluid/pybind/control_flow_api.h b/paddle/fluid/pybind/control_flow_api.h index 18905bdc09678..020904a6d999d 100644 --- a/paddle/fluid/pybind/control_flow_api.h +++ b/paddle/fluid/pybind/control_flow_api.h @@ -25,6 +25,22 @@ class PyIfOp : public dialect::IfOp { void UpdateOutput(); }; +class PyWhileOp : public dialect::WhileOp { + public: + explicit PyWhileOp(dialect::WhileOp while_op); + + /// + /// \brief Construct a new while_op to replace the original while_op. The + /// input, output, and parameters of the new while_op no longer contain the + /// variables that have not been modified in the loop. The size of the return + /// value is equal to the output size of the original while_op, where the + /// value of the read-only loop variable is the corresponding operand of the + /// original while_op, and the value of the non-read-only loop variable is the + /// corresponding output of the new while_op, + /// + std::vector OptimizeUpdate(); +}; + void BindControlFlowApi(pybind11::module *m); } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 330f5650caf1a..7e1d46b3364c8 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -527,14 +527,8 @@ void BindOperation(py::module *m) { }) .def("as_if_op", [](Operation &self) { return PyIfOp(self.dyn_cast()); }) - .def("as_while_op", [](Operation &self) -> WhileOp { - auto while_op = self.dyn_cast(); - if (!while_op) { - PADDLE_THROW(phi::errors::InvalidArgument( - "Can't cast non-while type Operation to WhileOp.")); - } - return while_op; - }); + .def("as_while_op", + [](Operation &self) { return PyWhileOp(self.dyn_cast()); }); py::class_ block_container( *m, "Operation_BlockContainer", R"DOC( The Operation_BlockContainer only use to walk all blocks in the operation. diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index d221c13968910..90987398057fe 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -1859,6 +1859,7 @@ void IncrementInferMeta(const MetaTensor& x, float value, MetaTensor* out) { product(x.dims()))); out->set_dims(x.dims()); out->share_lod(x); + out->set_layout(x.layout()); out->set_dtype(x.dtype()); } diff --git a/paddle/pir/core/block.cc b/paddle/pir/core/block.cc index 73902960c95ab..49389454545d1 100644 --- a/paddle/pir/core/block.cc +++ b/paddle/pir/core/block.cc @@ -32,6 +32,12 @@ void Block::push_back(Operation *op) { insert(ops_.end(), op); } void Block::push_front(Operation *op) { insert(ops_.begin(), op); } +void Block::pop_back() { + IR_ENFORCE(!ops_.empty(), "can't pop back from empty block."); + ops_.back()->Destroy(); + ops_.pop_back(); +} + Operation *Block::GetParentOp() const { return parent_ ? parent_->GetParent() : nullptr; } @@ -50,8 +56,7 @@ Block::Iterator Block::erase(ConstIterator position) { void Block::clear() { while (!empty()) { - ops_.back()->Destroy(); - ops_.pop_back(); + pop_back(); } } @@ -103,6 +108,13 @@ Value Block::AddArgument(Type type) { return argument; } +void Block::EraseArgument(uint32_t index) { + auto argument = arg(index); + IR_ENFORCE(argument.use_empty(), + "Erase a block argument that is still in use."); + argument.dyn_cast().Destroy(); + arguments_.erase(arguments_.begin() + index); +} bool Block::TopoOrderCheck(const OpListType &op_list) { std::unordered_set visited_values; for (Operation *op : op_list) { diff --git a/paddle/pir/core/block.h b/paddle/pir/core/block.h index a912676f7fb68..373f97e12c51e 100644 --- a/paddle/pir/core/block.h +++ b/paddle/pir/core/block.h @@ -69,6 +69,7 @@ class IR_API Block { void push_back(Operation *op); void push_front(Operation *op); + void pop_back(); Iterator insert(ConstIterator iterator, Operation *op); Iterator erase(ConstIterator position); void clear(); @@ -111,6 +112,7 @@ class IR_API Block { Type arg_type(uint32_t index) const { return arguments_[index].type(); } void ClearArguments(); Value AddArgument(Type type); + void EraseArgument(uint32_t index); template void AddArguments(TypeIter first, TypeIter last); template diff --git a/paddle/pir/core/interface_support.h b/paddle/pir/core/interface_support.h index f8fc83efa3172..60211a9437d7b 100644 --- a/paddle/pir/core/interface_support.h +++ b/paddle/pir/core/interface_support.h @@ -39,8 +39,8 @@ class ConstructInterfacesOrTraits { /// Placement new interface. template static void ConstrctInterface(InterfaceSet &interface_set) { // NOLINT - InterfaceValue val = InterfaceValue:: - Get>(); + InterfaceValue val = + InterfaceValue::Get>(); auto suceess = interface_set.insert(std::move(val)).second; IR_ENFORCE(suceess, "Interface: id[%u] is already registered. inset failed", diff --git a/paddle/pir/core/interface_value.h b/paddle/pir/core/interface_value.h index 3115dc47a365e..4c28e35c72ca2 100644 --- a/paddle/pir/core/interface_value.h +++ b/paddle/pir/core/interface_value.h @@ -22,7 +22,7 @@ namespace pir { class IR_API InterfaceValue { public: - template + template static InterfaceValue Get(); TypeId type_id() const { return type_id_; } void *model() const { return model_; } @@ -52,7 +52,7 @@ class IR_API InterfaceValue { void *model_{nullptr}; }; -template +template InterfaceValue InterfaceValue::Get() { InterfaceValue val; val.type_id_ = TypeId::get(); diff --git a/paddle/pir/core/region.cc b/paddle/pir/core/region.cc index 66e2e9d407f75..21a09198f1d79 100644 --- a/paddle/pir/core/region.cc +++ b/paddle/pir/core/region.cc @@ -70,6 +70,16 @@ void Region::clear() { } } +void Region::swap(Region &&other) { + blocks_.swap(other.blocks_); + for (auto iter = begin(); iter != end(); ++iter) { + iter->SetParent(this, iter); + } + for (auto iter = other.begin(); iter != other.end(); ++iter) { + iter->SetParent(&other, iter); + } +} + template void Region::Walk(FuncT &&callback) { for (auto &block : *this) { diff --git a/paddle/pir/core/region.h b/paddle/pir/core/region.h index 9a4675990c815..c8d4daadaa74c 100644 --- a/paddle/pir/core/region.h +++ b/paddle/pir/core/region.h @@ -55,7 +55,6 @@ class IR_API Region { Block &front() { return *blocks_.front(); } Block &back() { return *blocks_.back(); } - const Block &front() const { return *blocks_.front(); } const Block &back() const { return *blocks_.back(); } @@ -65,6 +64,7 @@ class IR_API Region { Iterator insert(ConstIterator position, Block *block); Iterator erase(ConstIterator position); void clear(); + void swap(Region &&other); /// Operation Walkers, walk the operations in this region. The callback method /// is called for each nested region, block or operation, @@ -77,7 +77,6 @@ class IR_API Region { void TakeBody(Region &&other); Operation *GetParent() const { return parent_; } - void set_parent(Operation *parent) { parent_ = parent; } // return the program which contains this region. // if region is not in a program, return nullptr. Program *parent_program() const; @@ -85,7 +84,7 @@ class IR_API Region { IrContext *ir_context() const; private: - Operation *parent_{nullptr}; // not owned - std::list blocks_; // owned + Operation *const parent_{nullptr}; // not owned + std::list blocks_; // owned }; } // namespace pir diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py index 5ba3a14469d8e..3d2f9858a1feb 100644 --- a/python/paddle/static/nn/control_flow.py +++ b/python/paddle/static/nn/control_flow.py @@ -687,21 +687,23 @@ def while_loop(cond, body, loop_vars, is_test=False, name=None): if in_pir_mode(): while_op = build_while_op(pre_cond, flatten(loop_vars)) with while_op.body() as cur_block: - args = cur_block.args() - next_var = body(*args) + args = pack_sequence_as(loop_vars, cur_block.args()) + next_vars = body(*args) try: assert_same_structure( - flatten(next_var), flatten(loop_vars), check_types=False + flatten(next_vars), flatten(loop_vars), check_types=False ) except ValueError as e: raise ValueError( "body in while_loop should return the same arity " f"(length and structure) as loop_vars: {e}" ) - next_cond = cond(*next_var) + if not isinstance(next_vars, (list, tuple)): + next_vars = [next_vars] + next_cond = cond(*next_vars) next_cond.stop_gradient = True - cf_yield([next_cond, *next_var]) - return while_op.as_operation().results() + cf_yield([next_cond, *flatten(next_vars)]) + return pack_sequence_as(loop_vars, while_op.optimize_update()) if in_dygraph_mode(): now_cond = pre_cond.item() diff --git a/test/ir/pir/test_ir_pybind.py b/test/ir/pir/test_ir_pybind.py index fda8236020b4d..9ae4a3ebbf633 100644 --- a/test/ir/pir/test_ir_pybind.py +++ b/test/ir/pir/test_ir_pybind.py @@ -42,7 +42,6 @@ def get_ir_program(): class TestPybind(unittest.TestCase): def test_program(self): pir_program = get_ir_program() - print(pir_program) block = pir_program.global_block() program = block.program @@ -152,7 +151,6 @@ def test_type(self): pir_program = get_ir_program() matmul_op = pir_program.global_block().ops[1] add_op = pir_program.global_block().ops[2] - print(matmul_op.result(0).type()) self.assertEqual( matmul_op.result(0).type() == add_op.result(0).type(), True ) @@ -184,7 +182,6 @@ def test_attr(self): ) pir_program = pir.translate_to_pir(main_program.desc) - print(pir_program) conv_attr = pir_program.global_block().ops[3].attrs() full_attr = pir_program.global_block().ops[8].attrs() self.assertEqual(conv_attr["stop_gradient"], [False]) diff --git a/test/ir/pir/test_while_api.py b/test/ir/pir/test_while_api.py index cc07cdbb58ad6..1a5ee3186d692 100644 --- a/test/ir/pir/test_while_api.py +++ b/test/ir/pir/test_while_api.py @@ -57,7 +57,7 @@ def test_while_base(self): out = last_op.results() self.assertEqual(out[0].stop_gradient, False) self.assertEqual(last_op.name(), "pd_op.while") - self.assertEqual(len(out), 2) + self.assertEqual(len(out), 1) def test_get_used_external_value(self): main_program = paddle.static.Program() @@ -177,20 +177,20 @@ def test_backward(self): ) self.assertEqual( main_program.global_block() - .ops[-1] + .ops[-3] .as_while_op() .body() - .ops[-2] + .ops[-4] .name(), "cf.has_elements", ) self.assertEqual( main_program.global_block() - .ops[-1] + .ops[-3] .as_while_op() .body() - .ops[-3] + .ops[-5] .name(), "pd_op.add_grad", ) diff --git a/test/legacy_test/test_while_loop_op.py b/test/legacy_test/test_while_loop_op.py index 42582d092fa6f..83fecc6b5ad7f 100644 --- a/test/legacy_test/test_while_loop_op.py +++ b/test/legacy_test/test_while_loop_op.py @@ -22,7 +22,6 @@ from paddle import base from paddle.base import core from paddle.base.backward import append_backward -from paddle.base.framework import program_guard from paddle.pir_utils import test_with_pir_api paddle.enable_static() @@ -98,6 +97,7 @@ def body(i, mem): np.testing.assert_allclose(np.asarray(res[1]), data, rtol=1e-05) @compare_legacy_with_pt + @test_with_pir_api def test_var_dict(self): def cond(i, ten, test_dict, test_list, test_list_dict): return paddle.less_than(i, ten) @@ -118,7 +118,7 @@ def body(i, ten, test_dict, test_list, test_list_dict): main_program = paddle.static.Program() startup_program = paddle.static.Program() - with program_guard(main_program, startup_program): + with paddle.static.program_guard(main_program, startup_program): i = paddle.zeros(shape=[1], dtype='int64') ten = paddle.tensor.fill_constant( shape=[1], dtype='int64', value=10 @@ -130,7 +130,7 @@ def body(i, ten, test_dict, test_list, test_list_dict): test_dict = {"test_key": test_data} test_list = [ paddle.tensor.fill_constant( - shape=[1, 2], dtype='int64', value=0 + shape=[2, 1], dtype='int64', value=0 ) ] test_list_dict = [ From de1fe4ba7b2ded5773b0a62aba09bd8b1a297ef2 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Thu, 28 Dec 2023 07:22:06 +0800 Subject: [PATCH 096/146] [Prim][PIR] decomp support Inference (#60141) * inference support decomp * polish code * add decomp base define * add decomp base define2 * change decomp infer * fix symbol overload * fix test case * debug --- .../fluid/inference/api/analysis_predictor.cc | 8 + .../tensor_operants_gen.py | 23 +-- paddle/fluid/primitive/base/decomp_trans.cc | 28 ++-- paddle/fluid/primitive/base/decomp_trans.h | 21 ++- paddle/fluid/pybind/pybind.cc | 3 +- .../test_decomp_inference_predictor_run.py | 155 ++++++++++++++++++ 6 files changed, 210 insertions(+), 28 deletions(-) create mode 100644 test/ir/inference/test_decomp_inference_predictor_run.py diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index c70ef74e94baa..4af55a7c6c933 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -56,6 +56,8 @@ #include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/prim/utils/utils.h" +#include "paddle/fluid/primitive/base/decomp_trans.h" #include "paddle/phi/api/include/context_pool.h" #include "paddle/phi/api/include/tensor.h" #include "paddle/phi/common/backend.h" @@ -786,6 +788,12 @@ bool AnalysisPredictor::PrepareExecutor() { pir_program_ = std::move( paddle::TranslateLegacyProgramToProgram(*inference_program_)); + if (paddle::prim::PrimCommonUtils::IsFwdPrimEnabled()) { + VLOG(4) << "[Prim] Decomp program in predictor begin."; + DecompProgram decomp_object(pir_program_.get()); + decomp_object.decomp_program(); + } + if (config_.use_gpu()) { ::pir::PassManager gpu_pm(::pir::IrContext::Instance(), 2); //----------------------------------------------------------------------------------------------// diff --git a/paddle/fluid/prim/api/auto_code_generated/tensor_operants_gen.py b/paddle/fluid/prim/api/auto_code_generated/tensor_operants_gen.py index 378f57a468cd4..6cf6615075282 100644 --- a/paddle/fluid/prim/api/auto_code_generated/tensor_operants_gen.py +++ b/paddle/fluid/prim/api/auto_code_generated/tensor_operants_gen.py @@ -216,6 +216,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { #include "paddle/fluid/primitive/type/lazy_tensor.h" PHI_DECLARE_bool(enable_pir_api); +PHI_DECLARE_bool(enable_pir_in_executor); """ @@ -228,7 +229,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { using LazyTensor = paddle::primitive::LazyTensor; Tensor StaticTensorOperants::add(const Tensor& x, const Scalar& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::add(x, paddle::primitive::backend::full(x.shape(), y, x.dtype(), x.place())); } else { return paddle::prim::add(x, paddle::prim::full(x.shape(), y, x.dtype(), x.place())); @@ -236,7 +237,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::subtract(const Tensor& x, const Scalar& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::subtract(x, paddle::primitive::backend::full(x.shape(), y, x.dtype(), x.place())); } else { return paddle::prim::subtract(x, paddle::prim::full(x.shape(), y, x.dtype(), x.place())); @@ -244,7 +245,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::multiply(const Tensor& x, const Scalar& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::scale(x, y, 0.0f, true); } else { return paddle::prim::scale(x, y, 0.0f, true); @@ -252,7 +253,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::divide(const Tensor& x, const Scalar& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::divide(x, paddle::primitive::backend::full(x.shape(), y, x.dtype(), x.place())); } else { return paddle::prim::divide(x, paddle::prim::full(x.shape(), y, x.dtype(), x.place())); @@ -260,7 +261,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::add(const Scalar& x, const Tensor& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::add(paddle::primitive::backend::full(y.shape(), x, y.dtype(), y.place()), y); } else { return paddle::prim::add(paddle::prim::full(y.shape(), x, y.dtype(), y.place()), y); @@ -269,7 +270,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { Tensor StaticTensorOperants::subtract(const Scalar& x, const Tensor& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::subtract(paddle::primitive::backend::full(y.shape(), x, y.dtype(), y.place()), y); } else { return paddle::prim::subtract(paddle::prim::full(y.shape(), x, y.dtype(), y.place()), y); @@ -277,7 +278,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::multiply(const Scalar& x, const Tensor& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::scale(y, x, 0.0f, true); } else { return paddle::prim::scale(y, x, 0.0f, true); @@ -285,7 +286,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::divide(const Scalar& x, const Tensor& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::divide(paddle::primitive::backend::full(y.shape(), x, y.dtype(), y.place()), y); } else { return paddle::prim::divide(paddle::prim::full(y.shape(), x, y.dtype(), y.place()), y); @@ -293,7 +294,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::pow(const Tensor& x, const Tensor& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::elementwise_pow(x, y); } else { return paddle::prim::elementwise_pow(x, y); @@ -301,7 +302,7 @@ class TEST_API StaticTensorOperants : public TensorOperantsBase { } Tensor StaticTensorOperants::pow(const Tensor& x, const Scalar& y) { - if (FLAGS_enable_pir_api) { + if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) { return paddle::primitive::backend::elementwise_pow(x, paddle::primitive::backend::full(x.shape(), y, x.dtype(), x.place())); } else { return paddle::prim::elementwise_pow(x, paddle::prim::full(x.shape(), y, x.dtype(), x.place())); @@ -394,7 +395,7 @@ def gene_static_tensor_func_call(self): ) static_func_parameters = self.get_func_args() - static_tensor_func_call = f"""if (FLAGS_enable_pir_api) {{ + static_tensor_func_call = f"""if (FLAGS_enable_pir_api || FLAGS_enable_pir_in_executor) {{ return {backend_static_func_name}({static_func_parameters}); }} else {{ return {prim_static_func_name}({static_func_parameters}); diff --git a/paddle/fluid/primitive/base/decomp_trans.cc b/paddle/fluid/primitive/base/decomp_trans.cc index 6dde6c8b94002..df0111d56f8af 100644 --- a/paddle/fluid/primitive/base/decomp_trans.cc +++ b/paddle/fluid/primitive/base/decomp_trans.cc @@ -124,8 +124,8 @@ void DecompProgram::check_decomp_outputs( for (size_t i = 0; i < orig_outs.size(); i++) { if (skip_invalid_op_check && paddle::dialect::IsEmptyValue(decomp_outs[i])) { - VLOG(0) << "[Prim] Decomp op skip check of output index " << i - << " of op " << op_name; + VLOG(4) << "[Prim] Decomp op skip check of " << i + << "-index output of op " << op_name; } else { PADDLE_ENFORCE( !paddle::dialect::IsEmptyValue(orig_outs[i]), @@ -238,6 +238,14 @@ std::vector DecompProgram::construct_dst_vars( return tar_vars; } +std::vector DecompProgram::get_dst_vars() { + if (!paddle::prim::PrimCommonUtils::IsFwdPrimEnabled()) { + return src_vars_; + } else { + return dst_vars_; + } +} + bool DecompProgram::enable_decomp_by_filter(const std::string& op_name) { bool flag = true; @@ -266,16 +274,7 @@ std::vector> call_decomp_rule(pir::Operation* op) { return decomp_res; } -DecompProgram::DecompProgram(pir::Program* program, - const std::vector& src_vars, - const std::set& blacklist, - const std::set& whitelist) - : program_(program), - src_vars_(src_vars), - blacklist_(blacklist), - whitelist_(whitelist) {} - -std::vector DecompProgram::decomp_program() { +void DecompProgram::decomp_program() { std::ostringstream orig_prog_stream; std::unordered_map orig_vars_dict; for (size_t i = 0; i < src_vars_.size(); i++) { @@ -285,7 +284,7 @@ std::vector DecompProgram::decomp_program() { VLOG(4) << "[Prim] Origin program bofore decomp :\n" << orig_prog_stream.str(); if (!paddle::prim::PrimCommonUtils::IsFwdPrimEnabled()) { - return src_vars_; + return; } std::vector tar_vars(src_vars_.size()); pir::Block* block = program_->block(); @@ -338,7 +337,8 @@ std::vector DecompProgram::decomp_program() { std::ostringstream decomp_prog_stream; program_->Print(decomp_prog_stream); VLOG(4) << "[Prim] New program after decomp :\n" << decomp_prog_stream.str(); - return tar_vars; + dst_vars_ = tar_vars; + return; } } // namespace paddle diff --git a/paddle/fluid/primitive/base/decomp_trans.h b/paddle/fluid/primitive/base/decomp_trans.h index 550d8beab8031..4f3a83d326b33 100644 --- a/paddle/fluid/primitive/base/decomp_trans.h +++ b/paddle/fluid/primitive/base/decomp_trans.h @@ -26,12 +26,18 @@ namespace paddle { class DecompProgram { public: + explicit DecompProgram(pir::Program* program) : program_(program) {} + DecompProgram(pir::Program* program, const std::vector& src_vars, const std::set& blacklist, - const std::set& whitelist); + const std::set& whitelist) + : program_(program), + src_vars_(src_vars), + blacklist_(blacklist), + whitelist_(whitelist) {} - std::vector decomp_program(); + void decomp_program(); bool check_decomp_dynamic_shape(pir::Operation* op); void check_decomp_outputs(const std::string& op_name, const std::vector& orig_outs, @@ -46,10 +52,21 @@ class DecompProgram { const std::vector& decomp_outs, std::unordered_map orig_vars_dict); bool enable_decomp_by_filter(const std::string& op_name); + void set_src_vars(const std::vector& src_vars) { + src_vars_ = src_vars; + } + void set_blacklist(const std::set& blacklist) { + blacklist_ = blacklist; + } + void set_whitelist(const std::set& whitelist) { + whitelist_ = whitelist; + } + std::vector get_dst_vars(); private: pir::Program* program_; std::vector src_vars_; + std::vector dst_vars_; std::set blacklist_; std::set whitelist_; }; diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index efeeb4855205e..53df4c25034ab 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -785,7 +785,8 @@ void BindDecomp(pybind11::module *m) { VLOG(4) << "[Prim] Bind Decomp sinking_decomp begin."; py::list res; DecompProgram decomp_object(program, src_vars, blacklist, whitelist); - auto tar_vars = decomp_object.decomp_program(); + decomp_object.decomp_program(); + std::vector tar_vars = decomp_object.get_dst_vars(); for (size_t i = 0; i < tar_vars.size(); ++i) { if (!tar_vars[i]) { res.append(nullptr); diff --git a/test/ir/inference/test_decomp_inference_predictor_run.py b/test/ir/inference/test_decomp_inference_predictor_run.py new file mode 100644 index 0000000000000..687f28c1bcf15 --- /dev/null +++ b/test/ir/inference/test_decomp_inference_predictor_run.py @@ -0,0 +1,155 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import unittest + +import numpy as np + +import paddle +from paddle.inference import Config, create_predictor + + +class TestNet(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.fc1 = paddle.nn.Linear(64, 32) + self.fc2 = paddle.nn.Linear(64, 32) + + def forward(self, x1, x2): + y1 = self.fc1(x1) + y2 = self.fc2(x2) + y3 = y1 + y2 + y4 = paddle.nn.functional.layer_norm(y3, y3.shape[1:]) + z = paddle.nn.functional.softmax(y4) + return z + + +class TestPredictorRunWithTensor(unittest.TestCase): + def setUp(self): + self.use_gpu = paddle.is_compiled_with_cuda() + np.random.seed(2023) + self.shape = [4, 8, 16, 64] + self.x = np.random.random(self.shape).astype(np.float32) + self.y = np.random.random(self.shape).astype(np.float32) + self.temp_dir = tempfile.TemporaryDirectory() + net = TestNet() + model = paddle.jit.to_static( + net, + input_spec=[ + paddle.static.InputSpec( + shape=self.shape, dtype='float32', name='input0' + ), + paddle.static.InputSpec( + shape=self.shape, dtype='float32', name='input1' + ), + ], + ) + paddle.jit.save( + model, + os.path.join( + self.temp_dir.name, 'test_predictor_run_model/inference' + ), + ) + + def tearDown(self): + self.temp_dir.cleanup() + + def enable_pir(self, flag: bool): + paddle.set_flags({'FLAGS_enable_pir_in_executor': flag}) + + def init_predictor(self): + config = Config( + os.path.join( + self.temp_dir.name, + 'test_predictor_run_model/inference.pdmodel', + ), + os.path.join( + self.temp_dir.name, + 'test_predictor_run_model/inference.pdiparams', + ), + ) + if self.use_gpu: + config.enable_use_gpu(256, 0) + config.switch_ir_optim(False) + config.enable_new_executor() + predictor = create_predictor(config) + return predictor + + def get_inputs(self): + input0_tensor = paddle.to_tensor(self.x) + input1_tensor = paddle.to_tensor(self.y) + + return [input0_tensor, input1_tensor] + + def get_disorder_output(self, predictor): + [input0_tensor, input1_tensor] = self.get_inputs() + + input_names = predictor.get_input_names() + input0_tensor.name = input_names[0] + input1_tensor.name = input_names[1] + + # disorder + inputs = [input1_tensor, input0_tensor] + outputs = predictor.run(inputs) + + return outputs[0] + + def get_inorder_output(self, predictor): + [input0_tensor, input1_tensor] = self.get_inputs() + + # inorder + inputs = [input0_tensor, input1_tensor] + outputs = predictor.run(inputs) + + return outputs[0] + + def test_output_prim_inorder(self): + self.enable_pir(False) + predictor = self.init_predictor() + output = self.get_inorder_output(predictor) + self.enable_pir(True) + paddle.core._set_prim_all_enabled(True) + pir_predictor = self.init_predictor() + pir_output = self.get_inorder_output(pir_predictor) + paddle.core._set_prim_all_enabled(False) + + np.testing.assert_allclose( + output.numpy().flatten(), + pir_output.numpy().flatten(), + rtol=1e-6, + atol=1e-6, + ) + + def test_output_prim_disorder(self): + self.enable_pir(False) + predictor = self.init_predictor() + output = self.get_disorder_output(predictor) + self.enable_pir(True) + paddle.core._set_prim_all_enabled(True) + pir_predictor = self.init_predictor() + pir_output = self.get_disorder_output(pir_predictor) + paddle.core._set_prim_all_enabled(False) + + np.testing.assert_allclose( + output.numpy().flatten(), + pir_output.numpy().flatten(), + rtol=1e-6, + atol=1e-6, + ) + + +if __name__ == '__main__': + unittest.main() From f1b736daa9474efb696620b4b639f10a3eedd6a6 Mon Sep 17 00:00:00 2001 From: Yuang Liu Date: Thu, 28 Dec 2023 08:11:55 +0800 Subject: [PATCH 097/146] [auto parallel] add recompute to pp ut (#60406) --- .../hybrid_strategy/test_semi_auto_parallel_llama_model.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py index 36b6c1d5d0e97..3ace2754c7123 100644 --- a/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py +++ b/test/auto_parallel/hybrid_strategy/test_semi_auto_parallel_llama_model.py @@ -74,9 +74,8 @@ def setUp(self): "backend": ["gpu"], "use_sp": ["true", "false"], "use_param_group": ["false", "true"], - # TODO(Yuang Liu): add recompute ut to pp after fixing pp probs - # "recompute": ["true", "false"], - # "recompute_granularity": ["full", "full_attn", "core_attn"], + "recompute": ["true", "false"], + "recompute_granularity": ["full", "full_attn", "core_attn"], } def test_simple_net_hybrid_strategy(self): @@ -103,6 +102,8 @@ def setUp(self): } self._changeable_envs = { "backend": ["gpu"], + "recompute": ["true", "false"], + "recompute_granularity": ["full", "full_attn", "core_attn"], } def test_simple_net_hybrid_strategy_acc(self): From a216f5b067c0a219b72f2972240afdc8bcaab90f Mon Sep 17 00:00:00 2001 From: Liujie0926 <44688141+Liujie0926@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:11:39 +0800 Subject: [PATCH 098/146] fix bug (#60354) --- tools/auto_parallel/ci_auto_parallel.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/auto_parallel/ci_auto_parallel.sh b/tools/auto_parallel/ci_auto_parallel.sh index 09095d7f6122b..848a5ca1b1bbd 100644 --- a/tools/auto_parallel/ci_auto_parallel.sh +++ b/tools/auto_parallel/ci_auto_parallel.sh @@ -160,8 +160,6 @@ if [[ ${#case_list[*]} -ne 0 ]];then elif [[ ${case} == "gpt-3_dygraph" ]];then bash /workspace/PaddleNLP/scripts/distribute/ci_case_dy.sh llm_gpt_case_list_dygraph $FLAGS_install_deps $FLAGS_download_data print_info $? `ls -lt ${log_path} | grep "llm_gpt" | head -n 1 | awk '{print $9}'` ${case} - export FLAGS_install_deps=1 - export FLAGS_download_data="llm_gpt ""$FLAGS_download_data" let case_num++ elif [[ ${case} == "dygraph_unit_test" ]];then bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh dygraph_unit_test From 95b5a6846c73bed0b0746ded0bf617a1f011ada3 Mon Sep 17 00:00:00 2001 From: zhink <33270771+zhink@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:22:16 +0800 Subject: [PATCH 099/146] [paddle inference]support tgt_mask in block_multihead_attention (#60389) [paddle inference]support tgt_mask in block_multihead_attention (#60389) --- paddle/phi/kernels/fusion/gpu/block_attn.h | 38 ++++++++++++++++++- .../test_block_multihead_attention.py | 10 ++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/paddle/phi/kernels/fusion/gpu/block_attn.h b/paddle/phi/kernels/fusion/gpu/block_attn.h index 73be0901c6f36..500ffe939870f 100644 --- a/paddle/phi/kernels/fusion/gpu/block_attn.h +++ b/paddle/phi/kernels/fusion/gpu/block_attn.h @@ -38,6 +38,10 @@ struct Block_AttN_params { // [bsz, 1, 1, time_step(cache_seq_length)+1] const T *attn_mask; + // mask_length is the 3th dimension of attn_mask. + int mask_length; + bool mask_broadcast_num_heads; + // k_cache [max_block_num, num_head, block_size, head_size] // v_cache [max_block_num, num_head, block_size, head_size] T *k_cache; @@ -312,6 +316,14 @@ __global__ __launch_bounds__(THREADS_PER_BLOCK) void block_attention_kernel( } if (tid == 0) { qk *= params.inv_sqrt_dh; + if (params.attn_mask) { + auto mask_bhi = bhi; + if (params.mask_broadcast_num_heads) { + mask_bhi = bi; + } + T mask = params.attn_mask[mask_bhi * params.mask_length + act_time_step]; + qk += static_cast(mask); + } qk_max = qk; qk_smem[act_time_step] = qk; } @@ -372,7 +384,14 @@ __global__ __launch_bounds__(THREADS_PER_BLOCK) void block_attention_kernel( } float qk = Qk_dot::dot(q, k, params.inv_sqrt_dh); - + if (params.attn_mask) { + auto mask_bhi = bhi; + if (params.mask_broadcast_num_heads) { + mask_bhi = bi; + } + T mask = params.attn_mask[mask_bhi * params.mask_length + ti]; + qk += static_cast(mask); + } if (ti < act_time_step && tid % THREADS_PER_KEY == 0) { qk_max = fmaxf(qk_max, qk); qk_smem[ti] = qk; @@ -786,8 +805,25 @@ void blha(const phi::GPUContext &dev_ctx, params.max_num_blocks_per_seq = max_num_blocks_per_seq; params.neox_rotary_style = neox_rotary_style; + params.attn_mask = nullptr; + bool mask_broadcast_num_heads = false; if (src_mask_tensor) { + if (src_mask_tensor->dims()[1] == 1) { + // all head share a mask. + mask_broadcast_num_heads = true; + } else if (src_mask_tensor->dims()[1] == num_head) { + mask_broadcast_num_heads = false; + } else { + PADDLE_THROW(errors::InvalidArgument( + "Unknow dimension for attn_mask, the num_head(2nd) " + "dimension is invalid, it should be 1 or num_head(%d), " + "but got %d", + num_head, + src_mask_tensor->dims()[1])); + } params.attn_mask = src_mask_tensor->data(); + params.mask_broadcast_num_heads = mask_broadcast_num_heads; + params.mask_length = src_mask_tensor->dims()[3]; } else { params.attn_mask = nullptr; } diff --git a/test/legacy_test/test_block_multihead_attention.py b/test/legacy_test/test_block_multihead_attention.py index 04919ca3d8240..7f3033044e1c5 100644 --- a/test/legacy_test/test_block_multihead_attention.py +++ b/test/legacy_test/test_block_multihead_attention.py @@ -306,6 +306,12 @@ def setUp(self): ] * self.batch_size, ) + + self.tgt_mask = paddle.randn( + [self.batch_size, self.num_head, 1, self.seq_len + 1], + dtype=self.dtype, + ) + self.scale = 1.0 / np.sqrt(self.shape[-1]) self.cache_k = paddle.zeros(shape=self.cache_shape, dtype=self.dtype) self.cache_v = paddle.zeros(shape=self.cache_shape, dtype=self.dtype) @@ -462,7 +468,7 @@ def test_all(self): naive_cache_v, None, None, - None, + self.tgt_mask, self.scale, ) .transpose([0, 2, 1, 3]) @@ -492,7 +498,7 @@ def test_all(self): None, # out_smooth None, # rotary_embs None, # attn_mask - None, # tgt_mask + self.tgt_mask, # tgt_mask 1, # seq_len, self.blocksize, False, # use_neox_rotary_style From b81deac6d3898d8ed09f5a639030353a7ce5a0b6 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Thu, 28 Dec 2023 10:23:45 +0800 Subject: [PATCH 100/146] [PIR] OneDNN Pir onednn instruction (#60257) * onednn dialect gend --- .gitignore | 2 + .../framework/new_executor/CMakeLists.txt | 10 + .../onednn_legacy_kernel_instruction.cc | 52 +++ .../onednn/onednn_legacy_kernel_instruction.h | 72 ++++ .../onednn_mixed_phi_kernel_instruction.cc | 61 +++ .../onednn_mixed_phi_kernel_instruction.h | 42 ++ .../onednn/onednn_phi_kernel_instruction.cc | 388 ++++++++++++++++++ .../onednn/onednn_phi_kernel_instruction.h | 82 ++++ .../framework/new_executor/pir_interpreter.cc | 19 + .../ir_adaptor/translator/op_translator.cc | 40 +- .../fluid/ir_adaptor/translator/translate.cc | 6 + paddle/fluid/ir_adaptor/translator/utils.cc | 6 + paddle/fluid/pir/dialect/CMakeLists.txt | 56 ++- .../pir/dialect/kernel/ir/kernel_dialect.cc | 103 +++++ .../pir/dialect/kernel/ir/kernel_dialect.h | 22 + .../fluid/pir/dialect/kernel/ir/kernel_op.cc | 127 ++++++ .../fluid/pir/dialect/kernel/ir/kernel_op.h | 43 ++ .../fluid/pir/dialect/op_generator/op_gen.py | 188 ++++++++- .../pir/dialect/op_generator/ops_api_gen.py | 1 + .../op_generator/ops_onednn_extra_parser.py | 86 ++++ .../fluid/pir/dialect/operator/ir/onednn.yaml | 9 + .../dialect/operator/ir/op_onednn_dialect.cc | 168 ++++++++ .../dialect/operator/ir/op_onednn_dialect.h | 44 ++ paddle/fluid/pir/dialect/operator/ir/ops.yaml | 9 + .../dialect/operator/ir/ops_onednn_extra.yaml | 33 ++ .../fluid/pir/dialect/operator/trait/onednn.h | 49 +++ .../fluid/pir/dialect/operator/trait/trait.cc | 10 +- .../operator/utils/op_yaml_info_util.h | 20 +- .../fluid/pir/dialect/operator/utils/utils.cc | 7 + .../fluid/pir/dialect/operator/utils/utils.h | 4 + .../pir/transforms/pd_op_to_kernel_pass.cc | 329 +++++++++++++-- paddle/phi/api/lib/data_transform.h | 5 + paddle/phi/api/yaml/op_compat.yaml | 9 + .../cpu/onednn_to_paddle_layout_kernel.cc | 94 +++++ .../kernels/onednn_to_paddle_layout_kernel.h | 28 ++ test/mkldnn/test_conv2d_mkldnn_op.py | 91 ++++ 36 files changed, 2257 insertions(+), 58 deletions(-) create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.cc create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.cc create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.cc create mode 100644 paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h create mode 100644 paddle/fluid/pir/dialect/op_generator/ops_onednn_extra_parser.py create mode 100644 paddle/fluid/pir/dialect/operator/ir/onednn.yaml create mode 100644 paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.cc create mode 100644 paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h create mode 100644 paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml create mode 100644 paddle/fluid/pir/dialect/operator/trait/onednn.h create mode 100644 paddle/phi/kernels/cpu/onednn_to_paddle_layout_kernel.cc create mode 100644 paddle/phi/kernels/onednn_to_paddle_layout_kernel.h diff --git a/.gitignore b/.gitignore index 232d8fa08b4bd..c4046a8d6b6e3 100644 --- a/.gitignore +++ b/.gitignore @@ -108,6 +108,8 @@ paddle/fluid/pir/dialect/operator/ir/pd_api.* paddle/fluid/pir/dialect/operator/ir/op_decomp.cc paddle/fluid/pir/dialect/operator/ir/pd_op_vjp.cc paddle/fluid/pir/dialect/operator/ir/pd_op.* +paddle/fluid/pir/dialect/operator/ir/pd_onednn_op.* +paddle/fluid/pir/dialect/operator/ir/pd_onednn_op_info.* paddle/fluid/pir/dialect/operator/ir/pd_op_bwd.* paddle/fluid/pir/dialect/operator/ir/pd_op_fused.* paddle/fluid/pir/dialect/operator/ir/pd_op_fused_bwd.* diff --git a/paddle/fluid/framework/new_executor/CMakeLists.txt b/paddle/fluid/framework/new_executor/CMakeLists.txt index df01de6d42491..990f82efa8ede 100644 --- a/paddle/fluid/framework/new_executor/CMakeLists.txt +++ b/paddle/fluid/framework/new_executor/CMakeLists.txt @@ -5,6 +5,16 @@ if(NOT (WITH_CINN AND NOT CINN_ONLY)) ${CMAKE_CURRENT_SOURCE_DIR}/instruction/cinn_jit_instruction.cc) endif() +if(NOT WITH_MKLDNN) + list( + REMOVE_ITEM + standalone_executor_srcs + ${CMAKE_CURRENT_SOURCE_DIR}/instruction/onednn/onednn_legacy_kernel_instruction.cc + ${CMAKE_CURRENT_SOURCE_DIR}/instruction/onednn/onednn_phi_kernel_instruction.cc + ${CMAKE_CURRENT_SOURCE_DIR}/instruction/onednn/onednn_mixed_phi_kernel_instruction.cc + ) +endif() + set(standalone_executor_deps pir program_translator diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.cc new file mode 100644 index 0000000000000..6d1944219a2dc --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.cc @@ -0,0 +1,52 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h" + +#include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" +#include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h" +#include "paddle/fluid/framework/new_executor/interpreter/stream_analyzer.h" +#include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" +#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h" + +#include "paddle/fluid/platform/device_context.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/core/meta_tensor.h" +#include "paddle/phi/core/type_defs.h" + +namespace paddle { +namespace framework { + +OneDNNLegacyKernelInstruction::OneDNNLegacyKernelInstruction( + size_t id, + const platform::Place& place, + pir::Operation* op, + const ValueExecutionInfo* value_exec_info) + : InstructionBase(id, place), value_exec_info_(value_exec_info) { + PADDLE_THROW(platform::errors::Unimplemented( + "OneDNNLegacyKernelInstruction not defined now.")); +} + +OneDNNLegacyKernelInstruction::~OneDNNLegacyKernelInstruction() {} + +void OneDNNLegacyKernelInstruction::Run() { + PADDLE_THROW(platform::errors::Unimplemented( + "OneDNNLegacyKernelInstruction not defined now.")); +} +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h new file mode 100644 index 0000000000000..e5c7b0cd15176 --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h @@ -0,0 +1,72 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/framework/new_executor/instruction/instruction_base.h" + +namespace pir { +class Operation; +} // namespace pir + +namespace paddle { +namespace framework { +class Scope; +class ValueExecutionInfo; + +class OneDNNLegacyKernelInstruction : public InstructionBase { + public: + OneDNNLegacyKernelInstruction(size_t id, + const platform::Place& place, + ::pir::Operation* op, + const ValueExecutionInfo* value_exec_info); + + ~OneDNNLegacyKernelInstruction(); + phi::Kernel* PhiKernel() const { return phi_kernel_; } + + const phi::InferMetaContext& InferMetaContext() const { + return infer_meta_context_; + } + + paddle::dialect::InferMetaInterface::Concept* InferMetaInterface() const { + return infer_meta_interface_; + } + + void Run() override; + + const std::string& Name() const override { return legacy_op_name_; } + + ::pir::Operation* Operation() const override { return op_; } + + private: + std::string legacy_op_name_; + + paddle::dialect::InferMetaInterface::Concept* infer_meta_interface_{ + nullptr}; // not owned + + phi::InferMetaContext infer_meta_context_; + + paddle::framework::ExecutionContext* kernel_context_{nullptr}; + std::shared_ptr runtime_context_; + std::shared_ptr operator_base_; + + phi::Kernel* phi_kernel_{nullptr}; // not owned + + ::pir::Operation* op_{nullptr}; // not owned + + const ValueExecutionInfo* value_exec_info_; // not owned +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.cc new file mode 100644 index 0000000000000..572c26eb42078 --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h" + +#include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h" +#include "paddle/fluid/framework/new_executor/interpreter/stream_analyzer.h" +#include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" +#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h" +#include "paddle/fluid/platform/collective_helper.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/core/meta_tensor.h" +#include "paddle/phi/core/type_defs.h" + +#include "paddle/pir/core/builtin_attribute.h" +#include "paddle/pir/core/operation.h" +#include "paddle/pir/core/value.h" + +#include "dnnl.hpp" // NOLINT +#include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" +#include "paddle/fluid/framework/type_defs.h" +#include "paddle/fluid/ir_adaptor/translator/op_compat_info.h" +#include "paddle/phi/backends/onednn/onednn_context.h" +#include "paddle/phi/backends/onednn/onednn_helper.h" +#include "paddle/phi/kernels/funcs/data_layout_transform.h" + +namespace paddle { +namespace framework { + +OneDNNMixedPhiKernelInstruction::OneDNNMixedPhiKernelInstruction( + size_t id, + const platform::Place& place, + pir::Operation* op, + const ValueExecutionInfo* value_exec_info) + : OneDNNPhiKernelInstruction(id, place, op, value_exec_info) {} + +void OneDNNMixedPhiKernelInstruction::Run() { + // Step1. Mixed Dynamic Choose Kernel + // todo if (input_tensor.layout() != phi::DataLayout::ONEDNN) + + OneDNNPhiKernelInstruction::Run(); +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h new file mode 100644 index 0000000000000..d39e5fa9d1fea --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h @@ -0,0 +1,42 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h" + +namespace pir { +class Operation; +} // namespace pir + +namespace paddle { +namespace framework { +class Scope; +class ValueExecutionInfo; + +using RuntimeAttribute = phi::Attribute; +using PIRAttribute = pir::Attribute; + +class OneDNNMixedPhiKernelInstruction : public OneDNNPhiKernelInstruction { + public: + OneDNNMixedPhiKernelInstruction(size_t id, + const platform::Place& place, + ::pir::Operation* op, + const ValueExecutionInfo* value_exec_info); + + void Run() override; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.cc b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.cc new file mode 100644 index 0000000000000..71385619cb958 --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.cc @@ -0,0 +1,388 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h" + +#include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h" +#include "paddle/fluid/framework/new_executor/interpreter/stream_analyzer.h" +#include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" +#include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/fluid/pir/dialect/operator/utils/op_yaml_info_parser.h" +#include "paddle/fluid/platform/collective_helper.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/core/meta_tensor.h" +#include "paddle/phi/core/type_defs.h" + +#include "paddle/pir/core/builtin_attribute.h" +#include "paddle/pir/core/operation.h" +#include "paddle/pir/core/value.h" + +#include "dnnl.hpp" // NOLINT +#include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" +#include "paddle/fluid/framework/type_defs.h" +#include "paddle/fluid/ir_adaptor/translator/op_compat_info.h" +#include "paddle/phi/backends/onednn/onednn_context.h" +#include "paddle/phi/backends/onednn/onednn_helper.h" +#include "paddle/phi/kernels/funcs/data_layout_transform.h" + +namespace paddle { +namespace framework { + +static RuntimeAttribute ConvertPirAttribute2RuntimeAttribute( + PIRAttribute attr, + const std::string& attr_name, + const paddle::dialect::OpYamlInfoParser& op_yaml_info) { + auto& attr_type_name = op_yaml_info.AttrTypeName(attr_name); + if (attr_type_name == "pir::Int32Attribute") { + return attr.dyn_cast().data(); + } else if (attr_type_name == "pir::FloatAttribute") { + return attr.dyn_cast().data(); + } else if (attr_type_name == "pir::BoolAttribute") { + return attr.dyn_cast().data(); + } else if (attr_type_name == "pir::StrAttribute") { + return attr.dyn_cast().AsString(); + } else if (attr_type_name == "pir::ArrayAttribute") { + auto array_list = attr.dyn_cast().AsVector(); + std::vector vec_res; + if (array_list.size() > 0) { + PADDLE_ENFORCE_EQ(array_list[0].isa(), + true, + phi::errors::Unimplemented( + "the 0th elementwise MUST be pir::Int32Attribute")); + for (size_t i = 0; i < array_list.size(); ++i) { + vec_res.push_back(array_list[i].dyn_cast().data()); + } + } + return vec_res; + } else if (attr_type_name == "pir::ArrayAttribute") { + auto array_list = attr.dyn_cast().AsVector(); + std::vector vec_res; + if (array_list.size() > 0) { + if (array_list[0].isa()) { + for (size_t i = 0; i < array_list.size(); ++i) { + vec_res.push_back( + array_list[i].dyn_cast().data()); + } + + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "ConvertPirAttribute2RuntimeAttribute not support [%s] ", + attr_type_name)); + } + } + return vec_res; + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "ConvertPirAttribute2RuntimeAttribute not support [%s] ", + attr_type_name)); + } +} + +void TensorNameMap(pir::Operation* op, + const ValueExecutionInfo& value_exec_info, + const paddle::dialect::OpYamlInfoParser& op_yaml_info, + std::map>& + inputs_tensor_name_map, // NOLINT + std::map>& + outputs_tensor_name_map) { // NOLINT + const Scope* inner_scope = value_exec_info.GetScope(); + VLOG(6) << "TensorNameMap in scope[" << inner_scope << "]"; + + auto& vec_kernel_fn_tensor_params = op_yaml_info.TensorParams(true); + + auto& name2id = op_yaml_info.InputName2Id(); + + std::string fluid_op_name = op_yaml_info.GetOriginOpName(); + + auto& op_normalizer = paddle::translator::OpNameNormalizer::instance(); + + for (auto& name : vec_kernel_fn_tensor_params) { + PADDLE_ENFORCE_EQ( + name2id.count(name), + true, + phi::errors::NotFound("param [%s] MUST in name2id map", name)); + auto index = name2id.at(name); + pir::Value ptr = op->operand_source(index); + + if (!IsInvalid(ptr)) { + continue; + } + + auto legacy_arg_name = op_normalizer.GetLegacyArgName(fluid_op_name, name); + auto in_var_name = value_exec_info.GetVarName(ptr); + PADDLE_ENFORCE_NOT_NULL(inner_scope->FindVar(in_var_name), + phi::errors::PreconditionNotMet( + "can not find var[%s] in scope", in_var_name)); + + auto type = ptr.type(); + if (type.isa() || + type.isa()) { + inputs_tensor_name_map[legacy_arg_name] = {in_var_name}; + } else if (type.isa()) { + auto var = inner_scope->FindVar(in_var_name); + auto var_ref = var->Get(); + std::vector vec_tmp; + vec_tmp.reserve(var_ref.size()); + for (size_t k = 0; k < var_ref.size(); ++k) { + vec_tmp.push_back(value_exec_info.GetVarName(var_ref[k])); + } + inputs_tensor_name_map[legacy_arg_name] = vec_tmp; + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "only support AllocatedDenseTensor, AllocatedSelectedRowsType and " + "pir::vector type")); + } + } + + auto& output_name_list = op_yaml_info.OutputNames(); + for (size_t i = 0; i < output_name_list.size(); ++i) { + auto name = output_name_list[i]; + pir::Value ptr = op->result(i); + auto legacy_arg_name = op_normalizer.GetLegacyArgName(fluid_op_name, name); + + if (!IsInvalid(ptr)) { + continue; + } + + auto out_var_name = value_exec_info.GetVarName(ptr); + + PADDLE_ENFORCE_NOT_NULL(inner_scope->FindVar(out_var_name), + phi::errors::PreconditionNotMet( + "can not find var[%s] in scope", out_var_name)); + + auto type = ptr.type(); + if (type.isa() || + type.isa()) { + outputs_tensor_name_map[legacy_arg_name] = {out_var_name}; + } else if (type.isa()) { + auto var = inner_scope->FindVar(out_var_name); + auto var_ref = var->Get(); + std::vector vec_tmp; + vec_tmp.reserve(var_ref.size()); + for (size_t k = 0; k < var_ref.size(); ++k) { + vec_tmp.push_back(value_exec_info.GetVarName(var_ref[k])); + } + outputs_tensor_name_map[legacy_arg_name] = vec_tmp; + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "only support AllocatedDenseTensor, AllocatedSelectedRowsType and " + "pir::vector type")); + } + } +} + +OneDNNPhiKernelInstruction::OneDNNPhiKernelInstruction( + size_t id, + const platform::Place& place, + pir::Operation* op, + const ValueExecutionInfo* value_exec_info) + : InstructionBase(id, place), value_exec_info_(value_exec_info) { + // Step1: build phi kernel instruction as PhiKernelInstruction + auto op_attributes = op->attributes(); + auto op_name = + op_attributes.at("op_name").dyn_cast().AsString(); + pir::OpInfo op_info = + pir::IrContext::Instance()->GetRegisteredOpInfo(op_name); + op_ = op; + phi_op_name_ = op_name; + VLOG(6) << "construct phi kernel instruction for: " << phi_op_name_; + + SetKernelType(AnalyseOpFuncType(op, place)); + VLOG(6) << "finish process analyse kernel type"; + + infer_meta_interface_ = + op_info.GetInterfaceImpl(); + VLOG(6) << "finish process infer_meta_interface_"; + + auto yaml_interface = + op_info.GetInterfaceImpl(); + PADDLE_ENFORCE_NOT_NULL( + yaml_interface, + phi::errors::PreconditionNotMet( + "can not find OpYamlInfoInterface from [%s]", phi_op_name_)); + paddle::dialect::OpYamlInfoParser yaml_info_parser( + yaml_interface->get_op_info_(), + paddle::dialect::IsOneDNNLegacyOp(op_name)); + VLOG(6) << "finish process yaml_info_parser"; + + if (infer_meta_interface_) { + BuildPhiContext< + phi::InferMetaContext, + phi::MetaTensor, + phi::MetaTensor, + paddle::small_vector, + paddle::small_vector, + false>(op, *value_exec_info_, yaml_info_parser, &infer_meta_context_); + } + VLOG(6) << "finish process infer meta context"; + + auto kernel_name = + op_attributes.at("kernel_name").dyn_cast().AsString(); + auto kernel_key = op_attributes.at("kernel_key") + .dyn_cast() + .data(); + + phi_kernel_ = new phi::Kernel( + phi::KernelFactory::Instance().SelectKernel(kernel_name, kernel_key)); + PADDLE_ENFORCE_EQ( + phi_kernel_->IsValid(), true, "not found kernel for [%s]", kernel_name); + VLOG(6) << "finish process select kernel"; + + BuildPhiContext, + paddle::small_vector, + true>( + op, *value_exec_info_, yaml_info_parser, &kernel_context_); + + kernel_context_.SetDeviceContext(phi::DeviceContextPool::Instance().Get( + phi::TransToPhiPlace(kernel_key.backend()))); + VLOG(6) << "finish process kernel context"; + + SetDeviceContext( + ParseDeviceContext(op, + phi::DeviceContextPool::Instance().Get( + phi::TransToPhiPlace(kernel_key.backend())), + place, + GetExecutionStream(), + GetStreamPriority())); + VLOG(6) << "finish process device context"; + + InitInputsOutputsIds(op, *value_exec_info); + VLOG(6) << "finish process inputs outputs index"; + + auto& no_need_buffer_ids = yaml_info_parser.NoNeedBufferIds(); + std::unordered_set no_need_buffer_values; + for (size_t id = 0; id < no_need_buffer_ids.size(); id++) { + no_need_buffer_values.insert(op->operand_source(no_need_buffer_ids[id])); + } + SetNoNeedBuffer(no_need_buffer_values); + VLOG(6) << "finish process no need buffer"; + + // Step2: build layout_transform information + if (op_attributes.count("layout_transform_arg")) { + auto layout_transform_arg = op_attributes.at("layout_transform_arg") + .dyn_cast() + .AsString(); + auto data_layout = op_attributes.at(layout_transform_arg) + .dyn_cast() + .AsString(); + input_layout_ = common::StringToDataLayout(data_layout); + std::vector layout_transform_inputs_attr = + op->attributes() + .at("layout_transform_inputs") + .dyn_cast() + .AsVector(); + std::vector layout_transform_inputs; + for (auto& attr : layout_transform_inputs_attr) { + auto pair = kernel_context_.InputRangeAt(value_exec_info_->GetIdByName( + attr.dyn_cast().AsString())); + for (int i = pair.first; i < pair.second; ++i) { + layout_transform_inputs_.insert(i); + } + } + } + + // Step3: build extra attr information + if (op_attributes.count("extra_args")) { + std::vector extra_args_attr = + op->attributes() + .at("extra_args") + .dyn_cast() + .AsVector(); + std::vector extra_args; + for (auto& attr : extra_args_attr) { + auto attr_name = attr.dyn_cast().AsString(); + extra_attr_[attr_name] = ConvertPirAttribute2RuntimeAttribute( + op_attributes.at(attr_name), attr_name, yaml_info_parser); + } + } + TensorNameMap(op, *value_exec_info_, yaml_info_parser, inputs_, outputs_); +} + +OneDNNPhiKernelInstruction::~OneDNNPhiKernelInstruction() { + if (phi_kernel_ != nullptr) { + delete phi_kernel_; + } +} + +void OneDNNPhiKernelInstruction::Run() { + // Step1. TransLayout + auto inputs = kernel_context_.InputsBetween( + size_t(0), kernel_context_.InputsSize()); + for (size_t i = 0; i < inputs.size(); ++i) { + auto input = inputs[i]; + if (input->layout() != phi::DataLayout::ONEDNN) { + phi::DataLayout from_layout = input->layout(); + + // Handle 'layout_transform' in + // ops_onednn_extra.yaml(GetKernelTypeForVar) + if (layout_transform_inputs_.count(i) && + input_layout_ != phi::DataLayout::kAnyLayout) { + from_layout = input_layout_; + } + + auto transed_tensor = const_cast(input); + + if (from_layout == DataLayout::kNHWC || + from_layout == DataLayout::kNDHWC) { + phi::funcs::MatchShapeToLayout( + transed_tensor, from_layout, phi::DataLayout::ONEDNN); + // We register only NHWC assuming that model is consistent e.g. either + // NHWC or NCHW + phi::OneDNNContext::tls().set_cur_paddle_data_layout(from_layout); + } + + if (from_layout == DataLayout::kAnyLayout) { + from_layout = phi::OneDNNContext::tls().get_cur_paddle_data_layout(); + } + + dnnl::memory::desc out_mem_desc = + phi::funcs::make_memory_desc(*input, from_layout); + transed_tensor->set_mem_desc(out_mem_desc); + } + } + + // Step2. Append extra information into ctx + // SetDnnAttrIntoDeviceContext + // SetInputsName SetOutputsName + auto one_dnn_ctx = const_cast( + &kernel_context_.GetDeviceContext()); + for (auto& attr : extra_attr_) { + one_dnn_ctx->SetDnnAttr(attr.first, attr.second); + } + one_dnn_ctx->SetInputsName(inputs_); + one_dnn_ctx->SetOutputsName(outputs_); + + // Step3. InferMeta + if (infer_meta_interface_) { + infer_meta_interface_->infer_meta_(&(infer_meta_context_)); + } + + // Step4. Run kernel + VLOG(6) << "Run op " << phi_op_name_ << " infer meta."; + (*(phi_kernel_))(&(kernel_context_)); + VLOG(6) << "Run op " << phi_op_name_ << " kernel."; + + // Step5. ClearDnnAttr + one_dnn_ctx->ClearDnnAttr(); +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h new file mode 100644 index 0000000000000..c15a69728f9c3 --- /dev/null +++ b/paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h @@ -0,0 +1,82 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/framework/new_executor/instruction/instruction_base.h" + +namespace pir { +class Operation; +} // namespace pir + +namespace paddle { +namespace framework { +class Scope; +class ValueExecutionInfo; + +using RuntimeAttribute = phi::Attribute; +using PIRAttribute = pir::Attribute; + +class OneDNNPhiKernelInstruction : public InstructionBase { + public: + OneDNNPhiKernelInstruction(size_t id, + const platform::Place& place, + ::pir::Operation* op, + const ValueExecutionInfo* value_exec_info); + + ~OneDNNPhiKernelInstruction(); + + phi::Kernel* PhiKernel() const { return phi_kernel_; } + + const phi::KernelContext& KernelContext() const { return kernel_context_; } + + const phi::InferMetaContext& InferMetaContext() const { + return infer_meta_context_; + } + + paddle::dialect::InferMetaInterface::Concept* InferMetaInterface() const { + return infer_meta_interface_; + } + + ::pir::Operation* Operation() const override { return op_; } + + void Run() override; + + const std::string& Name() const override { return phi_op_name_; } + + private: + paddle::dialect::InferMetaInterface::Concept* infer_meta_interface_{ + nullptr}; // not owned + + phi::InferMetaContext infer_meta_context_; + + phi::KernelContext kernel_context_; + + phi::Kernel* phi_kernel_{nullptr}; // not owned + + std::string phi_op_name_; + + ::pir::Operation* op_{nullptr}; // not owned + + const ValueExecutionInfo* value_exec_info_; // not owned + + std::set layout_transform_inputs_{}; + phi::DataLayout input_layout_{phi::DataLayout::kAnyLayout}; + std::map extra_attr_{}; + std::map> inputs_{}; + std::map> outputs_{}; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/new_executor/pir_interpreter.cc b/paddle/fluid/framework/new_executor/pir_interpreter.cc index 7dbb514513fc2..1cd1117d0ea1d 100644 --- a/paddle/fluid/framework/new_executor/pir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/pir_interpreter.cc @@ -34,6 +34,9 @@ #include "paddle/phi/core/sparse_csr_tensor.h" #ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_legacy_kernel_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_mixed_phi_kernel_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/onednn/onednn_phi_kernel_instruction.h" #include "paddle/fluid/platform/mkldnn_helper.h" #endif @@ -728,6 +731,22 @@ void PirInterpreter::BuildInstruction() { } else { CREATE_INSTR(PhiKernelInstruction); } +#ifdef PADDLE_WITH_DNNL + } else if (op.dialect()->name() == "pd_onednn_kernel") { + auto op_name = op.attributes() + .at("op_name") + .dyn_cast<::pir::StrAttribute>() + .AsString(); + VLOG(6) << "process " << op_name; + + if (op.isa()) { + CREATE_INSTR(OneDNNPhiKernelInstruction); + } else if (op.isa()) { + CREATE_INSTR(OneDNNMixedPhiKernelInstruction); + } else { + CREATE_INSTR(OneDNNLegacyKernelInstruction); + } +#endif #ifdef PADDLE_WITH_CINN } else if (op.dialect()->name() == "cinn_runtime") { CREATE_INSTR(CinnJitInstruction); diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 76a787cda64bf..626073d143e3e 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -44,6 +44,9 @@ #include "paddle/pir/core/operation.h" #include "paddle/pir/core/value.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/pd_onednn_op.h" +#endif // NOTE(zhangbo9674): File pd_op.h is generated by op_gen.py, see details in // paddle/fluid/pir/dialect/CMakeLists.txt. #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" @@ -77,7 +80,10 @@ using AttributeHandlerFn = std::function; using DenseTensorTypeStorage = paddle::dialect::DenseTensorTypeStorage; constexpr char kTargetDialectPrefix[] = "pd_op."; // NOLINT -constexpr char kEmptyVarName[] = "@EMPTY@"; // NOLINT +#ifdef PADDLE_WITH_DNNL +constexpr char kOneDNNTargetDialectPrefix[] = "pd_onednn_op."; // NOLINT +#endif +constexpr char kEmptyVarName[] = "@EMPTY@"; // NOLINT static const std::unordered_set SpecialNonInplaceOps = {}; @@ -223,12 +229,36 @@ inline pir::Operation* InsertCreateArrayOp(pir::IrContext* ctx, return create_array_op.operation(); } +inline std::string GetPrefix(pir::IrContext* ctx, const OpDesc& op_desc) { +#ifdef PADDLE_WITH_DNNL + if (op_desc.GetAttrIfExists("use_mkldnn")) { + std::string target_op_name = + kOneDNNTargetDialectPrefix + OpNameCompatibleMapping(op_desc.Type()); + if (IsInplace(op_desc) && *target_op_name.rbegin() != '_') { + target_op_name += "_"; + } + auto op_info = ctx->GetRegisteredOpInfo(target_op_name); + if (!op_info) { + VLOG(3) << op_desc.Type() + << "'s use_mkldnn == True, but PIR not support OneDNN for this " + "op right now."; + return kTargetDialectPrefix; + } else { + return kOneDNNTargetDialectPrefix; + } + } else { + return kTargetDialectPrefix; + } +#else + return kTargetDialectPrefix; +#endif +} } // namespace pir::OpInfo OpTranscriber::LoopkUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) { std::string target_op_name = - kTargetDialectPrefix + OpNameCompatibleMapping(op_desc.Type()); + GetPrefix(ctx, op_desc) + OpNameCompatibleMapping(op_desc.Type()); if (IsInplace(op_desc) && *target_op_name.rbegin() != '_') { target_op_name += "_"; } @@ -321,7 +351,7 @@ pir::OpInfo OpTranscriber::LoopkUpOpInfo(pir::IrContext* ctx, op_desc.Type(), target_op_name); - target_op_name = kTargetDialectPrefix + target_op_name; + target_op_name = GetPrefix(ctx, op_desc) + target_op_name; if (IsInplace(op_desc) && *target_op_name.rbegin() != '_') { target_op_name += "_"; } @@ -1054,7 +1084,7 @@ struct EmbeddingGradOpTranscriber : public OpTranscriber { pir::OpInfo LoopkUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { std::string target_op_name = - kTargetDialectPrefix + OpNameCompatibleMapping(op_desc.Type()); + GetPrefix(ctx, op_desc) + OpNameCompatibleMapping(op_desc.Type()); bool is_sparse = paddle::get(op_desc.GetAttr("is_sparse")); @@ -1307,7 +1337,7 @@ struct AddNOpTranscriber : public OpTranscriber { pir::OpInfo LoopkUpOpInfo(pir::IrContext* ctx, const OpDesc& op_desc) override { std::string target_op_name = - kTargetDialectPrefix + OpNameCompatibleMapping(op_desc.Type()); + GetPrefix(ctx, op_desc) + OpNameCompatibleMapping(op_desc.Type()); if (IsInplace(op_desc)) { target_op_name += "_"; } else { diff --git a/paddle/fluid/ir_adaptor/translator/translate.cc b/paddle/fluid/ir_adaptor/translator/translate.cc index 7a7081fe1acbf..04ddf1d13a5a8 100644 --- a/paddle/fluid/ir_adaptor/translator/translate.cc +++ b/paddle/fluid/ir_adaptor/translator/translate.cc @@ -22,6 +22,9 @@ #include "paddle/pir/core/builtin_dialect.h" #include "paddle/pir/core/program.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h" +#endif namespace paddle { using LegacyProgramDesc = ::paddle::framework::ProgramDesc; @@ -31,6 +34,9 @@ std::unique_ptr TranslateLegacyProgramToProgram( const LegacyProgramDesc& legacy_program) { pir::IrContext* ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); +#ifdef PADDLE_WITH_DNNL + ctx->GetOrRegisterDialect(); +#endif auto program = std::make_unique(ctx); translator::ProgramTranslator program_translator(&legacy_program, program.get()); diff --git a/paddle/fluid/ir_adaptor/translator/utils.cc b/paddle/fluid/ir_adaptor/translator/utils.cc index ebba4428220f7..dbd85292974bf 100644 --- a/paddle/fluid/ir_adaptor/translator/utils.cc +++ b/paddle/fluid/ir_adaptor/translator/utils.cc @@ -23,6 +23,9 @@ #include "paddle/pir/core/builtin_attribute.h" #include "paddle/pir/core/builtin_type.h" #include "paddle/pir/core/utils.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h" +#endif namespace paddle { namespace dialect { @@ -94,6 +97,9 @@ std::vector CheckUnregisteredOperationInBlock( std::vector CheckUnregisteredOperation( pir::IrContext* ctx, const framework::ProgramDesc& legacy_program) { ctx->GetOrRegisterDialect(); +#ifdef PADDLE_WITH_DNNL + ctx->GetOrRegisterDialect(); +#endif std::vector unregistered_ops; for (size_t block_idx = 0; block_idx < legacy_program.Size(); block_idx++) { diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index 2c812ccada69a..337841b227497 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -27,6 +27,7 @@ set(pir_op_fwd_src_yaml ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/ops.yaml) set(pir_op_bwd_src_yaml ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/ops_backward.yaml) + set(pir_update_op_fwd_src_yaml ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/update_ops.yaml) set(parsed_op_dir @@ -108,6 +109,44 @@ set(generated_files_pd_op "${pir_bwd_op_source_file}" "${pir_update_op_source_file}") +if(WITH_MKLDNN) + set(pir_op_onednn_yaml ${parsed_op_dir}/onednn.parsed.yaml) + + set(pd_onednn_op_yaml_file + ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/onednn.yaml) + + set(pd_ops_onednn_extra_yaml_file + ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml + ) + + set(op_onednn_info_file ${PD_DIALECT_SOURCE_DIR}/pd_onednn_op_info.cc) + set(op_onednn_info_file_tmp ${op_onednn_info_file}.tmp) + + set(onednn_op_namespace paddle,onednn,dialect) + set(onednn_dialect_name pd_onednn_op) + set(onednn_op_header_file ${PD_DIALECT_SOURCE_DIR}/pd_onednn_op.h) + set(onednn_op_source_file ${PD_DIALECT_SOURCE_DIR}/pd_onednn_op.cc) + set(onednn_op_header_file_tmp ${onednn_op_header_file}.tmp) + set(onednn_op_source_file_tmp ${onednn_op_source_file}.tmp) + + execute_process( + COMMAND ${PYTHON_EXECUTABLE} ${op_gen_parsed_yaml_file} --op_yaml_path + ${pd_onednn_op_yaml_file} --output_path ${pir_op_onednn_yaml}) + + execute_process( + COMMAND + ${PYTHON_EXECUTABLE} ${op_gen_file} --op_yaml_files ${op_yaml_files} + --op_compat_yaml_file ${op_compat_yaml_file} --namespaces + ${onednn_op_namespace} --dialect_name ${onednn_dialect_name} + --op_def_h_file ${onednn_op_header_file_tmp} --op_info_file + ${op_onednn_info_file_tmp} --op_def_cc_file ${onednn_op_source_file_tmp} + --onednn_yaml_file ${pir_op_onednn_yaml} --ops_onednn_extra_yaml_file + ${pd_ops_onednn_extra_yaml_file}) + + set(generated_files_onednn_pd_op + "${onednn_op_header_file}" "${onednn_op_source_file}" + "${op_onednn_info_file}") +endif() set(api_gen_yaml_files ${op_fwd_yaml},${op_bwd_yaml},${pir_op_fwd_yaml},${pir_op_bwd_yaml},${pir_update_op_fwd_yaml} ) @@ -159,8 +198,10 @@ execute_process( set(generated_files_ops_api "${ops_api_source_file}") -set(generated_files_pir ${generated_files_pd_op} ${generated_files_pd_api} - ${generated_files_python_c} ${generated_files_ops_api}) +set(generated_files_pir + ${generated_files_pd_op} ${generated_files_onednn_pd_op} + ${generated_files_pd_api} ${generated_files_python_c} + ${generated_files_ops_api}) foreach(generated_file ${generated_files_pir}) if(EXISTS "${generated_file}.tmp" AND EXISTS "${generated_file}") execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -206,6 +247,10 @@ set(op_dialect_srcs ${pir_update_op_source_file} ${api_source_file}) +if(WITH_MKLDNN) + set(op_dialect_srcs ${op_dialect_srcs} ${onednn_op_source_file}) +endif() + set(op_dialect_deps phi common pir type_info string_helper) cc_library( @@ -222,6 +267,13 @@ set(op_dialect_vjp_srcs ${op_decomp_source_file} ${op_vjp_source_file} ${PADDLE_SOURCE_DIR}/paddle/fluid/primitive/base/decomp_trans.cc) + +if(WITH_MKLDNN) + set(op_dialect_vjp_srcs + ${op_dialect_vjp_srcs} + ${CMAKE_CURRENT_SOURCE_DIR}/operator/ir/op_onednn_dialect.cc) +endif() + set(op_dialect_vjp_deps primitive_vjp_experimental op_dialect) cc_library( diff --git a/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.cc b/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.cc index 95e77ff6169c6..ecf04d4411397 100644 --- a/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.cc +++ b/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.cc @@ -122,7 +122,110 @@ void KernelDialect::PrintOperation(pir::Operation *op, } } +#ifdef PADDLE_WITH_DNNL +OneDNNKernelDialect::OneDNNKernelDialect(pir::IrContext *context) + : pir::Dialect(name(), context, pir::TypeId::get()) { + initialize(); +} + +void OneDNNKernelDialect::initialize() { + RegisterTypes(); + RegisterOps(); + RegisterAttributes(); +} + +void OneDNNKernelDialect::PrintType(pir::Type type, std::ostream &os) const { + if (type.isa()) { + AllocatedDenseTensorType tensor_type = + type.dyn_cast(); + + os << phi::AllocationTypeStr(tensor_type.place().GetType()) << "_"; + os << "tensor<"; + for (auto d : common::vectorize(tensor_type.dims())) { + os << d; + os << "x"; + } + tensor_type.dtype().Print(os); + os << ">"; + } else if (type.isa()) { + AllocatedSelectedRowsType tensor_type = + type.dyn_cast(); + + os << phi::AllocationTypeStr(tensor_type.place().GetType()) << "_"; + os << "tensor<"; + for (auto d : common::vectorize(tensor_type.dims())) { + os << d; + os << "x"; + } + tensor_type.dtype().Print(os); + os << ">"; + } else if (type.isa()) { + AllocatedDenseTensorArrayType tensor_array_type = + type.dyn_cast(); + + os << phi::AllocationTypeStr(tensor_array_type.place().GetType()) << "_"; + os << "tensor_array<"; + tensor_array_type.dtype().Print(os); + os << ">"; + } +} + +void OneDNNKernelDialect::PrintAttribute(pir::Attribute attr, + std::ostream &os) const { + phi::KernelKey kernel = attr.dyn_cast().data(); + + os << ""; +} + +void OneDNNKernelDialect::PrintOperation(pir::Operation *op, + pir::IrPrinter &printer) const { + if (op->dyn_cast() || op->dyn_cast()) { + auto &os = printer.os; + printer.PrintOpResult(op); + os << " ="; + if (auto phi_kernel_op = op->dyn_cast()) { + std::string kernel_name = phi_kernel_op.kernel_name(); + if (op->attributes().count("is_inplace") != 0 && + op->attributes() + .at("is_inplace") + .dyn_cast() + .data()) { + kernel_name = kernel_name + "_"; + } + os << " \"" << kernel_name << "(phi_kernel)\""; + } else { + auto legacy_kernel_op = op->dyn_cast(); + std::string kernel_name = legacy_kernel_op.kernel_name(); + if (op->attributes().count("is_inplace") != 0 && + op->attributes() + .at("is_inplace") + .dyn_cast() + .data()) { + kernel_name = kernel_name + "_"; + } + os << " \"" << kernel_name << "(legacy_kernel)\""; + } + printer.PrintOpOperands(op); + printer.PrintAttributeMap(op); + os << " :"; + printer.PrintOperandsType(op); + os << " -> "; + printer.PrintOpReturnType(op); + } else { + printer.PrintGeneralOperation(op); + } +} +#endif + } // namespace dialect } // namespace paddle IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::KernelDialect) +#ifdef PADDLE_WITH_DNNL +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNKernelDialect) +#endif diff --git a/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h b/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h index d2fbcadaf8cf2..fbdb53a40b183 100644 --- a/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h +++ b/paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h @@ -36,7 +36,29 @@ class KernelDialect : public pir::Dialect { void initialize(); }; +#ifdef PADDLE_WITH_DNNL +class OneDNNKernelDialect : public pir::Dialect { + public: + explicit OneDNNKernelDialect(pir::IrContext* context); + + static const char* name() { return "pd_onednn_kernel"; } + + void PrintType(pir::Type type, std::ostream& os) const override; + + void PrintAttribute(pir::Attribute attr, std::ostream& os) const override; + + void PrintOperation(pir::Operation* op, + pir::IrPrinter& printer) const override; // NOLINT + + private: + void initialize(); +}; +#endif + } // namespace dialect } // namespace paddle IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::KernelDialect) +#ifdef PADDLE_WITH_DNNL +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNKernelDialect) +#endif diff --git a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc index 8ad46bc8906ad..45f0a848fc174 100644 --- a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc +++ b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.cc @@ -98,8 +98,135 @@ phi::KernelKey LegacyKernelOp::kernel_key() { return attributes().at("kernel_key").dyn_cast().data(); } +#ifdef PADDLE_WITH_DNNL +const char* OneDNNPhiKernelOp::attributes_name[attributes_num] = { // NOLINT + "op_name", + "kernel_name", + "kernel_key"}; + +void OneDNNPhiKernelOp::VerifySig() { + VLOG(4) << "Verifying inputs, outputs and attributes for: OneDNNPhiKernelOp."; + + auto& attributes = this->attributes(); + + PADDLE_ENFORCE(attributes.count("op_name") > 0 && + attributes.at("op_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: op_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_name") > 0 && + attributes.at("kernel_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_key") > 0 && + attributes.at("kernel_key").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_key is not right.")); +} + +std::string OneDNNPhiKernelOp::op_name() { + return attributes().at("op_name").dyn_cast().AsString(); +} +std::string OneDNNPhiKernelOp::kernel_name() { + return attributes() + .at("kernel_name") + .dyn_cast() + .AsString(); +} +phi::KernelKey OneDNNPhiKernelOp::kernel_key() { + return attributes().at("kernel_key").dyn_cast().data(); +} + +const char* OneDNNMixedPhiKernelOp::attributes_name[attributes_num] = + { // NOLINT + "op_name", + "kernel_name", + "kernel_key"}; + +void OneDNNMixedPhiKernelOp::VerifySig() { + VLOG(4) << "Verifying inputs, outputs and attributes for: " + "OneDNNMixedPhiKernelOp."; + + auto& attributes = this->attributes(); + + PADDLE_ENFORCE(attributes.count("op_name") > 0 && + attributes.at("op_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: op_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_name") > 0 && + attributes.at("kernel_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_key") > 0 && + attributes.at("kernel_key").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_key is not right.")); +} + +std::string OneDNNMixedPhiKernelOp::op_name() { + return attributes().at("op_name").dyn_cast().AsString(); +} +std::string OneDNNMixedPhiKernelOp::kernel_name() { + return attributes() + .at("kernel_name") + .dyn_cast() + .AsString(); +} +phi::KernelKey OneDNNMixedPhiKernelOp::kernel_key() { + return attributes().at("kernel_key").dyn_cast().data(); +} + +const char* OneDNNLegacyKernelOp::attributes_name[attributes_num] = { // NOLINT + "op_name", + "kernel_name", + "kernel_key"}; + +void OneDNNLegacyKernelOp::VerifySig() { + VLOG(4) + << "Verifying inputs, outputs and attributes for: OneDNNLegacyKernelOp."; + + auto& attributes = this->attributes(); + + PADDLE_ENFORCE(attributes.count("op_name") > 0 && + attributes.at("op_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: op_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_name") > 0 && + attributes.at("kernel_name").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_name is not right.")); + + PADDLE_ENFORCE(attributes.count("kernel_key") > 0 && + attributes.at("kernel_key").isa(), + phi::errors::PreconditionNotMet( + "Type of attribute: kernel_key is not right.")); +} + +std::string OneDNNLegacyKernelOp::op_name() { + return attributes().at("op_name").dyn_cast().AsString(); +} +std::string OneDNNLegacyKernelOp::kernel_name() { + return attributes() + .at("kernel_name") + .dyn_cast() + .AsString(); +} +phi::KernelKey OneDNNLegacyKernelOp::kernel_key() { + return attributes().at("kernel_key").dyn_cast().data(); +} +#endif + } // namespace dialect } // namespace paddle IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::PhiKernelOp) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::LegacyKernelOp) +#ifdef PADDLE_WITH_DNNL +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNPhiKernelOp) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNMixedPhiKernelOp) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNLegacyKernelOp) +#endif diff --git a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.h b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.h index a96aa5732d580..df72315870208 100644 --- a/paddle/fluid/pir/dialect/kernel/ir/kernel_op.h +++ b/paddle/fluid/pir/dialect/kernel/ir/kernel_op.h @@ -44,8 +44,51 @@ class LegacyKernelOp : public pir::Op { void VerifySig(); }; +#ifdef PADDLE_WITH_DNNL +class OneDNNPhiKernelOp : public pir::Op { + public: + using Op::Op; + static const char *name() { return "pd_onednn_kernel.phi_kernel"; } + static constexpr uint32_t attributes_num = 3; + static const char *attributes_name[attributes_num]; + std::string op_name(); + std::string kernel_name(); + phi::KernelKey kernel_key(); + void VerifySig(); +}; + +class OneDNNMixedPhiKernelOp : public pir::Op { + public: + using Op::Op; + static const char *name() { return "pd_onednn_kernel.phi_mixed_kernel"; } + static constexpr uint32_t attributes_num = 3; + static const char *attributes_name[attributes_num]; + std::string op_name(); + std::string kernel_name(); + phi::KernelKey kernel_key(); + void VerifySig(); +}; + +class OneDNNLegacyKernelOp : public pir::Op { + public: + using Op::Op; + static const char *name() { return "pd_onednn_kernel.legacy_kernel"; } + static constexpr uint32_t attributes_num = 3; + static const char *attributes_name[attributes_num]; + std::string op_name(); + std::string kernel_name(); + phi::KernelKey kernel_key(); + void VerifySig(); +}; +#endif + } // namespace dialect } // namespace paddle IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::PhiKernelOp) IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::LegacyKernelOp) +#ifdef PADDLE_WITH_DNNL +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNPhiKernelOp) +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNMixedPhiKernelOp) +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNLegacyKernelOp) +#endif diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 7dd754e868f86..4cb54ada152b8 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -30,6 +30,7 @@ from op_kerneltype_gen import gen_kernel_type_for_var_str from op_member_func_gen import gen_op_get_inputs_outputs_str from op_verify_gen import gen_verify_func_str +from ops_onednn_extra_parser import parse_extra_args, parse_layout_transform from parse_kernel_key_gen import gen_parse_kernel_key_str from vjp_interface_black_list import vjp_interface_black_list @@ -67,6 +68,7 @@ #include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" #include "paddle/fluid/pir/dialect/operator/interface/decomp.h" #include "paddle/fluid/pir/dialect/operator/trait/inplace.h" +#include "paddle/fluid/pir/dialect/operator/trait/onednn.h" #include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" #include "paddle/fluid/framework/infershape_utils.h" #include "paddle/phi/core/infermeta_utils.h" @@ -213,6 +215,17 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ return std::make_tuple(inputs, attributes, outputs, run_time_info, "{origin_op_name}"); }} """ + +OP_INFO_ONEDNN_TEMPLATE = """ +OpInfoTuple {op_name}::GetOpInfo() {{ + std::vector inputs = {{ {inputs} }}; + std::vector attributes = {{ {attributes} }}; + std::vector outputs = {{ {outputs} }}; + paddle::dialect::OpRunTimeInfo run_time_info = paddle::dialect::OpRunTimeInfo("{infer_meta_func}", {{"{infer_meta_param}"}}, "{kernel_func}", {{"{kernel_param}"}}, {{{kernel_key_dtype}}}, {{{kernel_key_backend}}}, {{{inplace}}}, {{{view}}}, {{{extra_args}}}, "{layout_transform_arg}", {{{layout_transform_inputs}}}, {is_onednn_only}, {dynamic_fallback}); + return std::make_tuple(inputs, attributes, outputs, run_time_info, "{origin_op_name}"); +}} +""" + CONSTRUCT_INPUT_INFO_TEMPLATE = """paddle::dialect::OpInputInfo("{name}", "{typename}", {optional}, {no_need_buffer}, {is_mutable_attribute}, {with_grad_semantic})""" CONSTRUCT_OUTPUT_INFO_TEMPLATE = """paddle::dialect::OpOutputInfo("{name}", "{typename}", {optional}, {intermediate})""" CONSTRUCT_ATTRIBUTE_INFO_TEMPLATE = """paddle::dialect::OpAttributeInfo("{name}", "{typename}", "{data_type}")""" @@ -420,7 +433,7 @@ def __init__(self, op_yaml_item, op_compat_item): self.non_mutable_attribute_data_type_list, self.non_mutable_attribute_build_arg_type_list, self.non_mutable_attribute_default_value_list, - ) = self.parse_non_nutable_attribute() + ) = self.parse_non_mutable_attribute() # parse infermeta && kernel self.infer_meta_map = self.parse_infer_meta_map() @@ -462,6 +475,18 @@ def __init__(self, op_yaml_item, op_compat_item): # parse interfaces list self.interfaces_list = self.parse_op_interfaces() + # OneDNN info + if "extra_args" in self.op_yaml_item: + self.onednn_extra_args = self.op_yaml_item["extra_args"] + self.onednn_layout_transform = self.op_yaml_item["layout_transform"] + self.is_onednn_only = self.op_yaml_item["is_onednn_only"] + self.dynamic_fallback = self.op_yaml_item["dynamic_fallback"] + else: + self.onednn_extra_args = [] + self.onednn_layout_transform = None + self.is_onednn_only = False + self.dynamic_fallback = False + def parse_op_traits(self): if 'traits' in self.op_yaml_item: return self.op_yaml_item['traits'] @@ -633,7 +658,7 @@ def parse_mutable_attribute(self): sorted_mutable_attribute_type_list, ) - def parse_non_nutable_attribute(self): + def parse_non_mutable_attribute(self): op_non_mutable_attribute_name_list = [] op_non_mutable_attribute_type_list = [] op_non_mutable_attribute_data_type_list = [] @@ -1112,17 +1137,21 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): if ( op_info.backward_name and op_info.op_phi_name[0] not in vjp_interface_black_list + and dialect_name != "pd_onednn_op" ): op_interfaces += ["paddle::dialect::VjpInterface"] exclusive_interface_str = gen_exclusive_interface_str( op_info, op_info_items ) - if dialect_name == "pd_op": + if dialect_name == "pd_op" or dialect_name == "pd_onednn_op": op_interfaces += ["paddle::dialect::GetKernelTypeForVarInterface"] # if op has custom vjp rule, then append a CustomVjpTrait to it - if op_info.op_phi_name[0] in custom_vjp_op_name_list: + if ( + op_info.op_phi_name[0] in custom_vjp_op_name_list + and dialect_name != "pd_onednn_op" + ): op_traits += ["paddle::dialect::CustomVjpTrait"] # check op inputs and mutable_attributes grad semantics @@ -1143,6 +1172,15 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): if op_name[-1] == "_": op_traits += ["paddle::dialect::InplaceTrait"] + if dialect_name == "pd_onednn_op": + op_traits += ["paddle::dialect::OneDNNTrait"] + + if op_info.is_onednn_only: + op_traits += ["paddle::dialect::OneDNNOnlyTrait"] + + if op_info.dynamic_fallback: + op_traits += ["paddle::dialect::OneDNNDynamicFallbackTrait"] + op_traits_str = "" if len(op_traits) > 0: op_traits_str = "," + ",".join(op_traits) @@ -1158,6 +1196,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): if ( op_name in decomp_interface_declare_gen_op_list and kernel_func_name in decomp_interface_declare_gen_op_list + and dialect_name != "pd_onednn_op" ): op_interfaces = op_interfaces + [ "paddle::dialect::DecompInterface" @@ -1221,7 +1260,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): build_func_with_muta_attr_is_input = "" get_kernel_type_for_var_declare_str = "" - if dialect_name == "pd_op": + if dialect_name == "pd_op" or dialect_name == "pd_onednn_op": get_kernel_type_for_var_declare_str = ( get_kernel_type_for_var_declare_template ) @@ -1556,6 +1595,53 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): origin_op_name=op_info.op_yaml_item['name'], ) + if dialect_name == "pd_onednn_op": + if len(op_info.onednn_extra_args) > 0: + args_name = [] + for arg in op_info.onednn_extra_args: + args_name.append(arg["name"]) + + extra_args = '"' + '", "'.join(args_name) + '"' + else: + extra_args = "" + if op_info.onednn_layout_transform is None: + layout_transform_arg, layout_transform_inputs = ( + "", + "", + ) + else: + ( + layout_transform_arg, + layout_transform_inputs, + ) = op_info.onednn_layout_transform + layout_transform_inputs = ( + '"' + '", "'.join(layout_transform_inputs) + '"' + ) + + op_info_func_str = OP_INFO_ONEDNN_TEMPLATE.format( + op_name=op_class_name, + inputs=inputs_info_str, + attributes=attribute_info_str, + outputs=outputs_info_str, + infer_meta_func=infer_meta_func_str, + infer_meta_param=infer_meta_param_str, + kernel_func=kernel_func_str, + kernel_param=kernel_param_str, + kernel_key_dtype=kernel_key_dtype, + kernel_key_backend=kernel_key_backend, + inplace=inplace_str, + view=view_str, + origin_op_name=op_info.op_yaml_item['name'], + extra_args=extra_args, + layout_transform_arg=layout_transform_arg, + layout_transform_inputs=layout_transform_inputs, + is_onednn_only="true" + if op_info.is_onednn_only + else "false", + dynamic_fallback="true" + if op_info.dynamic_fallback + else "false", + ) # generate op verify function str op_verify_str = '' if not op_info.custom_verify: @@ -1600,7 +1686,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): # generate op GetKernelKeyForVar function str op_get_kernel_type_for_var_str = '' - if dialect_name == "pd_op": + if dialect_name == "pd_op" or dialect_name == "pd_onednn_op": op_get_kernel_type_for_var_str = ( gen_kernel_type_for_var_str( op_class_name, @@ -1629,6 +1715,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): op_info.backward_name and op_info.op_phi_name[0] not in vjp_interface_black_list + and dialect_name != "pd_onednn_op" ): op_vjp_str = gen_op_vjp_str( op_class_name, @@ -1659,7 +1746,7 @@ def AutoCodeGen(op_info_items, all_op_info_items, namespaces, dialect_name): ops_defined_list.append(infer_symbolic_shape_define_str) # NOTE(chenxi67)skip if dialect_name==cinn - if dialect_name == "cinn": + if dialect_name == "cinn" or dialect_name == "pd_onednn_op": pass else: ops_vjp_defined_list.append(op_vjp_str) @@ -1741,6 +1828,8 @@ def OpGenerator( op_info_file, op_def_cc_file, op_vjp_cc_file, + onednn_yaml_file, + ops_onednn_extra_yaml_file, ): # (1) Prepare: Delete existing old files: pd_op.h.tmp, pd_op.cc.tmp if os.path.exists(op_def_h_file): @@ -1754,8 +1843,32 @@ def OpGenerator( # (2) parse yaml files op_compat_parser = OpCompatParser(op_compat_yaml_file) + if dialect_name == "pd_onednn_op": + with open(ops_onednn_extra_yaml_file, "r") as f: + ops_onednn_extra = yaml.safe_load(f) + ops_onednn_extra_map = {} + for op in ops_onednn_extra: + op_name = op['op'] + item = {} + item["is_onednn_only"] = False + item["extra_args"] = parse_extra_args(op_name, op['extra_args']) + if 'layout_transform' in op: + item["layout_transform"] = parse_layout_transform( + op_name, op['layout_transform'] + ) + else: + item["layout_transform"] = None + if 'dynamic_fallback' in op: + item["dynamic_fallback"] = op['dynamic_fallback'] + else: + item["dynamic_fallback"] = False + item["attrs"] = parse_extra_args(op_name, op['extra_args']) + ops_onednn_extra_map[op_name] = item + op_yaml_files.insert(0, onednn_yaml_file) + op_infos = [] all_op_info_items = {} + first_file = True for yaml_file in op_yaml_files: op_yaml_items = [] with open(yaml_file, "r") as f: @@ -1765,7 +1878,7 @@ def OpGenerator( op_info_items = {} for op in op_yaml_items: op_compat_item = None - if dialect_name == "pd_op": + if dialect_name == "pd_op" or dialect_name == "pd_onednn_op": op_compat_item = op_compat_parser.get_compat(op['name']) if ( @@ -1791,11 +1904,26 @@ def OpGenerator( ) = op_compat_parser.parse_support_tensor(op) op_compat_item['scalar'] = scalar_item op_compat_item['int_array'] = int_array_item - - op_info_items[op['name']] = OpInfoParser(op, op_compat_item) - all_op_info_items[op['name']] = OpInfoParser(op, op_compat_item) + if dialect_name == "pd_onednn_op": + if first_file: + first_file = False + op["is_onednn_only"] = True + elif op['name'] in ops_onednn_extra_map: + onednn_item = ops_onednn_extra_map[op['name']] + op["is_onednn_only"] = onednn_item["is_onednn_only"] + op["extra_args"] = onednn_item["extra_args"] + op["layout_transform"] = onednn_item["layout_transform"] + op["dynamic_fallback"] = onednn_item["dynamic_fallback"] + op["attrs"] = op["attrs"] + onednn_item["attrs"] + else: + continue + item = OpInfoParser(op, op_compat_item) + op_info_items[op['name']] = item + all_op_info_items[op['name']] = item op_infos.append(op_info_items) + if dialect_name == "pd_onednn_op": + op_infos = [all_op_info_items] # (3) auto code gen op_list_strs = [] @@ -1867,14 +1995,15 @@ def OpGenerator( else: op_to_multi_kernels_map_str = "" - op_info_str = CC_OP_INFO_FILE_TEMPLATE.format( - op_declare=",".join(op_list_strs).replace("\n", ""), - op_to_multi_kernels_map=op_to_multi_kernels_map_str, - h_file=op_def_h_file[:-4], - ) + if op_info_file is not None: + op_info_str = CC_OP_INFO_FILE_TEMPLATE.format( + op_declare=",".join(op_list_strs).replace("\n", ""), + op_to_multi_kernels_map=op_to_multi_kernels_map_str, + h_file=op_def_h_file[:-4], + ) - with open(op_info_file, 'w') as f: - f.write(op_info_str) + with open(op_info_file, 'w') as f: + f.write(op_info_str) # (6) write to files for xx_op.cc.tmp for id in range(len(op_def_cc_file)): @@ -1883,8 +2012,17 @@ def OpGenerator( source_file_str = NAMESPACE_GARD_TEMPLATE.format( namespace=name, input=source_file_str ) # Add namespaces + + if dialect_name == "pd_onednn_op": + op_def_h_file_tmp = ( + "paddle/fluid/pir/dialect/operator/ir/pd_op.h\"\n#include \"" + + op_def_h_file + ) + else: + op_def_h_file_tmp = op_def_h_file + source_file_str = CC_FILE_TEMPLATE.format( - h_file=op_def_h_file[:-4], + h_file=op_def_h_file_tmp[:-4], input=source_file_str, define_type_id=define_type_id_strs[id], ) @@ -1896,7 +2034,11 @@ def OpGenerator( # and vjp is only avaible for pd dialect. vjp_source_file_str = "\n".join(vjp_source_file_strs) vjp_source_file_str = VJP_CC_FILE_TEMPLATE.format(input=vjp_source_file_str) - if dialect_name != 'cinn' and op_vjp_cc_file: + if ( + dialect_name != 'cinn' + and dialect_name != 'pd_onednn_op' + and op_vjp_cc_file + ): with open(op_vjp_cc_file, 'w') as f: f.write(vjp_source_file_str) @@ -1916,6 +2058,8 @@ def ParseArguments(): parser.add_argument('--op_info_file', type=str) parser.add_argument('--op_def_cc_file', type=str) parser.add_argument('--op_vjp_cc_file', type=str) + parser.add_argument('--onednn_yaml_file', type=str) + parser.add_argument('--ops_onednn_extra_yaml_file', type=str) return parser.parse_args() @@ -1935,6 +2079,8 @@ def ParseArguments(): op_info_file = args.op_info_file op_def_cc_files = args.op_def_cc_file.split(",") op_vjp_cc_file = args.op_vjp_cc_file + onednn_yaml_file = args.onednn_yaml_file + ops_onednn_extra_yaml_file = args.ops_onednn_extra_yaml_file # auto code generate OpGenerator( @@ -1946,4 +2092,6 @@ def ParseArguments(): op_info_file, op_def_cc_files, op_vjp_cc_file, + onednn_yaml_file, + ops_onednn_extra_yaml_file, ) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 9fd6bd4bfbd98..0a834bc7b0c2c 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -102,6 +102,7 @@ 'print', 'number_count', 'assign_value', + 'onednn_to_paddle_layout', ] NO_NEED_GEN_STATIC_ONLY_APIS = [ diff --git a/paddle/fluid/pir/dialect/op_generator/ops_onednn_extra_parser.py b/paddle/fluid/pir/dialect/op_generator/ops_onednn_extra_parser.py new file mode 100644 index 0000000000000..3296fa0d68829 --- /dev/null +++ b/paddle/fluid/pir/dialect/op_generator/ops_onednn_extra_parser.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Any, Dict, List, Tuple + + +def parse_plain_list(s: str, sep=",") -> List[str]: + if sep == ",": + patten = re.compile(r',(?![^{]*\})') # support "int[] a={1,2}" + items = re.split(patten, s.strip()) + items = [x.strip() for x in items] + return items + else: + return [item.strip() for item in s.strip().split(sep)] + + +def parse_arg(op_name: str, s: str) -> Dict[str, str]: + """parse an argument in following formats: + 1. typename name + 2. typename name = default_value + """ + typename, rest = (item.strip() for item in s.split(" ", 1)) + assert ( + len(typename) > 0 + ), f"The arg typename should not be empty. Please check the args of {op_name} in yaml." + + assert ( + rest.count("=") <= 1 + ), f"There is more than 1 = in an arg in {op_name}" + if rest.count("=") == 1: + name, default_value = (item.strip() for item in rest.split("=", 1)) + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {op_name} in yaml." + assert ( + len(default_value) > 0 + ), f"The default value should not be empty. Please check the args of {op_name} in yaml." + return { + "typename": typename, + "name": name, + "default_value": default_value, + } + else: + name = rest.strip() + assert ( + len(name) > 0 + ), f"The arg name should not be empty. Please check the args of {op_name} in yaml." + return {"typename": typename, "name": name} + + +def parse_extra_args(op_name: str, arguments: str) -> List: + if arguments is None: + return [] + args_str = arguments.strip() + args = parse_plain_list(args_str) + + attrs = [] + + for arg in args: + item = parse_arg(op_name, arg) + typename = item["typename"] + name = item["name"] + attrs.append(item) + return attrs + + +def parse_layout_transform( + op_name: str, layout_transform: Dict[str, Any] +) -> Tuple[str, List]: + if layout_transform is None: + return "", [] + return layout_transform["arg_name"], parse_plain_list( + layout_transform["tensors"] + ) diff --git a/paddle/fluid/pir/dialect/operator/ir/onednn.yaml b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml new file mode 100644 index 0000000000000..d7de4310d5781 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/onednn.yaml @@ -0,0 +1,9 @@ +- op : quantize + args : (Tensor input, bool is_negative_input=false, float scale=1.0, float shift=0.0, str output_format="NHWC", bool bfloat16=false) + output : Tensor(output) + infer_meta : + func : UnchangedInferMeta + param : [input] + kernel : + func : quantize + data_type : input diff --git a/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.cc new file mode 100644 index 0000000000000..0d65389cc4922 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.cc @@ -0,0 +1,168 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h" +#include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/fluid/pir/dialect/operator/ir/type_storage.h" +#include "paddle/fluid/pir/dialect/operator/transforms/param_to_variable.h" +#include "paddle/pir/core/builtin_type_interfaces.h" +#include "paddle/pir/core/interface_value.h" +#include "paddle/pir/core/ir_printer.h" +#include "paddle/pir/core/utils.h" +#include "paddle/pir/dialect/control_flow/ir/cf_dialect.h" +#include "paddle/pir/dialect/control_flow/ir/cf_op.h" + +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/pd_onednn_op.h" +#endif + +namespace paddle { +namespace dialect { + +OneDNNOperatorDialect::OneDNNOperatorDialect(pir::IrContext *ctx) + : pir::Dialect(name(), ctx, pir::TypeId::get()) { + initialize(); +} + +void OneDNNOperatorDialect::initialize() { + // NOTE(zhangbo9674): GET_OP_LIST is defined in pd_op.h which is + // generated by op_gen.py, see details in + // paddle/fluid/pir/dialect/CMakeLists.txt. + // NOTE(Ruting)GET_MANUAL_OP_LIST is define in manual_op.h" + // use RegisterOps when list has more than two ops. + RegisterOps< +#define GET_OP_LIST +#include "paddle/fluid/pir/dialect/operator/ir/pd_onednn_op_info.cc" // NOLINT + >(); +} + +void OneDNNOperatorDialect::PrintType(pir::Type type, std::ostream &os) const { + os << type.dialect().name(); + os << '.'; + if (auto tensor_type = type.dyn_cast()) { + os << "tensor<"; + for (auto d : common::vectorize(tensor_type.dims())) { + os << d; + os << "x"; + } + tensor_type.dtype().Print(os); + os << ">"; + } else if (auto selected_rows_type = type.dyn_cast()) { + os << "selectedrows<"; + for (auto d : common::vectorize(selected_rows_type.dims())) { + os << d; + os << "x"; + } + selected_rows_type.dtype().Print(os); + os << ">"; + } else if (auto tensor_array_type = type.dyn_cast()) { + os << "tensor_array<"; + tensor_array_type.dtype().Print(os); + os << ">"; + } +} + +void OneDNNOperatorDialect::PrintAttribute(pir::Attribute attr, + std::ostream &os) const { + os << "(" << attr.dialect().name(); + os << '.'; + if (auto int_array_attr = attr.dyn_cast()) { + phi::IntArray data = int_array_attr.data(); + os << "IntArray)" + << "["; + const auto &inner_data = data.GetData(); + pir::PrintInterleave( + inner_data.begin(), + inner_data.end(), + [&os](int64_t i) { os << i; }, + [&os]() { os << ","; }); + os << "]"; + } else if (auto data_type_attr = attr.dyn_cast()) { + os << "DataType)" << data_type_attr.data(); + } else if (auto place_type_attr = attr.dyn_cast()) { + os << "Place)" << place_type_attr.data(); + } else if (auto data_layout_attr = attr.dyn_cast()) { + os << "DataLayout)" << data_layout_attr.data(); + } else { + os << "<#AttrNotImplemented>"; + } +} + +pir::Type OneDNNOperatorDialect::ParseType(pir::IrParser &parser) { // NOLINT + parser.ConsumeAToken("pd_op.tensor"); + parser.ConsumeAToken("<"); + std::vector dim{}; + Token dim_token = parser.PeekToken(); + while (dim_token.token_type_ == DIGIT) { + dim_token = parser.ConsumeToken(); + dim.push_back(atoi(dim_token.val_.c_str())); + std::string peek_token_val = parser.PeekToken().val_; + if (peek_token_val[0] != 'x') { + break; + } + parser.ConsumeToken(); + parser.lexer->Unget(static_cast(peek_token_val.size() - 1)); + if (parser.PeekToken().token_type_ != DIGIT) { + break; + } + } + phi::DDim ddim = common::make_ddim(dim); + pir::Type dtype = parser.ParseType(); + std::vector> lod; + std::vector lodv; + lodv.push_back(0); + lod.push_back(lodv); + parser.ConsumeAToken(">"); + return DenseTensorType::get( + parser.ctx, dtype, ddim, phi::DataLayout::UNDEFINED, lod, 0); +} + +pir::Attribute OneDNNOperatorDialect::ParseAttribute( + pir::IrParser &parser) { // NOLINT + std::string type_name = parser.ConsumeToken().val_; + std::string attribute_name = + type_name.substr(type_name.find('.') + 1, std::string::npos); + parser.ConsumeAToken(")"); + if (attribute_name == "IntArray") { + return IntArrayAttribute::Parse(parser); + } else if (attribute_name == "DataType") { + return DataTypeAttribute::Parse(parser); + } else if (attribute_name == "Place") { + return PlaceAttribute::Parse(parser); + } else if (attribute_name == "DataLayout") { + return DataLayoutAttribute::Parse(parser); + } else { + IR_THROW("No function to parse " + attribute_name + " exists!" + + parser.GetErrorLocationInfo()); + } +} + +void OneDNNOperatorDialect::PrintOperation(pir::Operation *op, + pir::IrPrinter &printer) const { + if (auto if_op = op->dyn_cast()) { + if_op.Print(printer); + } else if (auto while_op = op->dyn_cast()) { + while_op.Print(printer); + } else { + printer.PrintGeneralOperation(op); + } +} + +} // namespace dialect +} // namespace paddle + +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNOperatorDialect) diff --git a/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h b/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h new file mode 100644 index 0000000000000..ac6483d4d53ec --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h @@ -0,0 +1,44 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/pir/core/dialect.h" + +namespace paddle { +namespace dialect { + +class OneDNNOperatorDialect : public pir::Dialect { + public: + explicit OneDNNOperatorDialect(pir::IrContext* context); + + static const char* name() { return "pd_onednn_op"; } + + pir::Type ParseType(pir::IrParser& parser) override; // NOLINT + pir::Attribute ParseAttribute(pir::IrParser& parser) override; // NOLINT + + void PrintType(pir::Type type, std::ostream& os) const override; + void PrintAttribute(pir::Attribute type, std::ostream& os) const override; + + void PrintOperation(pir::Operation* op, + pir::IrPrinter& printer) const override; // NOLINT + + private: + void initialize(); +}; + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNOperatorDialect) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 57d7857a2498c..0d571f8ef868a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1464,6 +1464,15 @@ func: number_count data_type: numbers +- op: onednn_to_paddle_layout + args: (Tensor x, int dst_layout) + output: Tensor(out) + infer_meta: + func : UnchangedInferMeta + param : [x] + kernel: + func: onednn_to_paddle_layout + - op: sparse_momentum args: (Tensor param, Tensor grad, Tensor velocity, Tensor index, Tensor learning_rate, Tensor master_param,float mu, Scalar axis=0, bool use_nesterov=false,str regularization_method="", float regularization_coeff=0.0f, bool multi_precision=false, float rescale_grad=1.0f) output: Tensor(param_out), Tensor(velocity_out), Tensor(master_param_out) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml new file mode 100644 index 0000000000000..58897216793dd --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/ir/ops_onednn_extra.yaml @@ -0,0 +1,33 @@ + +- op : conv2d + extra_args : bool is_test=false + layout_transform : + arg_name: data_format + tensors: input + +- op : conv2d_grad + extra_args : bool is_test=false + layout_transform : + arg_name: data_format + tensors: input, out_grad +# - op : matmul +# extra_args : str mkldnn_data_type="float32" +# layout_transform : +# arg_name: cur_paddle_data_layout +# tensors: x, y + +# - op : pad3d +# extra_args : +# layout_transform : +# arg_name: data_format +# tensors: x +# dynamic_fallback : True + +# - op : batch_norm +# extra_args : bool fuse_with_relu=false +# layout_transform : +# arg_name: data_layout +# tensors: x + +# - op : prelu +# extra_args : bool is_test=false, str mkldnn_data_type="float32" diff --git a/paddle/fluid/pir/dialect/operator/trait/onednn.h b/paddle/fluid/pir/dialect/operator/trait/onednn.h new file mode 100644 index 0000000000000..df810c6707df1 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/trait/onednn.h @@ -0,0 +1,49 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef PADDLE_WITH_DNNL + +#include "paddle/pir/core/op_base.h" + +namespace paddle { +namespace dialect { +class OneDNNTrait : public pir::OpTraitBase { + public: + explicit OneDNNTrait(pir::Operation *op) + : pir::OpTraitBase(op) {} +}; + +class OneDNNOnlyTrait : public pir::OpTraitBase { + public: + explicit OneDNNOnlyTrait(pir::Operation *op) + : pir::OpTraitBase(op) {} +}; + +class OneDNNDynamicFallbackTrait + : public pir::OpTraitBase { + public: + explicit OneDNNDynamicFallbackTrait(pir::Operation *op) + : pir::OpTraitBase(op) {} +}; + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNTrait) +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNOnlyTrait) +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNDynamicFallbackTrait) + +#endif diff --git a/paddle/fluid/pir/dialect/operator/trait/trait.cc b/paddle/fluid/pir/dialect/operator/trait/trait.cc index 2a5b7575959b9..9d828570d389a 100644 --- a/paddle/fluid/pir/dialect/operator/trait/trait.cc +++ b/paddle/fluid/pir/dialect/operator/trait/trait.cc @@ -14,6 +14,14 @@ #include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" #include "paddle/fluid/pir/dialect/operator/trait/inplace.h" - +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/trait/onednn.h" +#endif IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::InplaceTrait) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::CustomVjpTrait) + +#ifdef PADDLE_WITH_DNNL +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNTrait) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNOnlyTrait) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OneDNNDynamicFallbackTrait) +#endif diff --git a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h index 637de470675eb..662616bce773a 100644 --- a/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h +++ b/paddle/fluid/pir/dialect/operator/utils/op_yaml_info_util.h @@ -93,6 +93,12 @@ struct OpRunTimeInfo { std::vector kernel_key_backend; std::vector> inplace; std::vector> view; + std::vector extra_args; + std::string layout_transform_arg; + std::vector layout_transform_inputs; + bool is_onednn_only; + bool dynamic_fallback; + OpRunTimeInfo(const std::string& infer_meta_func, const std::vector& infer_meta_param, const std::string& kernel_func, @@ -100,7 +106,12 @@ struct OpRunTimeInfo { const std::vector& dtype, const std::vector& backend, const std::vector>& inplace, - const std::vector>& view) + const std::vector>& view, + const std::vector& extra_args = {}, + const std::string& layout_transform_arg = "", + const std::vector& layout_transform_inputs = {}, + bool is_onednn_only = false, + bool dynamic_fallback = false) : infer_meta_func(infer_meta_func), infer_meta_param(infer_meta_param), kernel_func(kernel_func), @@ -108,7 +119,12 @@ struct OpRunTimeInfo { kernel_key_dtype(dtype), kernel_key_backend(backend), inplace(inplace), - view(view) {} + view(view), + extra_args(extra_args), + layout_transform_arg(layout_transform_arg), + layout_transform_inputs(layout_transform_inputs), + is_onednn_only(is_onednn_only), + dynamic_fallback(dynamic_fallback) {} }; } // namespace dialect diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 6782b2f8bfd7c..722685fc3b510 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -60,6 +60,7 @@ const std::unordered_set LegacyOpList = { SoftReluOp::name(), SoftReluGradOp::name()}; +const std::unordered_set OneDNNLegacyOpList = {}; enum class AttrType { UNDEFINED = 0, BOOL, @@ -220,6 +221,12 @@ VariantType GetAttributeData(const pir::Attribute& attr) { bool IsLegacyOp(const std::string& name) { return LegacyOpList.count(name); } +#ifdef PADDLE_WITH_DNNL +bool IsOneDNNLegacyOp(const std::string& name) { + return OneDNNLegacyOpList.count(name); +} +#endif + bool IsEmptyValue(const pir::Value& value) { return !value.impl() || !value.type(); } diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.h b/paddle/fluid/pir/dialect/operator/utils/utils.h index 1ebe7d244affd..0e14077bb8559 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.h +++ b/paddle/fluid/pir/dialect/operator/utils/utils.h @@ -132,6 +132,10 @@ VariantType GetAttributeData(const pir::Attribute& attr); bool IsLegacyOp(const std::string& name); +#ifdef PADDLE_WITH_DNNL +bool IsOneDNNLegacyOp(const std::string& name); +#endif + bool IsEmptyValue(const pir::Value& value); std::vector GetInt64Vector(const pir::Attribute& attr); diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 91ca8a0d4b3f6..df7b8673d9ea8 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -17,6 +17,7 @@ #include #include "paddle/fluid/framework/op_kernel_type.h" +#include "paddle/fluid/framework/operator.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_attribute.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_op.h" @@ -44,6 +45,12 @@ #include "paddle/pir/dialect/control_flow/ir/cf_op.h" #include "paddle/utils/flags.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/fluid/pir/dialect/operator/ir/op_onednn_dialect.h" +#include "paddle/fluid/pir/dialect/operator/ir/pd_onednn_op.h" +#include "paddle/fluid/pir/dialect/operator/trait/onednn.h" +#endif + PHI_DECLARE_bool(print_ir); namespace paddle { namespace dialect { @@ -337,6 +344,49 @@ static pir::OpResult AddPlaceTransferOp(pir::Value in, return new_in; } +#ifdef PADDLE_WITH_DNNL +static pir::OpResult AddOneDNN2PaddleLayoutTransferOp( + pir::Value in, const phi::DataLayout& dst_layout, pir::Block* block) { + pir::IrContext* ctx = pir::IrContext::Instance(); + auto in_alloc_type = in.type().dyn_cast(); + + phi::KernelKey kernel_key; + kernel_key.set_backend(phi::Backend::CPU); + kernel_key.set_layout(phi::DataLayout::ANY); + kernel_key.set_dtype(dialect::TransToPhiDataType(in_alloc_type.dtype())); + + std::unordered_map op_attribute; + op_attribute = { + {"op_name", pir::StrAttribute::get(ctx, "pd_op.onednn_to_paddle_layout")}, + {"kernel_name", pir::StrAttribute::get(ctx, "onednn_to_paddle_layout")}, + {"kernel_key", KernelAttribute::get(ctx, kernel_key)}, + {"dst_layout", + pir::Int32Attribute::get(ctx, static_cast(dst_layout))}}; + + auto out_type = AllocatedDenseTensorType::get(ctx, + in_alloc_type.place(), + in_alloc_type.dtype(), + in_alloc_type.dims(), + dst_layout, + in_alloc_type.lod(), + in_alloc_type.offset()); + + pir::OpInfo kernel_op_info = ctx->GetRegisteredOpInfo(PhiKernelOp::name()); + pir::Operation* op = + pir::Operation::Create({in}, op_attribute, {out_type}, kernel_op_info); + + auto in_op = in.dyn_cast().owner(); + if (in_op && in_op->HasAttribute(kAttrIsPersisable)) { + op->set_attribute(kAttrIsPersisable, in_op->attribute(kAttrIsPersisable)); + } + + block->push_back(op); + auto new_in = op->result(0); + + return new_in; +} +#endif + static bool NeedTransformDataType(const phi::DataType& l, const phi::DataType& r) { return l != phi::DataType::ALL_DTYPE && r != phi::DataType::ALL_DTYPE && @@ -424,6 +474,46 @@ static pir::Type BuildOutputType(pir::Type type, } } +#ifdef PADDLE_WITH_DNNL +template +static pir::Type create_type(pir::Type type, + const phi::Place& place, + const phi::DataLayout& layout, + pir::Type out_dtype, + pir::IrContext* ctx) { + auto input_type = type.dyn_cast(); + return IrType2::get(ctx, + place, + out_dtype, + input_type.dims(), + layout, + input_type.lod(), + input_type.offset()); +} + +static pir::Type BuildOutputType(pir::Type type, + const phi::Place& place, + const phi::DataLayout& layout, + pir::IrContext* ctx) { + if (type.isa()) { + auto out_dtype = type.dyn_cast().dtype(); + return create_type( + type, place, layout, out_dtype, ctx); + } else if (type.isa()) { + auto out_dtype = type.dyn_cast().dtype(); + return create_type( + type, place, layout, out_dtype, ctx); + } else if (type.isa()) { + auto array_type = type.dyn_cast(); + return AllocatedDenseTensorArrayType::get( + ctx, place, array_type.dtype(), layout); + } else { + PADDLE_THROW(phi::errors::Unimplemented( + "BuildOutputType only support DenseTensorType and SelectedRowsType")); + } +} +#endif + pir::OpResult AddDtypeTransferOp(pir::Value in, pir::Block* block, const phi::KernelKey& kernel_key, @@ -666,6 +756,49 @@ std::string GetKernelName(const OpYamlInfoParser* op_info_parser, return kernel_fn_str; } +#ifdef PADDLE_WITH_DNNL +bool SupportsMKLDNN(const std::string& kernel_name, + const phi::DataType data_type) { + auto phi_kernels = + phi::KernelFactory::Instance().SelectKernelMap(kernel_name); + auto has_phi_kernel = + std::any_of(phi_kernels.begin(), + phi_kernels.end(), + [data_type](phi::KernelKeyMap::const_reference kern_pair) { + return kern_pair.first.backend() == phi::Backend::ONEDNN && + kern_pair.first.dtype() == data_type; + }); + if (has_phi_kernel) { + return true; + } else { + auto op_kernel_iter = + paddle::framework::OperatorWithKernel::AllOpKernels().find( + phi::TransToFluidOpName(kernel_name)); + if (op_kernel_iter == + paddle::framework::OperatorWithKernel::AllOpKernels().end()) { + return false; + } else { + auto& op_kernels = op_kernel_iter->second; + return std::any_of( + op_kernels.begin(), + op_kernels.end(), + [data_type](std::unordered_map< + paddle::framework::OpKernelType, + std::function, + paddle::framework::OpKernelType::Hash>::const_reference + kern_pair) { + return platform::is_cpu_place(kern_pair.first.place_) && + kern_pair.first.library_type_ == + paddle::framework::LibraryType::kMKLDNN && + kern_pair.first.data_type_ == + paddle::framework::TransToProtoVarType(data_type); + }); + } + } +} +#endif + phi::KernelKey GetKernelKey( pir::Operation* op, const phi::Place& place, @@ -899,6 +1032,13 @@ phi::KernelKey GetKernelKey( "to GPU"; } +#ifdef PADDLE_WITH_DNNL + if (op->HasTrait() && res.backend() == phi::Backend::CPU && + SupportsMKLDNN(kernel_fn_str, res.dtype())) { + res.set_backend(phi::Backend::ONEDNN); + res.set_layout(phi::DataLayout::ONEDNN); + } +#endif return res; } @@ -1375,7 +1515,17 @@ std::vector BuildOutputs(pir::Operation* op_item, } else if (result_type.isa() || result_type.isa() || result_type.isa()) { +#ifdef PADDLE_WITH_DNNL + if (kernel_key.backend() == phi::Backend::ONEDNN) { + op_output_types.push_back(BuildOutputType( + result_type, out_place, phi::DataLayout::ONEDNN, ctx)); + } else { + op_output_types.push_back(BuildOutputType(result_type, out_place, ctx)); + } +#else op_output_types.push_back(BuildOutputType(result_type, out_place, ctx)); +#endif + } else if (result_type.isa()) { std::vector vec_inner_types; auto base_types = result_type.dyn_cast().data(); @@ -1383,8 +1533,18 @@ std::vector BuildOutputs(pir::Operation* op_item, if (base_type) { if (base_type.isa() || base_type.isa()) { +#ifdef PADDLE_WITH_DNNL + if (kernel_key.backend() == phi::Backend::ONEDNN) { + vec_inner_types.push_back(BuildOutputType( + base_type, out_place, phi::DataLayout::ONEDNN, ctx)); + } else { + vec_inner_types.push_back( + BuildOutputType(base_type, out_place, ctx)); + } +#else vec_inner_types.push_back( BuildOutputType(base_type, out_place, ctx)); +#endif } else { PADDLE_THROW(phi::errors::Unimplemented( "only support dense tensor and selected rows in vector type " @@ -1395,6 +1555,11 @@ std::vector BuildOutputs(pir::Operation* op_item, pir::Type fp32_dtype = pir::Float32Type::get(ctx); phi::DDim dims = {}; phi::DataLayout data_layout = phi::DataLayout::NCHW; +#ifdef PADDLE_WITH_DNNL + if (kernel_key.backend() == phi::Backend::ONEDNN) { + data_layout = phi::DataLayout::ONEDNN; + } +#endif phi::LoD lod = {{}}; size_t offset = 0; auto dense_tensor_dtype = DenseTensorType::get( @@ -1463,7 +1628,21 @@ std::vector BuildInputs( } } - // 1.backend transfer + // 1. layout transfer(only for onednn) +#ifdef PADDLE_WITH_DNNL + if (kernel_key.backend() != phi::Backend::ONEDNN) { + auto new_in_type = new_in.type(); + if (new_in_type.isa()) { + if (new_in_type.dyn_cast().data_layout() == + phi::DataLayout::ONEDNN) { + new_in = AddOneDNN2PaddleLayoutTransferOp( + new_in, phi::DataLayout::ANY, block); + } + } + } +#endif + + // 2.backend transfer bool check_place_transfer = (op_item->isa<::pir::SetParameterOp>()) || (kernel.IsValid() && (!UnchangeOutputOps.count(op_item->name()))); @@ -1664,7 +1843,7 @@ std::vector BuildInputs( } } - // 2. dtype transfer + // 3. dtype transfer if (op_info_parser != nullptr) { std::string var_name = op_info_parser->InputNames()[i]; auto fake_tensors = PrepareFakeTensors(new_in); @@ -1694,6 +1873,7 @@ std::vector BuildInputs( } } } + vec_inputs.push_back(new_in); } return vec_inputs; @@ -1773,18 +1953,76 @@ pir::Operation* BuildKernelOp( op_attribute.emplace("is_inplace", pir::BoolAttribute::get(ctx, true)); } - pir::OpInfo phi_kernel_op_info = - ctx->GetRegisteredOpInfo(PhiKernelOp::name()); - - pir::OpInfo legacy_kernel_op_info = - ctx->GetRegisteredOpInfo(LegacyKernelOp::name()); pir::Operation* op = nullptr; - if (IsLegacyOp(op_item->name())) { - op = pir::Operation::Create( - vec_inputs, op_attribute, op_output_types, legacy_kernel_op_info); - } else { - op = pir::Operation::Create( - vec_inputs, op_attribute, op_output_types, phi_kernel_op_info); +#ifdef PADDLE_WITH_DNNL + if (op_item->HasTrait()) { + if (IsOneDNNLegacyOp(op_item->name())) { + VLOG(4) << "choose OneDNNLegacyKernelOp"; + pir::OpInfo legacy_kernel_op_info = + ctx->GetRegisteredOpInfo(OneDNNLegacyKernelOp::name()); + op = pir::Operation::Create( + vec_inputs, op_attribute, op_output_types, legacy_kernel_op_info); + } else { + auto op_info_parser = GetOpYamlInfoParser(op_item); + std::vector extra_args; + for (auto& arg : op_info_parser->OpRuntimeInfo().extra_args) { + extra_args.push_back(pir::StrAttribute::get(ctx, arg)); + } + op_attribute.emplace( + "extra_args", + pir::ArrayAttribute::get(pir::IrContext::Instance(), extra_args)); + op_attribute.emplace( + "layout_transform_arg", + pir::StrAttribute::get( + ctx, op_info_parser->OpRuntimeInfo().layout_transform_arg)); + std::vector layout_transform_inputs; + for (auto& input : + op_info_parser->OpRuntimeInfo().layout_transform_inputs) { + layout_transform_inputs.push_back(pir::StrAttribute::get(ctx, input)); + } + op_attribute.emplace("layout_transform_inputs", + pir::ArrayAttribute::get(pir::IrContext::Instance(), + layout_transform_inputs)); + op_attribute.emplace( + "is_onednn_only", + pir::BoolAttribute::get( + ctx, op_info_parser->OpRuntimeInfo().is_onednn_only)); + op_attribute.emplace( + "dynamic_fallback", + pir::BoolAttribute::get( + ctx, op_info_parser->OpRuntimeInfo().dynamic_fallback)); + if (op_item->HasTrait()) { + VLOG(4) << "choose OneDNNMixedPhiKernelOp"; + pir::OpInfo phi_kernel_op_info = + ctx->GetRegisteredOpInfo(OneDNNMixedPhiKernelOp::name()); + + op = pir::Operation::Create( + vec_inputs, op_attribute, op_output_types, phi_kernel_op_info); + } else { + VLOG(4) << "choose OneDNNPhiKernelOp"; + pir::OpInfo phi_kernel_op_info = + ctx->GetRegisteredOpInfo(OneDNNPhiKernelOp::name()); + + op = pir::Operation::Create( + vec_inputs, op_attribute, op_output_types, phi_kernel_op_info); + } + } + } else // NOLINT +#endif + { + if (IsLegacyOp(op_item->name())) { + pir::OpInfo legacy_kernel_op_info = + ctx->GetRegisteredOpInfo(LegacyKernelOp::name()); + + op = pir::Operation::Create( + vec_inputs, op_attribute, op_output_types, legacy_kernel_op_info); + } else { + pir::OpInfo phi_kernel_op_info = + ctx->GetRegisteredOpInfo(PhiKernelOp::name()); + + op = pir::Operation::Create( + vec_inputs, op_attribute, op_output_types, phi_kernel_op_info); + } } (*map_op_pair)[op_item] = op; @@ -1809,10 +2047,11 @@ void ProcessBlock( std::unordered_map* map_value_pair) { auto inputs_by_data_op = GetInputsByDataOp(block); - for (auto& op_item : *block) { - VLOG(6) << "op name " << op_item.name(); - if ((op_item.isa()) && - inputs_by_data_op.count(op_item.attributes() + for (auto iter = block->begin(); iter != block->end(); ++iter) { + pir::Operation* op_item = &(*iter); + VLOG(6) << "op name " << op_item->name(); + if ((op_item->isa()) && + inputs_by_data_op.count(op_item->attributes() .at("name") .dyn_cast() .AsString())) { @@ -1821,24 +2060,55 @@ void ProcessBlock( } // HandleSpecialOp - if (SpecialLowerOps.count(op_item.name())) { - VLOG(6) << "Handle Special Op: [" << op_item.name() + if (SpecialLowerOps.count(op_item->name())) { + VLOG(6) << "Handle Special Op: [" << op_item->name() << "] while lowering to kernel pass"; HandleForSpecialOp( - place, &op_item, new_block, ctx, map_op_pair, map_value_pair); + place, op_item, new_block, ctx, map_op_pair, map_value_pair); continue; } - auto op_info_parser = GetOpYamlInfoParser(&op_item); - auto kernel_name = GetKernelName(op_info_parser.get(), &op_item); + auto op_info_parser = GetOpYamlInfoParser(op_item); + auto kernel_name = GetKernelName(op_info_parser.get(), op_item); auto kernel_key = GetKernelKey( - &op_item, place, kernel_name, *map_value_pair, op_info_parser.get()); + op_item, place, kernel_name, *map_value_pair, op_info_parser.get()); VLOG(6) << "kernel type " << kernel_key; +#ifdef PADDLE_WITH_DNNL + if (op_item->HasTrait() && + kernel_key.backend() != phi::Backend::ONEDNN) { + std::vector op_item_inner_output_types; + if (op_item->num_results() > 0) { + for (size_t i = 0; i < op_item->num_results(); ++i) { + op_item_inner_output_types.push_back(op_item->result_type(i)); + } + } + std::string target_op_name = op_item->name(); + target_op_name.replace(0, 12, "pd_op"); + auto op_info = ctx->GetRegisteredOpInfo(target_op_name); + if (!op_info) { + IR_THROW("Ctx should have corresponding OpInfo %s", target_op_name); + } + pir::Operation* op_item_inner = + pir::Operation::Create(op_item->operands_source(), + op_item->attributes(), + op_item_inner_output_types, + op_info); + op_item->ReplaceAllUsesWith(op_item_inner->results()); + for (auto iter = block->begin(); iter != block->end(); ++iter) { + if (*iter == *op_item) { + block->Assign(iter, op_item_inner); + break; + } + } + op_item = op_item_inner; + op_info_parser = GetOpYamlInfoParser(op_item_inner); + } +#endif // build output type - auto op_output_types = BuildOutputs(&op_item, kernel_name, kernel_key, ctx); + auto op_output_types = BuildOutputs(op_item, kernel_name, kernel_key, ctx); // build input - auto vec_inputs = BuildInputs(&op_item, + auto vec_inputs = BuildInputs(op_item, kernel_name, kernel_key, place, @@ -1853,14 +2123,14 @@ void ProcessBlock( kernel_key, vec_inputs, op_output_types, - &op_item, + op_item, new_block, ctx, map_op_pair, map_value_pair); AddShadowFeedOpForDataOrFeed( - place, &op_item, op, new_block, ctx, map_op_pair, map_value_pair); + place, op_item, op, new_block, ctx, map_op_pair, map_value_pair); } } @@ -1877,7 +2147,10 @@ std::unique_ptr PdOpLowerToKernelPass(pir::Program* prog, pir::IrContext* ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); ctx->GetOrRegisterDialect(); - +#ifdef PADDLE_WITH_DNNL + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); +#endif std::unordered_map map_op_pair; std::unordered_map map_value_pair; diff --git a/paddle/phi/api/lib/data_transform.h b/paddle/phi/api/lib/data_transform.h index dd3166f05c3ef..e0509fa8582ae 100644 --- a/paddle/phi/api/lib/data_transform.h +++ b/paddle/phi/api/lib/data_transform.h @@ -178,6 +178,11 @@ inline bool NeedTransformPlace(const phi::Place& src_place, (target != Backend::ALL_BACKEND && phi::TransToPhiBackend(src_place) != (target != Backend::GPUDNN ? target : Backend::GPU)); +#ifdef PADDLE_WITH_DNNL + if (target == Backend::ONEDNN) { + ret = src_place.GetType() != AllocationType::CPU; + } +#endif return ret; } diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 556a713fdac30..d69e290bdbd14 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -2462,6 +2462,15 @@ outputs : {q : Q, r : R} +- op : quantize + backward : quantize_grad + inputs : + input : Input + outputs : + output : Output + attrs : + {scale : Scale, shift : Shift, include_self: Include_self} + - op : quantize_linear extra : attrs : [float moving_rate = 0.9] diff --git a/paddle/phi/kernels/cpu/onednn_to_paddle_layout_kernel.cc b/paddle/phi/kernels/cpu/onednn_to_paddle_layout_kernel.cc new file mode 100644 index 0000000000000..eba8b2b61f4d2 --- /dev/null +++ b/paddle/phi/kernels/cpu/onednn_to_paddle_layout_kernel.cc @@ -0,0 +1,94 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/phi/kernels/onednn_to_paddle_layout_kernel.h" + +#include +#include + +#include "glog/logging.h" + +#include "paddle/phi/backends/all_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/core/visit_type.h" +#include "paddle/phi/kernels/funcs/data_layout_transform.h" +#include "paddle/phi/kernels/funcs/math_function.h" +#include "paddle/phi/kernels/memcpy_kernel.h" +#ifdef PADDLE_WITH_DNNL +#include "paddle/phi/backends/onednn/onednn_helper.h" +#endif +namespace phi { + +template +void OneDNN2PaddleLayout(const Context& dev_ctx, + const DenseTensor& x, + int dst_layout, + DenseTensor* out) { +#ifdef PADDLE_WITH_DNNL + DataLayout src_layout = x.layout(); + VLOG(10) << "TransDataLayout from " << static_cast(src_layout) + << " -> " << static_cast(dst_layout); + + auto print_tensor_meta = [](const DenseTensor& x) { + std::ostringstream oss; + + oss << "["; + oss << "layout:" << x.layout() << " ,"; + oss << "dims:" << x.dims() << " ,"; + if (x.IsInitialized()) oss << "place:" << x.place(); + oss << "]"; + + return oss.str(); + }; + VLOG(10) << " x: " << print_tensor_meta(x); + VLOG(10) << " out: " << print_tensor_meta(*out) << " " << out; + + if (src_layout != DataLayout::ONEDNN) { + out->ShareDataWith(x); + out->ShareInplaceVersionCounterWith(x); + out->set_layout(static_cast(dst_layout)); + return; + } + + DataLayout tmp_layout = static_cast(dst_layout); + if (static_cast(dst_layout) == DataLayout::ANY) { + tmp_layout = phi::OneDNNContext::tls().get_cur_paddle_data_layout(); + } + + if (tmp_layout == DataLayout::ANY) { + tmp_layout = phi::OneDNNContext::tls().get_cur_paddle_data_layout(); + } + + // NOTE(zhiqiu): to handle the special case in ApplyDataTransform() in + // data_transfer.cc + if (!x.IsInitialized() && src_layout == DataLayout::ONEDNN && + tmp_layout == DataLayout::NHWC) { + VLOG(4) << src_layout << "->" << tmp_layout << " " << x.layout(); + out->Resize(x.dims()); + out->set_layout(tmp_layout); + funcs::MatchShapeToLayout(out, src_layout, tmp_layout); + return; + } + + funcs::TransDataLayoutFromOneDNN( + src_layout, tmp_layout, x, out, dev_ctx.GetPlace()); +#endif +} + +} // namespace phi + +PD_REGISTER_KERNEL_FOR_ALL_DTYPE(onednn_to_paddle_layout, + CPU, + ALL_LAYOUT, + phi::OneDNN2PaddleLayout) {} diff --git a/paddle/phi/kernels/onednn_to_paddle_layout_kernel.h b/paddle/phi/kernels/onednn_to_paddle_layout_kernel.h new file mode 100644 index 0000000000000..a6ddc280c4e3c --- /dev/null +++ b/paddle/phi/kernels/onednn_to_paddle_layout_kernel.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/infermeta/unary.h" +#include "paddle/phi/kernels/empty_kernel.h" + +namespace phi { + +template +void OneDNN2PaddleLayout(const Context& dev_ctx, + const DenseTensor& x, + int dst_layout, + DenseTensor* out); +} // namespace phi diff --git a/test/mkldnn/test_conv2d_mkldnn_op.py b/test/mkldnn/test_conv2d_mkldnn_op.py index 3c77581acf80d..2d6cafdbc3734 100644 --- a/test/mkldnn/test_conv2d_mkldnn_op.py +++ b/test/mkldnn/test_conv2d_mkldnn_op.py @@ -17,6 +17,9 @@ import numpy as np from op_test import OpTest, skip_check_grad_ci from test_conv2d_op import TestConv2DOp, TestConv2DOp_v2 +from utils import compare_legacy_with_pt + +from paddle.base import core def conv2d_bias_naive(out, bias): @@ -113,6 +116,94 @@ def setUp(self): self.outputs['Output'] = output +class TestConv2DMKLDNNOp2(TestConv2DOp): + def init_group(self): + self.groups = 1 + + def init_kernel_type(self): + self.data_format = "NCHW" + self.use_mkldnn = True + self._cpu_only = True + self.dtype = np.float32 + + def init_test_case(self): + self.pad = [0, 0] + self.stride = [1, 1] + self.input_size = [2, 3, 5, 5] # NCHW + assert np.mod(self.input_size[1], self.groups) == 0 + f_c = self.input_size[1] // self.groups + self.filter_size = [6, f_c, 3, 3] + + def setUp(self): + self.fuse_bias = False + self.bias_size = None + self.fuse_activation = "" + self.fuse_alpha = 0 + self.fuse_beta = 0 + self.fuse_residual_connection = False + self.input_residual_size = None + + TestConv2DOp.setUp(self) + + output = self.outputs['Output'] + + # mkldnn only support either conv-sum-relu, or conv-relu. + if self.fuse_bias and self.bias_size is not None: + bias = np.random.random(self.bias_size).astype(self.dtype) + output = conv2d_bias_naive(output, bias) + output = output.astype(self.dtype) + self.attrs['fuse_bias'] = self.fuse_bias + self.inputs['Bias'] = OpTest.np_dtype_to_base_dtype(bias) + + if ( + self.fuse_residual_connection + and self.input_residual_size is not None + ): + input_residual = np.random.random(self.input_residual_size).astype( + self.dtype + ) + output = conv2d_residual_naive(output, input_residual) + + self.attrs[ + 'fuse_residual_connection' + ] = self.fuse_residual_connection + self.inputs['ResidualData'] = OpTest.np_dtype_to_base_dtype( + input_residual + ) + + if self.fuse_activation == "relu": + output = np.maximum(output, 0).astype(self.dsttype) + + if self.fuse_activation == "relu6": + output = np.minimum(np.maximum(output, 0), self.fuse_beta).astype( + self.dsttype + ) + if ( + self.fuse_activation != "" + or self.fuse_bias + or self.fuse_residual_connection + ): + self.op_type = 'fused_conv2d' + + output = output.astype(self.dtype) + + self.attrs['fuse_bias'] = self.fuse_bias + self.attrs['fuse_activation'] = self.fuse_activation + self.attrs['fuse_alpha'] = self.fuse_alpha + self.attrs['fuse_beta'] = self.fuse_beta + self.attrs['fuse_residual_connection'] = self.fuse_residual_connection + + self.outputs['Output'] = output + + @compare_legacy_with_pt + def test_check_output(self): + place = core.CUDAPlace(0) if self.has_cuda() else core.CPUPlace() + # TODO(wangzhongpu): support mkldnn op in dygraph mode + self.check_output_with_place( + place, atol=1e-5, check_dygraph=(not self.use_mkldnn) + ) + + @skip_check_grad_ci( reason="Fusion is for inference only, check_grad is not required." ) From 1b5c02827f4cdc953110e6bda3d9cc6e52cf33e9 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:33:33 +0800 Subject: [PATCH 101/146] =?UTF-8?q?[cmake=E6=B2=BB=E7=90=86]Cmake=20optimi?= =?UTF-8?q?zation=20framework/details=20(#59478)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cmake optimization * cmake optimization * cmake optimization * cmake optimization --- paddle/fluid/eager/CMakeLists.txt | 2 +- paddle/fluid/framework/CMakeLists.txt | 14 +- paddle/fluid/framework/details/CMakeLists.txt | 363 ++++-------------- .../framework/details/build_strategy_test.cc | 308 --------------- paddle/fluid/framework/ir/CMakeLists.txt | 3 +- .../ir/memory_optimize_pass/CMakeLists.txt | 23 +- .../multi_devices_graph_pass/CMakeLists.txt | 29 +- paddle/fluid/imperative/CMakeLists.txt | 2 - paddle/fluid/inference/CMakeLists.txt | 2 +- paddle/fluid/pybind/CMakeLists.txt | 2 +- .../fluid/framework/details/CMakeLists.txt | 9 +- .../details/reduce_op_handle_test.cc | 0 12 files changed, 101 insertions(+), 656 deletions(-) delete mode 100644 paddle/fluid/framework/details/build_strategy_test.cc rename {paddle => test/cpp}/fluid/framework/details/reduce_op_handle_test.cc (100%) diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index dde3ed71bc8c9..5667a86876e19 100755 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -45,7 +45,7 @@ endif() cc_library( eager_nan_inf_utils SRCS nan_inf_utils.cc - DEPS phi common nan_inf_utils enforce) + DEPS phi common enforce) cc_library( grad_node_info SRCS grad_node_info.cc diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index e5aac824b753b..338130c64d9a0 100755 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -340,7 +340,7 @@ if(WITH_XPU) op_kernel_type op_call_stack unused_var_check - nan_inf_utils + detail_op_handle phi_utils infershape_utils phi @@ -367,7 +367,7 @@ else() op_kernel_type op_call_stack unused_var_check - nan_inf_utils + detail_op_handle phi_utils infershape_utils phi @@ -873,15 +873,7 @@ target_link_libraries( cc_library( parallel_executor SRCS parallel_executor.cc - DEPS threaded_ssa_graph_executor - scope_buffered_ssa_graph_executor - parallel_ssa_graph_executor - async_ssa_graph_executor - graph - build_strategy - bind_threaded_ssa_graph_executor - collective_helper - fast_threaded_ssa_graph_executor + DEPS ssa_graph_executor graph build_strategy collective_helper variable_helper) cc_library( diff --git a/paddle/fluid/framework/details/CMakeLists.txt b/paddle/fluid/framework/details/CMakeLists.txt index 2ee0da89fe980..d771a12411adb 100644 --- a/paddle/fluid/framework/details/CMakeLists.txt +++ b/paddle/fluid/framework/details/CMakeLists.txt @@ -1,55 +1,3 @@ -cc_library( - var_handle - SRCS var_handle.cc - DEPS place framework_proto node) -cc_library( - op_handle_base - SRCS op_handle_base.cc - DEPS var_handle device_context lod_tensor) - -cc_library( - scale_loss_grad_op_handle - SRCS scale_loss_grad_op_handle.cc - DEPS op_handle_base scope lod_tensor phi common) -cc_library( - fetch_op_handle - SRCS fetch_op_handle.cc - DEPS op_handle_base scope lod_tensor phi common) -cc_library( - fetch_async_op_handle - SRCS fetch_async_op_handle.cc - DEPS op_handle_base scope lod_tensor phi common) - -cc_library( - share_tensor_buffer_functor - SRCS share_tensor_buffer_functor.cc - DEPS framework_proto scope place operator op_registry) -cc_library( - computation_op_handle - SRCS computation_op_handle.cc - DEPS framework_proto scope place operator op_registry) -cc_library( - share_tensor_buffer_op_handle - SRCS share_tensor_buffer_op_handle.cc - DEPS op_handle_base scope computation_op_handle share_tensor_buffer_functor) -cc_library( - rpc_op_handle - SRCS rpc_op_handle.cc - DEPS framework_proto scope place operator op_registry) -cc_library( - fetch_barrier_op_handle - SRCS fetch_barrier_op_handle.cc - DEPS framework_proto scope place operator op_registry) -cc_library( - multi_devices_helper - SRCS multi_devices_helper.cc - DEPS graph graph_helper) - -cc_library( - variable_visitor - SRCS variable_visitor.cc - DEPS lod_tensor selected_rows_utils) - if(WITH_PSCORE) set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor" @@ -67,260 +15,101 @@ if(WITH_PSCORE) ${DISTRIBUTE_COMPILE_FLAGS}) endif() -if(WITH_GPU) - nv_library( - nan_inf_utils - SRCS nan_inf_utils_detail.cc - DEPS framework_proto scope place phi common) - nv_library( - all_reduce_op_handle - SRCS all_reduce_op_handle.cc - DEPS variable_visitor - op_handle_base - scope - lod_tensor - phi - common - fluid_memory - dynload_cuda) - nv_library( - fused_all_reduce_op_handle - SRCS fused_all_reduce_op_handle.cc - DEPS all_reduce_op_handle - variable_visitor - op_handle_base - scope - lod_tensor - phi - common - dynload_cuda - place) - nv_library( - grad_merge_all_reduce_op_handle - SRCS grad_merge_all_reduce_op_handle.cc - DEPS fused_all_reduce_op_handle - op_handle_base - scope - lod_tensor - phi - common - dynload_cuda - variable_visitor - place - all_reduce_op_handle) +set(op_handle_srcs + nan_inf_utils_detail.cc + all_reduce_op_handle.cc + fused_all_reduce_op_handle.cc + grad_merge_all_reduce_op_handle.cc + reduce_op_handle.cc + broadcast_op_handle.cc + fused_broadcast_op_handle.cc + var_handle.cc + op_handle_base.cc + scale_loss_grad_op_handle.cc + fetch_op_handle.cc + fetch_async_op_handle.cc + share_tensor_buffer_functor.cc + computation_op_handle.cc + share_tensor_buffer_op_handle.cc + rpc_op_handle.cc + fetch_barrier_op_handle.cc + multi_devices_helper.cc + variable_visitor.cc + gather_op_handle.cc + eager_deletion_op_handle.cc) + +if(WITH_DGC) + set(op_handle_srcs ${op_handle_srcs} sparse_all_reduce_op_handle.cc) +endif() - if(WITH_DGC) - nv_library( - sparse_all_reduce_op_handle - SRCS sparse_all_reduce_op_handle.cc - DEPS op_handle_base - scope - lod_tensor - phi - common - dynload_cuda - variable_visitor - dgc - all_reduce_op_handle) - endif() +set(op_handle_deps + pass + operator + place + framework_proto + node + device_context + op_registry + lod_tensor + selected_rows_utils + reference_count_pass_helper) - if(WITH_DISTRIBUTE) - nv_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common dynload_cuda) - else() - nv_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common dynload_cuda) - endif() - nv_library( - broadcast_op_handle - SRCS broadcast_op_handle.cc - DEPS op_handle_base scope phi common variable_visitor dynload_cuda) +if(WITH_MKLDNN) + set(op_handle_deps ${op_handle_deps} mkldnn) +endif() + +if(WITH_DGC) + set(op_handle_deps ${op_handle_deps} dgc) +endif() + +if(WITH_GPU) nv_library( - fused_broadcast_op_handle - SRCS fused_broadcast_op_handle.cc - DEPS broadcast_op_handle) + detail_op_handle + SRCS ${op_handle_srcs} + DEPS ${op_handle_deps}) elseif(WITH_ROCM) hip_library( - nan_inf_utils - SRCS nan_inf_utils_detail.cc - DEPS framework_proto scope place phi common) - hip_library( - all_reduce_op_handle - SRCS all_reduce_op_handle.cc - DEPS op_handle_base - scope - lod_tensor - phi - common - dynload_cuda - variable_visitor) - hip_library( - fused_all_reduce_op_handle - SRCS fused_all_reduce_op_handle.cc - DEPS all_reduce_op_handle - op_handle_base - variable_visitor - scope - lod_tensor - phi - common - dynload_cuda - place) - hip_library( - grad_merge_all_reduce_op_handle - SRCS grad_merge_all_reduce_op_handle.cc - DEPS fused_all_reduce_op_handle - op_handle_base - scope - lod_tensor - phi - common - dynload_cuda - variable_visitor - place - all_reduce_op_handle) - - if(WITH_DISTRIBUTE) - hip_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common dynload_cuda) - else() - hip_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common dynload_cuda) - endif() - hip_library( - broadcast_op_handle - SRCS broadcast_op_handle.cc - DEPS op_handle_base scope phi common variable_visitor dynload_cuda) - hip_library( - fused_broadcast_op_handle - SRCS fused_broadcast_op_handle.cc - DEPS broadcast_op_handle) + detail_op_handle + SRCS ${op_handle_srcs} + DEPS ${op_handle_deps}) else() cc_library( - nan_inf_utils - SRCS nan_inf_utils_detail.cc - DEPS framework_proto scope place phi common) - cc_library( - all_reduce_op_handle - SRCS all_reduce_op_handle.cc - DEPS op_handle_base scope lod_tensor phi common variable_visitor) - cc_library( - fused_all_reduce_op_handle - SRCS fused_all_reduce_op_handle.cc - DEPS all_reduce_op_handle - op_handle_base - scope - lod_tensor - phi - common - variable_visitor - place) - cc_library( - grad_merge_all_reduce_op_handle - SRCS grad_merge_all_reduce_op_handle.cc - DEPS fused_all_reduce_op_handle - op_handle_base - scope - lod_tensor - phi - common - variable_visitor - place - all_reduce_op_handle) - if(WITH_DISTRIBUTE) - cc_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common) - else() - cc_library( - reduce_op_handle - SRCS reduce_op_handle.cc - DEPS op_handle_base variable_visitor scope phi common) - endif() - cc_library( - broadcast_op_handle - SRCS broadcast_op_handle.cc - DEPS op_handle_base scope phi common variable_visitor) - cc_library( - fused_broadcast_op_handle - SRCS fused_broadcast_op_handle.cc - DEPS broadcast_op_handle) + detail_op_handle + SRCS ${op_handle_srcs} + DEPS ${op_handle_deps}) endif() -cc_library( - gather_op_handle - SRCS gather_op_handle.cc - DEPS op_handle_base scope phi common variable_visitor) - -cc_library( - eager_deletion_op_handle - SRCS eager_deletion_op_handle.cc - DEPS lod_tensor selected_rows_utils reference_count_pass_helper) - +add_dependencies(detail_op_handle framework_proto auto_parallel_proto xxhash) + +set(ssa_graph_executor_srcs + ssa_graph_executor.cc + threaded_ssa_graph_executor.cc + parallel_ssa_graph_executor.cc + async_ssa_graph_executor.cc + bind_threaded_ssa_graph_executor.cc + fast_threaded_ssa_graph_executor.cc + scope_buffered_ssa_graph_executor.cc + scope_buffered_monitor.cc) set(SSA_GRAPH_EXECUTOR_DEPS graph framework_proto - multi_devices_helper + detail_op_handle reference_count_pass eager_deletion_pass buffer_shared_inplace_op_pass buffer_shared_cross_op_memory_reuse_pass inplace_addto_op_pass - set_reader_device_info_utils) -cc_library( - ssa_graph_executor NOT_FOR_INFER - SRCS ssa_graph_executor.cc - DEPS ${SSA_GRAPH_EXECUTOR_DEPS}) - -cc_library( - threaded_ssa_graph_executor - SRCS threaded_ssa_graph_executor.cc - DEPS fetch_op_handle ssa_graph_executor scope simple_threadpool - device_context) + set_reader_device_info_utils + scope + simple_threadpool + device_context + profiler + selected_rows_utils) cc_library( - parallel_ssa_graph_executor - SRCS parallel_ssa_graph_executor.cc - DEPS threaded_ssa_graph_executor) - -set(ASYNC_SSA_GRAPH_EXECUTOR_DEPS threaded_ssa_graph_executor) - -cc_library( - async_ssa_graph_executor - SRCS async_ssa_graph_executor.cc - DEPS ${ASYNC_SSA_GRAPH_EXECUTOR_DEPS}) -cc_library( - scope_buffered_monitor - SRCS scope_buffered_monitor.cc - DEPS scope profiler selected_rows_utils) -cc_library( - scope_buffered_ssa_graph_executor - SRCS scope_buffered_ssa_graph_executor.cc - DEPS ssa_graph_executor scope_buffered_monitor) -cc_library( - bind_threaded_ssa_graph_executor - SRCS bind_threaded_ssa_graph_executor.cc - DEPS fetch_op_handle - phi - common - ssa_graph_executor - scope - simple_threadpool - device_context) -cc_library( - fast_threaded_ssa_graph_executor - SRCS fast_threaded_ssa_graph_executor.cc - DEPS fetch_async_op_handle ssa_graph_executor scope simple_threadpool - device_context) + ssa_graph_executor + SRCS ${ssa_graph_executor_srcs} + DEPS ${SSA_GRAPH_EXECUTOR_DEPS}) set(IR_PASS_DEPS graph_viz_pass diff --git a/paddle/fluid/framework/details/build_strategy_test.cc b/paddle/fluid/framework/details/build_strategy_test.cc deleted file mode 100644 index dc6a7e33e4f2f..0000000000000 --- a/paddle/fluid/framework/details/build_strategy_test.cc +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/framework/details/build_strategy.h" - -#include -#include -#include -#include -#include -#include - -#include "gtest/gtest-message.h" -#include "gtest/gtest-test-part.h" -#include "gtest/gtest.h" -#include "gtest/gtest_pred_impl.h" -#include "paddle/fluid/framework/op_proto_maker.h" -#include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/framework/var_type_inference.h" -#include "paddle/fluid/platform/place.h" - -PD_DECLARE_bool(convert_all_blocks); - -namespace paddle { -namespace framework { - -class SumOpMaker : public OpProtoAndCheckerMaker { - public: - void Make() { - AddInput("X", "").AsDuplicable(); - AddOutput("Out", "").AsDuplicable(); - AddComment(""); - } -}; - -class SumOpWithKernel : public OperatorWithKernel { - public: - using OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(framework::InferShapeContext *ctx) const override {} - phi::KernelKey GetExpectedKernelType( - const ExecutionContext &ctx) const override { - return phi::KernelKey(proto::VarType::FP32, - ctx.Input("X")->place()); - } -}; - -} // namespace framework -} // namespace paddle - -REGISTER_OP_WITHOUT_GRADIENT(fake_sum, - paddle::framework::SumOpWithKernel, - paddle::framework::SumOpMaker); - -namespace paddle { -namespace framework { -namespace details { - -static std::vector CreatePlaces(size_t num, bool use_cuda) { - std::vector result; - result.reserve(num); - for (size_t i = 0; i < num; ++i) { - if (use_cuda) { - result.emplace_back(platform::CUDAPlace(static_cast(i))); - } else { - result.emplace_back(platform::CPUPlace()); - } - } - return result; -} - -void BuildStrategyApply(BuildStrategy *build_strategy, ir::Graph *graph) { - std::string loss_name = ""; - Scope scope; - std::vector scopes = {&scope}; - - auto places = CreatePlaces(1, false); - auto device = platform::Place2DeviceType(places[0]); - -#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) - platform::NCCLCommunicator ctxs; -#elif defined(PADDLE_WITH_XPU) && defined(PADDLE_WITH_XPU_BKCL) - platform::BKCLCommunicator ctxs; -#endif - - build_strategy->Apply(graph, - places, - loss_name, - scopes, - 1, -#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL) - device, - &ctxs); -#elif defined(PADDLE_WITH_XPU) && defined(PADDLE_WITH_XPU_BKCL) - device, - &ctxs); -#else - device); -#endif -} - -std::unique_ptr CreateGraph() { - ProgramDesc prog; - auto *op = prog.MutableBlock(0)->AppendOp(); - op->SetType("fake_sum"); - op->SetInput("X", {"a1"}); - op->SetOutput("Out", {"b1"}); - op->SetAttr("op_role", 1); - - prog.MutableBlock(0)->Var("a1")->SetType(proto::VarType::LOD_TENSOR); - prog.MutableBlock(0)->Var("b1")->SetType(proto::VarType::LOD_TENSOR); - - std::unique_ptr g(new ir::Graph(prog)); - return g; -} - -std::unique_ptr CreateMultiGraph() { - ProgramDesc prog; - prog.AppendBlock(prog.Block(0)); - prog.AppendBlock(prog.Block(0)); - - // Set contents in block_0. - auto *op = prog.MutableBlock(0)->AppendOp(); - op->SetType("fake_sum"); - op->SetInput("X", {"test_a", "test_b", "test_c"}); - op->SetOutput("Out", {"test_out"}); - op->SetAttr("op_role", 1); - - prog.MutableBlock(0)->Var("test_a")->SetType(proto::VarType::SELECTED_ROWS); - prog.MutableBlock(0)->Var("test_b")->SetType(proto::VarType::SELECTED_ROWS); - prog.MutableBlock(0)->Var("test_c")->SetType(proto::VarType::SELECTED_ROWS); - prog.MutableBlock(0)->Var("test_out"); - op->InferVarType(prog.MutableBlock(0)); - - prog.MutableBlock(0)->Var("test_b")->SetType(proto::VarType::LOD_TENSOR); - op->InferVarType(prog.MutableBlock(0)); - - // Set contents in block_1. - op = prog.MutableBlock(1)->AppendOp(); - op->SetType("fake_sum"); - op->SetInput("X", {"a1"}); - op->SetOutput("Out", {"b1"}); - op->SetAttr("op_role", 1); - - prog.MutableBlock(1)->Var("a1")->SetType(proto::VarType::LOD_TENSOR); - prog.MutableBlock(1)->Var("b1")->SetType(proto::VarType::LOD_TENSOR); - - // Set contents in block_2. - op = prog.MutableBlock(2)->AppendOp(); - op->SetType("fake_sum"); - op->SetInput("X", {"a2"}); - op->SetOutput("Out", {"b2"}); - op->SetAttr("op_role", 1); - - prog.MutableBlock(2)->Var("a2")->SetType(proto::VarType::LOD_TENSOR); - prog.MutableBlock(2)->Var("b2")->SetType(proto::VarType::LOD_TENSOR); - - std::unique_ptr g(new ir::Graph(prog)); - return g; -} - -inline bool CheckSubGraphSame(ir::Graph *g1, ir::Graph *g2) { - const auto &g1_nodes_set = g1->Nodes(); - const auto &g2_nodes_set = g2->Nodes(); - - if (g1_nodes_set.size() != g2_nodes_set.size()) return false; - - std::vector g1_nodes(g1_nodes_set.begin(), g1_nodes_set.end()); - std::vector g2_nodes(g2_nodes_set.begin(), g2_nodes_set.end()); - - auto comp = [](ir::Node *n1, ir::Node *n2) { - return n1->Name() > n2->Name(); - }; - std::stable_sort(g1_nodes.begin(), g1_nodes.end(), comp); - std::stable_sort(g2_nodes.begin(), g2_nodes.end(), comp); - - for (size_t i = 0; i < g1_nodes.size(); ++i) { - const auto &n1 = g1_nodes[i]; - const auto &n2 = g2_nodes[i]; - - if (n1->NodeType() != n2->NodeType()) return false; - if (n1->Name() != n2->Name()) return false; - - auto n1_inputs = n1->inputs; - auto n2_inputs = n2->inputs; - if (n1_inputs.size() != n2_inputs.size()) return false; - - std::stable_sort(n1_inputs.begin(), n1_inputs.end(), comp); - std::stable_sort(n2_inputs.begin(), n2_inputs.end(), comp); - for (size_t i = 0; i < n1_inputs.size(); ++i) { - if (n1_inputs[i]->Name() != n2_inputs[i]->Name()) return false; - } - - auto n1_outputs = n1->outputs; - auto n2_outputs = n2->outputs; - if (n1_outputs.size() != n2_outputs.size()) return false; - - std::stable_sort(n1_outputs.begin(), n1_outputs.end(), comp); - std::stable_sort(n2_outputs.begin(), n2_outputs.end(), comp); - for (size_t i = 0; i < n1_outputs.size(); ++i) { - if (n1_outputs[i]->Name() != n2_outputs[i]->Name()) return false; - } - - if (n1->IsVar()) { - const auto &var1 = n1->Var(); - const auto &var2 = n2->Var(); - if ((var1 == nullptr) != (var2 == nullptr)) return false; - } - - if (n1->IsOp()) { - const auto &op1 = n1->Op(); - const auto &op2 = n2->Op(); - if ((op1 == nullptr) != (op2 == nullptr)) return false; - - const auto &op1_input = op1->InputNames(); - const auto &op2_input = op2->InputNames(); - if (op1_input.size() != op2_input.size()) return false; - if (op1_input != op2_input) return false; - - for (size_t i = 0; i < op1_input.size(); ++i) { - if (op1->Input(op1_input[i]) != op2->Input(op2_input[i])) return false; - } - - const auto &op1_output = op1->OutputNames(); - const auto &op2_output = op2->OutputNames(); - if (op1_output.size() != op2_output.size()) return false; - if (op1_output != op2_output) return false; - - for (size_t i = 0; i < op1_output.size(); ++i) { - if (op1->Output(op1_output[i]) != op2->Output(op2_output[i])) - return false; - } - } - } - return true; -} - -inline bool CheckGraphSame(ir::Graph *g1, ir::Graph *g2) { - if (g1 == nullptr || g2 == nullptr) return true; - - if (FLAGS_convert_all_blocks) { - if (g1->SubGraphsSize() != g2->SubGraphsSize()) return false; - - for (size_t i = 0; i < g1->SubGraphsSize(); ++i) { - if (!CheckSubGraphSame(g1->GetSubGraph(i), g2->GetSubGraph(i))) - return false; - } - } else { - if (!CheckSubGraphSame(g1, g2)) return false; - } - return true; -} - -TEST(BuildStrategy, Basic) { - BuildStrategy build_strategy; - - ProgramDesc prog; - ir::Graph old_graph(prog), graph(prog); - - BuildStrategyApply(&build_strategy, &graph); - - ASSERT_TRUE(CheckGraphSame(&old_graph, &graph)); -} - -TEST(BuildStrategy, TestSingleGraph) { - BuildStrategy build_strategy; - auto graph = CreateGraph(); - ir::Graph old_graph(graph->OriginProgram()); - - BuildStrategyApply(&build_strategy, graph.get()); - - // graph should not change for no pass here - ASSERT_TRUE(CheckGraphSame(&old_graph, graph.get())); -} - -TEST(BuildStrategy, TestMultiGraph) { - // Set FLAGS_convert_all_blocks to true to make sure this test works. - bool flag_temp = FLAGS_convert_all_blocks; - FLAGS_convert_all_blocks = true; - - BuildStrategy build_strategy; - auto graph = CreateMultiGraph(); - ir::Graph old_graph(graph->OriginProgram()); - - BuildStrategyApply(&build_strategy, graph.get()); - - // graph should not change for no pass here - ASSERT_TRUE(CheckGraphSame(&old_graph, graph.get())); - - // Recover FLAGS_convert_all_blocks. - FLAGS_convert_all_blocks = flag_temp; -} - -} // namespace details -} // namespace framework -} // namespace paddle diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index fa6c8a2583453..3c7560b69e332 100755 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -19,8 +19,7 @@ cc_library( cc_library( graph_helper SRCS graph_helper.cc - DEPS graph program_utils scale_loss_grad_op_handle - grad_merge_all_reduce_op_handle collective_helper) + DEPS graph program_utils collective_helper) # cc_library( pass SRCS pass.cc diff --git a/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt b/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt index d061861661903..85923aafc23a7 100644 --- a/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt +++ b/paddle/fluid/framework/ir/memory_optimize_pass/CMakeLists.txt @@ -1,36 +1,33 @@ cc_library( op_graph_view SRCS op_graph_view.cc - DEPS op_handle_base) + DEPS detail_op_handle) cc_library( conditional_block_op_eager_deletion_pass SRCS conditional_block_op_eager_deletion_pass.cc - DEPS conditional_block_op_helper graph_helper pass computation_op_handle) + DEPS conditional_block_op_helper graph_helper pass) cc_library( pylayer_op_eager_deletion_pass SRCS pylayer_op_eager_deletion_pass.cc - DEPS pylayer_op_helper graph_helper pass computation_op_handle) + DEPS pylayer_op_helper graph_helper pass) cc_library( while_op_eager_deletion_pass SRCS while_op_eager_deletion_pass.cc - DEPS while_op_helper graph_helper pass computation_op_handle) + DEPS while_op_helper graph_helper pass) cc_library( recurrent_op_eager_deletion_pass SRCS recurrent_op_eager_deletion_pass.cc - DEPS recurrent_op_helper graph_helper pass computation_op_handle) + DEPS recurrent_op_helper graph_helper pass) cc_library( reference_count_pass_helper SRCS reference_count_pass_helper.cc - DEPS garbage_collector computation_op_handle var_handle) + DEPS garbage_collector) # cc_library( reference_count_pass SRCS reference_count_pass.cc - DEPS computation_op_handle graph graph_helper pass op_graph_view - reference_count_pass_helper) + DEPS graph graph_helper pass op_graph_view reference_count_pass_helper) set(EAGER_DELETETION_PASS_DEPS - computation_op_handle - eager_deletion_op_handle graph graph_helper pass @@ -43,8 +40,7 @@ if(WITH_CINN) cc_library( share_varinfo_into_cinn_pass SRCS share_varinfo_into_cinn_pass.cc - DEPS pass enforce common graph_helper computation_op_handle - eager_deletion_op_handle) + DEPS pass enforce common graph_helper) cc_test( share_varinfo_into_cinn_pass_test SRCS share_varinfo_into_cinn_pass_test.cc @@ -61,8 +57,7 @@ cc_library( cc_library( memory_reuse_pass SRCS memory_reuse_pass.cc - DEPS computation_op_handle reference_count_pass_helper - share_tensor_buffer_op_handle graph pass multi_devices_helper) + DEPS reference_count_pass_helper graph pass) cc_library( buffer_shared_inplace_op_pass diff --git a/paddle/fluid/framework/ir/multi_devices_graph_pass/CMakeLists.txt b/paddle/fluid/framework/ir/multi_devices_graph_pass/CMakeLists.txt index e97331bc87a45..2aa76a8eb2214 100644 --- a/paddle/fluid/framework/ir/multi_devices_graph_pass/CMakeLists.txt +++ b/paddle/fluid/framework/ir/multi_devices_graph_pass/CMakeLists.txt @@ -1,36 +1,25 @@ cc_library( modify_op_lock_and_record_event_pass SRCS modify_op_lock_and_record_event_pass.cc - DEPS computation_op_handle scale_loss_grad_op_handle op_graph_view - multi_devices_helper) + DEPS detail_op_handle op_graph_view) cc_library( multi_devices_graph_print_pass SRCS multi_devices_graph_print_pass.cc - DEPS multi_devices_helper) + DEPS detail_op_handle) cc_library( multi_devices_graph_check_pass SRCS multi_devices_graph_check_pass.cc - DEPS multi_devices_helper) + DEPS detail_op_handle) -set(ALL_REDUCE_OP_HANDLES all_reduce_op_handle) -set(ALL_REDUCE_OP_HANDLES grad_merge_all_reduce_op_handle) if(WITH_GPU AND WITH_DGC) - list(APPEND ALL_REDUCE_OP_HANDLES sparse_all_reduce_op_handle) + list(APPEND ALL_REDUCE_OP_HANDLES detail_op_handle) endif() cc_library( multi_devices_graph_pass SRCS multi_devices_graph_pass.cc - DEPS multi_devices_helper - computation_op_handle - scale_loss_grad_op_handle - rpc_op_handle - fetch_barrier_op_handle - ${ALL_REDUCE_OP_HANDLES} - reduce_op_handle - broadcast_op_handle - fused_broadcast_op_handle) + DEPS detail_op_handle ${ALL_REDUCE_OP_HANDLES}) cc_library( sequential_execution_pass SRCS sequential_execution_pass.cc @@ -43,12 +32,11 @@ cc_library( cc_library( fuse_all_reduce_op_pass SRCS fuse_all_reduce_op_pass.cc - DEPS graph graph_helper fused_all_reduce_op_handle - grad_merge_all_reduce_op_handle) + DEPS graph graph_helper) cc_library( all_reduce_deps_pass SRCS all_reduce_deps_pass.cc - DEPS all_reduce_op_handle graph graph_helper pass) + DEPS graph graph_helper pass) cc_library( backward_optimizer_op_deps_pass SRCS backward_optimizer_op_deps_pass.cc @@ -60,5 +48,4 @@ cc_library( cc_library( fix_op_run_order_pass SRCS fix_op_run_order_pass.cc - DEPS graph graph_helper multi_devices_helper pass op_handle_base - eager_deletion_op_handle) + DEPS graph graph_helper pass) diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index b6d846e9a0c12..7a764f5302021 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -19,7 +19,6 @@ if(WITH_XPU) var_type_traits op_kernel_type data_transform - nan_inf_utils phi common var_helper @@ -37,7 +36,6 @@ else() var_type_traits op_kernel_type data_transform - nan_inf_utils phi common var_helper diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index 3f4e7a9344a30..1f353e2ba8409 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -107,7 +107,7 @@ set(SHARED_INFERENCE_SRCS list(REMOVE_ITEM fluid_modules cinn_op_dialect) # NOTE(Aurelisu84): Remove pir dialect related target DEPS for inference # shared library to prune library size. -list(REMOVE_ITEM fluid_modules ${not_infer_modules}) +# list(REMOVE_ITEM fluid_modules ${not_infer_modules}) set(SHARED_INFERENCE_DEPS phi common ${fluid_modules} analysis_predictor ${utils_modules}) diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 8965bcfbf234e..52eada6c5482f 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -44,7 +44,7 @@ set(PYBIND_DEPS new_profiler fluid_jit prim_utils - gather_op_handle + detail_op_handle static_tensor_operants type_info auto_parallel) diff --git a/test/cpp/fluid/framework/details/CMakeLists.txt b/test/cpp/fluid/framework/details/CMakeLists.txt index cb430109e286f..4a02fd08e0815 100644 --- a/test/cpp/fluid/framework/details/CMakeLists.txt +++ b/test/cpp/fluid/framework/details/CMakeLists.txt @@ -3,14 +3,7 @@ paddle_test(broadcast_op_test SRCS broadcast_op_handle_test.cc) cc_test( gather_op_test SRCS gather_op_handle_test.cc - DEPS var_handle - op_handle_base - scope - phi - common - fluid_memory - device_context - gather_op_handle) + DEPS detail_op_handle scope phi common fluid_memory device_context) paddle_test(fused_broadcast_op_test SRCS fused_broadcast_op_handle_test.cc) paddle_test(exception_holder_test SRCS exception_holder_test.cc) diff --git a/paddle/fluid/framework/details/reduce_op_handle_test.cc b/test/cpp/fluid/framework/details/reduce_op_handle_test.cc similarity index 100% rename from paddle/fluid/framework/details/reduce_op_handle_test.cc rename to test/cpp/fluid/framework/details/reduce_op_handle_test.cc From 51d25ec7d3b9f3c9a8534bc0a37c88de3bb1193e Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:40:27 +0800 Subject: [PATCH 102/146] fiox (#60404) --- test/dygraph_to_static/test_cache_program.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/dygraph_to_static/test_cache_program.py b/test/dygraph_to_static/test_cache_program.py index 2ad05f56b41e7..34744a6567cf0 100644 --- a/test/dygraph_to_static/test_cache_program.py +++ b/test/dygraph_to_static/test_cache_program.py @@ -18,9 +18,6 @@ import numpy as np from dygraph_to_static_utils import ( Dy2StTestBase, - IrMode, - ToStaticMode, - disable_test_case, enable_to_static_guard, test_ast_only, test_legacy_and_pt_and_pir, @@ -175,7 +172,6 @@ def sum_under_while(limit): return ret_sum -@disable_test_case((ToStaticMode.AST, IrMode.PT)) class TestToOutputWithCache(Dy2StTestBase): def test_output(self): ret = paddle.jit.to_static(sum_even_until_limit)(80, 10) From 61cf6b12537006f3efcbdde7e556104f3f0364c5 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Thu, 28 Dec 2023 10:54:43 +0800 Subject: [PATCH 103/146] sub block trace run in inference (#60419) * sub block trace run in inference * update --- .../new_executor/instruction/if_instruction.cc | 13 ++++++++----- .../new_executor/instruction/if_instruction.h | 3 ++- .../instruction/while_instruction.cc | 18 +++++++++++------- .../instruction/while_instruction.h | 3 ++- .../interpreter/interpreter_util.cc | 4 ++++ .../framework/new_executor/pir_interpreter.cc | 6 ++---- .../new_executor/program_interpreter.cc | 4 ++++ .../controlflow/conditional_block_op.cc | 3 +++ 8 files changed, 36 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/framework/new_executor/instruction/if_instruction.cc b/paddle/fluid/framework/new_executor/instruction/if_instruction.cc index c43eba69ed1f5..57146acdfb5df 100644 --- a/paddle/fluid/framework/new_executor/instruction/if_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/if_instruction.cc @@ -43,7 +43,7 @@ IfInstruction::IfInstruction(size_t id, const platform::Place& place, pir::Operation* op, ValueExecutionInfo* value_exec_info, - const std::set& skip_gc_vars) + interpreter::ExecutionConfig execution_config) : InstructionBase(id, place) { PADDLE_ENFORCE( op->isa(), @@ -124,12 +124,15 @@ IfInstruction::IfInstruction(size_t id, VLOG(6) << "finish process inputs outputs index"; Scope* true_scope = &(value_exec_info->GetScope()->NewScope()); + auto skip_gc_vars = execution_config.skip_gc_vars; + execution_config.skip_gc_vars.clear(); + execution_config.create_local_scope = true; true_branch_inter_ = new PirInterpreter(place, {}, &true_branch_block, true_scope, value_exec_info->NewChild(true_scope), - {}); + execution_config); std::set true_skip_gc_names_set; for (auto value : GetYiedOpInputs(&true_branch_block)) { @@ -143,7 +146,7 @@ IfInstruction::IfInstruction(size_t id, true_skip_gc_names_.push_back(true_branch_inter_->GetNameByValue(value)); true_skip_gc_names_set.insert(true_branch_inter_->GetNameByValue(value)); } - for (auto var_name : skip_gc_vars) { + for (const auto& var_name : skip_gc_vars) { true_skip_gc_names_.push_back(var_name); true_skip_gc_names_set.insert(var_name); } @@ -157,7 +160,7 @@ IfInstruction::IfInstruction(size_t id, &if_op.false_block(), false_scope, value_exec_info->NewChild(false_scope), - {}); + execution_config); std::set false_skip_gc_names_set; for (auto value : GetYiedOpInputs(&false_branch_block)) { false_branch_outputs_.push_back(false_branch_inter_->GetNameByValue(value)); @@ -168,7 +171,7 @@ IfInstruction::IfInstruction(size_t id, false_skip_gc_names_.push_back(false_branch_inter_->GetNameByValue(value)); false_skip_gc_names_set.insert(false_branch_inter_->GetNameByValue(value)); } - for (auto var_name : skip_gc_vars) { + for (const auto& var_name : skip_gc_vars) { false_skip_gc_names_.push_back(var_name); false_skip_gc_names_set.insert(var_name); } diff --git a/paddle/fluid/framework/new_executor/instruction/if_instruction.h b/paddle/fluid/framework/new_executor/instruction/if_instruction.h index e6d1fc4723c5d..b7b3ed6ac8f17 100644 --- a/paddle/fluid/framework/new_executor/instruction/if_instruction.h +++ b/paddle/fluid/framework/new_executor/instruction/if_instruction.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/fluid/framework/new_executor/instruction/instruction_base.h" +#include "paddle/fluid/framework/new_executor/interpreter/execution_config.h" namespace ir { class Operation; @@ -33,7 +34,7 @@ class IfInstruction : public InstructionBase { const platform::Place& place, ::pir::Operation* op, ValueExecutionInfo* value_exe_info, - const std::set& skip_gc_vars); + interpreter::ExecutionConfig execution_config); ~IfInstruction(); diff --git a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc b/paddle/fluid/framework/new_executor/instruction/while_instruction.cc index f2a6e92e2f4b2..b281e2b8a6cbe 100644 --- a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/while_instruction.cc @@ -40,11 +40,12 @@ namespace paddle { namespace framework { -WhileInstruction::WhileInstruction(size_t id, - const platform::Place& place, - pir::Operation* op, - ValueExecutionInfo* parent_exe_info, - const std::set& skip_gc_vars) +WhileInstruction::WhileInstruction( + size_t id, + const platform::Place& place, + pir::Operation* op, + ValueExecutionInfo* parent_exe_info, + interpreter::ExecutionConfig execution_config) : InstructionBase(id, place) { op_ = op; VLOG(6) << "finish process dist attributes"; @@ -108,8 +109,11 @@ WhileInstruction::WhileInstruction(size_t id, body_scope->Var(var_name); body_exe_info->Add(body_block_->arg(i), var_name); } + auto skip_gc_vars = execution_config.skip_gc_vars; + execution_config.skip_gc_vars.clear(); + execution_config.create_local_scope = true; body_inter_ = std::unique_ptr(new PirInterpreter( - place, {}, body_block_, body_scope, body_exe_info, {})); + place, {}, body_block_, body_scope, body_exe_info, execution_config)); std::set body_skip_gc_names_set; auto body_block_outputs = GetYiedOpInputs(body_block_); @@ -122,7 +126,7 @@ WhileInstruction::WhileInstruction(size_t id, body_skip_gc_names_.push_back(body_inter_->GetNameByValue(value)); body_skip_gc_names_set.insert(body_inter_->GetNameByValue(value)); } - for (auto var_name : skip_gc_vars) { + for (const auto& var_name : skip_gc_vars) { body_skip_gc_names_.push_back(var_name); body_skip_gc_names_set.insert(var_name); } diff --git a/paddle/fluid/framework/new_executor/instruction/while_instruction.h b/paddle/fluid/framework/new_executor/instruction/while_instruction.h index ae27c89b0051a..f8a98d3b03d6b 100644 --- a/paddle/fluid/framework/new_executor/instruction/while_instruction.h +++ b/paddle/fluid/framework/new_executor/instruction/while_instruction.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/fluid/framework/new_executor/instruction/instruction_base.h" +#include "paddle/fluid/framework/new_executor/interpreter/execution_config.h" namespace ir { class Operation; @@ -39,7 +40,7 @@ class WhileInstruction : public InstructionBase { const platform::Place& place, ::pir::Operation* op, ValueExecutionInfo* parent_exe_info, - const std::set& skip_gc_vars); + interpreter::ExecutionConfig execution_config); void Run() override; diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc index 006e9d5fc4603..614b97c26b7b0 100644 --- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc +++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc @@ -641,6 +641,10 @@ void BuildOpFuncList(const platform::Place& place, auto runtime_attrs = op->RuntimeAttrs(); runtime_attrs.insert(std::make_pair("used_for_inference", true)); op->SetRuntimeAttributeMap(runtime_attrs); + } else if (op->Type() == "conditional_block") { + auto runtime_attrs = op->RuntimeAttrs(); + runtime_attrs.insert(std::make_pair("used_for_inference", true)); + op->SetRuntimeAttributeMap(runtime_attrs); } } diff --git a/paddle/fluid/framework/new_executor/pir_interpreter.cc b/paddle/fluid/framework/new_executor/pir_interpreter.cc index 1cd1117d0ea1d..82bf2973345ad 100644 --- a/paddle/fluid/framework/new_executor/pir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/pir_interpreter.cc @@ -686,9 +686,8 @@ void PirInterpreter::BuildInstruction() { } } else if (op.dialect()->name() == "pd_op") { if (op.isa()) { - auto skip_gc_vars = execution_config_.skip_gc_vars; vec_instruction_base_.emplace_back(std::make_unique( - op_idx++, place_, &op, value_exe_info_.get(), skip_gc_vars)); + op_idx++, place_, &op, value_exe_info_.get(), execution_config_)); sub_blocks_.insert( {&op.dyn_cast().true_block(), dynamic_cast(vec_instruction_base_.back().get()) @@ -698,9 +697,8 @@ void PirInterpreter::BuildInstruction() { dynamic_cast(vec_instruction_base_.back().get()) ->FalseBranchInterpreter()}); } else if (op.isa()) { - auto skip_gc_vars = execution_config_.skip_gc_vars; vec_instruction_base_.emplace_back(std::make_unique( - op_idx++, place_, &op, value_exe_info_.get(), skip_gc_vars)); + op_idx++, place_, &op, value_exe_info_.get(), execution_config_)); sub_blocks_.insert( {&op.dyn_cast().body(), dynamic_cast(vec_instruction_base_.back().get()) diff --git a/paddle/fluid/framework/new_executor/program_interpreter.cc b/paddle/fluid/framework/new_executor/program_interpreter.cc index 442112033608d..9434e4fd81af6 100644 --- a/paddle/fluid/framework/new_executor/program_interpreter.cc +++ b/paddle/fluid/framework/new_executor/program_interpreter.cc @@ -921,6 +921,10 @@ void ProgramInterpreter::RunOperator(const Instruction& instr_node) { auto runtime_attrs = op->RuntimeAttrs(); runtime_attrs.insert(std::make_pair("used_for_inference", true)); op->SetRuntimeAttributeMap(runtime_attrs); + } else if (op->Type() == "conditional_block") { + auto runtime_attrs = op->RuntimeAttrs(); + runtime_attrs.insert(std::make_pair("used_for_inference", true)); + op->SetRuntimeAttributeMap(runtime_attrs); } } diff --git a/paddle/fluid/operators/controlflow/conditional_block_op.cc b/paddle/fluid/operators/controlflow/conditional_block_op.cc index 2ce1859346140..58e0114045db4 100644 --- a/paddle/fluid/operators/controlflow/conditional_block_op.cc +++ b/paddle/fluid/operators/controlflow/conditional_block_op.cc @@ -103,6 +103,9 @@ class ConditionalBlockOp : public ConditionalOp { dev_place); framework::interpreter::ExecutionConfig execution_config; + if (HasAttr("used_for_inference") && Attr("used_for_inference")) { + execution_config.used_for_inference = true; + } execution_config.create_local_scope = false; execution_config.used_for_control_flow_op = true; execution_config.skip_gc_vars = From 7d9f2a2c9efce4dfad797562d9e84c56aa795be2 Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Thu, 28 Dec 2023 10:55:20 +0800 Subject: [PATCH 104/146] [PIR] support while api for python api. (#60364) --- paddle/fluid/pybind/pir.cc | 6 +- paddle/pir/core/block.cc | 5 +- paddle/pir/core/block.h | 4 +- paddle/pir/core/region.cc | 12 ++-- python/paddle/static/nn/control_flow.py | 18 ++++-- python/paddle/tensor/math.py | 11 ++-- test/legacy_test/test_while_op.py | 76 +++++++++++++++++++------ 7 files changed, 91 insertions(+), 41 deletions(-) diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 7e1d46b3364c8..2103e7b7b660e 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -317,11 +317,15 @@ void BindBlock(py::module *m) { The constructor of Block should not be invoked directly. You can use `Program.block()` to get a block. )DOC"); - block + block.def("empty", &Block::empty) .def( "front", [](Block &self) { return &self.front(); }, return_value_policy::reference) + .def( + "back", + [](Block &self) { return &self.back(); }, + return_value_policy::reference) .def_property_readonly( "parent_op", [](Block &self) { return self.GetParentOp(); }, diff --git a/paddle/pir/core/block.cc b/paddle/pir/core/block.cc index 49389454545d1..e52e09258ab39 100644 --- a/paddle/pir/core/block.cc +++ b/paddle/pir/core/block.cc @@ -73,10 +73,7 @@ Operation *Block::Take(Operation *op) { return op; } -void Block::SetParent(Region *parent, Region::Iterator position) { - parent_ = parent; - position_ = position; -} +void Block::SetParent(Region *parent) { parent_ = parent; } Block::UseIterator Block::use_begin() const { return first_use_; } diff --git a/paddle/pir/core/block.h b/paddle/pir/core/block.h index 373f97e12c51e..3d7774f0be375 100644 --- a/paddle/pir/core/block.h +++ b/paddle/pir/core/block.h @@ -73,7 +73,6 @@ class IR_API Block { Iterator insert(ConstIterator iterator, Operation *op); Iterator erase(ConstIterator position); void clear(); - operator Region::Iterator() { return position_; } // Assign the operation underlying in position with parameter op, // meanwhile, destroy the original operation. @@ -145,7 +144,7 @@ class IR_API Block { // Allow access to 'SetParent'. friend class Region; - void SetParent(Region *parent, Region::Iterator position); + void SetParent(Region *parent); // Take out corresponding Operation and its ownershipe. friend class Operation; @@ -154,7 +153,6 @@ class IR_API Block { static bool TopoOrderCheck(const OpListType &op_list); private: - Region::Iterator position_; BlockOperand first_use_; OpListType ops_; // owned ArgListType arguments_; // owned diff --git a/paddle/pir/core/region.cc b/paddle/pir/core/region.cc index 21a09198f1d79..552df86861167 100644 --- a/paddle/pir/core/region.cc +++ b/paddle/pir/core/region.cc @@ -32,7 +32,7 @@ void Region::push_front(Block *block) { insert(blocks_.begin(), block); } Region::Iterator Region::insert(ConstIterator position, Block *block) { Region::Iterator iter = blocks_.insert(position, block); - block->SetParent(this, iter); + block->SetParent(this); return iter; } @@ -54,7 +54,7 @@ void Region::TakeBody(Region &&other) { clear(); blocks_.swap(other.blocks_); for (auto iter = blocks_.begin(); iter != blocks_.end(); ++iter) { - (*iter)->SetParent(this, iter); + (*iter)->SetParent(this); } } @@ -72,11 +72,11 @@ void Region::clear() { void Region::swap(Region &&other) { blocks_.swap(other.blocks_); - for (auto iter = begin(); iter != end(); ++iter) { - iter->SetParent(this, iter); + for (auto &block : *this) { + block.SetParent(this); } - for (auto iter = other.begin(); iter != other.end(); ++iter) { - iter->SetParent(&other, iter); + for (auto &block : other) { + block.SetParent(&other); } } diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py index 3d2f9858a1feb..a6a2027ac9a3d 100644 --- a/python/paddle/static/nn/control_flow.py +++ b/python/paddle/static/nn/control_flow.py @@ -153,14 +153,21 @@ class WhileGuard(BlockGuard): def __init__(self, while_op): if not isinstance(while_op, While): raise TypeError("WhileGuard takes a while op") - super().__init__(while_op.helper.main_program) + if not in_pir_mode(): + super().__init__(while_op.helper.main_program) self.while_op = while_op def __enter__(self): + if in_pir_mode(): + self.block = build_while_op(self.while_op.cond_var, []).body() + return self.block.__enter__() self.while_op.status = While.IN_WHILE_BLOCK return super().__enter__() def __exit__(self, exc_type, exc_val, exc_tb): + if in_pir_mode(): + cf_yield([self.while_op.cond_var]) + return self.block.__exit__(exc_type, exc_val, exc_tb) if exc_type is not None: return False self.while_op.status = While.AFTER_WHILE_BLOCK @@ -509,8 +516,7 @@ class While: AFTER_WHILE_BLOCK = 2 def __init__(self, cond, is_test=False, name=None): - self.helper = LayerHelper("while", name=name) - self.status = While.BEFORE_WHILE_BLOCK + self.cond_var = cond check_variable_and_dtype(cond, 'cond', ['bool'], 'static.nn.While') if reduce(lambda a, b: a * b, cond.shape, 1) != 1: raise TypeError( @@ -518,7 +524,10 @@ def __init__(self, cond, is_test=False, name=None): list(cond.shape) ) ) - self.cond_var = cond + if in_pir_mode(): + return + self.status = While.BEFORE_WHILE_BLOCK + self.helper = LayerHelper("while", name=name) self.is_test = is_test def block(self): @@ -1870,5 +1879,4 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.inside_scope = False if exc_type is not None: return False # re-raise exception - return True diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index 49fd425726cb5..acaa0905ce6f4 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -4719,12 +4719,15 @@ def increment(x, value=1.0, name=None): [1.]) """ - if in_dynamic_or_pir_mode(): + if in_dynamic_mode(): + return _C_ops.increment_(x, value) + + check_variable_and_dtype( + x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' + ) + if in_pir_mode(): return _C_ops.increment_(x, value) else: - check_variable_and_dtype( - x, 'x', ['float32', 'float64', 'int32', 'int64'], 'increment' - ) helper = LayerHelper("increment", **locals()) helper.append_op( type='increment', diff --git a/test/legacy_test/test_while_op.py b/test/legacy_test/test_while_op.py index 5ff7698b6b2bc..63affc80d7cf4 100644 --- a/test/legacy_test/test_while_op.py +++ b/test/legacy_test/test_while_op.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest import numpy from utils import compare_legacy_with_pt import paddle -from paddle import base +from paddle import base, set_flags from paddle.base import core from paddle.base.backward import append_backward from paddle.base.executor import Executor +from paddle.base.framework import in_pir_mode from paddle.incubate.layers.nn import shuffle_batch +from paddle.pir_utils import test_with_pir_api paddle.enable_static() @@ -70,7 +73,7 @@ def simple_net(self): prev2 = paddle.tensor.array_read(array=mem_array, i=j) result2 = paddle.add_n([d2, prev2]) - j = paddle.increment(x=j) + paddle.increment(x=j) paddle.tensor.array_write(result2, i=j, array=mem_array) paddle.assign(paddle.less_than(x=j, y=array_len2), cond2) @@ -79,7 +82,8 @@ def simple_net(self): loss = paddle.mean(sum_result) return loss, sum_result - # TODO(zhangbo): Support pir test(support write_to_array and read_from_array, support while_grad). + # TODO(winter-wang): Support pir test in (FLAGS_enable_pir_in_executor_trace_run = False && FLAGS_new_executor_serial_run == False). + @test_with_pir_api def test_simple_net(self): main_program = base.Program() startup_program = base.Program() @@ -88,6 +92,14 @@ def test_simple_net(self): append_backward(loss) + if in_pir_mode(): + flag_1 = "FLAGS_enable_pir_in_executor_trace_run" + flag_2 = "FLAGS_new_executor_serial_run" + os.environ[flag_1] = 'True' + os.environ[flag_2] = 'True' + set_flags({flag_1: True}) + set_flags({flag_2: True}) + cpu = core.CPUPlace() exe = Executor(cpu) d = [] @@ -99,15 +111,24 @@ def test_simple_net(self): feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}, fetch_list=[sum_result], ) + if in_pir_mode(): + del os.environ[flag_1] + del os.environ[flag_2] + set_flags({flag_1: False}) + set_flags({flag_2: False}) self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) - # TODO(zhangbo): Support pir test(support write_to_array and read_from_array) + # TODO(winter-wang): Support pir test in (FLAGS_enable_pir_in_executor_trace_run = False && FLAGS_new_executor_serial_run == False). + @test_with_pir_api def test_simple_net_forward(self): main_program = base.Program() startup_program = base.Program() with base.program_guard(main_program, startup_program): self.simple_net() - binary = base.compiler.CompiledProgram(main_program) + if in_pir_mode(): + binary = main_program + else: + binary = base.compiler.CompiledProgram(main_program) cpu = core.CPUPlace() exe = Executor(cpu) d = [] @@ -115,10 +136,23 @@ def test_simple_net_forward(self): for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) + if in_pir_mode(): + flag_1 = "FLAGS_enable_pir_in_executor_trace_run" + flag_2 = "FLAGS_new_executor_serial_run" + os.environ[flag_1] = 'True' + os.environ[flag_2] = 'True' + set_flags({flag_1: True}) + set_flags({flag_2: True}) for _ in range(2): exe.run(binary, feed={'d0': d[0], 'd1': d[1], 'd2': d[2]}) + if in_pir_mode(): + del os.environ[flag_1] + del os.environ[flag_2] + set_flags({flag_1: False}) + set_flags({flag_2: False}) @compare_legacy_with_pt + @test_with_pir_api def test_exceptions(self): i = paddle.zeros(shape=[2], dtype='int64') array_len = paddle.tensor.fill_constant( @@ -134,6 +168,7 @@ def test_exceptions(self): class BadInputTest(unittest.TestCase): @compare_legacy_with_pt + @test_with_pir_api def test_error(self): with base.program_guard(base.Program()): @@ -158,8 +193,9 @@ def body_func(i, ten, batch_info, origin_seq): x = paddle.static.data(name='x', shape=[-1, 1, 4], dtype='float32') y = paddle.static.data(name='y', shape=[-1, 1, 1], dtype='float32') - x.desc.set_need_check_feed(False) - y.desc.set_need_check_feed(False) + if not in_pir_mode(): + x.desc.set_need_check_feed(False) + y.desc.set_need_check_feed(False) temp = paddle.concat([x, y], axis=-1) i = paddle.tensor.fill_constant(shape=[1], value=0, dtype='int32') @@ -190,6 +226,7 @@ def body_func(i, ten, batch_info, origin_seq): class TestOutputsMustExistsInputs(unittest.TestCase): @compare_legacy_with_pt + @test_with_pir_api def test_outputs_exists_inputs(self): """ We guarantee that the output tensor must be in the input tensor, so that the output and input can correspond to each other, but the input can be greater than the number of outputs. It's required in paddle2onnx. @@ -218,17 +255,20 @@ def body(i, s, x): paddle.enable_static() x = paddle.static.data(shape=[-1], name='x', dtype='float32') func(x) - for op in main_program.block(0).ops: - if op.type == "while": - for out_name in op.output("Out"): - if out_name in op.input("Condition"): - continue - self.assertTrue( - out_name in op.input("X"), - "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition.".format( - out_name - ), - ) + + # NOTE(winter-wang): The while_op in pir mode doesn't need following constrait, so hre only check when in non-pir mode. + if not in_pir_mode(): + for op in main_program.block(0).ops: + if op.type == "while": + for out_name in op.output("Out"): + if out_name in op.input("Condition"): + continue + self.assertTrue( + out_name in op.input("X"), + "In while op, the variable in output(`Out`) must exists in inputs(`X`), but the variable with name `{}` not meet the precondition.".format( + out_name + ), + ) if __name__ == '__main__': From b989f8a16edfb0260e7aaaf21519768d022e7829 Mon Sep 17 00:00:00 2001 From: Galaxy1458 <55453380+Galaxy1458@users.noreply.github.com> Date: Thu, 28 Dec 2023 11:04:20 +0800 Subject: [PATCH 105/146] [compilation opt]change_cc_test (#60392) * change * update --- test/cpp/prim/CMakeLists.txt | 5 +---- test/legacy_test/CMakeLists.txt | 10 ++++++---- test/xpu/cpp/CMakeLists.txt | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/test/cpp/prim/CMakeLists.txt b/test/cpp/prim/CMakeLists.txt index 5be98e0a3b33d..cb9e2cdeae888 100644 --- a/test/cpp/prim/CMakeLists.txt +++ b/test/cpp/prim/CMakeLists.txt @@ -31,10 +31,7 @@ endif() # skip win32 since wget is not installed by default on windows machine. if(NOT WIN32) - cc_test( - test_vjp_pir - SRCS test_vjp.cc - DEPS op_dialect_vjp pir) + paddle_test(test_vjp_pir SRCS test_vjp.cc) endif() if(WITH_ONNXRUNTIME AND WIN32) # Copy onnxruntime for some c++ test in Windows, since the test will diff --git a/test/legacy_test/CMakeLists.txt b/test/legacy_test/CMakeLists.txt index 45bd253a5aa59..824d50d8a6aaf 100644 --- a/test/legacy_test/CMakeLists.txt +++ b/test/legacy_test/CMakeLists.txt @@ -857,10 +857,12 @@ if(WITH_HETERPS) endif() if(WIN32) - cc_test( - cc_imp_py_test - SRCS cc_imp_py_test.cc - DEPS python) + paddle_test(cc_imp_py_test SRCS cc_imp_py_test.cc) + if(WITH_ONNXRUNTIME) + # Copy onnxruntime for some c++ test in Windows, since the test will + # be build only in CI, so suppose the generator in Windows is Ninja. + copy_onnx(cc_imp_py_test) + endif() endif() set_tests_properties( diff --git a/test/xpu/cpp/CMakeLists.txt b/test/xpu/cpp/CMakeLists.txt index 7fd9278bfa7b4..8d1576446e9f3 100644 --- a/test/xpu/cpp/CMakeLists.txt +++ b/test/xpu/cpp/CMakeLists.txt @@ -1 +1 @@ -cc_test(enforce_xpu_test SRCS enforce_xpu_test.cc) +paddle_test(enforce_xpu_test SRCS enforce_xpu_test.cc) From 8d4fb21e532b096139057d8655b2b723e7b9568a Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Thu, 28 Dec 2023 11:15:22 +0800 Subject: [PATCH 106/146] Use DimExpr and change InferSymbolicShapeInterface (#60371) * Use DimExpr and change InferSymbolicShapeInterface * static infer lib --- paddle/cinn/hlir/dialect/operator/ir/ops.yaml | 1 + paddle/fluid/inference/CMakeLists.txt | 5 +- .../op_generator/infer_symbolic_shape_gen.py | 6 +- .../fluid/pir/dialect/op_generator/op_gen.py | 4 +- .../interface/infer_symbolic_shape.cc | 273 ++++++++------ .../operator/interface/infer_symbolic_shape.h | 85 +++-- .../pir/dialect/operator/ir/op_dialect.cc | 31 ++ paddle/fluid/pir/dialect/operator/ir/ops.yaml | 3 + .../pir/transforms/shape_optimization_pass.cc | 333 ++++-------------- paddle/phi/api/yaml/ops.yaml | 3 + paddle/pir/dialect/shape/utils/shape_utils.h | 11 +- 11 files changed, 335 insertions(+), 420 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml index 22006e1ae4570..2e42323782839 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml +++ b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml @@ -74,6 +74,7 @@ func : SliceRawInferMeta kernel : func : slice + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : uniform_random args : (int64_t[] shape, float min, float max, int seed, DataType dtype, int diag_num = 0, int diag_step=0, float diag_val=1.0) diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index 1f353e2ba8409..295e72c43ce8f 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -64,8 +64,9 @@ set(KERNEL_LIST # shared inference library deps list(REMOVE_DUPLICATES fluid_modules) -#windows GPU static library over the limit, so not create_static_lib, and cc_library is dummy -if(WIN32 AND WITH_GPU) +# windows static library（both CPU and GPU）over the limit, so no longer create_static_lib, +# and cc_library is dummy +if(WIN32) cc_library(paddle_inference DEPS ${fluid_modules} ${STATIC_INFERENCE_API} ${utils_modules}) else() diff --git a/paddle/fluid/pir/dialect/op_generator/infer_symbolic_shape_gen.py b/paddle/fluid/pir/dialect/op_generator/infer_symbolic_shape_gen.py index d85ed967418d5..ff2094a3df009 100644 --- a/paddle/fluid/pir/dialect/op_generator/infer_symbolic_shape_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/infer_symbolic_shape_gen.py @@ -13,11 +13,9 @@ # limitations under the License. OP_GET_KERNEL_TYPE_FOR_VAR_TEMPLATE = """ -bool {op_name}::InferSymbolicShape(pir::Builder &builder, - const std::vector &operands, - std::vector &reified_return_shapes) {{ +bool {op_name}::InferSymbolicShape(pir::ShapeConstraintIRAnalysis* shape_analysis) {{ VLOG(4) << "Infer symbolic shape for op: {op_name}"; - return {op_name}InferSymbolicShape(builder, operands, reified_return_shapes); + return {op_name}InferSymbolicShape(this->operation(), shape_analysis); }} """ diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 4cb54ada152b8..d29982d22e5f7 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -133,9 +133,7 @@ class {TEST_API} {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ """ infer_symbolic_shape_template = """ - static bool InferSymbolicShape(pir::Builder &builder, - const std::vector &operands, - std::vector &reified_return_shapes); + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis* shape_analysis); """ # ===================================== diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.cc b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.cc index 676e4b9d574b9..1b9ca43b7d9f1 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.cc +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.cc @@ -13,16 +13,15 @@ // limitations under the License. #include "paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.h" +#include "paddle/pir/core/builtin_attribute.h" +#include "paddle/pir/core/builtin_type.h" #include "paddle/pir/dialect/shape/ir/shape_op.h" namespace paddle::dialect { bool InferSymbolicShapeInterface::InferSymbolicShape( - pir::Builder &builder, - const std::vector &operands, - std::vector &reified_return_shapes) { - return impl_->infer_symbolic_shapes( - operation(), builder, operands, reified_return_shapes); + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return impl_->infer_symbolic_shapes(operation(), shape_analysis); } } // namespace paddle::dialect @@ -30,124 +29,176 @@ namespace paddle::dialect { namespace { -bool DeriveShapeFromOperand(pir::Builder *builder, - pir::Value operand, - std::vector *reified_return_shapes) { - auto shaped_type = operand.type().dyn_cast(); - if (!shaped_type) return false; - reified_return_shapes->assign( - {builder->Build(operand).result(0)}); +bool InferSymbolicShapeAllEqualUnary( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + std::string operand_source_id = pir::GetValueId(&operand_source); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + shape_analysis->value_id_to_shapeordata_[res_id] = + shape_analysis->value_id_to_shapeordata_[operand_source_id]; return true; } -// Returns a new scalar integer value having type `type`. -// Here `type` must be an integer or index type. -pir::Value MaybeCastTo(pir::Builder &builder, // NOLINT - pir::Value value, - pir::Type type) { - if (type == value.type()) return value; - // if (!type.IsIndex() && !value.type().IsIndex()) { - // Value casted = - // builder.Build(builder.index_type(), value) - // .result(0); - // return builder.Build(type, casted).result(0); - // } - // return builder.Build(type, value).result(0); +bool InferSymbolicShapeAllEqualBinary( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + std::string operand_source_id = pir::GetValueId(&operand_source); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + shape_analysis->value_id_to_shapeordata_[res_id] = + shape_analysis->value_id_to_shapeordata_[operand_source_id]; + return true; } + } // namespace -bool AbsOpInferSymbolicShape( - pir::Builder &builder, // NOLINT - const std::vector &operands, - std::vector &reified_return_shapes) { // NOLINT - return DeriveShapeFromOperand( - &builder, operands.front().source(), &reified_return_shapes); -} - -bool Abs_OpInferSymbolicShape( - pir::Builder &builder, // NOLINT - const std::vector &operands, - std::vector &reified_return_shapes) { // NOLINT - return DeriveShapeFromOperand( - &builder, operands.front().source(), &reified_return_shapes); -} - -bool TransposeOpInferSymbolicShape( - pir::Builder &builder, // NOLINT - const std::vector &operands, - std::vector &reified_return_shapes) { // NOLINT - // auto operand_type = operands[0].type().dyn_cast(); - // // Currently not support unranked type. - // if (!operand_type) return false; - // std::vector permutation = this->permutation(); - // std::vector shape_values(permutation.size()); - // Type shape_scalar_type = builder.index_type(); - // auto to_shape_scalar_type = [&](Value v) { - // return MaybeCastTo(builder, v, shape_scalar_type); - // }; - // auto shaped_type = operand_type.dyn_cast(); - // auto shape_vector = shaped_type.GetDyShape(); - // for (auto [idx, element] = std::tuple{0, shape_vector.begin()}; - // element != shape_vector.end(); - // ++idx, ++element) { - // auto it = std::find(permutation.begin(), permutation.end(), idx); - // // TODO(zhangbopd): Need BuildOrFold - // Value value_dim = to_shape_scalar_type( - // builder.Build(operands[0].source(), - // idx).result(0)); - // shape_values[std::distance(permutation.begin(), it)] = value_dim; - // } - // Value output_shape = - // builder.Build(shape_values).result(0); - // reified_return_shapes.push_back(output_shape); +bool AbsOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} + +bool Abs_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} + +bool CastOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} + +bool Cast_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} + +bool ExpOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} +bool Exp_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualUnary(op, shape_analysis); +} + +bool SubtractOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualBinary(op, shape_analysis); +} + +bool Subtract_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return InferSymbolicShapeAllEqualBinary(op, shape_analysis); +} + +bool ShapeOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + std::string operand_source_id = pir::GetValueId(&operand_source); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + + std::vector dims = + common::vectorize(res.type().dyn_cast().dims()); + + std::vector shapes; + for (int64_t dim : dims) { + symbol::DimExpr dim_expr; + if (dim == -1) { + symbol::DimExpr res_dim_expr(shape_analysis->GetNextSymName()); + dim_expr = res_dim_expr; + } else { + symbol::DimExpr res_dim_expr(dim); + dim_expr = res_dim_expr; + } + shapes.push_back(dim_expr); + } + + symbol::ShapeOrDataDimExprs shape_data{shapes}; + shape_analysis->value_id_to_shapeordata_[res_id] = shape_data; return true; } -bool ConcatOpInferSymbolicShape( - pir::Builder &builder, // NOLINT - const std::vector &operands, - std::vector &reified_return_shapes) { // NOLINT - // std::vector inputs = {x()}; - // auto operand_type = inputs[0].type().dyn_cast(); - // // Currently not support unranked type. - // if (!operand_type) return false; - // Type shapeScalarType = builder.index_type(); - // auto to_shape_scalar_type = [&](Value v) { - // return MaybeCastTo(builder, v, shapeScalarType); - // }; - // std::vector> all_shape_values; - // for (size_t inputId = 0; inputId < inputs.size(); ++inputId) { - // Value operand = inputs[inputId]; - // auto operand_type = operand.type().dyn_cast(); - // if (!operand_type) return false; - // std::vector shape_values; - // auto shaped_type = operand_type.dyn_cast(); - // auto shape_vector = shaped_type.GetDyShape(); - // for (auto [idx, element] = std::tuple{0, shape_vector.begin()}; - // element != shape_vector.end(); - // ++idx, ++element) { - // Value value_dim = to_shape_scalar_type( - // builder.Build(operand, idx).result(0)); - // shape_values.push_back(value_dim); - // } - // all_shape_values.emplace_back(std::move(shape_values)); - // } - // [[maybe_unused]] int axis = this->dimension(); - // auto &shape_values = all_shape_values[0]; - // for (size_t vecId = 1; vecId < all_shape_values.size(); ++vecId) { - // auto &otherShapeValues = all_shape_values[vecId]; - // if (otherShapeValues.size() != shape_values.size()) return false; - // TODO(zhangbopd): AddIOp - // shape_values[axis] = - // builder.Build(shape_values[axis], - // otherShapeValues[axis]); - // } - // Value output_shape = - // builder.Build(shape_values).result(0); - // reified_return_shapes.push_back(output_shape); +bool ShapeSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return ShapeOpInferSymbolicShape(op, shape_analysis); +} + +bool StackOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + std::string operand_source_id = pir::GetValueId(&operand_source); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + + symbol::ShapeOrDataDimExprs shape_data; + shape_data = shape_analysis->value_id_to_shapeordata_[operand_source_id]; + shape_analysis->value_id_to_shapeordata_[res_id] = shape_data; + return true; +} + +bool ReshapeOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source_1 = op->operand_source(1); + std::string operand_source_1_id = pir::GetValueId(&operand_source_1); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + + symbol::ShapeOrDataDimExprs shape_data; + + shape_data = shape_analysis->value_id_to_shapeordata_[operand_source_1_id]; + shape_analysis->value_id_to_shapeordata_[res_id] = shape_data; return true; } +bool Reshape_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return ReshapeOpInferSymbolicShape(op, shape_analysis); +} + } // namespace paddle::dialect +namespace cinn::dialect { + +bool SliceOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis) { + pir::Value operand_source = op->operand_source(0); + std::string operand_source_id = pir::GetValueId(&operand_source); + pir::OpResult res = op->result(0); + std::string res_id = pir::GetValueId(&res); + + std::vector dims = + common::vectorize(res.type().dyn_cast().dims()); + + std::vector shapes; + for (int64_t dim : dims) { + symbol::DimExpr dim_expr; + if (dim == -1) { + symbol::DimExpr res_dim_expr(shape_analysis->GetNextSymName()); + dim_expr = res_dim_expr; + } else { + symbol::DimExpr res_dim_expr(dim); + dim_expr = res_dim_expr; + } + shapes.push_back(dim_expr); + } + + // pir::AttributeMap attributes = op->attributes(); + + // auto attr_starts = + // attributes["starts"].dyn_cast().AsVector(); + // auto start = attr_starts[0].dyn_cast().data(); + + // auto attr_ends = + // attributes["ends"].dyn_cast().AsVector(); + // auto end = attr_ends[0].dyn_cast().data(); + + symbol::ShapeOrDataDimExprs shape_data{shapes}; + shape_analysis->value_id_to_shapeordata_[res_id] = shape_data; + return true; +} + +} // namespace cinn::dialect + IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::InferSymbolicShapeInterface) diff --git a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.h b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.h index 46ccf56183b2a..b1c72e3111df2 100644 --- a/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.h +++ b/paddle/fluid/pir/dialect/operator/interface/infer_symbolic_shape.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/pir/core/op_base.h" +#include "paddle/pir/dialect/shape/utils/shape_utils.h" // Type inference is currently modelled executionally for operation creation // using the `InferMetaInterface`. While `InferSymbolicShapeInterface` is used @@ -31,54 +32,82 @@ class InferSymbolicShapeInterface /// Defined these methods with the interface. struct Concept { explicit Concept(bool (*infer_symbolic_shapes)( - pir::Operation* op, - pir::Builder& builder, // NOLINT - const std::vector& operands, - std::vector& reified_return_shapes)) // NOLINT + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis)) : infer_symbolic_shapes(infer_symbolic_shapes) {} bool (*infer_symbolic_shapes)( - pir::Operation* op, - pir::Builder& builder, - const std::vector& operands, - std::vector& reified_return_shapes); // NOLINT + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); }; template struct Model : public Concept { static inline bool InferSymbolicShape( - pir::Operation* op, - pir::Builder& builder, // NOLINT - const std::vector& operands, - std::vector& reified_return_shapes) { // NOLINT - return op->dyn_cast().InferSymbolicShape( - builder, operands, reified_return_shapes); + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + return op->dyn_cast().InferSymbolicShape(shape_analysis); } Model() : Concept(InferSymbolicShape) {} }; /// Constructor - InferSymbolicShapeInterface(pir::Operation* op, Concept* impl) + InferSymbolicShapeInterface(pir::Operation *op, Concept *impl) : pir::OpInterfaceBase(op), impl_(impl) {} - bool InferSymbolicShape( - pir::Builder& builder, // NOLINT - const std::vector& operands, - std::vector& reified_return_shapes); // NOLINT + bool InferSymbolicShape(pir::ShapeConstraintIRAnalysis *shape_analysis); private: - Concept* impl_; + Concept *impl_; }; -bool AbsOpInferSymbolicShape( - pir::Builder& builder, // NOLINT - const std::vector& operands, - std::vector& reified_return_shapes); // NOLINT -bool Abs_OpInferSymbolicShape( - pir::Builder& builder, // NOLINT - const std::vector& operands, - std::vector& reified_return_shapes); // NOLINT +} // namespace paddle::dialect + +namespace paddle::dialect { + +bool AbsOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Abs_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool CastOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Cast_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool ExpOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Exp_OpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool SubtractOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Subtract_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool ShapeOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool ShapeSrOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool StackOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool ReshapeOpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); + +bool Reshape_OpInferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis); } // namespace paddle::dialect +namespace cinn::dialect { + +bool SliceOpInferSymbolicShape(pir::Operation *op, + pir::ShapeConstraintIRAnalysis *shape_analysis); + +} + IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::InferSymbolicShapeInterface) diff --git a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc index 7b5959a542e7a..6e2e105d9c18a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_dialect.cc @@ -29,6 +29,32 @@ namespace paddle { namespace dialect { +struct CombineOpInferSymbolicShapeInterfaceModel + : public InferSymbolicShapeInterface::Concept { + static inline bool InferSymbolicShape( + pir::Operation *op, pir::ShapeConstraintIRAnalysis *shape_analysis) { + symbol::ShapeOrDataDimExprs value_shape; + + // for (auto operand_source : op->operands_source()) { + // std::string operand_source_id = pir::GetValueId(&operand_source); + // auto source_shape_vec = + // shape_analysis->value_id_to_shapeordata_[operand_source_id]; + // for (int i = 0; i < source_shape_vec.size(); i++) { + // value_shape.second.emplace_back(source_shape_vec[i]); + // } + // } + + auto res = op->result(0); + auto res_id = pir::GetValueId(&res); + + shape_analysis->value_id_to_shapeordata_[res_id] = value_shape; + return true; + } + + CombineOpInferSymbolicShapeInterfaceModel() + : InferSymbolicShapeInterface::Concept(InferSymbolicShape) {} +}; + OperatorDialect::OperatorDialect(pir::IrContext *ctx) : pir::Dialect(name(), ctx, pir::TypeId::get()) { initialize(); @@ -36,6 +62,11 @@ OperatorDialect::OperatorDialect(pir::IrContext *ctx) auto info = ctx->GetRegisteredOpInfo(pir::TuplePushOp::name()); info.AttachInterface(std::move( pir::InterfaceValue::Get())); + + info = ctx->GetRegisteredOpInfo(pir::CombineOp::name()); + info.AttachInterface(std::move( + pir::InterfaceValue::Get())); } void OperatorDialect::initialize() { diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 0d571f8ef868a..ec68a17c9cb13 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -265,6 +265,7 @@ data_type : x inplace: (x -> out) backward : cast_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : channel_shuffle args : (Tensor x, int groups, str data_format="NCHW") @@ -1044,6 +1045,7 @@ view: (x -> out) intermediate : xshape backward: reshape_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : rnn args: (Tensor x, Tensor[] pre_state, Tensor[] weight_list, Tensor sequence_length, Tensor dropout_state_in, float dropout_prob=0.0, bool is_bidirec=false, int input_size=10, int hidden_size=100, int num_layers=1, str mode="RNN_TANH", int seed=0, bool is_test=false) @@ -1214,6 +1216,7 @@ func : subtract inplace : (x -> out) backward : subtract_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : sum args : (Tensor x, IntArray axis={}, DataType dtype=DataType::UNDEFINED, bool keepdim=false) diff --git a/paddle/fluid/pir/transforms/shape_optimization_pass.cc b/paddle/fluid/pir/transforms/shape_optimization_pass.cc index a7d32c6577906..5c6481110034e 100644 --- a/paddle/fluid/pir/transforms/shape_optimization_pass.cc +++ b/paddle/fluid/pir/transforms/shape_optimization_pass.cc @@ -111,8 +111,8 @@ class InferSymbolicShapePass : public pir::Pass { if (it != infer_sym_shape_map.end()) { it->second(op, shape_analysis_); } else { - VLOG(3) << "[" << op.name() - << "] is not supported for infer_symbolic_shape pass."; + LOG(WARNING) << "[" << op.name() + << "] is not supported for infer_symbolic_shape pass."; } } @@ -206,7 +206,7 @@ struct ExpandShapeOfOpPattern : public OpRewritePattern { bool MatchAndRewrite(shape::ShapeOfOp op, PatternRewriter& rewriter) const override { - VLOG(5) << "Apply ExpandShapeOfOpPattern..."; + VLOG(3) << "Apply ExpandShapeOfOpPattern..."; auto type = op.out().type().dyn_cast(); @@ -233,44 +233,6 @@ struct DimOfShapedTypeOpInterfacePattern : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; bool MatchAndRewrite(OpTy dim_op, PatternRewriter& rewriter) const override { - OpResult dim_value = dim_op.source().template dyn_cast(); - if (!dim_value) return false; - - auto shaped_type_op = - dim_value.owner() - ->dyn_cast(); - if (!shaped_type_op) return false; - - std::optional dim_index = dim_op.GetConstantIndex(); - if (!dim_index) return false; - - std::vector reified_result_shapes; - if (!shaped_type_op.InferSymbolicShape( - rewriter, shaped_type_op->operands(), reified_result_shapes)) - return false; - - if (reified_result_shapes.size() != shaped_type_op->num_results()) - return false; - - Value result_shape = reified_result_shapes[dim_value.index()]; - auto result_shape_type = result_shape.type().dyn_cast(); - auto shaped_type = result_shape_type.dyn_cast(); - if (!result_shape_type || !shaped_type.GetElementType().IsIntOrIndex()) - return false; - - // TODO(zhangbopd): BuildOrFold required. - std::vector indices; - indices.push_back(rewriter.Build(*dim_index).out()); - - Value new_value = - rewriter.Build(result_shape, indices).out(); - - if (!new_value.type().isa()) - new_value = - rewriter.Build(rewriter.index_type(), new_value) - .out(); - - rewriter.ReplaceOp(dim_op, {new_value}); return true; } }; @@ -349,19 +311,6 @@ bool ShapeComputationIRAnalysis::Run() { // Make sure only run once. if (initialized_) return false; initialized_ = true; - // auto build_shape_func = - // std::bind(&ShapeComputationIRAnalysis::BuildShapeOnOperation, - // this, - // std::placeholders::_1); - // if (!RunOnRegion(&(m_->region(0)), build_shape_func)) return false; - // auto apply_op_constraint_func = - // std::bind(&ShapeComputationIRAnalysis::ApplyOpConstraint, - // this, - // std::placeholders::_1); - // // TODO(zhangbopd): Delete the following 1 line and fix UT - // // `shape_optimization_test` - // return true; - // if (!RunOnRegion(&(m_->region(0)), apply_op_constraint_func)) return false; return true; } @@ -508,220 +457,81 @@ bool OptimizeShapeComputation(pir::ModuleOp m, PassPipelineRunner runner) { return true; } -void print_program(pir::ModuleOp m, std::string mgs) { +void PrintProgram(pir::ModuleOp m, std::string mgs) { std::ostringstream print_stream; print_stream << "\n\n"; m.program()->Print(print_stream); print_stream << "\n\n"; - VLOG(5) << "===================== " << mgs << "\n" << print_stream.str(); -} - -bool IsShapeSpecialOp(const pir::Operation& op) { - auto name = op.name(); - if (name == "pd_op.shape" || name == "cinn_op.slice") { - return true; - } - - return false; -} - -bool IsAllEqualUnaryOp(const pir::Operation& op) { - auto name = op.name(); - if (name == "pd_op.exp" || name == "pd_op.cast") { - return true; - } - - return false; -} - -void InferSymbolicShapeAllEqualUnary( - pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source = op->operand_source(0); - auto operand_source_id = pir::GetValueId(&operand_source); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - shape_analysis->value_to_valueshape_expr_[rst_id] = - shape_analysis->value_to_valueshape_expr_[operand_source_id]; -} - -bool IsAllEqualBinaryOp(const pir::Operation& op) { - auto name = op.name(); - if (name == "pd_op.subtract") { - return true; - } - - return false; -} - -void InferSymbolicShapeAllEqualBinary( - pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source = op->operand_source(0); - auto operand_source_id = pir::GetValueId(&operand_source); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - shape_analysis->value_to_valueshape_expr_[rst_id] = - shape_analysis->value_to_valueshape_expr_[operand_source_id]; -} - -void InferSymbolicShapePdShape(pir::Operation* op, - pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source = op->operand_source(0); - auto operand_source_id = pir::GetValueId(&operand_source); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - std::pair, std::vector> value_shape; - - auto type = rst.type(); - auto tensor_type = type.dyn_cast(); - auto ddim_vec = common::vectorize(tensor_type.dims()); - for (auto dim : ddim_vec) { - std::string sym_name = ""; - if (dim == -1) { - sym_name = shape_analysis->GetNextSymName(); - } else { - sym_name = std::to_string(dim); - } - value_shape.first.emplace_back(sym_name); - } - - value_shape.second = - shape_analysis->value_to_valueshape_expr_[operand_source_id].first; - shape_analysis->value_to_valueshape_expr_[rst_id] = value_shape; -} - -void InferSymbolicShapeCinnSlice( - pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source = op->operand_source(0); - auto operand_source_id = pir::GetValueId(&operand_source); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - std::pair, std::vector> value_shape; - - auto type = rst.type(); - auto tensor_type = type.dyn_cast(); - auto ddim_vec = common::vectorize(tensor_type.dims()); - for (auto dim : ddim_vec) { - std::string sym_name = ""; - if (dim == -1) { - sym_name = shape_analysis->GetNextSymName(); - } else { - sym_name = std::to_string(dim); - } - value_shape.first.emplace_back(sym_name); - } - - auto attributes = op->attributes(); - - auto attr_starts = attributes["starts"].dyn_cast().AsVector(); - auto start = attr_starts[0].dyn_cast().data(); - - auto attr_ends = attributes["ends"].dyn_cast().AsVector(); - auto end = attr_ends[0].dyn_cast().data(); - - auto source_shape_vec = - shape_analysis->value_to_valueshape_expr_[operand_source_id].second; - for (int i = start; i < end; i++) { - value_shape.second.emplace_back(source_shape_vec[i]); - } - - shape_analysis->value_to_valueshape_expr_[rst_id] = value_shape; + VLOG(3) << "===================== " << mgs << " =====================\n" + << print_stream.str(); } -void InferSymbolicShapeBuiltinCombine( - pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis) { - std::pair, std::vector> value_shape; - for (auto operand_source : op->operands_source()) { - auto operand_source_id = pir::GetValueId(&operand_source); - auto source_shape_vec = - shape_analysis->value_to_valueshape_expr_[operand_source_id].second; - for (int i = 0; i < source_shape_vec.size(); i++) { - value_shape.second.emplace_back(source_shape_vec[i]); - } - } - - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - - shape_analysis->value_to_valueshape_expr_[rst_id] = value_shape; -} - -void InferSymbolicShapeStack(pir::Operation* op, - pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source = op->operand_source(0); - auto operand_source_id = pir::GetValueId(&operand_source); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - std::pair, std::vector> value_shape; - - value_shape.second = - shape_analysis->value_to_valueshape_expr_[operand_source_id].second; - shape_analysis->value_to_valueshape_expr_[rst_id] = value_shape; -} - -void InferSymbolicShapeReshape(pir::Operation* op, - pir::ShapeConstraintIRAnalysis* shape_analysis) { - auto operand_source_1 = op->operand_source(1); - auto operand_source_1_id = pir::GetValueId(&operand_source_1); - auto rst = op->result(0); - auto rst_id = pir::GetValueId(&rst); - - std::pair, std::vector> value_shape; - - value_shape.first = - shape_analysis->value_to_valueshape_expr_[operand_source_1_id].second; - shape_analysis->value_to_valueshape_expr_[rst_id] = value_shape; -} - -void debug_print_op_info( +void DebugPrintOpInfo( pir::Operation* op, pir::ShapeConstraintIRAnalysis* shape_analysis = nullptr) { - VLOG(5) << op->name() << ", num_operands: " << op->num_operands(); - for (auto& rst : op->results()) { - auto type = rst.type(); - auto value_id = pir::GetValueId(&rst); + VLOG(0) << op->name() << ", num_operands: " << op->num_operands(); + for (auto& res : op->results()) { + auto value_id = pir::GetValueId(&res); std::ostringstream print_stream; - print_stream << ">>>> result(" << rst.index() << ") 's ID: " << value_id; - if (shape_analysis != nullptr) { - auto value_shape = shape_analysis->value_to_valueshape_expr_[value_id]; - print_stream << ", value_shape.first: ["; - for (auto str : value_shape.first) { - print_stream << str << ", "; + print_stream << ">>>> result(" << res.index() << ") 's ID: " << value_id; + if (shape_analysis != nullptr) { + auto shape_data = shape_analysis->value_id_to_shapeordata_[value_id]; + print_stream << ", ShapeOrData.shape: ["; + + for (auto str : shape_data.shape()) { + int64_t* i = std::get_if(&str); + std::string* s = std::get_if(&str); + if (i) { + print_stream << *i << ", "; + } else if (s) { + print_stream << *s << ", "; + } } - print_stream << "], second: ["; - for (auto str : value_shape.second) { - print_stream << str << ", "; + + print_stream << "], ShapeOrData.data: ["; + if (shape_data.data().has_value()) { + for (auto str : shape_data.data().value()) { + int64_t* i = std::get_if(&str); + std::string* s = std::get_if(&str); + if (i) { + print_stream << *i << ", "; + } else if (s) { + print_stream << *s << ", "; + } + } } print_stream << "]\n"; } - VLOG(5) << print_stream.str(); + VLOG(0) << print_stream.str(); } } -void InferSymExprForAllValues(pir::ModuleOp module_op) { - auto shape_analysis_mgr = pir::ShapeAnalysisManager::Instance(); - pir::ShapeConstraintIRAnalysis& shape_analysis = +void InferSymExprForAllValues(ModuleOp module_op) { + auto shape_analysis_mgr = ShapeAnalysisManager::Instance(); + ShapeConstraintIRAnalysis& shape_analysis = shape_analysis_mgr.Get(module_op.program()); for (int i = 0; i < module_op->num_regions(); i++) { for (auto& block : module_op->region(i)) { for (auto& op : block) { if (op.num_operands() == 0) { - // Need new syms for -1s - for (auto& rst : op.results()) { - auto value_id = pir::GetValueId(&rst); - std::pair, std::vector> - value_shape; - auto type = rst.type(); - auto tensor_type = type.dyn_cast(); - auto ddim_vec = common::vectorize(tensor_type.dims()); - for (auto dim : ddim_vec) { - std::string sym_name = ""; + for (auto& res : op.results()) { + auto value_id = pir::GetValueId(&res); + + std::vector dims = common::vectorize( + res.type().dyn_cast().dims()); + + std::vector shapes; + for (int64_t dim : dims) { + symbol::DimExpr dim_expr; if (dim == -1) { - sym_name = shape_analysis.GetNextSymName(); + symbol::DimExpr res_dim_expr(shape_analysis.GetNextSymName()); + dim_expr = res_dim_expr; } else { - sym_name = std::to_string(dim); + symbol::DimExpr res_dim_expr(dim); + dim_expr = res_dim_expr; } - value_shape.first.emplace_back(sym_name); + shapes.push_back(dim_expr); } if (op.name() == "pd_op.full_int_array") { @@ -730,28 +540,23 @@ void InferSymExprForAllValues(pir::ModuleOp module_op) { auto arr = attr.dyn_cast(); const auto& vec = arr.AsVector(); for (auto item : vec) { - auto i = item.dyn_cast(); - value_shape.second.emplace_back(std::to_string(i.data())); + int64_t i = item.dyn_cast().data(); + shapes.push_back(symbol::DimExpr(i)); } } - shape_analysis.value_to_valueshape_expr_[value_id] = value_shape; + symbol::ShapeOrDataDimExprs shape_data{shapes}; + shape_analysis.value_id_to_shapeordata_[value_id] = shape_data; + } + } else { + auto infer_symbolic_shape_interface = + op.dyn_cast(); + if (infer_symbolic_shape_interface) { + PADDLE_ENFORCE(infer_symbolic_shape_interface.InferSymbolicShape( + &shape_analysis)); } - } else if (IsAllEqualUnaryOp(op)) { - InferSymbolicShapeAllEqualUnary(&op, &shape_analysis); - } else if (IsAllEqualBinaryOp(op)) { - InferSymbolicShapeAllEqualBinary(&op, &shape_analysis); - } else if (op.name() == "pd_op.shape") { - InferSymbolicShapePdShape(&op, &shape_analysis); - } else if (op.name() == "cinn_op.slice") { - InferSymbolicShapeCinnSlice(&op, &shape_analysis); - } else if (op.name() == "builtin.combine") { - InferSymbolicShapeBuiltinCombine(&op, &shape_analysis); - } else if (op.name() == "pd_op.stack") { - InferSymbolicShapeStack(&op, &shape_analysis); - } else if (op.name() == "pd_op.reshape") { - InferSymbolicShapeReshape(&op, &shape_analysis); } - debug_print_op_info(&op, &shape_analysis); + + DebugPrintOpInfo(&op, &shape_analysis); } } } @@ -762,11 +567,11 @@ class ShapeOptimizationPass : public pir::Pass { ShapeOptimizationPass() : pir::Pass("shape_optimization_pass", 0) {} void Run(pir::Operation* op) override { - VLOG(5) << "===================== ShapeOptimizationPass Run start... " + VLOG(3) << "===================== ShapeOptimizationPass Run start... " "============================="; auto module_op = op->dyn_cast(); IR_ENFORCE(module_op, "ShapeOptimizationPass should run on module op."); - print_program(module_op, "Origin Program:"); + PrintProgram(module_op, "Origin Program"); InferSymExprForAllValues(module_op); MaterializeShapeComputation(module_op); @@ -777,7 +582,7 @@ class ShapeOptimizationPass : public pir::Pass { // if (!OptimizeShapeComputation(module_op, runner)) { // return; // } - VLOG(5) << "===================== ShapeOptimizationPass Run End. " + VLOG(3) << "===================== ShapeOptimizationPass Run End. " "============================="; } diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index de7c49250ea16..de4d700cdf80e 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -841,6 +841,7 @@ func : exp inplace : (x -> out) backward : exp_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : expand args : (Tensor x, IntArray shape = {}) @@ -2355,6 +2356,7 @@ shape_sr {selected_rows -> dense} data_transform: skip_transform : input + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : shard_index args : (Tensor input, int index_num, int nshards, int shard_id, int ignore_value=-1) @@ -2538,6 +2540,7 @@ kernel : func : stack backward : stack_grad + interfaces : paddle::dialect::InferSymbolicShapeInterface - op : stanh args : (Tensor x, float scale_a=0.67f, float scale_b=1.7159f) diff --git a/paddle/pir/dialect/shape/utils/shape_utils.h b/paddle/pir/dialect/shape/utils/shape_utils.h index 717b05eb8fede..ac72c0bae88c7 100644 --- a/paddle/pir/dialect/shape/utils/shape_utils.h +++ b/paddle/pir/dialect/shape/utils/shape_utils.h @@ -76,11 +76,6 @@ class IR_API ShapeConstraintIRAnalysis : public ShapeAnalysis { Value rhs, std::vector rhs_dim_idxs) override; - std::unordered_map< - std::string, - std::pair, std::vector>> - value_to_valueshape_expr_; - inline const std::string GetNextSymName() { return "S" + std::to_string(next_sym_idx_++); } @@ -89,6 +84,9 @@ class IR_API ShapeConstraintIRAnalysis : public ShapeAnalysis { symbol::DimExprBuilder CreateDimExprBuilder() override; + std::unordered_map + value_id_to_shapeordata_; + private: // The operation this analysis runs on. ModuleOp m_; @@ -99,9 +97,6 @@ class IR_API ShapeConstraintIRAnalysis : public ShapeAnalysis { std::unordered_map> value_to_sym_dims_; - std::unordered_map - value_id_to_shapeordata; - int64_t next_sym_idx_ = 0; std::vector constraints_; From 4551f5f8de03a2886eb8d9b3d68df4d855f7bf6e Mon Sep 17 00:00:00 2001 From: xingmingyyj <135400902+xingmingyyj@users.noreply.github.com> Date: Thu, 28 Dec 2023 11:34:32 +0800 Subject: [PATCH 107/146] =?UTF-8?q?=E3=80=90PIR=20OpTest=20Fix=20No.39?= =?UTF-8?q?=E3=80=91=20fix=20test=5Fc=5Freduce=5Fmin=5Ftranslate=20(#60236?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add test_c_reduce_min_translate * fix * fix * fix * fix * fix * fix * fix --- .../pir/dialect/op_generator/ops_api_gen.py | 2 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 10 ++++ .../fluid/pir/dialect/operator/utils/utils.cc | 3 +- paddle/phi/api/yaml/op_compat.yaml | 6 +++ test/ir/pir/CMakeLists.txt | 2 + test/ir/pir/translator/CMakeLists.txt | 15 ++++++ .../translator/test_c_reduce_min_translate.py | 42 ++++++++++++++++ test/ir/pir/translator/test_op_transcriber.py | 48 +++++++++++++++++++ 8 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 test/ir/pir/translator/CMakeLists.txt create mode 100644 test/ir/pir/translator/test_c_reduce_min_translate.py create mode 100644 test/ir/pir/translator/test_op_transcriber.py diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 0a834bc7b0c2c..d541f34a890dc 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -136,6 +136,8 @@ 'sparse_momentum', 'soft_relu', 'uniform_random_batch_size_like', + 'c_reduce_min', + 'c_reduce_min_', ] diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index ec68a17c9cb13..5bdcadc3cca03 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -213,6 +213,16 @@ func : c_identity inplace : (x -> out) +- op : c_reduce_min + args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) + output : Tensor(out) + infer_meta : + func : DistReduceInferMeta + param : [x] + kernel : + func : c_reduce_min + inplace : (x -> out) + - op : c_reduce_sum args : (Tensor x, int ring_id, int root_id, bool use_calc_stream) output : Tensor(out) diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 722685fc3b510..ebc1615a16d51 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -58,7 +58,8 @@ const std::unordered_set LegacyOpList = { RowConvOp::name(), RowConvGradOp::name(), SoftReluOp::name(), - SoftReluGradOp::name()}; + SoftReluGradOp::name(), + CReduceMinOp::name()}; const std::unordered_set OneDNNLegacyOpList = {}; enum class AttrType { diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index d69e290bdbd14..e605dab154337 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -3376,6 +3376,12 @@ outputs : out: Out +- op: c_reduce_min + inputs : + x : X + outputs : + out: Out + - op: c_reduce_sum inputs : x : X diff --git a/test/ir/pir/CMakeLists.txt b/test/ir/pir/CMakeLists.txt index 61d69ee4816f3..0b8d91aed1761 100644 --- a/test/ir/pir/CMakeLists.txt +++ b/test/ir/pir/CMakeLists.txt @@ -39,3 +39,5 @@ py_test_modules( FLAGS_pir_subgraph_saving_dir=${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(fused_pass) + +add_subdirectory(translator) diff --git a/test/ir/pir/translator/CMakeLists.txt b/test/ir/pir/translator/CMakeLists.txt new file mode 100644 index 0000000000000..108615b0c204e --- /dev/null +++ b/test/ir/pir/translator/CMakeLists.txt @@ -0,0 +1,15 @@ +file( + GLOB TEST_INTERP_CASES + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "test_*.py") +string(REPLACE ".py" "" TEST_INTERP_CASES "${TEST_INTERP_CASES}") + +set(DISTRIBUTED_OP_TRANSLATION_TEST test_c_reduce_min_translate) + +if(NOT WITH_DISTRIBUTE) + list(REMOVE_ITEM TEST_INTERP_CASES ${DISTRIBUTED_OP_TRANSLATION_TEST}) +endif() + +foreach(target ${TEST_INTERP_CASES}) + py_test_modules(${target} MODULES ${target}) +endforeach() diff --git a/test/ir/pir/translator/test_c_reduce_min_translate.py b/test/ir/pir/translator/test_c_reduce_min_translate.py new file mode 100644 index 0000000000000..63c4e8271c2e1 --- /dev/null +++ b/test/ir/pir/translator/test_c_reduce_min_translate.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import test_op_transcriber + +import paddle +from paddle.base.layer_helper import LayerHelper + + +class TestCReduceMinOpTranscriber(test_op_transcriber.TestOpTranscriber): + def append_op(self): + self.op_type = "c_reduce_min" + x = paddle.ones(shape=(100, 2, 3), dtype='float32') + y = paddle.ones(shape=(100, 2, 3), dtype='float32') + attrs = {'ring_id': 0, 'root_id': 0, 'use_calc_stream': False} + helper = LayerHelper(self.op_type) + helper.append_op( + type=self.op_type, + inputs={"X": x}, + outputs={"Out": y}, + attrs=attrs, + ) + + def test_translator(self): + self.check() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ir/pir/translator/test_op_transcriber.py b/test/ir/pir/translator/test_op_transcriber.py new file mode 100644 index 0000000000000..dfb8fa63a1870 --- /dev/null +++ b/test/ir/pir/translator/test_op_transcriber.py @@ -0,0 +1,48 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import paddle +from paddle import pir +from paddle.base import core + +paddle.enable_static() + + +class TestOpTranscriber(unittest.TestCase): + def setUp(self): + self.place = core.Place() + self.place.set_place(paddle.CPUPlace()) + self.new_scope = paddle.static.Scope() + self.main_program = paddle.static.Program() + + def append_op(self): + raise Exception("Define the op to be tested here!") + + def build_model(self): + with paddle.static.scope_guard(self.new_scope): + with paddle.static.program_guard(self.main_program): + self.append_op() + + def check(self): + self.build_model() + l = pir.translate_to_pir(self.main_program.desc) + assert hasattr(self, "op_type"), "Op_type should be specified!" + assert self.op_type in str(l), ( + self.op_type + + " should be translated to pd_op." + + self.op_type + + '!' + ) From 4c975499456ca37cbeafda232a94fbfb97daf854 Mon Sep 17 00:00:00 2001 From: RuohengMa <120699764+RuohengMa@users.noreply.github.com> Date: Thu, 28 Dec 2023 11:55:13 +0800 Subject: [PATCH 108/146] [PHI] add new supported datatype for tile and sigmoid_grad (#60119) --- paddle/phi/backends/xpu/xpu2_op_list.cc | 3 ++- .../phi/kernels/xpu/activation_grad_kernel.cc | 8 +++++++- paddle/phi/kernels/xpu/tile_kernel.cc | 19 ------------------- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index 253f0a8c1b87f..31d16aaf5c0a3 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -835,7 +835,8 @@ XPUOpMap& get_kl2_ops() { phi::DataType::BFLOAT16})}, {"sigmoid", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, - {"sigmoid_grad", XPUKernelSet({phi::DataType::FLOAT32})}, + {"sigmoid_grad", + XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"sign", XPUKernelSet({phi::DataType::FLOAT32})}, {"slice_grad", XPUKernelSet({phi::DataType::FLOAT32, diff --git a/paddle/phi/kernels/xpu/activation_grad_kernel.cc b/paddle/phi/kernels/xpu/activation_grad_kernel.cc index 7cada9005c33e..48ff73d247720 100644 --- a/paddle/phi/kernels/xpu/activation_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/activation_grad_kernel.cc @@ -734,6 +734,13 @@ PD_REGISTER_KERNEL(swish_grad, phi::dtype::float16, phi::dtype::bfloat16) {} +PD_REGISTER_KERNEL(sigmoid_grad, + XPU, + ALL_LAYOUT, + phi::SigmoidGradKernel, + float, + phi::dtype::float16) {} + PD_REGISTER_ACTIVATION_GRAD_KERNEL(exp_grad, ExpGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(log_grad, LogGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(leaky_relu_grad, LeakyReluGradKernel) @@ -741,7 +748,6 @@ PD_REGISTER_ACTIVATION_GRAD_KERNEL(hardsigmoid_grad, HardSigmoidGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(hardswish_grad, HardSwishGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(reciprocal_grad, ReciprocalGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(relu6_grad, Relu6GradKernel) -PD_REGISTER_ACTIVATION_GRAD_KERNEL(sigmoid_grad, SigmoidGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(sqrt_grad, SqrtGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(mish_grad, MishGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(softplus_grad, SoftplusGradKernel) diff --git a/paddle/phi/kernels/xpu/tile_kernel.cc b/paddle/phi/kernels/xpu/tile_kernel.cc index cce230c970bf9..d90232b6767e7 100644 --- a/paddle/phi/kernels/xpu/tile_kernel.cc +++ b/paddle/phi/kernels/xpu/tile_kernel.cc @@ -29,7 +29,6 @@ void TileKernel(const Context& dev_ctx, const DenseTensor& x, const IntArray& repeat_times_arr, DenseTensor* out) { - using XPUType = typename XPUTypeTrait::Type; auto rank = x.dims().size(); std::vector repeat_times = repeat_times_arr.GetData(); int repeat_times_size = repeat_times.size(); @@ -123,24 +122,6 @@ void TileKernel(const Context& dev_ctx, vec_in_dims, vec_out_dims); - } else if (std::is_same::value) { - float* x_t = RAII_GUARD.alloc_l3_or_gm(x.numel()); - float* y_t = RAII_GUARD.alloc_l3_or_gm(out->numel()); - int r = - xpu::cast(dev_ctx.x_context(), - reinterpret_cast(x.data()), - x_t, - x.numel()); - PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast"); - ret = xpu::broadcast( - dev_ctx.x_context(), x_t, y_t, vec_in_dims, vec_out_dims); - PADDLE_ENFORCE_XDNN_SUCCESS(ret, "broadcast"); - r = xpu::cast(dev_ctx.x_context(), - y_t, - reinterpret_cast(out->data()), - out->numel()); - PADDLE_ENFORCE_XDNN_SUCCESS(r, "cast"); - } else { ret = xpu::broadcast(dev_ctx.x_context(), x.data(), From cfa74f5a316117821a0e98f4845316e2d6083496 Mon Sep 17 00:00:00 2001 From: freeliuzc Date: Thu, 28 Dec 2023 12:57:07 +0800 Subject: [PATCH 109/146] Fix build bug for V100 (#60418) --- paddle/phi/kernels/funcs/weight_only_gemv.cu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/paddle/phi/kernels/funcs/weight_only_gemv.cu b/paddle/phi/kernels/funcs/weight_only_gemv.cu index ff9285693b55f..2a14c4c9fb9d8 100644 --- a/paddle/phi/kernels/funcs/weight_only_gemv.cu +++ b/paddle/phi/kernels/funcs/weight_only_gemv.cu @@ -649,6 +649,7 @@ struct WeightOnlyConverter { } }; +#ifdef PADDLE_CUDA_BF16 template <> struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int8b> { static __device__ inline void convert(__nv_bfloat16 halves[4], @@ -689,6 +690,7 @@ struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int8b> { #endif } }; +#endif template <> struct WeightOnlyConverter { @@ -766,6 +768,7 @@ struct WeightOnlyConverter { } }; +#ifdef PADDLE_CUDA_BF16 template <> struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int4b> { static __device__ inline void convert(__nv_bfloat16 halves[8], @@ -817,6 +820,7 @@ struct WeightOnlyConverter<__nv_bfloat16, WeightOnlyQuantType::Int4b> { #endif } }; +#endif template __device__ __forceinline__ void load(T0* dst, T1* src, size_t offset = 0) { @@ -1401,7 +1405,7 @@ template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, const std::string& weight_only_type, const std::string& act_method, phi::dtype::float16* output); - +#ifdef PADDLE_CUDA_BF16 template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, const phi::dtype::bfloat16* input, const int8_t* weight, @@ -1415,4 +1419,6 @@ template void WeightOnlyGemvWrapper(const phi::GPUContext& ctx, const std::string& weight_only_type, const std::string& act_method, phi::dtype::bfloat16* output); +#endif + } // namespace phi From db27fe4e38ef24af461516ea627bf66e33f6730a Mon Sep 17 00:00:00 2001 From: zhangyikun02 <48021248+zhangyk0314@users.noreply.github.com> Date: Thu, 28 Dec 2023 13:58:25 +0800 Subject: [PATCH 110/146] elementwise_pow, square, sin and cos support bfloat16 for xpu (#60402) --- paddle/phi/backends/xpu/xpu3_op_list.cc | 18 +++-- .../kernels/legacy/xpu/elementwise_kernel.cc | 3 +- paddle/phi/kernels/xpu/activation_kernel.cc | 27 ++++++-- paddle/phi/kernels/xpu/elementwise_kernel.cc | 3 +- test/xpu/test_activation_op_xpu.py | 63 +++++++++--------- test/xpu/test_elementwise_pow_op_xpu.py | 65 ++++++++----------- 6 files changed, 96 insertions(+), 83 deletions(-) diff --git a/paddle/phi/backends/xpu/xpu3_op_list.cc b/paddle/phi/backends/xpu/xpu3_op_list.cc index 016e5ef917af5..20c649ee4ba97 100644 --- a/paddle/phi/backends/xpu/xpu3_op_list.cc +++ b/paddle/phi/backends/xpu/xpu3_op_list.cc @@ -296,7 +296,9 @@ XPUOpMap& get_kl3_ops() { phi::DataType::INT32, phi::DataType::INT64})}, {"elementwise_pow", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"elementwise_sub_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, @@ -891,7 +893,9 @@ XPUOpMap& get_kl3_ops() { {"square_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"square", - XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"squared_l2_norm", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16, @@ -1142,9 +1146,15 @@ XPUOpMap& get_kl3_ops() { phi::DataType::FLOAT32, phi::DataType::FLOAT16, phi::DataType::BFLOAT16})}, - {"sin", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"sin", + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"sin_grad", XPUKernelSet({phi::DataType::FLOAT32})}, - {"cos", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"cos", + XPUKernelSet({phi::DataType::FLOAT32, + phi::DataType::FLOAT16, + phi::DataType::BFLOAT16})}, {"cos_grad", XPUKernelSet({phi::DataType::FLOAT32})}, {"linspace", XPUKernelSet({phi::DataType::FLOAT32, diff --git a/paddle/phi/kernels/legacy/xpu/elementwise_kernel.cc b/paddle/phi/kernels/legacy/xpu/elementwise_kernel.cc index 2e4bf779d26cd..96ad9bb1f5684 100644 --- a/paddle/phi/kernels/legacy/xpu/elementwise_kernel.cc +++ b/paddle/phi/kernels/legacy/xpu/elementwise_kernel.cc @@ -153,4 +153,5 @@ PD_REGISTER_KERNEL(elementwise_pow_raw, ALL_LAYOUT, phi::ElementwisePowRawKernel, float, - phi::dtype::float16) {} + phi::dtype::float16, + phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/xpu/activation_kernel.cc b/paddle/phi/kernels/xpu/activation_kernel.cc index e76fded263f7c..449be30474193 100644 --- a/paddle/phi/kernels/xpu/activation_kernel.cc +++ b/paddle/phi/kernels/xpu/activation_kernel.cc @@ -624,8 +624,13 @@ PD_REGISTER_KERNEL(sqrt, PD_REGISTER_KERNEL( tanh, XPU, ALL_LAYOUT, phi::TanhKernel, float, phi::dtype::float16) {} -PD_REGISTER_KERNEL( - square, XPU, ALL_LAYOUT, phi::SquareKernel, float, phi::dtype::float16) {} +PD_REGISTER_KERNEL(square, + XPU, + ALL_LAYOUT, + phi::SquareKernel, + float, + phi::dtype::float16, + phi::dtype::bfloat16) {} PD_REGISTER_KERNEL( log, XPU, ALL_LAYOUT, phi::LogKernel, float, phi::dtype::float16) {} @@ -633,10 +638,20 @@ PD_REGISTER_KERNEL( PD_REGISTER_KERNEL( relu6, XPU, ALL_LAYOUT, phi::Relu6Kernel, float, phi::dtype::float16) {} -PD_REGISTER_KERNEL( - sin, XPU, ALL_LAYOUT, phi::SinKernel, float, phi::dtype::float16) {} -PD_REGISTER_KERNEL( - cos, XPU, ALL_LAYOUT, phi::CosKernel, float, phi::dtype::float16) {} +PD_REGISTER_KERNEL(sin, + XPU, + ALL_LAYOUT, + phi::SinKernel, + float, + phi::dtype::float16, + phi::dtype::bfloat16) {} +PD_REGISTER_KERNEL(cos, + XPU, + ALL_LAYOUT, + phi::CosKernel, + float, + phi::dtype::float16, + phi::dtype::bfloat16) {} #define PD_REGISTER_ACTIVATION_KERNEL(name, func) \ PD_REGISTER_KERNEL(name, XPU, ALL_LAYOUT, phi::func, float) {} diff --git a/paddle/phi/kernels/xpu/elementwise_kernel.cc b/paddle/phi/kernels/xpu/elementwise_kernel.cc index 83dce5437c9ec..a4b1385393d69 100644 --- a/paddle/phi/kernels/xpu/elementwise_kernel.cc +++ b/paddle/phi/kernels/xpu/elementwise_kernel.cc @@ -114,4 +114,5 @@ PD_REGISTER_KERNEL(elementwise_pow, ALL_LAYOUT, phi::ElementwisePowKernel, float, - phi::dtype::float16) {} + phi::dtype::float16, + phi::dtype::bfloat16) {} diff --git a/test/xpu/test_activation_op_xpu.py b/test/xpu/test_activation_op_xpu.py index 9ea61f229822e..3952217a301f2 100644 --- a/test/xpu/test_activation_op_xpu.py +++ b/test/xpu/test_activation_op_xpu.py @@ -521,6 +521,11 @@ def set_case(self): self.op_type = "square" self.dtype = self.in_type self.init_config() + if self.dtype == np.uint16: + # bfloat16 actually + self.x = convert_float_to_uint16(self.tmp_x) + else: + self.x = self.tmp_x.astype(self.dtype) out = np.square(self.x) self.attrs = {'use_xpu': True} @@ -528,27 +533,27 @@ def set_case(self): self.outputs = {'Out': out} def init_config(self): - self.x = np.random.uniform(-1, 1, [11, 17]).astype(self.dtype) + self.tmp_x = np.random.uniform(-1, 1, [11, 17]) class XPUTestSquare_ZeroDim(XPUTestSquare): def init_config(self): - self.x = np.random.uniform(-2, 2, []).astype(self.dtype) + self.tmp_x = np.random.uniform(-2, 2, []) class XPUTestSquare2(XPUTestSquare): def init_config(self): - self.x = np.random.uniform(-2, 2, [100]).astype(self.dtype) + self.tmp_x = np.random.uniform(-2, 2, [100]) class XPUTestSquare3(XPUTestSquare): def init_config(self): - self.x = np.random.uniform(-2, 2, [1, 15, 19]).astype(self.dtype) + self.tmp_x = np.random.uniform(-2, 2, [1, 15, 19]) class XPUTestSquare4(XPUTestSquare): def init_config(self): - self.x = np.random.uniform(-2, 2, [100, 10]).astype(self.dtype) + self.tmp_x = np.random.uniform(-2, 2, [100, 10]) class XPUTestSquare5(XPUTestSquare): def init_config(self): - self.x = np.random.uniform(-2, 2, [1, 2, 5, 17]).astype(self.dtype) + self.tmp_x = np.random.uniform(-2, 2, [1, 2, 5, 17]) support_types = get_xpu_op_support_types('square') @@ -1297,6 +1302,11 @@ def set_case(self): self.dtype = self.in_type self.init_config() + if self.dtype == np.uint16: + # bfloat16 actually + self.x = convert_float_to_uint16(self.tmp_x) + else: + self.x = self.tmp_x.astype(self.dtype) out = np.sin(self.x) self.inputs = {'X': self.x} @@ -1304,31 +1314,23 @@ def set_case(self): self.attrs = {'use_xpu': True} def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [11, 17]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [11, 17]) class XPUTestSin_ZeroDim(XPUTestSinBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, []).astype(self.dtype) + self.tmp_x = np.random.uniform(-np.pi, np.pi, []) class XPUTestSin2(XPUTestSinBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [1024, 8]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [1024, 8]) class XPUTestSin3(XPUTestSinBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [4, 512, 15, 15]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [4, 512, 15, 15]) class XPUTestSin4(XPUTestSinBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [4, 256, 22, 22]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [4, 256, 22, 22]) support_types = get_xpu_op_support_types('sin') @@ -1347,6 +1349,11 @@ def set_case(self): self.dtype = self.in_type self.init_config() + if self.dtype == np.uint16: + # bfloat16 actually + self.x = convert_float_to_uint16(self.tmp_x) + else: + self.x = self.tmp_x.astype(self.dtype) out = np.cos(self.x) self.inputs = {'X': self.x} @@ -1354,31 +1361,23 @@ def set_case(self): self.attrs = {'use_xpu': True} def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [11, 17]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [11, 17]) class XPUTestCos_ZeroDim(XPUTestCosBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, []).astype(self.dtype) + self.tmp_x = np.random.uniform(-np.pi, np.pi, []) class XPUTestCos2(XPUTestCosBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [1024, 8]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [1024, 8]) class XPUTestCos3(XPUTestCosBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [4, 512, 15, 15]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [4, 512, 15, 15]) class XPUTestCos4(XPUTestCosBase): def init_config(self): - self.x = np.random.uniform(-np.pi, np.pi, [4, 256, 22, 22]).astype( - self.dtype - ) + self.tmp_x = np.random.uniform(-np.pi, np.pi, [4, 256, 22, 22]) support_types = get_xpu_op_support_types('cos') diff --git a/test/xpu/test_elementwise_pow_op_xpu.py b/test/xpu/test_elementwise_pow_op_xpu.py index ddcf64fb9d405..a63e403ca50d5 100644 --- a/test/xpu/test_elementwise_pow_op_xpu.py +++ b/test/xpu/test_elementwise_pow_op_xpu.py @@ -20,7 +20,7 @@ create_test_class, get_xpu_op_support_types, ) -from op_test import OpTest, skip_check_grad_ci +from op_test import OpTest, convert_float_to_uint16, skip_check_grad_ci from op_test_xpu import XPUOpTest import paddle @@ -40,14 +40,23 @@ def setUp(self): self.dtype = self.in_type self.__class__.no_need_check_grad = True self.compute_input_output() - - def compute_input_output(self): + if self.dtype == np.uint16: + # bfloat16 actually + self.x = convert_float_to_uint16(self.tmp_x) + self.y = convert_float_to_uint16(self.tmp_y) + else: + self.x = self.tmp_x.astype(self.dtype) + self.y = self.tmp_y.astype(self.dtype) self.inputs = { - 'X': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), - 'Y': np.random.uniform(1, 2, [20, 5]).astype(self.dtype), + 'X': self.x, + 'Y': self.y, } self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + def compute_input_output(self): + self.tmp_x = np.random.uniform(1, 2, [20, 5]) + self.tmp_y = np.random.uniform(1, 2, [20, 5]) + def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) @@ -55,58 +64,36 @@ def test_check_output(self): class TestElementwisePowOp_big_shape_1(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [10, 10]).astype(self.dtype), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(1, 2, [10, 10]) + self.tmp_y = np.random.uniform(0.1, 1, [10, 10]) class TestElementwisePowOp_big_shape_2(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(1, 2, [10, 10]).astype(self.dtype), - 'Y': np.random.uniform(0.2, 2, [10, 10]).astype(self.dtype), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(1, 2, [10, 10]) + self.tmp_y = np.random.uniform(0.2, 2, [10, 10]) @skip_check_grad_ci( reason="[skip shape check] Use y_shape(1) to test broadcast." ) class TestElementwisePowOp_scalar(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(0.1, 1, [3, 3, 4]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(0.1, 1, [3, 3, 4]) + self.tmp_y = np.random.uniform(0.1, 1, [1]) class TestElementwisePowOp_tensor(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(1, 3, [100]).astype(self.dtype), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(0.1, 1, [100]) + self.tmp_y = np.random.uniform(1, 3, [100]) class TestElementwisePowOp_broadcast_0(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 1, 100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(0.1, 1, [2, 1, 100]) + self.tmp_y = np.random.uniform(0.1, 1, [100]) class TestElementwisePowOp_broadcast_4(TestElementwisePowOp): def compute_input_output(self): - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 10, 3, 5]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [2, 10, 1, 5]).astype( - self.dtype - ), - } - self.outputs = {'Out': np.power(self.inputs['X'], self.inputs['Y'])} + self.tmp_x = np.random.uniform(0.1, 1, [2, 10, 3, 5]) + self.tmp_y = np.random.uniform(0.1, 1, [2, 10, 1, 5]) class TestElementwisePowOpInt(OpTest): def setUp(self): From bae368752ab884300dfe5f55524b8df26ff26d3f Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 28 Dec 2023 14:10:30 +0800 Subject: [PATCH 111/146] [Dy2St] Replace all astor usage with `ast_to_source_code` (#60302) --- .../jit/dy2static/transformers/basic_api_transformer.py | 7 +++---- python/paddle/jit/dy2static/utils.py | 7 +++---- python/paddle/jit/dy2static/utils_helper.py | 5 ++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/python/paddle/jit/dy2static/transformers/basic_api_transformer.py b/python/paddle/jit/dy2static/transformers/basic_api_transformer.py index 1d9c865bf75b2..0902a3558b2b0 100644 --- a/python/paddle/jit/dy2static/transformers/basic_api_transformer.py +++ b/python/paddle/jit/dy2static/transformers/basic_api_transformer.py @@ -13,11 +13,10 @@ # limitations under the License. -import astor - from paddle.utils import gast from .. import utils +from ..ast_utils import ast_to_source_code from .base import BaseTransformer __all__ = [] @@ -63,7 +62,7 @@ def visit_Expr(self, node): def _visit_Call(self, node): assert isinstance(node, gast.Call) - func_name = astor.to_source(gast.gast_to_ast(node.func)) + func_name = ast_to_source_code(node.func) if self._is_dygraph_forward(func_name): class_node = self._get_class_node(func_name) @@ -91,7 +90,7 @@ def _update_class_node_dict(self, node): return False utils.update_args_of_func(node_value, node_value, "__init__") - target_str = astor.to_source(gast.gast_to_ast(node.targets[0])) + target_str = ast_to_source_code(node.targets[0]) self.class_node_dict[target_str] = node_value return True # TODO: node.value is not dygraph class diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py index 8079a9a527168..fc18ee5883e9c 100644 --- a/python/paddle/jit/dy2static/utils.py +++ b/python/paddle/jit/dy2static/utils.py @@ -27,7 +27,6 @@ import warnings from importlib.machinery import SourceFileLoader -import astor import numpy as np import paddle @@ -320,7 +319,7 @@ def in_white_list(module, func_name): def _delete_keywords_from(node): assert isinstance(node, gast.Call) - func_src = astor.to_source(gast.gast_to_ast(node.func)) + func_src = ast_to_source_code(node.func) full_args = eval(f"inspect.getfullargspec({func_src})") full_args_name = full_args[0] @@ -398,7 +397,7 @@ def update_args_of_func(node, dygraph_node, method_name): "The method name of class to update args should be '__init__' or 'forward'" ) - class_src = astor.to_source(gast.gast_to_ast(dygraph_node.func)) + class_src = ast_to_source_code(dygraph_node.func) if method_name == "__init__" or eval( f"issubclass({class_src}, paddle.nn.Layer)" @@ -454,7 +453,7 @@ def get_attribute_full_name(node): assert isinstance( node, gast.Attribute ), "Input non-Attribute node to get attribute full name" - return astor.to_source(gast.gast_to_ast(node)).strip() + return ast_to_source_code(node).strip() def generate_name_node(name_ids, ctx=gast.Load(), gen_tuple_if_single=False): diff --git a/python/paddle/jit/dy2static/utils_helper.py b/python/paddle/jit/dy2static/utils_helper.py index 5f9c8c506aca7..4f1ae01450739 100644 --- a/python/paddle/jit/dy2static/utils_helper.py +++ b/python/paddle/jit/dy2static/utils_helper.py @@ -15,7 +15,6 @@ import inspect -import astor import numpy as np # noqa: F401 import paddle @@ -62,7 +61,7 @@ def is_api_in_module(node, module_prefix): while isinstance(func_node, gast.Call): func_node = func_node.func - func_str = astor.to_source(gast.gast_to_ast(func_node)).strip() + func_str = ast_to_source_code(func_node).strip() try: import paddle.jit.dy2static as _jst # noqa: F401 from paddle import to_tensor # noqa: F401 @@ -80,7 +79,7 @@ def _is_api_in_module_helper(obj, module_prefix): # Is numpy_api cannot reuse is_api_in_module because of numpy module problem def is_numpy_api(node): assert isinstance(node, gast.Call), "Input non-Call node for is_numpy_api" - func_str = astor.to_source(gast.gast_to_ast(node.func)) + func_str = ast_to_source_code(node.func) try: module_result = eval( "_is_api_in_module_helper({}, '{}')".format(func_str, "numpy") From 54ee802ee4757a7681a54b943960d50332bf741a Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Thu, 28 Dec 2023 14:18:23 +0800 Subject: [PATCH 112/146] test test_dist_fuse_resunit_pass (#60393) --- tools/gpups_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/gpups_test.sh b/tools/gpups_test.sh index 883604ef6685e..91cc6627dd7e2 100644 --- a/tools/gpups_test.sh +++ b/tools/gpups_test.sh @@ -27,10 +27,11 @@ function collect_failed_tests() { done } -# disable test: test_dist_fuse_resunit_pass +# disable test: serial_list="^test_conv2d_op$|\ ^test_conv2d_transpose_op$|\ +^test_dist_fuse_resunit_pass$|\ ^test_dygraph_dataparallel_bf16$|\ ^test_dygraph_sharding_stage1_fp16$|\ ^test_dygraph_sharding_stage1_bf16$|\ From 8710cb794f4e149ff93a045f1c3fcbc04ead03ed Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Thu, 28 Dec 2023 14:31:56 +0800 Subject: [PATCH 113/146] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.271?= =?UTF-8?q?=E3=80=91Migrate=20LogNormal=20to=20pir=20(#60318)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/distribution/distribution.py | 7 +++-- python/paddle/distribution/normal.py | 20 +++++++++++-- .../test_distribution_lognormal_static.py | 30 +++++++++++++++++-- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/python/paddle/distribution/distribution.py b/python/paddle/distribution/distribution.py index 0bc1a70a4c854..130a5c300a64d 100644 --- a/python/paddle/distribution/distribution.py +++ b/python/paddle/distribution/distribution.py @@ -150,7 +150,7 @@ def _validate_args(self, *args): is_variable = False is_number = False for arg in args: - if isinstance(arg, Variable): + if isinstance(arg, (Variable, paddle.pir.Value)): is_variable = True else: is_number = True @@ -176,7 +176,10 @@ def _to_tensor(self, *args): tmp = 0.0 for arg in args: - if not isinstance(arg, (float, list, tuple, np.ndarray, Variable)): + if not isinstance( + arg, + (float, list, tuple, np.ndarray, Variable, paddle.pir.Value), + ): raise TypeError( "Type of input args must be float, list, tuple, numpy.ndarray or Tensor, but received type {}".format( type(arg) diff --git a/python/paddle/distribution/normal.py b/python/paddle/distribution/normal.py index 53155c49287e6..aacf8ffa635a2 100644 --- a/python/paddle/distribution/normal.py +++ b/python/paddle/distribution/normal.py @@ -99,13 +99,29 @@ def __init__(self, loc, scale, name=None): check_type( loc, 'loc', - (int, float, np.ndarray, Variable, list, tuple), + ( + int, + float, + np.ndarray, + Variable, + paddle.pir.Value, + list, + tuple, + ), 'Normal', ) check_type( scale, 'scale', - (int, float, np.ndarray, Variable, list, tuple), + ( + int, + float, + np.ndarray, + Variable, + paddle.pir.Value, + list, + tuple, + ), 'Normal', ) diff --git a/test/distribution/test_distribution_lognormal_static.py b/test/distribution/test_distribution_lognormal_static.py index b2d61e6ddc68c..ac4b4d428cfc9 100644 --- a/test/distribution/test_distribution_lognormal_static.py +++ b/test/distribution/test_distribution_lognormal_static.py @@ -33,9 +33,10 @@ ('one-dim', xrand((2,)), xrand((2,)), xrand((2,))), ('multi-dim', xrand((3, 3)), xrand((3, 3)), xrand((3, 3))), ], + test_pir=True, ) class TestLogNormal(unittest.TestCase): - def setUp(self): + def run_program(self): paddle.enable_static() startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -67,6 +68,13 @@ def setUp(self): self.log_prob, ] = executor.run(main_program, feed=self.feeds, fetch_list=fetch_list) + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.run_program() + else: + self.run_program() + def test_mean(self): np_mean = self.np_lognormal.mean self.assertEqual(str(self.mean.dtype).split('.')[-1], self.scale.dtype) @@ -122,9 +130,10 @@ def test_log_prob(self): @parameterize_cls( (TEST_CASE_NAME, 'loc', 'scale'), [('sample', xrand((4,)), xrand((4,), min=0, max=1))], + test_pir=True, ) class TestLogNormalSample(unittest.TestCase): - def setUp(self): + def run_program(self): paddle.enable_static() startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -150,6 +159,13 @@ def setUp(self): main_program, feed=self.feeds, fetch_list=fetch_list ) + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.run_program() + else: + self.run_program() + def test_sample(self): samples_mean = self.samples.mean(axis=0) samples_var = self.samples.var(axis=0) @@ -196,9 +212,10 @@ def _kstest(self, loc, scale, samples): xrand((2, 2)), ), ], + test_pir=True, ) class TestLogNormalKL(unittest.TestCase): - def setUp(self): + def run_program(self): paddle.enable_static() startup_program = paddle.static.Program() main_program = paddle.static.Program() @@ -236,6 +253,13 @@ def setUp(self): main_program, feed=self.feeds, fetch_list=fetch_list ) + def setUp(self): + if self.test_pir: + with paddle.pir_utils.IrGuard(): + self.run_program() + else: + self.run_program() + def test_kl_divergence(self): np.testing.assert_allclose( self.kl0, From a773f32ddaf10565dc33d5e481b25115a7b9b1ee Mon Sep 17 00:00:00 2001 From: Yuang Liu Date: Thu, 28 Dec 2023 14:34:09 +0800 Subject: [PATCH 114/146] [auto parallel] Lazy init with random control. (#60316) --- paddle/fluid/pybind/eager.cc | 15 ++++- .../paddle/distributed/auto_parallel/api.py | 38 ++++++++++-- .../distributed/auto_parallel/random.py | 40 +++++++++++- python/paddle/nn/initializer/Bilinear.py | 3 + python/paddle/nn/initializer/assign.py | 3 + python/paddle/nn/initializer/dirac.py | 3 + python/paddle/nn/initializer/initializer.py | 12 +++- python/paddle/nn/initializer/kaiming.py | 3 + python/paddle/nn/initializer/normal.py | 3 + python/paddle/nn/initializer/orthogonal.py | 3 + python/paddle/nn/initializer/uniform.py | 3 + python/paddle/nn/initializer/xavier.py | 17 +++-- test/auto_parallel/CMakeLists.txt | 4 ++ .../semi_auto_parallel_lazy_init.py | 62 +++++++++++++++++++ .../test_semi_auto_parallel_lazy_init.py | 44 +++++++++++++ 15 files changed, 236 insertions(+), 17 deletions(-) create mode 100644 test/auto_parallel/semi_auto_parallel_lazy_init.py create mode 100644 test/auto_parallel/test_semi_auto_parallel_lazy_init.py diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc index 99ceed6b2b309..3cb3ccf964ec8 100644 --- a/paddle/fluid/pybind/eager.cc +++ b/paddle/fluid/pybind/eager.cc @@ -244,9 +244,18 @@ void InitDistTensorWithTensor(TensorObject* self, std::make_shared(tensor, process_mesh, placements)); VLOG(4) << "Same place, do ShareDataWith for DistTensor."; } else { - std::shared_ptr tensor = - std::static_pointer_cast( - src.copy_to(place, true).impl()); + std::shared_ptr tensor; + if (src.initialized()) { + tensor = std::static_pointer_cast( + src.copy_to(place, true).impl()); + } else { + // lazy init branch. The src tensor is on undefined place. + PADDLE_ENFORCE( + src.place().GetType() == phi::AllocationType::UNDEFINED, + phi::errors::InvalidArgument("Only undefined place is support for " + "uninitialized input tensor.")); + tensor = std::static_pointer_cast(src.impl()); + } self->tensor.set_impl( std::make_shared(tensor, process_mesh, placements)); VLOG(4) << "Different place, do TensorCopy for DistTensor."; diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index f8eb3f71f89b9..d3f19baded5e6 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -45,6 +45,7 @@ from paddle.framework import core from .placement_type import check_placements_equal, get_shard_spec +from .random import determinate_rng, rng_state # There are the auto parallel API of the unified version of dynamic and static mode. # Some APIs have the same name with the previous APIs implementation, which are @@ -171,19 +172,48 @@ def shard_tensor( # `paddle.to_tensor` supports both dynamic and static mode if stop_gradient is None: stop_gradient = getattr(data, "stop_gradient", True) - tensor = paddle.to_tensor( - data, dtype=dtype, place=place, stop_gradient=stop_gradient - ) + if isinstance(data, EagerParamBase) and not data._is_initialized(): + assert ( + data._init_func is not None + ), "Get an uninitialized param with an unregistered init_func." + tensor = data + else: + tensor = paddle.to_tensor( + data, dtype=dtype, place=place, stop_gradient=stop_gradient + ) if paddle.in_dynamic_mode(): # here the dist tensor is deep copy constructed if isinstance(data, EagerParamBase): - return EagerParamBase.from_tensor( + + def lazy_init_hook(param, origin_hook): + # lazy init hook with randomness controlling + def _init_func(var, block): + # get the unique rng name + rng_name = determinate_rng( + dist.get_rank(), + process_mesh=param.process_mesh, + placements=param.placements, + ) + # real call the init function + with rng_state(rng_name): + origin_hook(var, block) + + return _init_func + + dist_param = EagerParamBase.from_tensor( tensor, process_mesh=mesh, placements=placements, **tensor.__dict__, ) + if tensor._init_func is not None: + origin_init_func = tensor._init_func + dist_param.set_init_func( + lazy_init_hook(dist_param, origin_init_func) + ) + + return dist_param else: return paddle.Tensor( tensor, process_mesh=mesh, placements=placements, place=place diff --git a/python/paddle/distributed/auto_parallel/random.py b/python/paddle/distributed/auto_parallel/random.py index d79f94e166524..4f27d3f7cc5ed 100644 --- a/python/paddle/distributed/auto_parallel/random.py +++ b/python/paddle/distributed/auto_parallel/random.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import logging import paddle @@ -22,6 +23,7 @@ _logger = get_logger(logging.INFO) _rng_name_to_seed = {} +_rng_name_to_states = {} _inited_rng_name_to_seed = {} _enable_random_control = False _basic_seed = 42 @@ -71,7 +73,16 @@ def parallel_manual_seed(seed, name=""): _basic_name = name -def determinate_rng(rank, dims_mapping, process_mesh): +def determinate_rng( + rank, dims_mapping=None, process_mesh=None, placements=None +): + assert process_mesh is not None, "Must provide process mesh" + assert ( + dims_mapping is not None or placements is not None + ), "Must provide one of dims mapping or placements." + assert not ( + dims_mapping is not None and placements is not None + ), "Cannot provide dims mapping and placements at same time." # TODO(JZ-LIANG) Support Mesh with any high rank # use a string to unique integer hashing algorithm for seed computation. # instead of using offsets to coodinate seed across devices. @@ -98,7 +109,9 @@ def determinate_rng(rank, dims_mapping, process_mesh): seed_ += _mesh_offset * (unique_id + 1) for i in range(len(process_mesh.shape)): - if i not in dims_mapping: + if (dims_mapping is not None and i not in dims_mapping) or ( + placements is not None and not placements[i].is_shard() + ): relative_idx = -1 else: relative_idx = _get_idx_in_axis( @@ -112,6 +125,7 @@ def determinate_rng(rank, dims_mapping, process_mesh): seed_ += _dim_offsets[i] * (relative_idx + 1) global _rng_name_to_seed + global _rng_name_to_states if sharding_expr in _rng_name_to_seed: assert _rng_name_to_seed[sharding_expr] == seed_ else: @@ -121,10 +135,30 @@ def determinate_rng(rank, dims_mapping, process_mesh): seed_, sharding_expr, _rng_name_to_seed ) _rng_name_to_seed[sharding_expr] = seed_ - + if paddle.in_dynamic_mode(): + # for dygraph, just init the seed when meeting a new seed + orig_rng_state = paddle.get_rng_state() + paddle.seed(seed_) + _rng_name_to_states[sharding_expr] = paddle.get_rng_state() + paddle.set_rng_state(orig_rng_state) return sharding_expr +@contextlib.contextmanager +def rng_state(name): + global _rng_name_to_states + assert ( + name in _rng_name_to_states + ), f"The rng state name {name} haven't been init. " + orig_rng_state = paddle.get_rng_state() + paddle.set_rng_state(_rng_name_to_states[name]) + try: + yield + finally: + _rng_name_to_states[name] = paddle.get_rng_state() + paddle.set_rng_state(orig_rng_state) + + def init_auto_parallel_rng(): if not is_enable_auto_rand_ctrl(): return diff --git a/python/paddle/nn/initializer/Bilinear.py b/python/paddle/nn/initializer/Bilinear.py index a12393e2e2872..cfb18dac02c2a 100644 --- a/python/paddle/nn/initializer/Bilinear.py +++ b/python/paddle/nn/initializer/Bilinear.py @@ -89,6 +89,9 @@ def forward(self, var, block=None): Returns: The initialization op """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, Bilinear initializer not support lazy init for dist param." block = self._check_block(block) if not isinstance(var, (framework.Variable, pir.core.ParameterMeta)): diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 62cbcf6179f9a..9274ff5275df0 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -56,6 +56,9 @@ def forward(self, var, block=None): Returns: The initialization op """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, assign initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance( diff --git a/python/paddle/nn/initializer/dirac.py b/python/paddle/nn/initializer/dirac.py index 8ec63f64bbc02..7da5cd15b54f7 100644 --- a/python/paddle/nn/initializer/dirac.py +++ b/python/paddle/nn/initializer/dirac.py @@ -106,6 +106,9 @@ def __call__(self, var, block=None): Returns: The most critical OP(scatter) in this initializer, which contains 7~8 ops in total. """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, dirac initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance(var, (framework.Variable, pir.core.ParameterMeta)) assert isinstance(block, (framework.Block, pir.Block)) diff --git a/python/paddle/nn/initializer/initializer.py b/python/paddle/nn/initializer/initializer.py index 6f37e95a79816..7b3901613f9e3 100644 --- a/python/paddle/nn/initializer/initializer.py +++ b/python/paddle/nn/initializer/initializer.py @@ -17,7 +17,11 @@ import numpy as np -from ...base.framework import default_main_program, in_dygraph_mode +from ...base.framework import ( + EagerParamBase, + default_main_program, + in_dygraph_mode, +) from .lazy_init import lazy_init_helper __all__ = [] @@ -86,7 +90,11 @@ def _compute_fans(self, var): Returns: tuple of two integers (fan_in, fan_out). """ - shape = var.shape + shape = ( + var._local_shape + if (isinstance(var, EagerParamBase) and var.is_dist()) + else var.shape + ) if not shape or len(shape) == 0: fan_in = fan_out = 1 elif len(shape) == 1: diff --git a/python/paddle/nn/initializer/kaiming.py b/python/paddle/nn/initializer/kaiming.py index 14e3d726c8736..39329acaf7da1 100644 --- a/python/paddle/nn/initializer/kaiming.py +++ b/python/paddle/nn/initializer/kaiming.py @@ -91,6 +91,9 @@ def forward(self, var, block=None): Returns: The initialization op. """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, kaiming initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance( var, (framework.Variable, paddle.pir.core.ParameterMeta) diff --git a/python/paddle/nn/initializer/normal.py b/python/paddle/nn/initializer/normal.py index 3983f270e60a6..4ca0a0902246c 100644 --- a/python/paddle/nn/initializer/normal.py +++ b/python/paddle/nn/initializer/normal.py @@ -56,6 +56,9 @@ def forward(self, var, block=None): Returns: The initialization op. """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, normal initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance(block, (framework.Block, pir.Block)) diff --git a/python/paddle/nn/initializer/orthogonal.py b/python/paddle/nn/initializer/orthogonal.py index 0dc2bd2aede47..486a68bcd5d0f 100644 --- a/python/paddle/nn/initializer/orthogonal.py +++ b/python/paddle/nn/initializer/orthogonal.py @@ -81,6 +81,9 @@ def __call__(self, var, block=None): Returns: The last initialization op, it contain 8 ops in orthogonal initializer. """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, orthogonal initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance(var, (framework.Variable, pir.core.ParameterMeta)) assert isinstance(block, (framework.Block, pir.Block)) diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py index 86ef5aedbf1af..f30ef1b38402d 100644 --- a/python/paddle/nn/initializer/uniform.py +++ b/python/paddle/nn/initializer/uniform.py @@ -73,6 +73,9 @@ def forward(self, var, block=None): Returns: The initialization op """ + assert not ( + isinstance(var, framework.EagerParamBase) and var.is_dist() + ), "Currently, uniform initializer not support lazy init for dist param." block = self._check_block(block) assert isinstance(block, (framework.Block, pir.Block)) diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py index 13a2c8cdce28f..58d73d21dfe86 100644 --- a/python/paddle/nn/initializer/xavier.py +++ b/python/paddle/nn/initializer/xavier.py @@ -114,7 +114,9 @@ def forward(self, var, block=None): name=unique_name.generate( ".".join(['xavier_init', var.name, 'tmp']) ), - shape=var.shape, + shape=var._local_shape + if (isinstance(var, framework.EagerParamBase) and var.is_dist()) + else var.shape, dtype=out_dtype, type=core.VarDesc.VarType.LOD_TENSOR, persistable=False, @@ -151,10 +153,15 @@ def forward(self, var, block=None): if var.dtype == core.VarDesc.VarType.FP16 or ( var.dtype == core.VarDesc.VarType.BF16 and not self._uniform ): - var_tmp = _C_ops.cast(out_var, var.dtype) - var_tmp._share_underline_tensor_to(var) - else: - out_var._share_underline_tensor_to(var) + out_var = _C_ops.cast(out_var, var.dtype) + if isinstance(var, framework.EagerParamBase) and var.is_dist(): + # lazy init for dist tensor + out_var = ( + paddle.distributed.auto_parallel.api.dtensor_from_local( + out_var, var.process_mesh, var.placements + ) + ) + out_var._share_underline_tensor_to(var) return None elif in_pir_mode(): if self._uniform: diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index 04d6219c5946e..774dc3d2023b9 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -162,6 +162,10 @@ if(WITH_DISTRIBUTE AND WITH_GPU) test_semi_auto_parallel_single_strategy) set_tests_properties(test_semi_auto_parallel_single_strategy PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 400) + py_test_modules(test_semi_auto_parallel_lazy_init MODULES + test_semi_auto_parallel_lazy_init) + set_tests_properties(test_semi_auto_parallel_lazy_init + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120) py_test_modules(test_semi_auto_parallel_in_framework MODULES test_semi_auto_parallel_in_framework) set_tests_properties(test_semi_auto_parallel_in_framework diff --git a/test/auto_parallel/semi_auto_parallel_lazy_init.py b/test/auto_parallel/semi_auto_parallel_lazy_init.py new file mode 100644 index 0000000000000..52016c358ea35 --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_lazy_init.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import paddle +import paddle.distributed as dist +from paddle import LazyGuard + + +class TestSemiAutoParallelLazyInit: + def __init__(self): + self._backend = os.getenv("backend") + self._seed = eval(os.getenv("seed")) + self._mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) + + def test_replicate(self): + paddle.distributed.auto_parallel.parallel_manual_seed(self._seed) + with LazyGuard(): + linear = paddle.nn.Linear(10, 10) + linear.weight = dist.shard_tensor( + linear.weight, self._mesh, [dist.Replicate()] + ) + linear.bias = dist.shard_tensor( + linear.bias, self._mesh, [dist.Replicate()] + ) + for param in linear.parameters(): + assert not param._is_initialized() + param.initialize() + assert param._is_initialized() + + local_weight_md5 = linear.weight._local_value()._md5sum() + mesh0 = dist.ProcessMesh([0], dim_names=["x"]) + mesh1 = dist.ProcessMesh([1], dim_names=["x"]) + tmp = paddle.distributed.auto_parallel.api.dtensor_from_local( + linear.weight._local_value(), + mesh0 if dist.get_rank() == 0 else mesh1, + [dist.Replicate()], + ) + tmp = dist.reshard( + tmp, mesh1 if dist.get_rank() == 0 else mesh0, [dist.Replicate()] + ) + tmp_md5 = tmp._local_value()._md5sum() + assert local_weight_md5 == tmp_md5 + + def run_test_case(self): + self.test_replicate() + + +if __name__ == '__main__': + TestSemiAutoParallelLazyInit().run_test_case() diff --git a/test/auto_parallel/test_semi_auto_parallel_lazy_init.py b/test/auto_parallel/test_semi_auto_parallel_lazy_init.py new file mode 100644 index 0000000000000..d0c09749af53d --- /dev/null +++ b/test/auto_parallel/test_semi_auto_parallel_lazy_init.py @@ -0,0 +1,44 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import collective.test_communication_api_base as test_base + + +class TestSemiAutoParallelLazyInit(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp( + num_of_devices=2, + timeout=120, + ) + self._default_envs = { + "dtype": "float32", + "seed": "2023", + } + self._changeable_envs = {"backend": ["cpu", "gpu"]} + + def test_lazy_init(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + self.run_test_case( + "semi_auto_parallel_lazy_init.py", + user_defined_envs=envs, + ) + + +if __name__ == "__main__": + unittest.main() From 180ded554f73baa1e8a401a7979bf9e4b9038492 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 28 Dec 2023 14:35:16 +0800 Subject: [PATCH 115/146] [Dy2St] Unify PT flags in dy2st and run PT in AST (#60410) --- .../eager/to_static/run_program_op_node.h | 120 ++++++++++++------ paddle/fluid/framework/executor_cache.cc | 4 +- paddle/fluid/framework/executor_cache.h | 18 ++- .../paddle/jit/dy2static/partial_program.py | 64 ++++------ test/custom_runtime/CMakeLists.txt | 8 +- .../test_custom_cpu_to_static.py | 4 +- test/dygraph_to_static/CMakeLists.txt | 2 +- 7 files changed, 130 insertions(+), 90 deletions(-) diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index 257b249e51600..b409c0f7067e5 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -488,8 +488,11 @@ inline void PirRunProgramAPI( paddle::framework::InterpreterCoreInfoCache::Instance(); std::shared_ptr interpreter_core = nullptr; - if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/false)) { + if (!interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/true)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -555,8 +558,12 @@ inline void PirRunProgramAPI( 1); VLOG(2) << "Get interpretercore cache by program:" << program_id; // Step 1. get cache interpretercore - auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/false); + auto &cached_value = + interpretercore_info_cache.GetMutable(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/true); interpreter_core = cached_value.core_; // Step 2. update scope for cache interpretercore details::ShareTensorsIntoScopeByValue( @@ -631,6 +638,12 @@ inline void RunProgramAPI( int64_t program_id = PADDLE_GET_CONST(int64_t, attrs.at("program_id")); auto place = egr::Controller::Instance().GetExpectedPlace(); + bool in_pir_pt_mode = FLAGS_enable_pir_with_pt_in_dy2st; + if (attrs.count("in_pir_pt_mode")) { + in_pir_pt_mode = PADDLE_GET_CONST(bool, attrs.at("in_pir_pt_mode")); + } + in_pir_pt_mode = in_pir_pt_mode || FLAGS_enable_pir_in_executor; + // NOTE(chenweihang): In order not to add new variable type, use vector // here. Originally, here can use scope directly. auto *out_scope_vec = &step_scope; @@ -688,8 +701,11 @@ inline void RunProgramAPI( paddle::framework::InterpreterCoreInfoCache::Instance(); std::shared_ptr interpreter_core = nullptr; - if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/false)) { + if (!interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/in_pir_pt_mode)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -702,12 +718,7 @@ inline void RunProgramAPI( details::ShareTensorsIntoScope(params, global_inner_scope); // Step 2. create new interpretercore - bool in_pir_pt_mode = FLAGS_enable_pir_with_pt_in_dy2st; - if (attrs.count("in_pir_pt_mode")) { - in_pir_pt_mode = PADDLE_GET_CONST(bool, attrs.at("in_pir_pt_mode")); - } - - if (FLAGS_enable_pir_in_executor || in_pir_pt_mode) { + if (in_pir_pt_mode) { // build new ir program auto ir_program = paddle::framework::ConstructFowardIrProgram(forward_global_block, @@ -765,6 +776,7 @@ inline void RunProgramAPI( global_inner_scope, place_hash_key, false, + in_pir_pt_mode, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); } else { @@ -774,8 +786,12 @@ inline void RunProgramAPI( 1); VLOG(2) << "Get interpretercore cahce by program:" << program_id; // Step 1. get cache interpretercore - auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/false); + auto &cached_value = + interpretercore_info_cache.GetMutable(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/in_pir_pt_mode); interpreter_core = cached_value.core_; // Step 2. update scope for cache interpretercore details::ShareTensorsIntoScopeWithName(x, input_names, global_inner_scope); @@ -840,6 +856,12 @@ inline void RunProgramGradAPI( int64_t program_id = PADDLE_GET_CONST(int64_t, attrs.at("program_id")); + bool in_pir_pt_mode = FLAGS_enable_pir_with_pt_in_dy2st; + if (attrs.count("in_pir_pt_mode")) { + in_pir_pt_mode = PADDLE_GET_CONST(bool, attrs.at("in_pir_pt_mode")); + } + in_pir_pt_mode = in_pir_pt_mode || FLAGS_enable_pir_in_executor; + auto place = egr::Controller::Instance().GetExpectedPlace(); VLOG(2) << "RunProgramGradOp use interpretercore to execute program."; @@ -858,8 +880,11 @@ inline void RunProgramGradAPI( paddle::framework::InterpreterCoreInfoCache::Instance(); std::shared_ptr interpreter_core = nullptr; - if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/true)) { + if (!interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/true, + /*in_pir_mode=*/in_pir_pt_mode)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -869,12 +894,7 @@ inline void RunProgramGradAPI( << program_id; details::ShareTensorsIntoScope(out_grad, global_inner_scope); - bool in_pir_pt_mode = FLAGS_enable_pir_with_pt_in_dy2st; - if (attrs.count("in_pir_pt_mode")) { - in_pir_pt_mode = PADDLE_GET_CONST(bool, attrs.at("in_pir_pt_mode")); - } - - if (FLAGS_enable_pir_in_executor || in_pir_pt_mode) { + if (in_pir_pt_mode) { auto res = paddle::framework::ConstructBackwardIrProgram(backward_global_block, out_grad, @@ -904,14 +924,19 @@ inline void RunProgramGradAPI( // share threadpool // NOTE(zhiqiu): this only works interpreter_core is executed strictly // after the related fwd_interpreter_core. - if (interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, false)) { - auto fwd_interpreter_core = interpretercore_info_cache - .GetMutable(program_id, - global_inner_scope, - place_hash_key, - /*is_grad=*/false) - .core_; + if (interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/in_pir_pt_mode)) { + auto fwd_interpreter_core = + interpretercore_info_cache + .GetMutable(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/in_pir_pt_mode) + .core_; interpreter_core->ShareWorkQueueFrom(fwd_interpreter_core); VLOG(4) << "Share workqueue from " << fwd_interpreter_core.get() << " to " << interpreter_core.get(); @@ -938,6 +963,7 @@ inline void RunProgramGradAPI( global_inner_scope, place_hash_key, /*is_grad=*/true, + in_pir_pt_mode, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); } else { @@ -946,8 +972,12 @@ inline void RunProgramGradAPI( paddle::platform::TracerEventType::UserDefined, 1); VLOG(2) << "Get interpretercore cahce by program:" << program_id; - auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/true); + auto &cached_value = + interpretercore_info_cache.GetMutable(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/true, + /*in_pir_mode=*/in_pir_pt_mode); interpreter_core = cached_value.core_; // update scope @@ -1054,8 +1084,11 @@ inline void PirRunProgramGradAPI( paddle::framework::InterpreterCoreInfoCache::Instance(); std::shared_ptr interpreter_core = nullptr; - if (!interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/true)) { + if (!interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/true, + /*in_pir_mode=*/true)) { paddle::platform::RecordEvent record_event( "create_new_interpretercore", paddle::platform::TracerEventType::UserDefined, @@ -1080,13 +1113,17 @@ inline void PirRunProgramGradAPI( // share threadpool // NOTE(zhiqiu): this only works interpreter_core is executed strictly // after the related fwd_interpreter_core. - if (interpretercore_info_cache.Has( - program_id, global_inner_scope, place_hash_key, false)) { + if (interpretercore_info_cache.Has(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/false, + /*in_pir_mode=*/true)) { auto fwd_interpreter_core = interpretercore_info_cache .GetMutable(program_id, global_inner_scope, place_hash_key, - /*is_grad=*/false) + /*is_grad=*/false, + /*in_pir_mode=*/true) .core_; interpreter_core->ShareWorkQueueFrom(fwd_interpreter_core); VLOG(4) << "Share workqueue from " << fwd_interpreter_core.get() << " to " @@ -1107,6 +1144,7 @@ inline void PirRunProgramGradAPI( global_inner_scope, place_hash_key, /*is_grad=*/true, + /*in_pir_mode=*/true, skip_eager_delete_vars); VLOG(2) << "Get skip GC vars size is: " << skip_eager_delete_vars.size(); details::print_collection(skip_eager_delete_vars); @@ -1116,8 +1154,12 @@ inline void PirRunProgramGradAPI( paddle::platform::TracerEventType::UserDefined, 1); VLOG(2) << "Get interpretercore cahce by program:" << program_id; - auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, global_inner_scope, place_hash_key, /*is_grad=*/true); + auto &cached_value = + interpretercore_info_cache.GetMutable(program_id, + global_inner_scope, + place_hash_key, + /*is_grad=*/true, + /*in_pir_mode=*/true); interpreter_core = cached_value.core_; if (interpreter_core->GetVariableScope()->GetMutableScope() != diff --git a/paddle/fluid/framework/executor_cache.cc b/paddle/fluid/framework/executor_cache.cc index 6af7443358361..97e4d386ea9aa 100644 --- a/paddle/fluid/framework/executor_cache.cc +++ b/paddle/fluid/framework/executor_cache.cc @@ -326,7 +326,7 @@ std::shared_ptr CreateProgramInterpreterCoreInfoToCache( place, program_desc.Block(0), scope, execution_config)); auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, scope, place_hash_key, is_grad); + program_id, scope, place_hash_key, is_grad, /*in_pir_mode=*/false); cached_value.core_ = core; return core; } @@ -355,7 +355,7 @@ std::shared_ptr CreatePirInterpreterCoreInfoToCache( place, {}, ir_program->block(), scope, execution_config)); auto &cached_value = interpretercore_info_cache.GetMutable( - program_id, scope, place_hash_key, is_grad); + program_id, scope, place_hash_key, is_grad, /*in_pir_mode=*/true); cached_value.core_ = core; cached_value.ir_prog_ = std::move(ir_program); return core; diff --git a/paddle/fluid/framework/executor_cache.h b/paddle/fluid/framework/executor_cache.h index 57d9b06d92b0e..bd8b82180cbac 100644 --- a/paddle/fluid/framework/executor_cache.h +++ b/paddle/fluid/framework/executor_cache.h @@ -196,8 +196,9 @@ class InterpreterCoreInfoCache { bool Has(int64_t program_id, const framework::Scope* scope, const int64_t& place_hash_key, - bool is_grad) { - if (FLAGS_enable_pir_in_executor || FLAGS_enable_pir_with_pt_in_dy2st) { + bool is_grad, + bool in_pir_mode) { + if (in_pir_mode) { int64_t scope_i = reinterpret_cast(scope); program_id = hash_with_seed(program_id, scope_i); program_id = hash_with_seed(program_id, place_hash_key); @@ -209,8 +210,9 @@ class InterpreterCoreInfoCache { InterpreterCoreInfo::CacheValue& GetMutable(int64_t program_id, const framework::Scope* scope, const int64_t& place_hash_key, - bool is_grad) { - if (FLAGS_enable_pir_in_executor || FLAGS_enable_pir_with_pt_in_dy2st) { + bool is_grad, + bool in_pir_mode) { + if (in_pir_mode) { int64_t scope_i = reinterpret_cast(scope); program_id = hash_with_seed(program_id, scope_i); program_id = hash_with_seed(program_id, place_hash_key); @@ -222,16 +224,20 @@ class InterpreterCoreInfoCache { const framework::Scope* scope, const int64_t& place_hash_key, bool is_grad, + bool in_pir_mode, const std::set& skip_vars) { - auto& cached_value = GetMutable(program_id, scope, place_hash_key, is_grad); + auto& cached_value = + GetMutable(program_id, scope, place_hash_key, is_grad, in_pir_mode); cached_value.skip_eager_delete_vars_ = std::move(skip_vars); } std::set& GetSkipEagerDeleteVars(int64_t program_id, const framework::Scope* scope, const int64_t& place_hash_key, + bool in_pir_mode, bool is_grad) { - auto& cached_value = GetMutable(program_id, scope, place_hash_key, is_grad); + auto& cached_value = + GetMutable(program_id, scope, place_hash_key, is_grad, in_pir_mode); return cached_value.skip_eager_delete_vars_; } diff --git a/python/paddle/jit/dy2static/partial_program.py b/python/paddle/jit/dy2static/partial_program.py index ef567d193b85c..84719c3eee792 100644 --- a/python/paddle/jit/dy2static/partial_program.py +++ b/python/paddle/jit/dy2static/partial_program.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from copy import deepcopy import numpy as np @@ -229,10 +228,7 @@ def __call__(self, inputs): in_vars, in_var_names = self._prepare_inputs(inputs) out_vars = self._prepare_outputs() self._cast_fp16_if_pure_fp16(in_vars) - # TODO(dev): Currently AST + PT has some issues in control flow, so we only - # enable SOT + PT in 2.6, we will fix it later. - is_dy2st_test = os.environ.get("DY2ST_TEST", None) == "True" - attrs = self._prepare_attributes(force_not_use_pt=(not is_dy2st_test)) + attrs = self._prepare_attributes() attrs.extend(["x_names", in_var_names]) self._sync_lr_value_with_scheduler() @@ -259,7 +255,7 @@ def sot_call(self, inputs): """ out_vars = self._prepare_outputs() self._cast_fp16_if_pure_fp16(inputs) - attrs = self._prepare_attributes(force_not_use_pt=False) + attrs = self._prepare_attributes() attrs.extend(["x_names", self._in_var_names]) self._sync_lr_value_with_scheduler() @@ -296,14 +292,7 @@ def set_hooker(self, hooker): self._hooker = hooker def _get_scope(self, program_id=None, use_scope_cache=False): - if ( - get_flags('FLAGS_enable_pir_in_executor')[ - 'FLAGS_enable_pir_in_executor' - ] - or get_flags('FLAGS_enable_pir_with_pt_in_dy2st')[ - 'FLAGS_enable_pir_with_pt_in_dy2st' - ] - ): + if self._in_pir_pt_mode or self._enable_pir_in_executor: _scope_cache = self._pir_scope_cache else: _scope_cache = self._legacy_scope_cache @@ -768,7 +757,28 @@ def _cast_fp16_if_pure_fp16(self, in_vars): in_vars[i] = var.astype('float16') in_vars[i].name = name - def _prepare_attributes(self, force_not_use_pt=False): + @property + def _in_pir_pt_mode(self): + pir_dy2st_flag = 'FLAGS_enable_pir_with_pt_in_dy2st' + in_pir_pt_mode = get_flags(pir_dy2st_flag)[pir_dy2st_flag] + is_prim_enabled = ( + core._is_fwd_prim_enabled() or core._is_bwd_prim_enabled() + ) + in_cinn_backend = self._backend == "CINN" + is_cinn_enabled = self._build_strategy.build_cinn_pass + if is_prim_enabled or in_cinn_backend or is_cinn_enabled: + in_pir_pt_mode = False + return in_pir_pt_mode + + @property + def _enable_pir_in_executor(self): + enable_pir_in_executor_flag = 'FLAGS_enable_pir_in_executor' + enable_pir_in_executor = get_flags(enable_pir_in_executor_flag)[ + enable_pir_in_executor_flag + ] + return enable_pir_in_executor + + def _prepare_attributes(self): attrs = [ 'forward_global_block', self.forward_program.desc.block(0), @@ -804,17 +814,7 @@ def _prepare_attributes(self, force_not_use_pt=False): ) ) - pir_dy2st_flag = 'FLAGS_enable_pir_with_pt_in_dy2st' - in_pir_pt_mode = get_flags(pir_dy2st_flag)[pir_dy2st_flag] - is_prim_enabled = ( - core._is_fwd_prim_enabled() or core._is_bwd_prim_enabled() - ) - in_cinn_backend = self._backend == "CINN" - is_cinn_enabled = self._build_strategy.build_cinn_pass - if is_prim_enabled or in_cinn_backend or is_cinn_enabled: - in_pir_pt_mode = False - if force_not_use_pt: - in_pir_pt_mode = False + in_pir_pt_mode = self._in_pir_pt_mode attrs.extend(['in_pir_pt_mode', in_pir_pt_mode]) return attrs @@ -901,21 +901,13 @@ def _apply_inplace_pass(self, forward_program, backward_program): forward_program, backward_program ) backward_mem_opt_skip_vars = self._parse_skip_gc_vars(forward_program) - in_pir_pt_mode = ( - get_flags('FLAGS_enable_pir_in_executor')[ - 'FLAGS_enable_pir_in_executor' - ] - or get_flags('FLAGS_enable_pir_with_pt_in_dy2st')[ - 'FLAGS_enable_pir_with_pt_in_dy2st' - ] - ) if forward_program: attrs = { "use_cuda": use_cuda, "mem_opt_skip_vars": forward_mem_opt_skip_vars, "for_partial_block": True, } - if not in_pir_pt_mode: + if not (self._in_pir_pt_mode or self._enable_pir_in_executor): _apply_pass( forward_program, empty_startup_program, @@ -929,7 +921,7 @@ def _apply_inplace_pass(self, forward_program, backward_program): "mem_opt_skip_vars": backward_mem_opt_skip_vars, "for_partial_block": True, } - if not in_pir_pt_mode: + if not (self._in_pir_pt_mode or self._enable_pir_in_executor): _apply_pass( backward_program, empty_startup_program, diff --git a/test/custom_runtime/CMakeLists.txt b/test/custom_runtime/CMakeLists.txt index e8b14445278be..b0b162c19d6ed 100644 --- a/test/custom_runtime/CMakeLists.txt +++ b/test/custom_runtime/CMakeLists.txt @@ -9,9 +9,11 @@ if(WITH_CUSTOM_DEVICE AND NOT WITH_GPU) string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") foreach(TEST_OP ${TEST_OPS}) - py_test(${TEST_OP} - SRCS ${TEST_OP}.py ENVS FLAGS_allocator_strategy=naive_best_fit - PLUGIN_URL=${PLUGIN_URL} PLUGIN_TAG=${PLUGIN_TAG}) + py_test( + ${TEST_OP} + SRCS ${TEST_OP}.py ENVS FLAGS_allocator_strategy=naive_best_fit + PLUGIN_URL=${PLUGIN_URL} PLUGIN_TAG=${PLUGIN_TAG} + FLAGS_enable_pir_with_pt_in_dy2st=False) endforeach() bash_test_modules( diff --git a/test/custom_runtime/test_custom_cpu_to_static.py b/test/custom_runtime/test_custom_cpu_to_static.py index 78978e9175310..b365f8ab39811 100644 --- a/test/custom_runtime/test_custom_cpu_to_static.py +++ b/test/custom_runtime/test_custom_cpu_to_static.py @@ -164,9 +164,7 @@ def forward(self, x): # convert to static model build_strategy = paddle.static.BuildStrategy() - mnist = paddle.jit.to_static( - model, build_strategy=build_strategy, full_graph=True - ) + mnist = paddle.jit.to_static(model, build_strategy=build_strategy) # data loader transform = paddle.vision.transforms.Compose( diff --git a/test/dygraph_to_static/CMakeLists.txt b/test/dygraph_to_static/CMakeLists.txt index e9ae745681017..f54bd5f714b9e 100644 --- a/test/dygraph_to_static/CMakeLists.txt +++ b/test/dygraph_to_static/CMakeLists.txt @@ -4,7 +4,7 @@ file( "test_*.py") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") set(SOT_ENVS SOT_LOG_LEVEL=0 COST_MODEL=False MIN_GRAPH_SIZE=0 - STRICT_MODE=False DY2ST_TEST=True) + STRICT_MODE=False) set(GC_ENVS FLAGS_eager_delete_tensor_gb=0.0) list(REMOVE_ITEM TEST_OPS test_lac) From 875fbfb4b733856e3a4a452358e6d7d6047dbbc0 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Thu, 28 Dec 2023 15:15:55 +0800 Subject: [PATCH 116/146] [Dy2St] Use `ShadowOutputOp` to get dy2st output (#60363) --- .../pir_adaptor/pir_adaptor_util.cc | 4 ++ paddle/fluid/pybind/pir.cc | 44 +++++++++---------- .../jit/dy2static/pir_partial_program.py | 35 ++++++++------- .../jit/pir_dy2static/parameter_recorder.py | 2 +- .../test_tensor_memcpy_on_cpu.py | 3 +- 5 files changed, 48 insertions(+), 40 deletions(-) diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 7f110b49b218f..a06abb197de5f 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -542,6 +542,10 @@ void HandleForSpecialOp(pir::Operation* op, // change opreand name to param_name auto orig_name = value_exe_info->GetValue2VarName().at(value); + if (var_name == orig_name) { + return; + } + if (value_exe_info->GetScope()->FindVar(var_name) != nullptr) { const_cast(value_exe_info->GetScope())->EraseVars({var_name}); VLOG(1) << "var " << var_name << " has been removed from scope"; diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 2103e7b7b660e..9e87a3f39459d 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1057,14 +1057,14 @@ std::pair, OpResultMap> CloneProgram( std::make_pair(associated_array_key, associated_array_value)); } -void AppendSetParameter(Program *forward_program, +void AppendShadowOutput(Program *forward_program, const pir::OpResult &result, const std::string &name, size_t start_point) { pir::IrContext *ctx = pir::IrContext::Instance(); - auto op_info = ctx->GetRegisteredOpInfo(pir::SetParameterOp::name()); + auto op_info = ctx->GetRegisteredOpInfo(pir::ShadowOutputOp::name()); pir::AttributeMap attribute_map = { - {"parameter_name", pir::StrAttribute::get(ctx, name)}, + {"output_name", pir::StrAttribute::get(ctx, name)}, }; pir::Operation *operation = pir::Operation::Create({result}, attribute_map, {}, op_info); @@ -1077,7 +1077,7 @@ void AppendSetParameter(Program *forward_program, } } -int AppendSetParameters(Program *forward_program, +int AppendShadowOutputs(Program *forward_program, const std::vector &outputs_op_result, int start_point, std::string name_prefix) { @@ -1086,9 +1086,9 @@ int AppendSetParameters(Program *forward_program, for (const auto &result : outputs_op_result) { if (!added_op_result.count(result) || IsFakeOpResult(result)) { - std::string parameter_name = name_prefix + std::to_string(counter); - AppendSetParameter( - forward_program, result, parameter_name, start_point + counter); + std::string shadow_output_name = name_prefix + std::to_string(counter); + AppendShadowOutput( + forward_program, result, shadow_output_name, start_point + counter); counter += 1; added_op_result.insert(result); } @@ -1204,20 +1204,20 @@ SplitedResult SplitForwardBackward( if (v.impl() == nullptr) { return; } - // NOTE(Aurelius84): we should skip insert SetParameterOp repeatly by + // NOTE(Aurelius84): we should skip insert ShadowOutputOp repeatly by // calling SplitForwardBackward multi-times. - std::string parameter_name = + std::string shadow_output_name = std::string("output_") + std::to_string(counter); std::unordered_set inserted_value; for (auto it = forward_program->block()->rbegin(); it != forward_program->block()->rend(); ++it) { - if (it->isa()) { + if (it->isa()) { auto out_name = - it->attribute("parameter_name").AsString(); - if (out_name == parameter_name) { + it->attribute("output_name").AsString(); + if (out_name == shadow_output_name) { VLOG(4) << out_name - << " has been inserted SetParameterOp, skip it now."; + << " has been inserted ShadowOutputOp, skip it now."; return; } @@ -1228,9 +1228,9 @@ SplitedResult SplitForwardBackward( if (inserted_value.count(forward_value_map[v])) { return; } - auto op_info = ctx->GetRegisteredOpInfo(pir::SetParameterOp::name()); + auto op_info = ctx->GetRegisteredOpInfo(pir::ShadowOutputOp::name()); pir::AttributeMap attribute_map = { - {"parameter_name", pir::StrAttribute::get(ctx, parameter_name)}, + {"output_name", pir::StrAttribute::get(ctx, shadow_output_name)}, }; pir::Operation *operation = pir::Operation::Create( {forward_value_map[v]}, attribute_map, {}, op_info); @@ -1245,9 +1245,9 @@ SplitedResult SplitForwardBackward( if (v.impl() == nullptr) { return; } - auto op_info = ctx->GetRegisteredOpInfo(pir::SetParameterOp::name()); + auto op_info = ctx->GetRegisteredOpInfo(pir::ShadowOutputOp::name()); pir::AttributeMap attribute_map = { - {"parameter_name", + {"output_name", pir::StrAttribute::get( ctx, std::string("output_") + std::to_string(counter))}, }; @@ -1372,10 +1372,10 @@ pir::Type CreateSelectedRowsTypeByDenseTensor(pir::Type dense_tensor_type) { } } -void ResetParameterName(pir::Operation *op, const std::string &name) { +void ResetShadowOutputName(pir::Operation *op, const std::string &name) { pir::IrContext *ctx = pir::IrContext::Instance(); - if (op->isa()) { - op->set_attribute("parameter_name", pir::StrAttribute::get(ctx, name)); + if (op->isa()) { + op->set_attribute("output_name", pir::StrAttribute::get(ctx, name)); } } @@ -1410,9 +1410,9 @@ std::map GetOpInplaceInfo(const pir::Operation *op) { void BindUtils(pybind11::module *m) { m->def("clone_program", CloneProgram); m->def("get_op_inplace_info", GetOpInplaceInfo); - m->def("reset_parameter_name", ResetParameterName); + m->def("reset_shadow_output_name", ResetShadowOutputName); m->def("split_program", SplitForwardBackward); - m->def("append_set_parameters", AppendSetParameters); + m->def("append_shadow_outputs", AppendShadowOutputs); m->def("fake_op_result", FakeOpResult); m->def("is_fake_op_result", IsFakeOpResult); m->def("get_current_insertion_point", []() -> PyInsertionPoint { diff --git a/python/paddle/jit/dy2static/pir_partial_program.py b/python/paddle/jit/dy2static/pir_partial_program.py index 2b1f6c6b47874..a5858df1886e8 100644 --- a/python/paddle/jit/dy2static/pir_partial_program.py +++ b/python/paddle/jit/dy2static/pir_partial_program.py @@ -103,7 +103,7 @@ def union(self, x, y): self.father[father_x] = father_y def find_root(self, x): - if not self.father.__contains__(x): + if x not in self.father: self.father[x] = x if self.father[x].is_same(x): return x @@ -135,24 +135,29 @@ def _get_value_name_map_from_program(cls, program): ret = ValueDict() ret[fake_op_result()] = "FakeVar" for op in program.global_block().ops: - if op.name() == "pd_op.data": - ret[op.result(0)] = op.attrs()["name"] if op.name() == "builtin.set_parameter": ret[op.operand(0).source()] = op.attrs()["parameter_name"] - if op.name() == "builtin.parameter": + elif op.name() == "builtin.parameter": ret[op.result(0)] = op.attrs()["parameter_name"] + elif op.name() == "builtin.shadow_output": + ret[op.operand(0).source()] = op.attrs()["output_name"] + elif op.name() == "pd_op.data": + ret[op.result(0)] = op.attrs()["name"] return ret @classmethod def _get_name_defining_op(cls, program, value): for op in program.global_block().ops: - if op.name() == "pd_op.data": + if op.name() == "builtin.set_parameter": + if value.is_same(op.operand(0).source()): + return op + elif op.name() == "builtin.parameter": if value.is_same(op.result(0)): return op - if op.name() == "builtin.set_parameter": + elif op.name() == "builtin.shadow_output": if value.is_same(op.operand(0).source()): return op - if op.name() == "builtin.parameter": + elif op.name() == "pd_op.data": if value.is_same(op.result(0)): return op return None @@ -291,7 +296,7 @@ def _forward_backward_program(self): def program_attr(self): assert ( self.finish_pass is False - ), "program_attr() is called by PartialProgramLayer, don't call it matually, use program_name_attr instead." + ), "program_attr() is called by PartialProgramLayer, don't call it manually, use program_name_attr instead." # can't apply pass after call this function. self.finish_pass = True fwd_map = { @@ -346,7 +351,7 @@ def has_name(value): if has_name(ufset.find_root(value)): name_defining_op = self._get_name_defining_op(program, value) if name_defining_op: - paddle.core.pir.reset_parameter_name( + paddle.core.pir.reset_shadow_output_name( name_defining_op, value2name[ufset.find_root(value)] ) @@ -384,8 +389,8 @@ class PirPassContext: """ INPUT_OP_NAME = "pd_op.data" - PARM_OP_NAME = "builtin.parameter" - OUTPUT_OP_NAME = "builtin.set_parameter" + PARAM_OP_NAME = "builtin.parameter" + OUTPUT_OP_NAME = "builtin.shadow_output" @classmethod def apply(cls, runable_program, build_strategy): @@ -419,7 +424,7 @@ def _prepare_attr(cls, program): op_name = op.name() if op_name == cls.INPUT_OP_NAME: inputs.append(op.result(0)) - elif op_name == cls.PARM_OP_NAME: + elif op_name == cls.PARAM_OP_NAME: params.append(op.result(0)) elif op_name == cls.OUTPUT_OP_NAME: outputs.append(op.operand(0).source()) @@ -546,7 +551,7 @@ def origin_runable_program(self): inputs = list(self._inputs.var_list) outputs = list(self._outputs.var_list) params = self._param_values - paddle.base.libpaddle.pir.append_set_parameters( + paddle.base.libpaddle.pir.append_shadow_outputs( self._origin_main_program, outputs, len(self._origin_main_program.global_block().ops), @@ -796,7 +801,7 @@ def _append_backward_desc(self, train_runnable_program: RunableProgram): dtype=out_op_result.dtype, ) forward_outputs_grads.append(value) - paddle.base.libpaddle.pir.append_set_parameters( + paddle.base.libpaddle.pir.append_shadow_outputs( program, forward_outputs_grads, len(program.global_block().ops), @@ -861,7 +866,7 @@ def _append_backward_desc(self, train_runnable_program: RunableProgram): ) ) backward_end_op_index = len(program.global_block().ops) - paddle.base.libpaddle.pir.append_set_parameters( + paddle.base.libpaddle.pir.append_shadow_outputs( program, output_grads_to_append, backward_end_op_index, diff --git a/python/paddle/jit/pir_dy2static/parameter_recorder.py b/python/paddle/jit/pir_dy2static/parameter_recorder.py index 565dad78f394d..538ec04f265a9 100644 --- a/python/paddle/jit/pir_dy2static/parameter_recorder.py +++ b/python/paddle/jit/pir_dy2static/parameter_recorder.py @@ -81,7 +81,7 @@ def get(self, program, value): return None root_var = inplace_dict[value] saved = [] - while inplace_dict.__contains__(root_var): + while root_var in inplace_dict: saved.append(root_var) root_var = inplace_dict[root_var] for var in saved: diff --git a/test/dygraph_to_static/test_tensor_memcpy_on_cpu.py b/test/dygraph_to_static/test_tensor_memcpy_on_cpu.py index 0b92fae0556bb..ccf0b35ee4d29 100644 --- a/test/dygraph_to_static/test_tensor_memcpy_on_cpu.py +++ b/test/dygraph_to_static/test_tensor_memcpy_on_cpu.py @@ -18,7 +18,6 @@ from dygraph_to_static_utils import ( Dy2StTestBase, enable_to_static_guard, - test_legacy_and_pt, test_legacy_and_pt_and_pir, ) @@ -69,7 +68,7 @@ def _run(self): x2 = paddle.jit.to_static(tensor_copy_to_cuda)(x1) return x1.place, x2.place, x2.numpy() - @test_legacy_and_pt + @test_legacy_and_pt_and_pir def test_tensor_cuda_on_default_cpu(self): if not paddle.is_compiled_with_cuda(): return From beba862cd2aa4dd2b14cdd0c6c4c08be33df62f2 Mon Sep 17 00:00:00 2001 From: Wang Xin Date: Thu, 28 Dec 2023 15:39:18 +0800 Subject: [PATCH 117/146] =?UTF-8?q?=E3=80=90Hackathon=205th=20No.25?= =?UTF-8?q?=E3=80=91add=20`gammaln`=20api=20(#59311)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paddle/phi/api/yaml/backward.yaml | 10 ++ paddle/phi/api/yaml/ops.yaml | 10 ++ paddle/phi/kernels/cpu/gammaln_grad_kernel.cc | 22 +++ paddle/phi/kernels/cpu/gammaln_kernel.cc | 22 +++ paddle/phi/kernels/gammaln_grad_kernel.h | 27 +++ paddle/phi/kernels/gammaln_kernel.h | 26 +++ paddle/phi/kernels/gpu/gammaln_grad_kernel.cu | 30 ++++ paddle/phi/kernels/gpu/gammaln_kernel.cu | 29 ++++ .../kernels/impl/gammaln_grad_kernel_impl.h | 92 ++++++++++ paddle/phi/kernels/impl/gammaln_kernel_impl.h | 49 ++++++ python/paddle/__init__.py | 4 + python/paddle/tensor/__init__.py | 4 + python/paddle/tensor/math.py | 45 +++++ test/legacy_test/test_gammaln_op.py | 160 ++++++++++++++++++ test/legacy_test/test_inplace.py | 8 + 15 files changed, 538 insertions(+) create mode 100644 paddle/phi/kernels/cpu/gammaln_grad_kernel.cc create mode 100644 paddle/phi/kernels/cpu/gammaln_kernel.cc create mode 100644 paddle/phi/kernels/gammaln_grad_kernel.h create mode 100644 paddle/phi/kernels/gammaln_kernel.h create mode 100644 paddle/phi/kernels/gpu/gammaln_grad_kernel.cu create mode 100644 paddle/phi/kernels/gpu/gammaln_kernel.cu create mode 100644 paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h create mode 100644 paddle/phi/kernels/impl/gammaln_kernel_impl.h create mode 100644 test/legacy_test/test_gammaln_op.py diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index 938ea9d500046..d5748145ffe49 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -922,6 +922,16 @@ kernel : func : frame_grad +- backward_op : gammaln_grad + forward : gammaln(Tensor x) -> Tensor(out) + args : (Tensor x, Tensor out_grad) + output : Tensor(x_grad) + infer_meta : + func : UnchangedInferMeta + param: [x] + kernel : + func : gammaln_grad + - backward_op : gather_grad forward : gather(Tensor x, Tensor index, Scalar axis=0) -> Tensor(out) args : (Tensor x, Tensor index, Tensor out_grad, Scalar axis=0) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index de4d700cdf80e..dc545b7a2da54 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -1042,6 +1042,16 @@ data_type : dtype backend : place +- op : gammaln + args : (Tensor x) + output : Tensor(out) + infer_meta : + func : UnchangedInferMeta + kernel : + func : gammaln + inplace: (x -> out) + backward : gammaln_grad + - op : gather args : (Tensor x, Tensor index, Scalar axis=0) output : Tensor(out) diff --git a/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc b/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc new file mode 100644 index 0000000000000..c52ee8b3848e9 --- /dev/null +++ b/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/gammaln_grad_kernel.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h" + +PD_REGISTER_KERNEL( + gammaln_grad, CPU, ALL_LAYOUT, phi::GammalnGradKernel, float, double) {} diff --git a/paddle/phi/kernels/cpu/gammaln_kernel.cc b/paddle/phi/kernels/cpu/gammaln_kernel.cc new file mode 100644 index 0000000000000..ff62f86d2522f --- /dev/null +++ b/paddle/phi/kernels/cpu/gammaln_kernel.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/gammaln_kernel.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" +#include "paddle/phi/kernels/impl/gammaln_kernel_impl.h" + +PD_REGISTER_KERNEL( + gammaln, CPU, ALL_LAYOUT, phi::GammalnKernel, float, double) {} diff --git a/paddle/phi/kernels/gammaln_grad_kernel.h b/paddle/phi/kernels/gammaln_grad_kernel.h new file mode 100644 index 0000000000000..440dca72a9d46 --- /dev/null +++ b/paddle/phi/kernels/gammaln_grad_kernel.h @@ -0,0 +1,27 @@ + +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void GammalnGradKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& d_out, + DenseTensor* d_x); +} // namespace phi diff --git a/paddle/phi/kernels/gammaln_kernel.h b/paddle/phi/kernels/gammaln_kernel.h new file mode 100644 index 0000000000000..db3015c4a747d --- /dev/null +++ b/paddle/phi/kernels/gammaln_kernel.h @@ -0,0 +1,26 @@ + +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void GammalnKernel(const Context& dev_ctx, + const DenseTensor& x, + DenseTensor* out); +} // namespace phi diff --git a/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu b/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu new file mode 100644 index 0000000000000..b2513d9e3f25c --- /dev/null +++ b/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu @@ -0,0 +1,30 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/gammaln_grad_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/core/kernel_registry.h" + +#include "paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h" + +PD_REGISTER_KERNEL(gammaln_grad, + GPU, + ALL_LAYOUT, + phi::GammalnGradKernel, + float, + double, + phi::dtype::float16, + phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/gpu/gammaln_kernel.cu b/paddle/phi/kernels/gpu/gammaln_kernel.cu new file mode 100644 index 0000000000000..3d57be7b27733 --- /dev/null +++ b/paddle/phi/kernels/gpu/gammaln_kernel.cu @@ -0,0 +1,29 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/gammaln_kernel.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +#include "paddle/phi/kernels/impl/gammaln_kernel_impl.h" + +PD_REGISTER_KERNEL(gammaln, + GPU, + ALL_LAYOUT, + phi::GammalnKernel, + float, + double, + phi::dtype::float16, + phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h b/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h new file mode 100644 index 0000000000000..50c73cff27ce4 --- /dev/null +++ b/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h @@ -0,0 +1,92 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/kernels/funcs/for_range.h" + +namespace phi { +template +HOSTDEVICE T digamma(T x) { + static T c = T{8.5}; + static T euler_mascheroni = T{0.57721566490153286060}; + T r; + T value; + T x2; + + if (x <= T{0.0}) { + value = T{0.0}; + return value; + } + + if (x <= T{0.000001}) { + value = -euler_mascheroni - T{1.0} / x + T{1.6449340668482264365} * x; + return value; + } + + value = T{0.0}; + x2 = x; + while (x2 < c) { + value = value - T{1.0} / x2; + x2 = x2 + T{1.0}; + } + + r = T{1.0} / x2; + value = value + std::log(x2) - T{0.5} * r; + + r = r * r; + + value = value - + r * (T{1.0} / T{12.0} - + r * (T{1.0} / T{120.0} - + r * (T{1.0} / T{252.0} - + r * (T{1.0} / T{240.0} - r * (T{1.0} / T{132.0}))))); + + return value; +} + +template +struct GammalnGradFunctor { + GammalnGradFunctor(const T* dout, const T* x, T* output, int64_t numel) + : dout_(dout), x_(x), output_(output), numel_(numel) {} + + HOSTDEVICE void operator()(int64_t idx) const { + using MT = typename phi::dtype::MPTypeTrait::Type; + const MT mp_dout = static_cast(dout_[idx]); + const MT mp_x = static_cast(x_[idx]); + output_[idx] = static_cast(mp_dout * digamma(mp_x)); + } + + private: + const T* dout_; + const T* x_; + T* output_; + int64_t numel_; +}; +template +void GammalnGradKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& d_out, + DenseTensor* d_x) { + auto numel = d_out.numel(); + auto* dout_data = d_out.data(); + auto* x_data = x.data(); + auto* dx_data = + dev_ctx.template Alloc(d_x, static_cast(numel * sizeof(T))); + phi::funcs::ForRange for_range(dev_ctx, numel); + GammalnGradFunctor functor(dout_data, x_data, dx_data, numel); + for_range(functor); +} +} // namespace phi diff --git a/paddle/phi/kernels/impl/gammaln_kernel_impl.h b/paddle/phi/kernels/impl/gammaln_kernel_impl.h new file mode 100644 index 0000000000000..38385610de0de --- /dev/null +++ b/paddle/phi/kernels/impl/gammaln_kernel_impl.h @@ -0,0 +1,49 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/phi/common/amp_type_traits.h" +#include "paddle/phi/kernels/funcs/for_range.h" + +namespace phi { +template +struct GammalnFunctor { + GammalnFunctor(const T* x, T* output, int64_t numel) + : x_(x), output_(output), numel_(numel) {} + + HOSTDEVICE void operator()(int64_t idx) const { + using MT = typename phi::dtype::MPTypeTrait::Type; + const MT mp_x = static_cast(x_[idx]); + output_[idx] = static_cast(std::lgamma(mp_x)); + } + + private: + const T* x_; + T* output_; + int64_t numel_; +}; + +template +void GammalnKernel(const Context& dev_ctx, + const DenseTensor& x, + DenseTensor* out) { + auto numel = x.numel(); + auto* x_data = x.data(); + auto* out_data = dev_ctx.template Alloc(out); + phi::funcs::ForRange for_range(dev_ctx, numel); + GammalnFunctor functor(x_data, out_data, numel); + for_range(functor); +} +} // namespace phi diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index fc7b2a3533f89..1f0017562ebad 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -398,6 +398,8 @@ frac, frac_, frexp, + gammaln, + gammaln_, gcd, gcd_, heaviside, @@ -773,6 +775,8 @@ 'square_', 'divide', 'divide_', + 'gammaln', + 'gammaln_', 'ceil', 'atan', 'atan_', diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index b26798892a2b2..b718910348d8f 100644 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -278,6 +278,8 @@ frac, frac_, frexp, + gammaln, + gammaln_, gcd, gcd_, heaviside, @@ -668,6 +670,8 @@ 'real', 'imag', 'is_floating_point', + 'gammaln', + 'gammaln_', 'digamma', 'digamma_', 'diagonal', diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index acaa0905ce6f4..6d75d41b4949c 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -5003,6 +5003,51 @@ def conj(x, name=None): return out +def gammaln(x, name=None): + r""" + Calculates the logarithm of the absolute value of the gamma function elementwisely. + + Args: + x (Tensor): Input Tensor. Must be one of the following types: float16, float32, float64, bfloat16. + name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Tensor, The values of the logarithm of the absolute value of the gamma at the given tensor x. + + Examples: + .. code-block:: python + + >>> import paddle + + >>> x = paddle.arange(1.5, 4.5, 0.5) + >>> out = paddle.gammaln(x) + >>> print(out) + Tensor(shape=[6], dtype=float32, place=Place(cpu), stop_gradient=True, + [-0.12078224, 0. , 0.28468287, 0.69314718, 1.20097363, + 1.79175949]) + """ + if in_dynamic_or_pir_mode(): + return _C_ops.gammaln(x) + else: + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64', 'bfloat16'], 'gammaln' + ) + helper = LayerHelper('gammaln', **locals()) + out = helper.create_variable_for_type_inference(x.dtype) + helper.append_op(type='gammaln', inputs={'x': x}, outputs={'out': out}) + return out + + +@inplace_apis_in_dygraph_only +def gammaln_(x, name=None): + r""" + Inplace version of ``gammaln`` API, the output Tensor will be inplaced with input ``x``. + Please refer to :ref:`api_paddle_gammaln`. + """ + if in_dynamic_mode(): + return _C_ops.gammaln_(x) + + def digamma(x, name=None): r""" Calculates the digamma of the given input tensor, element-wise. diff --git a/test/legacy_test/test_gammaln_op.py b/test/legacy_test/test_gammaln_op.py new file mode 100644 index 0000000000000..50331af5c7a34 --- /dev/null +++ b/test/legacy_test/test_gammaln_op.py @@ -0,0 +1,160 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +from op_test import OpTest, convert_float_to_uint16 +from scipy import special + +import paddle +from paddle.base import core + + +def ref_gammaln(x): + return special.gammaln(x) + + +def ref_gammaln_grad(x, dout): + return dout * special.polygamma(0, x) + + +class TestGammalnOp(OpTest): + def setUp(self): + self.op_type = 'gammaln' + self.python_api = paddle.gammaln + self.init_dtype_type() + self.shape = (3, 40) + self.x = np.random.random(self.shape).astype(self.dtype) + 1 + self.inputs = {'x': self.x} + out = ref_gammaln(self.x) + self.outputs = {'out': out} + + def init_dtype_type(self): + self.dtype = np.float64 + + def test_check_output(self): + self.check_output(check_pir=True) + + def test_check_grad(self): + self.check_grad(['x'], 'out', check_pir=True) + + +class TestGammalnOpFp32(TestGammalnOp): + def init_dtype_type(self): + self.dtype = np.float32 + + +class TestGammalnFP16Op(TestGammalnOp): + def init_dtype_type(self): + self.dtype = np.float16 + + +class TestGammalnBigNumberOp(TestGammalnOp): + def setUp(self): + self.op_type = 'gammaln' + self.python_api = paddle.gammaln + self.init_dtype_type() + self.shape = (100, 1) + self.x = np.random.random(self.shape).astype(self.dtype) + 1 + self.x[:5, 0] = np.array([1e5, 1e10, 1e20, 1e40, 1e80]) + self.inputs = {'x': self.x} + out = ref_gammaln(self.x) + self.outputs = {'out': out} + + def init_dtype_type(self): + self.dtype = np.float64 + + def test_check_grad(self): + d_out = self.outputs['out'] + d_x = ref_gammaln_grad(self.x, d_out) + self.check_grad( + ['x'], + 'out', + user_defined_grads=[ + d_x, + ], + user_defined_grad_outputs=[ + d_out, + ], + check_pir=True, + ) + + +@unittest.skipIf( + not core.is_compiled_with_cuda() + or not core.is_bfloat16_supported(core.CUDAPlace(0)), + "core is not compiled with CUDA or not support bfloat16", +) +class TestGammalnBF16Op(OpTest): + def setUp(self): + self.op_type = 'gammaln' + self.python_api = paddle.gammaln + self.dtype = np.uint16 + self.shape = (5, 30) + x = np.random.random(self.shape).astype("float32") + 1 + self.inputs = {'x': convert_float_to_uint16(x)} + out = ref_gammaln(x) + self.outputs = {'out': convert_float_to_uint16(out)} + + def test_check_output(self): + self.check_output_with_place(core.CUDAPlace(0), check_pir=True) + + def test_check_grad(self): + self.check_grad_with_place( + core.CUDAPlace(0), ['x'], 'out', check_pir=True + ) + + +class TestGammalnOpApi(unittest.TestCase): + def setUp(self): + self.shape = [2, 3, 4, 5] + self.init_dtype_type() + self.x_np = np.random.random(self.shape).astype(self.dtype) + 1 + self.place = ( + paddle.CUDAPlace(0) + if core.is_compiled_with_cuda() + else paddle.CPUPlace() + ) + + def init_dtype_type(self): + self.dtype = "float64" + + def test_static_api(self): + paddle.enable_static() + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data('x', self.x_np.shape, self.x_np.dtype) + out = paddle.gammaln(x) + exe = paddle.static.Executor(self.place) + (res,) = exe.run(feed={'x': self.x_np}, fetch_list=[out]) + out_ref = ref_gammaln(self.x_np) + np.testing.assert_allclose(out_ref, res, rtol=1e-5, atol=1e-5) + + def test_dygraph_api(self): + paddle.disable_static(self.place) + x = paddle.to_tensor(self.x_np) + out = paddle.gammaln(x) + out_ref = ref_gammaln(self.x_np) + np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-5, atol=1e-5) + paddle.enable_static() + + +class TestGammalnOpApiFp32(TestGammalnOpApi): + def init_dtype_type(self): + self.dtype = "float32" + + +if __name__ == "__main__": + paddle.enable_static() + unittest.main() diff --git a/test/legacy_test/test_inplace.py b/test/legacy_test/test_inplace.py index 42f9a46cfb910..38fbac0357d6d 100644 --- a/test/legacy_test/test_inplace.py +++ b/test/legacy_test/test_inplace.py @@ -869,6 +869,14 @@ def test_leaf_inplace_var_error(self): pass +class TestDygraphInplaceGammaln(TestDygraphInplaceWithContinuous): + def inplace_api_processing(self, var): + return paddle.gammaln_(var) + + def non_inplace_api_processing(self, var): + return paddle.gammaln(var) + + class TestDygraphInplaceNeg(TestDygraphInplaceWithContinuous): def inplace_api_processing(self, var): return paddle.neg_(var) From b03482a24c8b5e6f1e44329c0f6a397d370f6061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=82=85=E5=89=91=E5=AF=92?= Date: Thu, 28 Dec 2023 15:47:30 +0800 Subject: [PATCH 118/146] complete dsl test case for dynamic schedule primitive (#60428) --- .../ir/schedule/impl/loop_transformation.cc | 3 +- .../ir/test_llir_schedule_cache_read_write.py | 61 +++++++++++- test/cinn/ir/test_llir_schedule_compute_at.py | 93 +++++++++++++++++++ .../ir/test_llir_schedule_compute_inline.py | 77 +++++++++++++++ test/cinn/ir/test_llir_schedule_fuse_split.py | 84 +++++++++++++++++ test/cinn/ir/test_llir_schedule_reorder.py | 66 +++++++++++++ 6 files changed, 379 insertions(+), 5 deletions(-) diff --git a/paddle/cinn/ir/schedule/impl/loop_transformation.cc b/paddle/cinn/ir/schedule/impl/loop_transformation.cc index 4577db7770a73..c3a3ad448f536 100644 --- a/paddle/cinn/ir/schedule/impl/loop_transformation.cc +++ b/paddle/cinn/ir/schedule/impl/loop_transformation.cc @@ -114,7 +114,8 @@ std::vector DyScheduleImpl::Split(const Expr& loop, for (auto factor : factors) prod_size = prod_size * Expr(factor); std::for_each(factors.begin(), factors.end(), [&](int factor) { if (factor == -1) { - process_factors.push_back(tot_extent / prod_size + Expr(1)); + process_factors.push_back( + cinn::common::AutoSimplify(tot_extent / prod_size + Expr(1))); } else { process_factors.push_back(Expr(factor)); } diff --git a/test/cinn/ir/test_llir_schedule_cache_read_write.py b/test/cinn/ir/test_llir_schedule_cache_read_write.py index 41f1fc8d342ab..7dd8cb488e918 100644 --- a/test/cinn/ir/test_llir_schedule_cache_read_write.py +++ b/test/cinn/ir/test_llir_schedule_cache_read_write.py @@ -28,6 +28,7 @@ def elementwise_add_cache_read( Y: DataArray((128, 128)), A: DataArray((128, 128)), A_local_temp_buffer: DataArray((128, 128)), + N: ir.Var(), ): for i in range(128): for j in range(128): @@ -49,6 +50,7 @@ def elementwise_add_cache_read( Y: DataArray((128, 128)), A: DataArray((128, 128)), A_local_temp_buffer: DataArray((128, 128)), + N: ir.Var(), ): for i in range(128): for j in range(128): @@ -68,10 +70,6 @@ def elementwise_add_cache_read( i1, j1 = ir.AxisMap("SS", [i3, j3]) Y[i1, j1] = -A_local_temp_buffer[i1, j1] + 3.0 - assert str(origin.elementwise_add_cache_read) == str( - expected.elementwise_add_cache_read - ) - def test_cache_write_elementwise(): @to_cinn_llir @@ -98,6 +96,61 @@ def elementwise_add_cache_write( # assert_llir_equal(elementwise_add_cache_write, elementwise_add_cache_write) +def test_cache_read_elementwise_dynamic(): + class origin: + @to_cinn_llir + def elementwise_add_cache_read( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + A_local_temp_buffer: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for i3 in range(N): + for j3 in range(128): + with ir.ScheduleBlockContext("B") as B_block: + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A[i1, j1] + 3.0 + + cached_b = sch.cache_read(B_block.block, 0, "local") + + class expected: + @to_cinn_llir + def elementwise_add_cache_read( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + A_local_temp_buffer: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for cache_ax0 in range(N): + for cache_ax1 in range(128): + with ir.ScheduleBlockContext( + "A_local_temp_buffer" + ) as A_local_temp_buffer_block: + v0, v1 = ir.AxisMap("SS", [cache_ax0, cache_ax1]) + A_local_temp_buffer[v0, v1] = A[v0, v1] + for i3 in range(N): + for j3 in range(128): + with ir.ScheduleBlockContext("B") as B_block: + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A_local_temp_buffer[i1, j1] + 3.0 + + assert str(origin.elementwise_add_cache_read) == str( + expected.elementwise_add_cache_read + ) + + if __name__ == "__main__": test_cache_read_elementwise() test_cache_write_elementwise() diff --git a/test/cinn/ir/test_llir_schedule_compute_at.py b/test/cinn/ir/test_llir_schedule_compute_at.py index 0f82786935b41..4c96ff23436ae 100644 --- a/test/cinn/ir/test_llir_schedule_compute_at.py +++ b/test/cinn/ir/test_llir_schedule_compute_at.py @@ -106,6 +106,99 @@ def reverse_compute_at_tiled_gt( assert_llir_equal(reverse_compute_at_tiled, reverse_compute_at_tiled_gt) +def test_compute_at_elementwise_dynamic(): + @to_cinn_llir + def elementwise_add( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("Y"): + i1, j1 = ir.AxisMap("SS", [i, j]) + sch.compute_at(A_block.block, i, False) + Y[i1, j1] = A[i1, j1] + 2.0 + + @to_cinn_llir + def elementwise_add_gt( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A"): + i1, j1 = ir.AxisMap("SS", [i, 0 + j]) + A[i1, j1] = X[i1, j1] * 2.0 + for k in range(128): + with ir.ScheduleBlockContext("Y"): + i2, k1 = ir.AxisMap("SS", [i, k]) + Y[i2, k1] = A[i2, k1] + 2.0 + + assert_llir_equal(elementwise_add, elementwise_add_gt) + + +def test_reverse_compute_at_dynamic(): + @to_cinn_llir + def reverse_compute_at_tiled( + A: DataArray((-1, 128)), + B: DataArray((-1, 128)), + C: DataArray((-1, 128)), + N: ir.Var(), + ): + for i0 in range(N / 16): + for j0 in range(8): + for i1 in range(16): + for j1 in range(16): + with ir.ScheduleBlockContext("B") as B_block: + vi, vj = ir.AxisMap( + "SS", [i0 * 16 + i1, j0 * 16 + j1] + ) + B[vi, vj] = A[vi, vj] * 2.0 + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("C") as C_block: + vi, vj = ir.AxisMap("SS", [i, j]) + C[vi, vj] = B[vi, vj] + 1.0 + + sch.reverse_compute_at(C_block.block, B_block.i1) + + @to_cinn_llir + def reverse_compute_at_tiled_gt( + A: DataArray((-1, 128)), + B: DataArray((-1, 128)), + C: DataArray((-1, 128)), + N: ir.Var(), + ): + for i0 in range(N / 16): + for j0 in range(8): + for i1 in range(16): + for j1 in range(16): + with ir.ScheduleBlockContext("B") as B_block: + vi, vj = ir.AxisMap( + "SS", [i0 * 16 + i1, j0 * 16 + j1] + ) + B[vi, vj] = A[vi, vj] * 2.0 + for j2 in range(16): + with ir.ScheduleBlockContext("C") as C_block: + vi, vj = ir.AxisMap( + "SS", [16 * i0 + i1, 16 * j0 + j2] + ) + C[vi, vj] = B[vi, vj] + 1.0 + + assert_llir_equal(reverse_compute_at_tiled, reverse_compute_at_tiled_gt) + + if __name__ == '__main__': test_compute_at_elementwise() test_reverse_compute_at() + test_compute_at_elementwise_dynamic() + test_reverse_compute_at_dynamic() diff --git a/test/cinn/ir/test_llir_schedule_compute_inline.py b/test/cinn/ir/test_llir_schedule_compute_inline.py index a95d1dd817449..113c0b7dfe621 100644 --- a/test/cinn/ir/test_llir_schedule_compute_inline.py +++ b/test/cinn/ir/test_llir_schedule_compute_inline.py @@ -90,6 +90,83 @@ def elementwise_add_inline_gt( assert_llir_equal(elementwise_add_inline, elementwise_add_inline_gt) +def test_compute_inline_elementwise_dynamic(): + @to_cinn_llir + def elementwise_add_inline( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for i3 in range(N): + for j3 in range(128): + with ir.ScheduleBlockContext("Y"): + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A[i1, j1] + 3.0 + + block_a = sch.get_block("A") + sch.compute_inline(block_a) + + @to_cinn_llir + def elementwise_add_inline_gt( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("Y"): + i1, j1 = ir.AxisMap("SS", [i, j]) + Y[i1, j1] = -(X[i1, j1] * 2.0) + 3.0 + + assert_llir_equal(elementwise_add_inline, elementwise_add_inline_gt) + + +def test_reverse_compute_inline_elementwise_dynamic(): + @to_cinn_llir + def elementwise_add_inline( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A") as A_block: + i1, j1 = ir.AxisMap("SS", [i, j]) + A[i1, j1] = X[i1, j1] * 2.0 + for i3 in range(-1): + for j3 in range(128): + with ir.ScheduleBlockContext("Y") as Y_block: + i1, j1 = ir.AxisMap("SS", [i3, j3]) + Y[i1, j1] = -A[i1, j1] + 3.0 + + sch.reverse_compute_inline(Y_block.block) + + @to_cinn_llir + def elementwise_add_inline_gt( + X: DataArray((-1, 128)), + Y: DataArray((-1, 128)), + A: DataArray((-1, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + with ir.ScheduleBlockContext("A"): + i1, j1 = ir.AxisMap("SS", [i, j]) + Y[i1, j1] = -(X[i1, j1] * 2.0) + 3.0 + + assert_llir_equal(elementwise_add_inline, elementwise_add_inline_gt) + + if __name__ == "__main__": test_compute_inline_elementwise() test_reverse_compute_inline_elementwise() + test_compute_inline_elementwise_dynamic() + test_reverse_compute_inline_elementwise_dynamic() diff --git a/test/cinn/ir/test_llir_schedule_fuse_split.py b/test/cinn/ir/test_llir_schedule_fuse_split.py index 07712590b9ac1..362cb81f87b96 100644 --- a/test/cinn/ir/test_llir_schedule_fuse_split.py +++ b/test/cinn/ir/test_llir_schedule_fuse_split.py @@ -125,7 +125,91 @@ def elementwise_split_predicate_gt( ) +def test_fuse_dynamic(): + class origin: + @to_cinn_llir + def elementwise_fuse_assign_loop( + X: DataArray((-1, 128, 128)), + Y: DataArray((-1, 128, 128)), + N: ir.Var(), + ): + for i in range(N): + for j in range(128): + for k in range(128): + with ir.ScheduleBlockContext("Y") as block_y: + sch.fuse([i, j, k]) + i1, j1, k1 = ir.AxisMap("SSS", [i, j, k]) + Y[i1, j1, k1] = X[i1, j1, k1] * 2.0 + + class expected: + @to_cinn_llir + def elementwise_fuse_assign_loop( + X: DataArray((-1, 128, 128)), + Y: DataArray((-1, 128, 128)), + N: ir.Var(), + ): + for i_j_k_fused in range(((1 * N) * 128) * 128): + with ir.ScheduleBlockContext("Y") as block_y: + i1, j1, k1 = ir.AxisMap( + "SSS", + [ + (i_j_k_fused / 128) / 128, + (i_j_k_fused / 128) % 128, + i_j_k_fused % 128, + ], + ) + Y[i1, j1, k1] = 2.0 * X[i1, j1, k1] + + assert str(origin.elementwise_fuse_assign_loop) == str( + expected.elementwise_fuse_assign_loop + ) + + +def test_split_dynamic(): + class origin: + @to_cinn_llir + def elementwise_split( + X: DataArray((128, 128, -1)), + Y: DataArray((128, 128, -1)), + N: ir.Var(), + ): + for i in range(128): + for j in range(128): + for k in range(N): + with ir.ScheduleBlockContext("Y") as Y_block: + i1, j1, k1 = ir.AxisMap("SSS", [i, j, k]) + sch.split(Y_block.k, factors=[16, -1]) + Y[i1, j1, k1] = X[i1, j1, k1] * 2.0 + + class expected: + @to_cinn_llir + def elementwise_split( + X: DataArray((128, 128, -1)), + Y: DataArray((128, 128, -1)), + N: ir.Var(), + ): + for i in range(128): + for j in range(128): + for k_7 in range(16): + for k_8 in range((N / 16) + 1): + if (((N / 16) * k_7) + (k_7 + k_8)) < N: + with ir.ScheduleBlockContext("Y") as Y_block: + i1, j1, k1 = ir.AxisMap( + "SSS", + [ + i, + j, + (((N / 16) * k_7) + (k_7 + k_8)), + ], + ) + Y[i1, j1, k1] = X[i1, j1, k1] * 2.0 + + assert_llir_equal(origin.elementwise_split, expected.elementwise_split) + + if __name__ == "__main__": test_fuse() test_split() test_split_predicate() + test_fuse_dynamic() + test_split_dynamic() diff --git a/test/cinn/ir/test_llir_schedule_reorder.py b/test/cinn/ir/test_llir_schedule_reorder.py index 00ca99388ba94..254197beb222a 100644 --- a/test/cinn/ir/test_llir_schedule_reorder.py +++ b/test/cinn/ir/test_llir_schedule_reorder.py @@ -75,6 +75,72 @@ def reorder_overlapped_gt(X: DataArray((28, 8)), Y: DataArray((28, 8))): assert_llir_equal(reorder_overlapped, reorder_overlapped_gt) +def test_reorder_elementwise_dynamic(): + @to_cinn_llir + def reorder_elementwise( + X: DataArray((-1, 64, 64, 64)), + Y: DataArray((-1, 64, 64, 64)), + N: ir.Var(), + ): + for i in range(N): + for j in range(64): + for k in range(64): + for l in range(8): + with ir.ScheduleBlockContext("Y") as Y_block: + vi, vj, vk, vl = ir.AxisMap( + "SSSS", [i, j, k, 8 * l] + ) + Y[vi, vj, vk, vl] = X[vi, vj, vk, vl] * 2.0 + sch.reorder([Y_block.k, Y_block.l, Y_block.i]) + + @to_cinn_llir + def reorder_elementwise_gt( + X: DataArray((-1, 64, 64, 64)), + Y: DataArray((-1, 64, 64, 64)), + N: ir.Var(), + ): + for k in range(64): + for j in range(64): + for l in range(8): + for i in range(N): + with ir.ScheduleBlockContext("Y"): + vi, vj, vk, vl = ir.AxisMap( + "SSSS", [i, j, k, 8 * l] + ) + Y[vi, vj, vk, vl] = X[vi, vj, vk, vl] * 2.0 + + assert_llir_equal(reorder_elementwise, reorder_elementwise_gt) + + +def test_reorder_overlapped_dynamic(): + @to_cinn_llir + def reorder_overlapped( + X: DataArray((-1, 8)), Y: DataArray((-1, 8)), N: ir.Var() + ): + for i in range(N / 4): + for j in range(4): + for k in range(4): + with ir.ScheduleBlockContext("Y"): + vi, vj = ir.AxisMap("SS", [i, j]) + sch.reorder([i, k, j]) + Y[vi, vj] = X[vi, vj] + 1.0 + + @to_cinn_llir + def reorder_overlapped_gt( + X: DataArray((-1, 8)), Y: DataArray((-1, 8)), N: ir.Var() + ): + for i in range(N / 4): + for k in range(4): + for j in range(4): + with ir.ScheduleBlockContext("Y"): + vi, vj = ir.AxisMap("SS", [i, j]) + Y[vi, vj] = X[vi, vj] + 1.0 + + assert_llir_equal(reorder_overlapped, reorder_overlapped_gt) + + if __name__ == '__main__': test_reorder_elementwise() test_reorder_overlapped() + test_reorder_elementwise_dynamic() + test_reorder_overlapped_dynamic() From 65e2d934caf47a4ecb81730ff6172d57982fbe6a Mon Sep 17 00:00:00 2001 From: Ryan <44900829+DrRyanHuang@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:28:54 +0800 Subject: [PATCH 119/146] [Dy2St] Remove `NodeVarType` (#60381) --- python/paddle/jit/dy2static/__init__.py | 2 +- .../paddle/jit/dy2static/static_analysis.py | 62 ++++---- .../transformers/loop_transformer.py | 14 +- python/paddle/jit/dy2static/utils.py | 1 - python/paddle/jit/dy2static/utils_helper.py | 132 +++++++----------- .../dygraph_to_static/test_static_analysis.py | 72 +++++----- 6 files changed, 115 insertions(+), 168 deletions(-) diff --git a/python/paddle/jit/dy2static/__init__.py b/python/paddle/jit/dy2static/__init__.py index 83535ac17aee6..d2c90a2c852db 100644 --- a/python/paddle/jit/dy2static/__init__.py +++ b/python/paddle/jit/dy2static/__init__.py @@ -30,7 +30,7 @@ unpack_by_structure as Unpack, ) from .program_translator import convert_to_static # noqa: F401 -from .static_analysis import NodeVarType, StaticAnalysisVisitor # noqa: F401 +from .static_analysis import StaticAnalysisVisitor # noqa: F401 from .transformers import DygraphToStaticAst # noqa: F401 from .utils import UndefinedVar, ast_to_source_code, saw # noqa: F401 from .variable_trans_func import ( # noqa: F401 diff --git a/python/paddle/jit/dy2static/static_analysis.py b/python/paddle/jit/dy2static/static_analysis.py index 81bfa589b018f..c239e8aaacf48 100644 --- a/python/paddle/jit/dy2static/static_analysis.py +++ b/python/paddle/jit/dy2static/static_analysis.py @@ -15,11 +15,12 @@ from paddle.utils import gast from .utils_helper import ( - NodeVarType, + binary_op_output_type, index_in_list, is_dygraph_api, is_numpy_api, is_paddle_api, + type_from_annotation, ) __all__ = [] @@ -37,7 +38,7 @@ def __init__(self, node): self.node = node self.parent = None self.children = [] - self.node_var_type = {NodeVarType.UNKNOWN} + self.node_var_type = {"UNKNOWN"} class StaticAnalysisVisitor: @@ -87,7 +88,7 @@ def get_node_to_wrapper_map(self): return self.node_to_wrapper_map def is_tensor_node(self, node): - tensor_types = {NodeVarType.TENSOR, NodeVarType.PADDLE_RETURN_TYPES} + tensor_types = {"TENSOR", "PADDLE_RETURN_TYPES"} node_wrapper = self.node_to_wrapper_map.get(node, None) if node_wrapper is None: return False @@ -101,17 +102,17 @@ def _get_constant_node_type(self, node): ) # singleton: None, True or False if node.value is None: - return {NodeVarType.NONE} + return {"NONE"} if isinstance(node.value, bool): - return {NodeVarType.BOOLEAN} + return {"BOOLEAN"} if isinstance(node.value, int): - return {NodeVarType.INT} + return {"INT"} if isinstance(node.value, float): - return {NodeVarType.FLOAT} + return {"FLOAT"} if isinstance(node.value, str): - return {NodeVarType.STRING} + return {"STRING"} - return {NodeVarType.UNKNOWN} + return {"UNKNOWN"} def _get_node_var_type(self, cur_wrapper): node = cur_wrapper.node @@ -119,14 +120,14 @@ def _get_node_var_type(self, cur_wrapper): return self._get_constant_node_type(node) if isinstance(node, gast.BoolOp): - return {NodeVarType.BOOLEAN} + return {"BOOLEAN"} if isinstance(node, gast.Compare): - return {NodeVarType.BOOLEAN} + return {"BOOLEAN"} if isinstance(node, gast.Dict): - return {NodeVarType.DICT} + return {"DICT"} if isinstance(node, gast.Set): - return {NodeVarType.SET} + return {"SET"} if isinstance(node, gast.UnaryOp): return self.node_to_wrapper_map[node.operand].node_var_type @@ -137,7 +138,7 @@ def _get_node_var_type(self, cur_wrapper): result_type = set() for l in left_type: for r in right_type: - result_type.add(NodeVarType.binary_op_output_type(l, r)) + result_type.add(binary_op_output_type(l, r)) return result_type if isinstance(node, gast.Assign): @@ -157,16 +158,13 @@ def _get_node_var_type(self, cur_wrapper): if isinstance(node, gast.AnnAssign): # TODO(0x45f): To determine whether need to support assignment statements # like `self.x: float = 2.1`. - ret_type = {NodeVarType.type_from_annotation(node.annotation)} + ret_type = {type_from_annotation(node.annotation)} # if annotation and value(Constant) are diffent type, we use value type if node.value: node_value_type = self.node_to_wrapper_map[ node.value ].node_var_type - if not ( - node_value_type - & {NodeVarType.UNKNOWN, NodeVarType.STATEMENT} - ): + if not (node_value_type & {"UNKNOWN", "STATEMENT"}): ret_type = node_value_type if isinstance(node.target, gast.Name): self.node_to_wrapper_map[node.target].node_var_type = ret_type @@ -174,9 +172,9 @@ def _get_node_var_type(self, cur_wrapper): if isinstance(node, gast.Name): if node.id == "None": - return {NodeVarType.NONE} + return {"NONE"} if node.id in {"True", "False"}: - return {NodeVarType.BOOLEAN} + return {"BOOLEAN"} # If node is child of functionDef.arguments parent_node_wrapper = cur_wrapper.parent if parent_node_wrapper and isinstance( @@ -184,33 +182,33 @@ def _get_node_var_type(self, cur_wrapper): ): return self._get_func_argument_type(parent_node_wrapper, node) - return {NodeVarType.UNKNOWN} + return {"UNKNOWN"} if isinstance(node, gast.Return): # If return nothing: if node.value is None: - return {NodeVarType.NONE} + return {"NONE"} - return {NodeVarType.UNKNOWN} + return {"UNKNOWN"} if isinstance(node, gast.Call): if is_dygraph_api(node): if isinstance(node.func, gast.Attribute): if node.func.attr == "to_variable": - return {NodeVarType.TENSOR} + return {"TENSOR"} if is_paddle_api(node): - return {NodeVarType.PADDLE_RETURN_TYPES} + return {"PADDLE_RETURN_TYPES"} if is_numpy_api(node): # In this simple version we assume numpy api returns nd-array - return {NodeVarType.NUMPY_NDARRAY} + return {"NUMPY_NDARRAY"} if isinstance(node.func, gast.Name): - return {NodeVarType.UNKNOWN} + return {"UNKNOWN"} if isinstance(node, gast.Subscript): if self.is_tensor_node(node.value): - return {NodeVarType.TENSOR} + return {"TENSOR"} - return {NodeVarType.STATEMENT} + return {"STATEMENT"} def _get_func_argument_type(self, parent_node_wrapper, node): """ @@ -232,9 +230,9 @@ def _get_func_argument_type(self, parent_node_wrapper, node): assert isinstance(node, gast.Name) parent_node = parent_node_wrapper.node - var_type = {NodeVarType.UNKNOWN} + var_type = {"UNKNOWN"} if node.annotation is not None: - var_type = {NodeVarType.type_from_annotation(node.annotation)} + var_type = {type_from_annotation(node.annotation)} # if annotation and value(Constant) are diffent type, we use value type if parent_node.defaults: diff --git a/python/paddle/jit/dy2static/transformers/loop_transformer.py b/python/paddle/jit/dy2static/transformers/loop_transformer.py index 42c2a40a5ca98..2d2cfee1f97b0 100644 --- a/python/paddle/jit/dy2static/transformers/loop_transformer.py +++ b/python/paddle/jit/dy2static/transformers/loop_transformer.py @@ -18,7 +18,7 @@ from paddle.base import unique_name from paddle.utils import gast -from ..static_analysis import NodeVarType, StaticAnalysisVisitor +from ..static_analysis import StaticAnalysisVisitor from ..utils import ( FOR_BODY_PREFIX, FOR_CONDITION_PREFIX, @@ -344,18 +344,6 @@ def _var_node_to_name(self, node): elif isinstance(node, gast.Attribute): return get_attribute_full_name(node) - def _node_var_type_is_basic(self, node_var_type): - basic_types = { - NodeVarType.BOOLEAN, - NodeVarType.INT, - NodeVarType.FLOAT, - NodeVarType.STRING, - } - for t in node_var_type: - if t in basic_types: - return True - return False - def _is_call_func_name_node(self, node): parent_node = self._get_parent_node(node) if isinstance(parent_node, gast.Call) and parent_node.func == node: diff --git a/python/paddle/jit/dy2static/utils.py b/python/paddle/jit/dy2static/utils.py index fc18ee5883e9c..3061e9f47b7e8 100644 --- a/python/paddle/jit/dy2static/utils.py +++ b/python/paddle/jit/dy2static/utils.py @@ -42,7 +42,6 @@ DYGRAPH_MODULE_PREFIX, DYGRAPH_TO_STATIC_MODULE_PREFIX, PADDLE_MODULE_PREFIX, - NodeVarType, _is_api_in_module_helper, index_in_list, is_api_in_module, diff --git a/python/paddle/jit/dy2static/utils_helper.py b/python/paddle/jit/dy2static/utils_helper.py index 4f1ae01450739..9a55f23cf46db 100644 --- a/python/paddle/jit/dy2static/utils_helper.py +++ b/python/paddle/jit/dy2static/utils_helper.py @@ -97,91 +97,53 @@ def is_paddle_api(node): return is_api_in_module(node, PADDLE_MODULE_PREFIX) -class NodeVarType: - """ - Enum class of python variable types. We have to know some variable types - during compile time to transfer AST. For example, a string variable and a - tensor variable in if clause may lead to different conversion from dygraph - to static graph. - """ - - ERROR = -1 # Returns when static analysis gets error - UNKNOWN = 0 # Reserve for AST nodes have not known the type - STATEMENT = 1 # For nodes representing statement (non-variable type) - CALLABLE = 2 - - # python data types - NONE = 100 - BOOLEAN = 101 - INT = 102 - FLOAT = 103 - STRING = 104 - TENSOR = 105 - NUMPY_NDARRAY = 106 - - # python collections - LIST = 200 - SET = 201 - DICT = 202 - - PADDLE_DYGRAPH_API = 300 - PADDLE_CONTROL_IF = 301 - PADDLE_CONTROL_WHILE = 302 - PADDLE_CONTROL_FOR = 303 - # Paddle API may not be visible to get source code. - # We use this enum value to denote the type return by a Paddle API - PADDLE_RETURN_TYPES = 304 - - # If node.node_var_type in TENSOR_TYPES, it can be considered as tensor-dependent. - TENSOR_TYPES = {TENSOR, PADDLE_RETURN_TYPES} - - Annotation_map = { - "Tensor": TENSOR, - "paddle.Tensor": TENSOR, - "int": INT, - "float": FLOAT, - "bool": BOOLEAN, - "str": STRING, - } - - @staticmethod - def binary_op_output_type(in_type1, in_type2): - if in_type1 == in_type2: - return in_type1 - - if in_type1 == NodeVarType.UNKNOWN: - return in_type2 - if in_type2 == NodeVarType.UNKNOWN: - return in_type1 - - supported_types = [ - NodeVarType.BOOLEAN, - NodeVarType.INT, - NodeVarType.FLOAT, - NodeVarType.NUMPY_NDARRAY, - NodeVarType.TENSOR, - NodeVarType.PADDLE_RETURN_TYPES, - ] - - if in_type1 not in supported_types: - return NodeVarType.UNKNOWN - if in_type2 not in supported_types: - return NodeVarType.UNKNOWN - - forbidden_types = [NodeVarType.NUMPY_NDARRAY, NodeVarType.TENSOR] - if in_type1 in forbidden_types and in_type2 in forbidden_types: - return NodeVarType.UNKNOWN - return max(in_type1, in_type2) - - @staticmethod - def type_from_annotation(annotation): - annotation_str = ast_to_source_code(annotation).strip() - if annotation_str in NodeVarType.Annotation_map: - return NodeVarType.Annotation_map[annotation_str] - - # raise warning if not found - warn("Currently we don't support annotation: %s" % annotation_str) - return NodeVarType.UNKNOWN +def binary_op_output_type(in_type1, in_type2): + if in_type1 == in_type2: + return in_type1 + + if in_type1 == "UNKNOWN": + return in_type2 + if in_type2 == "UNKNOWN": + return in_type1 + + supported_types = [ + "BOOLEAN", + "INT", + "FLOAT", + "NUMPY_NDARRAY", + "TENSOR", + "PADDLE_RETURN_TYPES", + ] + + if in_type1 not in supported_types: + return "UNKNOWN" + if in_type2 not in supported_types: + return "UNKNOWN" + + forbidden_types = ["NUMPY_NDARRAY", "TENSOR"] + if in_type1 in forbidden_types and in_type2 in forbidden_types: + return "UNKNOWN" + return max(in_type1, in_type2) + + +Annotation_map = { + "Tensor": "TENSOR", + "paddle.Tensor": "TENSOR", + "int": "INT", + "float": "FLOAT", + "bool": "BOOLEAN", + "str": "STRING", +} + + +def type_from_annotation(annotation): + annotation_str = ast_to_source_code(annotation).strip() + if annotation_str in Annotation_map: + return Annotation_map[annotation_str] + + # raise warning if not found + warn("Currently we don't support annotation: %s" % annotation_str) + return "UNKNOWN" def set_dynamic_shape(variable, shape_list): diff --git a/test/dygraph_to_static/test_static_analysis.py b/test/dygraph_to_static/test_static_analysis.py index ea44992a04844..889bf183d079c 100644 --- a/test/dygraph_to_static/test_static_analysis.py +++ b/test/dygraph_to_static/test_static_analysis.py @@ -19,7 +19,7 @@ import paddle from paddle import base -from paddle.jit.dy2static import NodeVarType, StaticAnalysisVisitor +from paddle.jit.dy2static import StaticAnalysisVisitor from paddle.utils import gast @@ -42,7 +42,7 @@ def func_to_test2(x): return x -result_var_type2 = {'m': {NodeVarType.INT}} +result_var_type2 = {'m': {"INT"}} def func_to_test3(): @@ -61,18 +61,18 @@ def func_to_test3(): result_var_type3 = { - 'a': {NodeVarType.INT}, - 'b': {NodeVarType.FLOAT}, - 'c': {NodeVarType.FLOAT}, - 'd': {NodeVarType.FLOAT}, - 'e': {NodeVarType.BOOLEAN}, - 'f': {NodeVarType.INT}, - 'g': {NodeVarType.STRING}, - 'h': {NodeVarType.NONE}, - 'i': {NodeVarType.BOOLEAN}, - 'j': {NodeVarType.UNKNOWN}, - 'k': {NodeVarType.FLOAT}, - 'l': {NodeVarType.PADDLE_RETURN_TYPES}, + 'a': {"INT"}, + 'b': {"FLOAT"}, + 'c': {"FLOAT"}, + 'd': {"FLOAT"}, + 'e': {"BOOLEAN"}, + 'f': {"INT"}, + 'g': {"STRING"}, + 'h': {"NONE"}, + 'i': {"BOOLEAN"}, + 'j': {"UNKNOWN"}, + 'k': {"FLOAT"}, + 'l': {"PADDLE_RETURN_TYPES"}, } @@ -85,10 +85,10 @@ def func_to_test4(): result_var_type4 = { - 'a': {NodeVarType.NUMPY_NDARRAY}, - 'b': {NodeVarType.NUMPY_NDARRAY}, - 'c': {NodeVarType.TENSOR}, - 'd': {NodeVarType.TENSOR}, + 'a': {"NUMPY_NDARRAY"}, + 'b': {"NUMPY_NDARRAY"}, + 'c': {"TENSOR"}, + 'd': {"TENSOR"}, } @@ -112,13 +112,13 @@ def inner_unknown_func(x): result_var_type5 = { - 'a': {NodeVarType.INT}, - 'b': {NodeVarType.FLOAT, NodeVarType.BOOLEAN}, - 'c': {NodeVarType.UNKNOWN}, - 'd': {NodeVarType.PADDLE_RETURN_TYPES}, - 'inner_int_func': {NodeVarType.INT}, - 'inner_bool_float_func': {NodeVarType.FLOAT, NodeVarType.BOOLEAN}, - 'inner_unknown_func': {NodeVarType.UNKNOWN}, + 'a': {"INT"}, + 'b': {"FLOAT", "BOOLEAN"}, + 'c': {"UNKNOWN"}, + 'd': {"PADDLE_RETURN_TYPES"}, + 'inner_int_func': {"INT"}, + 'inner_bool_float_func': {"FLOAT", "BOOLEAN"}, + 'inner_unknown_func': {"UNKNOWN"}, } @@ -136,10 +136,10 @@ def add(x, y): result_var_type6 = { - 'i': {NodeVarType.INT}, - 'x': {NodeVarType.INT}, - 'y': {NodeVarType.INT}, - 'add': {NodeVarType.INT}, + 'i': {"INT"}, + 'x': {"INT"}, + 'y': {"INT"}, + 'add': {"INT"}, } @@ -150,13 +150,13 @@ def func_to_test7(a: int, b: float, c: paddle.Tensor, d: float = 'diff'): result_var_type7 = { - 'a': {NodeVarType.BOOLEAN}, - 'b': {NodeVarType.FLOAT}, - 'c': {NodeVarType.TENSOR}, - 'd': {NodeVarType.STRING}, - 'e': {NodeVarType.PADDLE_RETURN_TYPES}, - 'f': {NodeVarType.PADDLE_RETURN_TYPES}, - 'g': {NodeVarType.TENSOR}, + 'a': {"BOOLEAN"}, + 'b': {"FLOAT"}, + 'c': {"TENSOR"}, + 'd': {"STRING"}, + 'e': {"PADDLE_RETURN_TYPES"}, + 'f': {"PADDLE_RETURN_TYPES"}, + 'g': {"TENSOR"}, } test_funcs = [ From 76ce9bb2de84ee6ea052cfde6ce269be1e4d8baf Mon Sep 17 00:00:00 2001 From: pangengzheng <117730991+pangengzheng@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:52:06 +0800 Subject: [PATCH 120/146] support save load optimizer master_weights (#60027) * exclude xpu * dedup tensor in state_dict * polish * support flatten and unflatten state_dict * test flatten * rename test * fix dedup tensor test * fix test * fix load state dict * rename * fix test * support save load optimizer master weights * add comment --- .../distributed/checkpoint/load_state_dict.py | 36 +++--- .../paddle/distributed/checkpoint/metadata.py | 1 + .../distributed/checkpoint/save_state_dict.py | 57 ++++++++-- python/paddle/distributed/checkpoint/utils.py | 44 +++++++- python/paddle/optimizer/optimizer.py | 30 +---- test/auto_parallel/CMakeLists.txt | 3 + ...i_auto_parallel_checkpoint_dedup_tensor.py | 68 ++++++++++++ ...uto_parallel_checkpoint_flatten_mapping.py | 74 ++++++++++++ .../semi_auto_parallel_shard_optimizer_api.py | 55 +++++++++ .../test_dist_checkpoint_utils.py | 105 ++++++++++++++++++ 10 files changed, 416 insertions(+), 57 deletions(-) create mode 100644 test/auto_parallel/semi_auto_parallel_checkpoint_dedup_tensor.py create mode 100644 test/auto_parallel/semi_auto_parallel_checkpoint_flatten_mapping.py create mode 100644 test/auto_parallel/test_dist_checkpoint_utils.py diff --git a/python/paddle/distributed/checkpoint/load_state_dict.py b/python/paddle/distributed/checkpoint/load_state_dict.py index fda6b6f9174b5..4ae82398713ae 100644 --- a/python/paddle/distributed/checkpoint/load_state_dict.py +++ b/python/paddle/distributed/checkpoint/load_state_dict.py @@ -405,9 +405,9 @@ def load_state_dict( assert isinstance( state_dict, dict ), "The state_dict should be a dictionary." - state_dict = flatten_state_dict(state_dict) - if len(state_dict) > 0: - for val in state_dict.values(): + flat_state_dict, mapping = flatten_state_dict(state_dict) + if len(flat_state_dict) > 0: + for val in flat_state_dict.values(): assert isinstance( val, paddle.Tensor ), f"Only support dygraph Tensor now, but is {val}" @@ -423,7 +423,7 @@ def load_state_dict( paddle.distributed.barrier(process_group) rank_to_files = get_rank_to_files( - path, state_dict, process_group, use_dist + path, flat_state_dict, process_group, use_dist ) if len(rank_to_files) <= 0: return @@ -434,16 +434,18 @@ def load_state_dict( ) # read_items: [ReadItem(local_tensor_index, rank, cur_offsets, storage_offsets, lengths)], # slice the storage local tensor in (storage_offsets, lengths) to assign the current tensor in (cur_offsets, lengths) in rank. - read_items = get_read_items(path, state_dict, process_group, use_dist) + read_items = get_read_items( + path, flat_state_dict, process_group, use_dist + ) storage_file_to_state_dict = {} logger.debug( - f"before load, state_dict:{state_dict},\n load_infos:{load_infos},\n read_items:{read_items}" + f"before load, state_dict:{flat_state_dict},\n load_infos:{load_infos},\n read_items:{read_items}" ) state_dict_in_cpu = [] - for k, v in state_dict.items(): + for k, v in flat_state_dict.items(): if v.place.is_cpu_place(): state_dict_in_cpu.append(k) - state_dict[k] = v.cuda() + flat_state_dict[k] = v.cuda() for item in read_items: assert ( item.local_tensor_index in load_infos @@ -484,15 +486,17 @@ def load_state_dict( # The read item rank need to be assigned if item.rank == paddle.distributed.get_rank(): assert ( - item.local_tensor_index.tensor_key in state_dict - ), f"item:{item}, state_dict:{state_dict}" + item.local_tensor_index.tensor_key in flat_state_dict + ), f"item:{item}, state_dict:{flat_state_dict}" cur_local_tensor = ( - state_dict[ + flat_state_dict[ item.local_tensor_index.tensor_key ]._local_value() if use_dist - and state_dict[item.local_tensor_index.tensor_key].is_dist() - else state_dict[item.local_tensor_index.tensor_key] + and flat_state_dict[ + item.local_tensor_index.tensor_key + ].is_dist() + else flat_state_dict[item.local_tensor_index.tensor_key] ) cur_offsets = item.cur_offset cur_lengths = item.lengths @@ -513,7 +517,9 @@ def load_state_dict( else: cur_chunk_tensor = paddle.zeros( item.lengths, - dtype=state_dict[item.local_tensor_index.tensor_key].dtype, + dtype=flat_state_dict[ + item.local_tensor_index.tensor_key + ].dtype, ) if src_rank == item.rank: @@ -530,6 +536,6 @@ def load_state_dict( cur_chunk_tensor, src=src_rank, group=process_group ) - for k, v in state_dict.items(): + for k, v in flat_state_dict.items(): if k in state_dict_in_cpu: state_dict[k] = v.cpu() diff --git a/python/paddle/distributed/checkpoint/metadata.py b/python/paddle/distributed/checkpoint/metadata.py index 4eb5d559a9c0c..d1f3a3fdb66c0 100644 --- a/python/paddle/distributed/checkpoint/metadata.py +++ b/python/paddle/distributed/checkpoint/metadata.py @@ -40,3 +40,4 @@ class LocalTensorIndex: class Metadata: state_dict_metadata: Dict[str, List[LocalTensorMetadata]] = None storage_metadata: Dict[LocalTensorIndex, str] = None + flat_mapping: Dict[str, Tuple[str]] = None diff --git a/python/paddle/distributed/checkpoint/save_state_dict.py b/python/paddle/distributed/checkpoint/save_state_dict.py index b2c380c66ba2f..86047e637e360 100644 --- a/python/paddle/distributed/checkpoint/save_state_dict.py +++ b/python/paddle/distributed/checkpoint/save_state_dict.py @@ -13,7 +13,6 @@ # limitations under the License. import os -from typing import List import paddle from paddle.distributed.communication.group import is_initialized @@ -50,7 +49,7 @@ def check_file_name(file_name, process_group): def merge_state_dict_metadata(global_state_dict_metadata): assert isinstance( - global_state_dict_metadata, List + global_state_dict_metadata, list ), "The global_state_dict should be a list." out = {} for state_dict in global_state_dict_metadata: @@ -64,7 +63,7 @@ def merge_state_dict_metadata(global_state_dict_metadata): return out -def dedup_storage_metadata(global_storage_metadata): +def dedup_key_in_dict(global_storage_metadata): out = {} for storage_metadata in global_storage_metadata: for key, val in storage_metadata.items(): @@ -74,6 +73,34 @@ def dedup_storage_metadata(global_storage_metadata): return out +def dedup_tensor( + local_state_dict, local_storage_metadata, global_storage_metadata +): + """ + Dedup the replicated tensor in local state_dict. + + Args: + local_state_dict(Dict[str, paddle.Tensor]): The state_dict of current rank. + local_storage_metadata(Dict[LocalTensorIndex, str]): The storage metadata of current rank. + global_storage_metadata(Dict[LocalTensorIndex, str]): The final storage metadata of all ranks. + + Examples: + In rank0, local_state_dict:{"w1": t1_0, "w2": t2}, local_storage_metadata:{LocalTensorIndex("w1", (0,0)): "0_0.distcp", LocalTensorIndex("w2", (0,0)): "0_0.distcp"}, + in rank1, local_state_dict:{"w1": t1_1, "w2": t2}, local_storage_metadata:{LocalTensorIndex("w1", (1,0)): "1_0.distcp", LocalTensorIndex("w2", (0,0)): "1_0.distcp"}, + global_storage_metadata:{LocalTensorIndex("w1", (0,0)): "0_0.distcp", LocalTensorIndex("w1", (1,0)): "1_0.distcp", LocalTensorIndex("w2", (0, 0)): "0_0.distcp"}. + w2 is replicated in rank0 and rank1. We save it in rank0 as default thus need to remove it in other ranks. + Finally, the local_state_dict:{"w1": t1_1, "w2": t2} in rank1 update to {"w1": t1_1}. + """ + + for tensor_index, file_name in global_storage_metadata.items(): + rank = int(file_name.split(".")[0].split("_")[0]) + if ( + tensor_index in local_storage_metadata + and rank != paddle.distributed.get_rank() + ): + local_state_dict.pop(tensor_index.tensor_key) + + def save_state_dict( state_dict, path, @@ -107,9 +134,9 @@ def save_state_dict( assert isinstance( state_dict, dict ), "The state_dict should be a dictionary." - state_dict = flatten_state_dict(state_dict) - if len(state_dict) > 0: - for val in state_dict.values(): + flat_state_dict, mapping = flatten_state_dict(state_dict) + if len(flat_state_dict) > 0: + for val in flat_state_dict.values(): assert isinstance( val, paddle.Tensor ), "Only support dygraph Tensor now, support static DistributedTensor later" @@ -134,12 +161,12 @@ def save_state_dict( if use_dist: check_file_name(file_name, process_group) # the parameter_name and order in state_dict should be the same - check_state_dict(state_dict, process_group) + check_state_dict(flat_state_dict, process_group) metadata = Metadata() local_state_dict = {} local_state_dict_metadata = {} local_storage_metadata = {} - for key, val in state_dict.items(): + for key, val in flat_state_dict.items(): if isinstance(val, paddle.Tensor): # Case1: not initialized means this tensor is placed in another mesh which do not contain this rank if not val._is_initialized(): @@ -178,6 +205,7 @@ def save_state_dict( ] = file_name global_state_dict_metadata = [] global_storage_metadata = [] + global_flatten_mapping = [] if use_dist: paddle.distributed.all_gather_object( global_state_dict_metadata, @@ -187,19 +215,24 @@ def save_state_dict( paddle.distributed.all_gather_object( global_storage_metadata, local_storage_metadata, process_group ) + paddle.distributed.all_gather_object( + global_flatten_mapping, mapping, process_group + ) else: global_state_dict_metadata.append(local_state_dict_metadata) global_storage_metadata.append(local_storage_metadata) + global_flatten_mapping.append(mapping) metadata.state_dict_metadata = merge_state_dict_metadata( global_state_dict_metadata ) - metadata.storage_metadata = dedup_storage_metadata( - global_storage_metadata - ) + metadata.storage_metadata = dedup_key_in_dict(global_storage_metadata) + metadata.flat_mapping = dedup_key_in_dict(global_flatten_mapping) if coordinator_rank == paddle.distributed.get_rank(): logger.debug(f"metadata:{metadata}") paddle.save(metadata, os.path.join(path, f"{unique_id}.metadata")) logger.debug(f"local_state_dict:{local_state_dict}") - # TODO(pangengzheng): del the replicated tensor in local_state_dict, now different might save the replicated tensor + dedup_tensor( + local_state_dict, local_storage_metadata, metadata.storage_metadata + ) paddle.save(local_state_dict, os.path.join(path, file_name)) diff --git a/python/paddle/distributed/checkpoint/utils.py b/python/paddle/distributed/checkpoint/utils.py index cb0f069984c3a..d592d6ebcb97b 100644 --- a/python/paddle/distributed/checkpoint/utils.py +++ b/python/paddle/distributed/checkpoint/utils.py @@ -63,5 +63,47 @@ def compute_local_shape_and_global_offset( def flatten_state_dict(state_dict): - # TODO, {"model": {"w0": xxx}} -> {model.w0: xxx} + """ + Flatten the nested dict to a flat dict. + {"model": {"w0": xxx}} -> {model.w0: xxx} + """ + flatten_state_dict = {} + mapping = {} + + def _flatten(key, value): + if isinstance(value, dict): + for k, v in value.items(): + assert isinstance(k, str), f"The key should be str, but is {k}" + _flatten(key + (k,), v) + elif isinstance(value, paddle.Tensor): + flatten_key_str = ".".join(key) + flatten_state_dict[flatten_key_str] = value + mapping[flatten_key_str] = key + else: + raise ValueError( + f"The value should be dict or paddle.Tensor, but is {value}" + ) + + _flatten((), state_dict) + + return flatten_state_dict, mapping + + +def unflatten_state_dict(flat_state_dict, mapping): + """ + Unflatten the flat dict to a nested dict. + {model.w0: xxx} -> {"model": {"w0": xxx}} + """ + state_dict = {} + for key, value in flat_state_dict.items(): + key_tuple = mapping[key] + assert isinstance( + key_tuple, tuple + ), f"The key should be tuple, but is {key_tuple}" + tmp = state_dict + for i in range(len(key_tuple) - 1): + key = key_tuple[i] + tmp = tmp.setdefault(key, {}) + tmp[key_tuple[-1]] = value + return state_dict diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 3a64f2095f30a..134b164409a95 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -406,35 +406,7 @@ def set_state_dict(self, state_dict): tensor.set_xpu_scale_value( state_dict.get(var_tmp.name + ".SCALE_VALUE", -1.0) ) - - model_np = np.array(tensor) - - load_para = state_dict[var_tmp.name] - - if isinstance(load_para, Variable): - load_para_np = np.array(load_para) - elif isinstance(load_para, core.eager.Tensor): - load_para_np = np.array(load_para) - elif isinstance(load_para, np.ndarray): - load_para_np = load_para - else: - raise RuntimeError( - f"State dict type {str(type(load_para))} not supprt" - ) - - assert ( - model_np.shape == load_para_np.shape - ), "Parameter shape not match, Dygraph Parameter [ {} ] need tensor with shape {} but load tensor with shape {}".format( - model_np.name, model_np.shape, load_para_np.shape - ) - - assert ( - model_np.dtype == load_para_np.dtype - ), "Parameter dtype not match, Dygraph Parameter [ {} ] need tensor with dtype {} but load tensor with dtype {}".format( - model_np.name, model_np.dtype, load_para_np.dtype - ) - - tensor.set(load_para_np, framework._current_expected_place()) + var.set_value(state_dict[var_tmp.name]) def get_opti_var_name_list(self): return self._opti_name_list diff --git a/test/auto_parallel/CMakeLists.txt b/test/auto_parallel/CMakeLists.txt index 774dc3d2023b9..a735762cce658 100644 --- a/test/auto_parallel/CMakeLists.txt +++ b/test/auto_parallel/CMakeLists.txt @@ -194,6 +194,9 @@ if(WITH_DISTRIBUTE AND WITH_GPU) py_test_modules(test_gpt_with_prim MODULES test_gpt_with_prim) set_tests_properties(test_gpt_with_prim PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 200) + py_test_modules(test_dist_checkpoint_utils MODULES test_dist_checkpoint_utils) + set_tests_properties(test_dist_checkpoint_utils + PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120) py_test_modules(test_semi_auto_parallel_unshard_dtensor MODULES test_semi_auto_parallel_unshard_dtensor) set_tests_properties(test_semi_auto_parallel_unshard_dtensor diff --git a/test/auto_parallel/semi_auto_parallel_checkpoint_dedup_tensor.py b/test/auto_parallel/semi_auto_parallel_checkpoint_dedup_tensor.py new file mode 100644 index 0000000000000..7f8884156aa7e --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_checkpoint_dedup_tensor.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +import paddle +import paddle.distributed as dist + + +class TestSaveStateDict: + def __init__(self): + self._ckpt_path = os.getenv("ckpt_path") + + def test_dedup_tesnor(self): + w1 = paddle.arange(32).reshape([4, 8]) + w2 = paddle.arange(32, 36).reshape([2, 2]) + mesh = dist.ProcessMesh([0, 1]) + dist_w1 = dist.shard_tensor(w1, mesh, [dist.Replicate()]) + dist_w2 = dist.shard_tensor(w2, mesh, [dist.Shard(0)]) + state_dict = {"w1": dist_w1, "w2": dist_w2} + # w1 is replicated in rank0 and ran1, it will only save in rank0. + # Therefore, rank0 save state_dict:{"w1": dist_w1, "w2": dist_w2}, rank1 save state_dict:{"w2": dist_w2} + dist.save_state_dict(state_dict, self._ckpt_path) + paddle.distributed.barrier() + # check + expect_local_state_dict = {} + for k, v in state_dict.items(): + if k == "w1" and paddle.distributed.get_rank() != 0: + continue + expect_local_state_dict[k] = v._local_value() + data_file_path = os.path.join( + self._ckpt_path, f"{paddle.distributed.get_rank()}_0.distcp" + ) + metadata_file_path = os.path.join(self._ckpt_path, "0.metadata") + assert os.path.exists(data_file_path) and os.path.exists( + metadata_file_path + ) + local_state_dict = paddle.load(data_file_path) + metadata = paddle.load(metadata_file_path) + + for k, local_tensor in local_state_dict.items(): + assert k in expect_local_state_dict + expect_tensor = expect_local_state_dict[k] + np.testing.assert_equal(expect_tensor.numpy(), local_tensor.numpy()) + for tensor_index, file_name in metadata.storage_metadata.items(): + rank = int(file_name.split(".")[0].split("_")[0]) + if tensor_index.tensor_key == "w1": + assert rank == 0 + + def run_test_case(self): + self.test_dedup_tesnor() + + +if __name__ == '__main__': + TestSaveStateDict().run_test_case() diff --git a/test/auto_parallel/semi_auto_parallel_checkpoint_flatten_mapping.py b/test/auto_parallel/semi_auto_parallel_checkpoint_flatten_mapping.py new file mode 100644 index 0000000000000..c8cfdb22d8598 --- /dev/null +++ b/test/auto_parallel/semi_auto_parallel_checkpoint_flatten_mapping.py @@ -0,0 +1,74 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import paddle +import paddle.distributed as dist + + +class TestSemiautoSaveLoad: + def __init__(self): + self._ckpt_path = os.getenv("ckpt_path") + + def test_flatten_mapping(self): + if paddle.distributed.get_rank() == 0: + state_dict = { + "model": { + "a": paddle.to_tensor([1, 2]), + "b": paddle.to_tensor([3, 4]), + }, + "optimizer": { + "c": paddle.to_tensor([5, 6]), + "d": paddle.to_tensor([7, 8]), + }, + } + else: + state_dict = { + "model": { + "a": paddle.to_tensor([10, 20]), + "b": paddle.to_tensor([30, 40]), + }, + "optimizer": { + "c": paddle.to_tensor([50, 60]), + "d": paddle.to_tensor([70, 80]), + }, + } + expected_mapping = { + "model.a": ("model", "a"), + "model.b": ("model", "b"), + "optimizer.c": ("optimizer", "c"), + "optimizer.d": ("optimizer", "d"), + } + dist.save_state_dict(state_dict, self._ckpt_path) + metadata_path = os.path.join(self._ckpt_path, "0.metadata") + assert os.path.exists(metadata_path) + metadata = paddle.load(metadata_path) + assert len(metadata.flat_mapping) == len( + expected_mapping + ), f"expect {len(expected_mapping)}, but got {len(metadata.flat_mapping)}" + for key in metadata.flat_mapping: + assert ( + key in expected_mapping + ), f"expect {key} in flatten_mapping, but not found" + assert ( + metadata.flat_mapping[key] == expected_mapping[key] + ), f"expect {metadata.flat_mapping[key]} == {expected_mapping[key]}, but not equal" + + def run_test_case(self): + self.test_flatten_mapping() + + +if __name__ == '__main__': + TestSemiautoSaveLoad().run_test_case() diff --git a/test/auto_parallel/semi_auto_parallel_shard_optimizer_api.py b/test/auto_parallel/semi_auto_parallel_shard_optimizer_api.py index f4d22a16c41bd..0153d3bd21216 100644 --- a/test/auto_parallel/semi_auto_parallel_shard_optimizer_api.py +++ b/test/auto_parallel/semi_auto_parallel_shard_optimizer_api.py @@ -179,6 +179,61 @@ def test_shard_optimizer_master_params(self): assert v.is_dist() assert v.shape[-1] == v._local_shape[-1] * 2 + # save load + ckpt_state_dict = opt.state_dict() + dist.save_state_dict(ckpt_state_dict, self._ckpt_path) + paddle.distributed.barrier() + expected_local_state_dict = {} + expected_local_state_dict.setdefault("master_weights", {}) + need_load_state_dict = {} + need_load_state_dict.setdefault("master_weights", {}) + for k, v in ckpt_state_dict.items(): + if k == "LR_Scheduler": + continue + elif k == "master_weights": + assert isinstance(v, dict), v + for mk, mv in v.items(): + expected_local_state_dict[k][mk] = mv._local_value().clone() + need_load_state_dict[k][mk] = paddle.zeros_like(mv) + else: + expected_local_state_dict[k] = v._local_value().clone() + need_load_state_dict[k] = paddle.zeros_like(v) + opt.set_state_dict(need_load_state_dict) + after_set_state_dict = opt.state_dict() + for k, v in after_set_state_dict.items(): + if k == "master_weights": + assert isinstance(v, dict), v + for mk, mv in v.items(): + assert ( + mv.numpy().sum() == 0.0 + ), f"state_dict {k} in master_weights is not zero" + assert ( + need_load_state_dict[k][mk].numpy().sum() == 0.0 + ), f"state_dict {k} in master_weights is not zero" + else: + assert v.numpy().sum() == 0.0, f"state_dict {k} is not zero" + assert k in need_load_state_dict, f"state_dict {k} is not found" + assert ( + need_load_state_dict[k].numpy().sum() == 0.0 + ), f"state_dict {k} is not zero" + dist.load_state_dict(need_load_state_dict, self._ckpt_path) + opt.set_state_dict(need_load_state_dict) + new_state_dict = opt.state_dict() + assert "master_weights" in new_state_dict, new_state_dict + for k, v in new_state_dict.items(): + assert k in expected_local_state_dict + if k == "master_weights": + for mk, mv in v.items(): + np.testing.assert_equal( + mv._local_value().numpy(), + expected_local_state_dict[k][mk].numpy(), + ) + else: + np.testing.assert_equal( + v._local_value().numpy(), + expected_local_state_dict[k].numpy(), + ) + def test_shard_optimizer_params_group(self): paddle.seed(self._seed) linear = paddle.nn.Linear(10, 10) diff --git a/test/auto_parallel/test_dist_checkpoint_utils.py b/test/auto_parallel/test_dist_checkpoint_utils.py new file mode 100644 index 0000000000000..5a51f73f0fa56 --- /dev/null +++ b/test/auto_parallel/test_dist_checkpoint_utils.py @@ -0,0 +1,105 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import collective.test_communication_api_base as test_base +import numpy as np + +import paddle +from paddle.distributed.checkpoint.utils import ( + flatten_state_dict, + unflatten_state_dict, +) + + +class TestDistCheckpointUtils(test_base.CommunicationTestDistBase): + def setUp(self): + super().setUp(num_of_devices=2, timeout=120, nnode=1) + self._default_envs = {} + self._changeable_envs = {"backend": ["gpu"]} + + def test_flatten_mapping(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + ckpt_path_tmp = tempfile.TemporaryDirectory() + ckpt_path = ckpt_path_tmp.name + envs["ckpt_path"] = ckpt_path + self.run_test_case( + "semi_auto_parallel_checkpoint_flatten_mapping.py", + user_defined_envs=envs, + ) + ckpt_path_tmp.cleanup() + + def test_dedup_tensor(self): + envs_list = test_base.gen_product_envs_list( + self._default_envs, self._changeable_envs + ) + for envs in envs_list: + ckpt_path_tmp = tempfile.TemporaryDirectory() + ckpt_path = ckpt_path_tmp.name + envs["ckpt_path"] = ckpt_path + self.run_test_case( + "semi_auto_parallel_checkpoint_dedup_tensor.py", + user_defined_envs=envs, + ) + ckpt_path_tmp.cleanup() + + def test_flatten_state_dict(self): + state_dict = { + "model": { + "a.0": paddle.to_tensor([1, 2]), + "b": paddle.to_tensor([3, 4]), + }, + "optimizer": { + "c": paddle.to_tensor([5, 6]), + "d.2": paddle.to_tensor([7, 8]), + }, + } + expected_flat_state_dict = { + "model.a.0": paddle.to_tensor([1, 2]), + "model.b": paddle.to_tensor([3, 4]), + "optimizer.c": paddle.to_tensor([5, 6]), + "optimizer.d.2": paddle.to_tensor([7, 8]), + } + flat_state_dict, mapping = flatten_state_dict(state_dict) + self.assertTrue(len(expected_flat_state_dict) == len(flat_state_dict)) + for k, v in flat_state_dict.items(): + self.assertTrue(isinstance(v, paddle.Tensor)) + self.assertTrue(k in expected_flat_state_dict) + np.testing.assert_equal( + v.numpy(), expected_flat_state_dict[k].numpy() + ) + recover_state_dict = unflatten_state_dict(flat_state_dict, mapping) + + def check_state_dict(d1, d2): + self.assertTrue(len(d1) == len(d2)) + self.assertTrue(type(d1) == type(d2)) + if isinstance(d1, dict): + for k in d1: + self.assertTrue(k in d2) + check_state_dict(d1[k], d2[k]) + elif isinstance(d1, paddle.Tensor): + np.testing.assert_equal(d1.numpy(), d2.numpy()) + else: + raise ValueError(f"Invalid type of state_dict:{d1} != {d2}") + + check_state_dict(recover_state_dict, state_dict) + + +if __name__ == "__main__": + unittest.main() From c1d78603ca6d818aec775733521e04db9c145716 Mon Sep 17 00:00:00 2001 From: zyt1024 <42999008+zyt1024@users.noreply.github.com> Date: Thu, 28 Dec 2023 17:14:26 +0800 Subject: [PATCH 121/146] =?UTF-8?q?=E3=80=90Complex=20op=E3=80=91add=20com?= =?UTF-8?q?plex=20support=20for=20assign=5Fvalue=20=20(#59536)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * support_complex_for_assign_value * add test complex test for test_program_converter * add complex test for assign_value xpu * solve conflict * fix timeout * fix CE infer bug * fix program convert bug * fix program convert bug for assign_value --------- Co-authored-by: zyt1024 <1522064645@qq.com> --- paddle/fluid/framework/op_version_proto.cc | 1 + paddle/fluid/framework/program_converter.cc | 85 +++++++- .../ir_adaptor/translator/op_translator.cc | 17 +- .../ops_signature/assign_value_sig.cc | 26 +-- .../pir/dialect/operator/ir/op_attribute.cc | 4 + .../pir/dialect/operator/ir/op_attribute.h | 4 +- .../fluid/pir/dialect/operator/utils/utils.h | 6 + paddle/fluid/pybind/op_function_common.cc | 21 +- paddle/phi/api/yaml/op_version.yaml | 15 ++ paddle/phi/api/yaml/static_ops.yaml | 2 +- paddle/phi/kernels/assign_kernel.cc | 12 +- paddle/pir/core/builder.h | 5 + paddle/pir/core/builtin_attribute.cc | 10 + paddle/pir/core/builtin_attribute.h | 23 +++ paddle/pir/core/builtin_attribute_storage.h | 40 ++++ paddle/pir/core/builtin_dialect.cc | 4 +- paddle/pir/core/ir_printer.cc | 4 + python/paddle/nn/initializer/Bilinear.py | 2 +- python/paddle/nn/initializer/assign.py | 6 +- python/paddle/nn/initializer/dirac.py | 4 +- python/paddle/tensor/creation.py | 58 ++---- .../test_program_translator.py | 14 +- test/ir/inference/CMakeLists.txt | 4 +- test/ir/inference/test_mul_gru_fuse_pass.py | 2 +- test/ir/inference/test_mul_lstm_fuse_pass.py | 2 +- .../inference/test_seq_concat_fc_fuse_pass.py | 4 +- test/legacy_test/test_assign_value_op.py | 101 +++++++-- test/legacy_test/test_initializer.py | 3 +- test/legacy_test/test_initializer_nn.py | 4 +- test/legacy_test/test_program_converter.py | 193 ++++++++++++++++++ test/xpu/test_assign_value_op_xpu.py | 61 +++++- 31 files changed, 614 insertions(+), 123 deletions(-) diff --git a/paddle/fluid/framework/op_version_proto.cc b/paddle/fluid/framework/op_version_proto.cc index 2a93e755b085b..8be9323098c97 100644 --- a/paddle/fluid/framework/op_version_proto.cc +++ b/paddle/fluid/framework/op_version_proto.cc @@ -21,6 +21,7 @@ namespace pb { const std::unordered_map& GetLegacyOpVersions() { static std::unordered_map op_versions = { {"not_equal", 1}, + {"assign_value", 0}, {"fake_channel_wise_dequantize_max_abs", 2}, {"yolo_box", 1}, {"data_norm", 1}, diff --git a/paddle/fluid/framework/program_converter.cc b/paddle/fluid/framework/program_converter.cc index fc60a0abf676e..82739e788bba3 100644 --- a/paddle/fluid/framework/program_converter.cc +++ b/paddle/fluid/framework/program_converter.cc @@ -117,6 +117,41 @@ void ConvertSetValueOp(OpDesc* op) { } } +void ConvertAssignValueOp(OpDesc* op) { + std::vector values = PADDLE_GET_CONST( + std::vector, op->GetAttr("values", false)); + op->RemoveAttr("values"); + op->SetAttr("bool_values", std::vector()); + op->SetAttr("fp32_values", std::vector()); + op->SetAttr("int32_values", std::vector()); + op->SetAttr("int64_values", std::vector()); + + phi::DataType dtype = phi::DataType::FLOAT32; + if (values.size()) { + dtype = values.at(0).dtype(); + } + + switch (dtype) { + case phi::DataType::BOOL: + op->SetAttr("bool_values", ExtractPlainVector(values)); + break; + case phi::DataType::FLOAT32: + op->SetAttr("fp32_values", ExtractPlainVector(values)); + break; + case phi::DataType::FLOAT64: + op->SetAttr("fp32_values", ExtractPlainVector(values)); + break; + case phi::DataType::INT32: + op->SetAttr("int32_values", ExtractPlainVector(values)); + break; + case phi::DataType::INT64: + op->SetAttr("int64_values", ExtractPlainVector(values)); + break; + default: + PD_THROW("Invalid data type `", dtype, "`."); + } +} + void ConvertProgram(ProgramDesc* program) { PADDLE_ENFORCE_NOT_NULL( program, @@ -144,6 +179,9 @@ void ConvertProgram(ProgramDesc* program) { if (op_type == "set_value" || op_type == "set_value_grad") { ConvertSetValueOp(op); } + if (op_type == "assign_value") { + ConvertAssignValueOp(op); + } } } } @@ -204,6 +242,45 @@ void ConvertSetValueOp(OpDesc* op) { op->SetAttr("values", values); } +void ConvertAssignValueOp(OpDesc* op) { + VLOG(3) << "convert old assign value op to new"; + std::vector values; + + if (op->HasAttr("bool_values")) { + std::vector bool_values = + PADDLE_GET_CONST(std::vector, op->GetAttr("bool_values", false)); + if (bool_values.size()) { + values = WrapAsScalars(bool_values); + } + op->RemoveAttr("bool_values"); + } + if (op->HasAttr("fp32_values")) { + std::vector fp32_values = + PADDLE_GET_CONST(std::vector, op->GetAttr("fp32_values", false)); + if (fp32_values.size()) { + values = WrapAsScalars(fp32_values); + } + op->RemoveAttr("fp32_values"); + } + if (op->HasAttr("int32_values")) { + std::vector int32_values = + PADDLE_GET_CONST(std::vector, op->GetAttr("int32_values", false)); + if (int32_values.size()) { + values = WrapAsScalars(int32_values); + } + op->RemoveAttr("int32_values"); + } + if (op->HasAttr("int64_values")) { + std::vector int64_values = PADDLE_GET_CONST( + std::vector, op->GetAttr("int64_values", false)); + if (int64_values.size()) { + values = WrapAsScalars(int64_values); + } + op->RemoveAttr("int64_values"); + } + op->SetAttr("values", values); +} + void ConvertProgram(ProgramDesc* program) { PADDLE_ENFORCE_NOT_NULL( program, @@ -214,6 +291,7 @@ void ConvertProgram(ProgramDesc* program) { const std::unordered_map& legacy_op_versions = legacy_op_results.second; + VLOG(3) << "is_legacy_program : " << is_legacy_program; if (!is_legacy_program) return; VLOG(3) << "Updating Program Version and OpVersionMap"; @@ -232,10 +310,15 @@ void ConvertProgram(ProgramDesc* program) { for (size_t j = 0; j < num_ops; j++) { OpDesc* op = block->Op(static_cast(j)); const std::string op_type = op->Type(); + + if (op_type == "assign_value") { + VLOG(3) << "Converting program from old to new, op_type=" << op_type; + ConvertAssignValueOp(op); + } if (!legacy_op_versions.count(op_type)) { continue; } - + VLOG(3) << "Converting program from old to new, op_type=" << op_type; if (op_type == "set_value" || op_type == "set_value_grad") { ConvertSetValueOp(op); } diff --git a/paddle/fluid/ir_adaptor/translator/op_translator.cc b/paddle/fluid/ir_adaptor/translator/op_translator.cc index 626073d143e3e..c64004c7191dd 100644 --- a/paddle/fluid/ir_adaptor/translator/op_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/op_translator.cc @@ -972,19 +972,20 @@ struct AssignValueOpTranscriber : public OpTranscriber { ctx, phi::Place(phi::AllocationType::UNDEFINED)); attribute_map["place"] = attr_place; - int dtype = paddle::get(op_desc.GetAttr("dtype")); - - if (dtype == /*BOOL*/ 0) { + if (op_desc.HasAttr("bool_values")) { legacy_attr = op_desc.GetAttr("bool_values"); - } else if (dtype == /*INT32*/ 2) { - legacy_attr = op_desc.GetAttr("int32_values"); - } else if (dtype == /*FP32*/ 5) { + } else if (op_desc.HasAttr("fp32_values")) { legacy_attr = op_desc.GetAttr("fp32_values"); - } else if (dtype == /*INT64*/ 3) { + } else if (op_desc.HasAttr("int32_values")) { + legacy_attr = op_desc.GetAttr("int32_values"); + } else if (op_desc.HasAttr("int64_values")) { legacy_attr = op_desc.GetAttr("int64_values"); + } else if (op_desc.HasAttr("values")) { + legacy_attr = op_desc.GetAttr("values"); } else { IR_THROW( - "Op assign_value should have attribute `**_values` but not find"); + "Op assign_value should have attribute `**_values` or `values` but " + "not find"); } pir::Attribute attr_values = attribute_translator( diff --git a/paddle/fluid/operators/ops_signature/assign_value_sig.cc b/paddle/fluid/operators/ops_signature/assign_value_sig.cc index 977c2260e59b9..ae14c5a9d7879 100644 --- a/paddle/fluid/operators/ops_signature/assign_value_sig.cc +++ b/paddle/fluid/operators/ops_signature/assign_value_sig.cc @@ -18,30 +18,8 @@ namespace phi { KernelSignature AssignValueOpArgumentMapping( const ArgumentMappingContext& ctx) { - // Here we must use `dtype` attr to determine which attr to use, we can't - // judge by whether the attr is empty, some unittests will failed - int dtype = paddle::any_cast(ctx.Attr("dtype")); - // heer we can't depend on the fluid proto::VarType, so we use the dtype enum - // value directly, If the enum value is updated, the code also needs to be - // updated here, but the probability of updating the enum value is very low - if (dtype == /*BOOL*/ 0) { - return KernelSignature( - "assign_value", {}, {"shape", "dtype", "bool_values"}, {"Out"}); - } else if (dtype == /*INT32*/ 2) { - return KernelSignature( - "assign_value", {}, {"shape", "dtype", "int32_values"}, {"Out"}); - } else if (dtype == /*FP32*/ 5) { - return KernelSignature( - "assign_value", {}, {"shape", "dtype", "fp32_values"}, {"Out"}); - } else if (dtype == /*FP64*/ 6) { - return KernelSignature( - "assign_value", {}, {"shape", "dtype", "fp64_values"}, {"Out"}); - } else if (dtype == /*INT64*/ 3) { - return KernelSignature( - "assign_value", {}, {"shape", "dtype", "int64_values"}, {"Out"}); - } else { - return KernelSignature("unregistered", {}, {}, {}); - } + return KernelSignature( + "assign_value", {}, {"shape", "dtype", "values"}, {"Out"}); } } // namespace phi diff --git a/paddle/fluid/pir/dialect/operator/ir/op_attribute.cc b/paddle/fluid/pir/dialect/operator/ir/op_attribute.cc index 3134214cf9029..10ae5a77d9f4a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_attribute.cc +++ b/paddle/fluid/pir/dialect/operator/ir/op_attribute.cc @@ -43,6 +43,10 @@ phi::Scalar ScalarAttribute::data() { return phi::Scalar(dyn_cast().data()); } else if (isa()) { return phi::Scalar(dyn_cast().AsString()); + } else if (isa()) { + return phi::Scalar(dyn_cast().data()); + } else if (isa()) { + return phi::Scalar(dyn_cast().data()); } else { PADDLE_THROW(phi::errors::Unimplemented( "Unsupported ir attribute when casting it into " diff --git a/paddle/fluid/pir/dialect/operator/ir/op_attribute.h b/paddle/fluid/pir/dialect/operator/ir/op_attribute.h index 0b0973a5205c8..f58803fa20002 100644 --- a/paddle/fluid/pir/dialect/operator/ir/op_attribute.h +++ b/paddle/fluid/pir/dialect/operator/ir/op_attribute.h @@ -50,7 +50,9 @@ class ScalarAttribute : public pir::Attribute { (val.type_id() == pir::Int32Attribute::type_id()) || (val.type_id() == pir::IndexAttribute::type_id()) || (val.type_id() == pir::Int64Attribute::type_id()) || - (val.type_id() == pir::StrAttribute::type_id()); + (val.type_id() == pir::StrAttribute::type_id()) || + (val.type_id() == pir::Complex64Attribute::type_id()) || + (val.type_id() == pir::Complex128Attribute::type_id()); } static pir::Attribute get(pir::IrContext *ctx, phi::Scalar scalar) { diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.h b/paddle/fluid/pir/dialect/operator/utils/utils.h index 0e14077bb8559..7a8a5083a3dae 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.h +++ b/paddle/fluid/pir/dialect/operator/utils/utils.h @@ -120,6 +120,12 @@ static inline pir::Attribute TransToIrAttribute(phi::Scalar scalar, return pir::Int64Attribute::get(ctx, scalar.to()); case phi::DataType::BOOL: return pir::BoolAttribute::get(ctx, scalar.to()); + case phi::DataType::COMPLEX64: + return pir::Complex64Attribute::get( + ctx, scalar.to>()); + case phi::DataType::COMPLEX128: + return pir::Complex128Attribute::get( + ctx, scalar.to>()); default: PADDLE_THROW(phi::errors::Unimplemented( "Unsupported phi data type `%s` when casting it into " diff --git a/paddle/fluid/pybind/op_function_common.cc b/paddle/fluid/pybind/op_function_common.cc index 489b25f35867c..0555724a49cfa 100644 --- a/paddle/fluid/pybind/op_function_common.cc +++ b/paddle/fluid/pybind/op_function_common.cc @@ -77,7 +77,7 @@ bool PyObject_CheckLongOrToLong(PyObject** obj) { } if (std::string(((PyTypeObject*)(*obj)->ob_type)->tp_name) // NOLINT - .find("numpy") != std::string::npos) { + .find("numpy.int") != std::string::npos) { auto to = PyNumber_Long(*obj); if (to) { *obj = to; @@ -95,8 +95,12 @@ bool PyObject_CheckFloatOrToFloat(PyObject** obj) { (((TensorObject*)(*obj))->tensor.numel() == 1))) { // NOLINT return true; } - if (std::string(((PyTypeObject*)(*obj)->ob_type)->tp_name) // NOLINT - .find("numpy") != std::string::npos) { + auto type_name = + std::string(reinterpret_cast((*obj)->ob_type)->tp_name); + VLOG(4) << "type_name: " << type_name; + + if (type_name.find("numpy") != std::string::npos && + type_name.find("numpy.complex") == std::string::npos) { auto to = PyNumber_Float(*obj); if (to) { *obj = to; @@ -107,11 +111,15 @@ bool PyObject_CheckFloatOrToFloat(PyObject** obj) { } bool PyObject_CheckComplexOrToComplex(PyObject** obj) { - if (PyComplex_Check(*obj) || PyLong_Check(*obj) || PyFloat_Check(*obj) || + if (PyComplex_Check(*obj) || PyObject_TypeCheck(*obj, g_vartype_pytype) || // NOLINT PyObject_TypeCheck(*obj, p_tensor_type)) { // NOLINT return true; } + if (std::string(((PyTypeObject*)(*obj)->ob_type)->tp_name) // NOLINT + .find("numpy.complex") != std::string::npos) { + return true; + } // consider numpy cfloat & numpy cdouble? return false; } @@ -242,10 +250,15 @@ double CastPyArg2Double(PyObject* obj, phi::dtype::complex CastPyArg2Complex(PyObject* obj, const std::string& op_type, ssize_t arg_pos) { + PyTypeObject* type = obj->ob_type; + auto type_name = std::string(type->tp_name); if (PyComplex_Check(obj)) { double real = PyComplex_RealAsDouble(obj); double imag = PyComplex_ImagAsDouble(obj); return phi::dtype::complex(real, imag); // NOLINT + } else if (type_name == "numpy.complex64") { + Py_complex v = PyComplex_AsCComplex(obj); + return phi::dtype::complex(v.real, v.imag); } else { PADDLE_THROW(platform::errors::InvalidArgument( "%s(): argument (position %d) must be " diff --git a/paddle/phi/api/yaml/op_version.yaml b/paddle/phi/api/yaml/op_version.yaml index 7c9618f52b17b..2bd09abd311ae 100644 --- a/paddle/phi/api/yaml/op_version.yaml +++ b/paddle/phi/api/yaml/op_version.yaml @@ -55,6 +55,21 @@ - delete_attr : atol comment : The attribute 'atol' is deleted. The reason why it is deleted is that attributes do not support a float64 value and it is changed to a tensor. +- op : assign_value + version : + - checkpoint : Upgrade assign_value, remove plain attributes in favor of generic attribute. + action : + - add_attr : values + comment : replace generic types with scalar. + default : std::vector() + - delete_attr : bool_values + comment : remove plain attributes. + - delete_attr : fp32_values + comment : remove plain attributes. + - delete_attr : int32_values + comment : remove plain attributes. + - delete_attr : int64_values + comment : remove plain attributes. - op : auc version : diff --git a/paddle/phi/api/yaml/static_ops.yaml b/paddle/phi/api/yaml/static_ops.yaml index 5fe9ea4260d40..6ff2bfe427122 100755 --- a/paddle/phi/api/yaml/static_ops.yaml +++ b/paddle/phi/api/yaml/static_ops.yaml @@ -90,7 +90,7 @@ backward : assign_grad - op : assign_value - args : (int[] shape, DataType dtype, int[] bool_values = {}, float[] fp32_values = {}, double[] fp64_values = {}, int[] int32_values = {}, int64_t[] int64_values = {}) + args : (int[] shape, DataType dtype, Scalar[] values = {}) output : Tensor(out) infer_meta : func : AssignValueInferMeta diff --git a/paddle/phi/kernels/assign_kernel.cc b/paddle/phi/kernels/assign_kernel.cc index b4504f83818d7..f54dfec2f6ad2 100644 --- a/paddle/phi/kernels/assign_kernel.cc +++ b/paddle/phi/kernels/assign_kernel.cc @@ -137,7 +137,9 @@ PD_REGISTER_KERNEL(assign_value, float, double, int8_t, - int64_t) {} + int64_t, + phi::dtype::complex, + phi::dtype::complex) {} #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) PD_REGISTER_KERNEL_FOR_ALL_DTYPE(assign, @@ -165,7 +167,9 @@ PD_REGISTER_KERNEL(assign_value, float, double, int8_t, - int64_t) {} + int64_t, + phi::dtype::complex, + phi::dtype::complex) {} #endif #ifdef PADDLE_WITH_XPU @@ -193,5 +197,7 @@ PD_REGISTER_KERNEL(assign_value, int, float, double, - int64_t) {} + int64_t, + phi::dtype::complex, + phi::dtype::complex) {} #endif diff --git a/paddle/pir/core/builder.h b/paddle/pir/core/builder.h index c5e3472bb070a..158d82f3fbcbe 100644 --- a/paddle/pir/core/builder.h +++ b/paddle/pir/core/builder.h @@ -16,6 +16,7 @@ #include +#include "paddle/phi/common/complex.h" #include "paddle/pir/core/block.h" #include "paddle/pir/core/ir_context.h" #include "paddle/pir/core/operation.h" @@ -44,6 +45,8 @@ class Int64Attribute; class ArrayAttribute; class PointerAttribute; class TensorNameAttribute; +class Complex64Attribute; +class Complex128Attribute; using InsertionPoint = std::pair; /// @@ -150,6 +153,8 @@ class Builder { IR_API ArrayAttribute array_attr(const std::vector &value); IR_API PointerAttribute pointer_attr(void *value); IR_API TensorNameAttribute tensor_name_attr(const std::string &value); + IR_API Complex64Attribute complex64_attr(phi::dtype::complex value); + IR_API Complex128Attribute complex128_attr(phi::dtype::complex value); private: Operation *Insert(Operation *op); diff --git a/paddle/pir/core/builtin_attribute.cc b/paddle/pir/core/builtin_attribute.cc index a817fb48c55fc..32136371d5780 100644 --- a/paddle/pir/core/builtin_attribute.cc +++ b/paddle/pir/core/builtin_attribute.cc @@ -32,6 +32,14 @@ void* PointerAttribute::data() const { return storage()->data(); } Type TypeAttribute::data() const { return storage()->data(); } +phi::dtype::complex Complex64Attribute::data() const { + return storage()->data(); +} + +phi::dtype::complex Complex128Attribute::data() const { + return storage()->data(); +} + bool StrAttribute::operator<(const StrAttribute& right) const { return storage() < right.storage(); } @@ -109,3 +117,5 @@ IR_DEFINE_EXPLICIT_TYPE_ID(pir::ArrayAttribute) IR_DEFINE_EXPLICIT_TYPE_ID(pir::PointerAttribute) IR_DEFINE_EXPLICIT_TYPE_ID(pir::TypeAttribute) IR_DEFINE_EXPLICIT_TYPE_ID(pir::TensorNameAttribute) +IR_DEFINE_EXPLICIT_TYPE_ID(pir::Complex64Attribute) +IR_DEFINE_EXPLICIT_TYPE_ID(pir::Complex128Attribute) diff --git a/paddle/pir/core/builtin_attribute.h b/paddle/pir/core/builtin_attribute.h index a1751a8c248b8..59345c9e1b4f6 100644 --- a/paddle/pir/core/builtin_attribute.h +++ b/paddle/pir/core/builtin_attribute.h @@ -14,6 +14,7 @@ #pragma once +#include "paddle/phi/common/complex.h" #include "paddle/pir/core/attribute.h" #include "paddle/pir/core/builtin_attribute_storage.h" #include "paddle/pir/core/utils.h" @@ -28,6 +29,26 @@ class IR_API BoolAttribute : public Attribute { bool data() const; }; +class IR_API Complex64Attribute : public Attribute { + public: + using Attribute::Attribute; + + DECLARE_ATTRIBUTE_UTILITY_FUNCTOR(Complex64Attribute, + Complex64AttributeStorage); + + phi::dtype::complex data() const; +}; + +class IR_API Complex128Attribute : public Attribute { + public: + using Attribute::Attribute; + + DECLARE_ATTRIBUTE_UTILITY_FUNCTOR(Complex128Attribute, + Complex128AttributeStorage); + + phi::dtype::complex data() const; +}; + class IR_API FloatAttribute : public Attribute { public: using Attribute::Attribute; @@ -157,3 +178,5 @@ IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::ArrayAttribute) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::PointerAttribute) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::TypeAttribute) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::TensorNameAttribute) +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::Complex64Attribute) +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::Complex128Attribute) diff --git a/paddle/pir/core/builtin_attribute_storage.h b/paddle/pir/core/builtin_attribute_storage.h index 533b0a4ad03e9..9e66fb6b010c9 100644 --- a/paddle/pir/core/builtin_attribute_storage.h +++ b/paddle/pir/core/builtin_attribute_storage.h @@ -19,6 +19,7 @@ #include #include "paddle/common/enforce.h" +#include "paddle/phi/common/complex.h" #include "paddle/pir/core/attribute.h" #include "paddle/pir/core/attribute_base.h" #include "paddle/pir/core/type.h" @@ -149,4 +150,43 @@ struct ArrayAttributeStorage : public AttributeStorage { const size_t size_; }; +struct Complex64AttributeStorage : public AttributeStorage { + using ParamKey = phi::dtype::complex; + explicit Complex64AttributeStorage(const ParamKey &key) { data_ = key; } + static Complex64AttributeStorage *Construct(const ParamKey &key) { + return new Complex64AttributeStorage(key); + } + static std::size_t HashValue(const ParamKey &key) { + std::stringstream complex_str; + complex_str << key.real << "+" << key.imag << "i"; + return std::hash{}(complex_str.str()); + } + + bool operator==(ParamKey key) const { return data_ == key; } + + phi::dtype::complex data() const { return data_; } + + private: + phi::dtype::complex data_; +}; + +struct Complex128AttributeStorage : public AttributeStorage { + using ParamKey = phi::dtype::complex; + explicit Complex128AttributeStorage(const ParamKey &key) { data_ = key; } + static Complex128AttributeStorage *Construct(const ParamKey &key) { + return new Complex128AttributeStorage(key); + } + static std::size_t HashValue(const ParamKey &key) { + std::stringstream complex_str; + complex_str << key.real << "+" << key.imag << "i"; + return std::hash{}(complex_str.str()); + } + + bool operator==(ParamKey key) const { return data_ == key; } + + phi::dtype::complex data() const { return data_; } + + private: + phi::dtype::complex data_; +}; } // namespace pir diff --git a/paddle/pir/core/builtin_dialect.cc b/paddle/pir/core/builtin_dialect.cc index 4bba7185384a3..91835c3029dc7 100644 --- a/paddle/pir/core/builtin_dialect.cc +++ b/paddle/pir/core/builtin_dialect.cc @@ -50,7 +50,9 @@ void BuiltinDialect::initialize() { Int64Attribute, ArrayAttribute, TypeAttribute, - TensorNameAttribute>(); + TensorNameAttribute, + Complex64Attribute, + Complex128Attribute>(); RegisterOps()) { os << "(Pointer)" << p.data(); + } else if (auto p = attr.dyn_cast()) { + os << "(Complex64)" << p.data(); + } else if (auto p = attr.dyn_cast()) { + os << "(Complex128)" << p.data(); } else if (auto arr = attr.dyn_cast()) { const auto& vec = arr.AsVector(); os << "["; diff --git a/python/paddle/nn/initializer/Bilinear.py b/python/paddle/nn/initializer/Bilinear.py index cfb18dac02c2a..1da82cbeee970 100644 --- a/python/paddle/nn/initializer/Bilinear.py +++ b/python/paddle/nn/initializer/Bilinear.py @@ -148,7 +148,7 @@ def forward(self, var, block=None): out_var = var if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32): - value_name = "fp32_values" + value_name = "values" values = [float(v) for v in weight.flat] else: raise TypeError("Unsupported dtype %s", var.dtype) diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py index 9274ff5275df0..3988f9f14859d 100644 --- a/python/paddle/nn/initializer/assign.py +++ b/python/paddle/nn/initializer/assign.py @@ -89,13 +89,13 @@ def forward(self, var, block=None): np_value = self._value if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32): - value_name = "fp32_values" + value_name = "values" values = [float(v) for v in np_value.flat] elif out_dtype in (core.VarDesc.VarType.FP64, core.DataType.FLOAT64): - value_name = "fp64_values" + value_name = "values" values = [float(v) for v in np_value.flat] elif out_dtype in (core.VarDesc.VarType.INT32, core.DataType.INT32): - value_name = "int32_values" + value_name = "values" values = [int(v) for v in np_value.flat] elif out_dtype in ( core.VarDesc.VarType.INT8, diff --git a/python/paddle/nn/initializer/dirac.py b/python/paddle/nn/initializer/dirac.py index 7da5cd15b54f7..4aea131684f21 100644 --- a/python/paddle/nn/initializer/dirac.py +++ b/python/paddle/nn/initializer/dirac.py @@ -255,7 +255,7 @@ def __call__(self, var, block=None): attrs={ 'dtype': VarDesc.VarType.INT64, 'shape': [len(idx_list)], - 'int64_values': idx_list, + 'values': idx_list, }, stop_gradient=True, ) @@ -298,7 +298,7 @@ def __call__(self, var, block=None): attrs={ 'dtype': VarDesc.VarType.FP32, 'shape': [len(value_list)], - 'fp32_values': value_list, + 'values': value_list, }, stop_gradient=True, ) diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index 1fb067edcbb6e..5fbf1f0fbc468 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -16,7 +16,6 @@ import math import re -import warnings import numpy as np @@ -2361,6 +2360,8 @@ def assign(x, output=None): 'uint8', 'int8', 'bool', + 'complex64', + 'complex128', ], 'assign', '(When the type of input in assign is Variable.)', @@ -2408,44 +2409,23 @@ def convert_scalar(x): ) dtype = convert_np_dtype_to_dtype_(input.dtype) - if dtype == core.VarDesc.VarType.FP64: - # Setting FP64 numpy data is not supported in Paddle, so we - # use FP32 here - warnings.warn( - "paddle.assign doesn't support float64 input now due " - "to current platform protobuf data limitation, we convert " - "it to float32" - ) - dtype = core.VarDesc.VarType.FP32 - - if dtype == core.DataType.FLOAT64: - # Setting FP64 numpy data is not supported in Paddle, so we - # use FP32 here - warnings.warn( - "paddle.assign doesn't support float64 input now due " - "to current platform protobuf data limitation, we convert " - "it to float32" - ) - dtype = core.DataType.FLOAT32 - - if dtype in [core.VarDesc.VarType.BOOL, core.DataType.BOOL]: - value_name = "bool_values" - values = [int(v) for v in input.flat] - elif dtype in [core.VarDesc.VarType.FP32, core.DataType.FLOAT32]: - value_name = "fp32_values" - values = [float(v) for v in input.flat] - elif dtype in [core.VarDesc.VarType.INT32, core.DataType.INT32]: - value_name = "int32_values" - values = [int(v) for v in input.flat] - elif dtype in [core.VarDesc.VarType.INT64, core.DataType.INT64]: - value_name = "int64_values" - values = [int(v) for v in input.flat] - else: - raise TypeError( - "When the type of 'input' in assign is numpy.ndarray, " - "the data type of 'input' must be bool, float32, int32 or int64, but " - "received %s." % convert_dtype(dtype) - ) + check_dtype( + dtype, + 'input', + [ + 'float32', + 'float64', + 'int32', + 'int64', + 'bool', + 'complex64', + 'complex128', + ], + 'assign', + '(When the type of input in assign is numpy array.)', + ) + value_name = "values" + values = input.ravel().tolist() if input.size > 1024 * 1024: raise ValueError( "The size of input is too big. Please consider " diff --git a/test/dygraph_to_static/test_program_translator.py b/test/dygraph_to_static/test_program_translator.py index d384c7ad649d9..d6addfe3400bc 100644 --- a/test/dygraph_to_static/test_program_translator.py +++ b/test/dygraph_to_static/test_program_translator.py @@ -314,14 +314,24 @@ def test_ifelse_early_return1(self): answer = np.zeros([2, 2]) + 1 static_func = paddle.jit.to_static(dyfunc_with_if_else_early_return1) out = static_func() - np.testing.assert_allclose(answer, out[0].numpy(), rtol=1e-05) + if isinstance(out, paddle.Tensor): + np.testing.assert_allclose( + paddle.to_tensor(answer), out, rtol=1e-05 + ) + elif isinstance(out, tuple): + np.testing.assert_allclose(answer, out[0].numpy(), rtol=1e-05) @disable_test_case((ToStaticMode.AST, IrMode.PT)) def test_ifelse_early_return2(self): answer = np.zeros([2, 2]) + 3 static_func = paddle.jit.to_static(dyfunc_with_if_else_early_return2) out = static_func() - np.testing.assert_allclose(answer, out[0].numpy(), rtol=1e-05) + if isinstance(out, paddle.Tensor): + np.testing.assert_allclose( + paddle.to_tensor(answer), out, rtol=1e-05 + ) + elif isinstance(out, tuple): + np.testing.assert_allclose(answer, out[0].numpy(), rtol=1e-05) class TestRemoveCommentInDy2St(Dy2StTestBase): diff --git a/test/ir/inference/CMakeLists.txt b/test/ir/inference/CMakeLists.txt index 020b84b4fd32a..185ca22f897f6 100755 --- a/test/ir/inference/CMakeLists.txt +++ b/test/ir/inference/CMakeLists.txt @@ -168,8 +168,8 @@ if(NOT WITH_MKLDNN set_tests_properties(${target} PROPERTIES LABELS "RUN_TYPE=INFER") endforeach() - set_tests_properties(test_mul_lstm_fuse_pass PROPERTIES TIMEOUT 300) - set_tests_properties(test_mul_gru_fuse_pass PROPERTIES TIMEOUT 300) + set_tests_properties(test_mul_lstm_fuse_pass PROPERTIES TIMEOUT 1000) + set_tests_properties(test_mul_gru_fuse_pass PROPERTIES TIMEOUT 600) endif() if(WITH_GPU AND TENSORRT_FOUND) diff --git a/test/ir/inference/test_mul_gru_fuse_pass.py b/test/ir/inference/test_mul_gru_fuse_pass.py index 91c8058c54ec5..0ccbe46724608 100644 --- a/test/ir/inference/test_mul_gru_fuse_pass.py +++ b/test/ir/inference/test_mul_gru_fuse_pass.py @@ -134,7 +134,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis( - quant=False, max_duration=300, passes=["mul_gru_fuse_pass"] + quant=False, max_duration=600, passes=["mul_gru_fuse_pass"] ) diff --git a/test/ir/inference/test_mul_lstm_fuse_pass.py b/test/ir/inference/test_mul_lstm_fuse_pass.py index f6304404c3694..fec34311604ee 100644 --- a/test/ir/inference/test_mul_lstm_fuse_pass.py +++ b/test/ir/inference/test_mul_lstm_fuse_pass.py @@ -120,7 +120,7 @@ def sample_predictor_configs(self, program_config): def test(self): self.run_and_statis( - quant=False, max_duration=300, passes=["mul_lstm_fuse_pass"] + quant=False, max_duration=1000, passes=["mul_lstm_fuse_pass"] ) diff --git a/test/ir/inference/test_seq_concat_fc_fuse_pass.py b/test/ir/inference/test_seq_concat_fc_fuse_pass.py index 4f1a0cbb7af83..68e446c5a6469 100644 --- a/test/ir/inference/test_seq_concat_fc_fuse_pass.py +++ b/test/ir/inference/test_seq_concat_fc_fuse_pass.py @@ -140,7 +140,9 @@ def teller1(program_config, predictor_config): ) def test(self): - self.run_and_statis(quant=False, passes=["seq_concat_fc_fuse_pass"]) + self.run_and_statis( + quant=False, passes=["seq_concat_fc_fuse_pass"], max_duration=1000 + ) if __name__ == "__main__": diff --git a/test/legacy_test/test_assign_value_op.py b/test/legacy_test/test_assign_value_op.py index 6ff4282d9fc55..10ff186e2e966 100644 --- a/test/legacy_test/test_assign_value_op.py +++ b/test/legacy_test/test_assign_value_op.py @@ -22,24 +22,24 @@ from paddle.base import framework -def assign_value_wrapper( - shape=[], dtype=base.core.VarDesc.VarType.FP32, values=0.0 -): - if paddle.framework.in_dynamic_mode(): - tensor = paddle.Tensor() - else: - np_type = paddle.base.data_feeder._PADDLE_DTYPE_2_NUMPY_DTYPE[dtype] - tensor = paddle.zeros(list(shape), np_type) - dtype = paddle.pir.core.convert_np_dtype_to_dtype_(np_type) - return paddle._C_ops.assign_value_( - tensor, shape, dtype, values, framework._current_expected_place() - ) +def wrap_assign_value_wrapper(dtype=base.core.VarDesc.VarType.FP32): + def assign_value_wrapper(shape=[], dtype=dtype, values=0.0): + if paddle.framework.in_dynamic_mode(): + tensor = paddle.Tensor() + else: + np_type = paddle.base.data_feeder._PADDLE_DTYPE_2_NUMPY_DTYPE[dtype] + tensor = paddle.zeros(list(shape), np_type) + dtype = paddle.pir.core.convert_np_dtype_to_dtype_(np_type) + return paddle._C_ops.assign_value_( + tensor, shape, dtype, values, framework._current_expected_place() + ) + + return assign_value_wrapper class TestAssignValueOp(op_test.OpTest): def setUp(self): self.op_type = "assign_value" - self.python_api = assign_value_wrapper self.inputs = {} self.attrs = {} self.init_data() @@ -47,11 +47,12 @@ def setUp(self): self.attrs["dtype"] = framework.convert_np_dtype_to_dtype_( self.value.dtype ) + self.python_api = wrap_assign_value_wrapper(self.attrs["dtype"]) self.outputs = {"Out": self.value} def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.float32) - self.attrs["fp32_values"] = [float(v) for v in self.value.flat] + self.attrs["values"] = [float(v) for v in self.value.flat] def test_forward(self): self.check_output(check_cinn=True, check_pir=True) @@ -60,13 +61,13 @@ def test_forward(self): class TestAssignValueOp2(TestAssignValueOp): def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) - self.attrs["int32_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) - self.attrs["int64_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): @@ -74,7 +75,29 @@ def init_data(self): self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( np.bool_ ) - self.attrs["bool_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] + + +class TestAssignValueOp5(TestAssignValueOp): + def init_data(self): + self.value = np.random.random(size=(2, 5)).astype(np.float64) + self.attrs["values"] = [float(v) for v in self.value.flat] + + +class TestAssignValueOp6(TestAssignValueOp): + def init_data(self): + self.value = ( + np.random.random(size=(2, 5)) + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex64) + self.attrs["values"] = list(self.value.flat) + + +class TestAssignValueOp7(TestAssignValueOp): + def init_data(self): + self.value = ( + np.random.random(size=(2, 5)) + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex128) + self.attrs["values"] = list(self.value.flat) class TestAssignApi(unittest.TestCase): @@ -97,8 +120,7 @@ def test_assign(self): with op_test.paddle_static_guard(): main_program = base.Program() with base.program_guard(main_program): - x = paddle.tensor.create_tensor(dtype=self.dtype) - paddle.assign(self.value, output=x) + x = paddle.assign(self.value) exe = base.Executor(self.place) [fetched_x] = exe.run(main_program, feed={}, fetch_list=[x]) @@ -145,5 +167,46 @@ def init_dtype(self): self.dtype = "bool" +class TestAssignApi5(TestAssignApi): + def init_dtype(self): + self.dtype = "float64" + + +class TestAssignApi6(TestAssignApi): + def setUp(self): + with op_test.paddle_static_guard(): + self.init_dtype() + self.value = ( + np.random.random(size=(2, 5)) + + 1j * (np.random.random(size=(2, 5))) + ).astype(np.complex64) + self.place = ( + base.CUDAPlace(0) + if base.is_compiled_with_cuda() + else base.CPUPlace() + ) + + def init_dtype(self): + self.dtype = "complex64" + + +class TestAssignApi7(TestAssignApi): + def setUp(self): + with op_test.paddle_static_guard(): + self.init_dtype() + self.value = ( + np.random.random(size=(2, 5)) + + 1j * (np.random.random(size=(2, 5))) + ).astype(np.complex128) + self.place = ( + base.CUDAPlace(0) + if base.is_compiled_with_cuda() + else base.CPUPlace() + ) + + def init_dtype(self): + self.dtype = "complex128" + + if __name__ == '__main__': unittest.main() diff --git a/test/legacy_test/test_initializer.py b/test/legacy_test/test_initializer.py index 5170207284459..ac612d2b2bee3 100644 --- a/test/legacy_test/test_initializer.py +++ b/test/legacy_test/test_initializer.py @@ -1354,7 +1354,8 @@ def test_numpy_array_initializer(self, dtype="float32"): self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'assign_value') - assert (init_op.attr('fp32_values') == np_array).all() + values = framework.extract_plain_list(init_op.attr('values')) + assert values == np_array.ravel().tolist() return block def test_numpy_array_initializer_fp16(self): diff --git a/test/legacy_test/test_initializer_nn.py b/test/legacy_test/test_initializer_nn.py index 95c64ac648290..1d9d8b08cf16d 100644 --- a/test/legacy_test/test_initializer_nn.py +++ b/test/legacy_test/test_initializer_nn.py @@ -664,8 +664,8 @@ def test_assign_initializer(self, dtype="float32"): self.assertEqual(len(block.ops), num_ops) init_op = block.ops[0] self.assertEqual(init_op.type, 'assign_value') - assert (init_op.attr('fp32_values') == np_array).all() - + values = framework.extract_plain_list(init_op.attr('values')) + assert values == np_array.ravel().tolist() paddle.disable_static() return block diff --git a/test/legacy_test/test_program_converter.py b/test/legacy_test/test_program_converter.py index 3894ca930ee0f..3ba1e7f33ad57 100644 --- a/test/legacy_test/test_program_converter.py +++ b/test/legacy_test/test_program_converter.py @@ -301,3 +301,196 @@ def test_complex128(self): legacy_program_bytes = mp._get_desc().serialize_to_string( legacy_format=True ) + + +class TestAssignValue(unittest.TestCase): + def setUp(self): + paddle.enable_static() + + def _test_for_new_program_format(self, program_bytes): + restored_prog_as_is = framework_pb2.ProgramDesc.FromString( + program_bytes + ) + for block in restored_prog_as_is.blocks: + for op in block.ops: + if op.type in ("assign_value"): + attr_names = [attr.name for attr in op.attrs] + self.assertTrue("values" in attr_names) + self.assertFalse("bool_values" in attr_names) + self.assertFalse("int32_values" in attr_names) + self.assertFalse("int64_values" in attr_names) + self.assertFalse("fp32_values" in attr_names) + + def _test_for_legacy_program_format(self, program_bytes): + restored_prog_as_is = framework_pb2.ProgramDesc.FromString( + program_bytes + ) + for block in restored_prog_as_is.blocks: + for op in block.ops: + if op.type in ("set_value", "set_value_grad"): + attr_names = [attr.name for attr in op.attrs] + self.assertFalse("values" in attr_names) + self.assertTrue("bool_values" in attr_names) + self.assertTrue("int32_values" in attr_names) + self.assertTrue("int64_values" in attr_names) + self.assertTrue("fp32_values" in attr_names) + + def _test_equivalence( + self, + new_program_bytes, + legacy_program_bytes, + fetch_list, + expected_outputs, + ): + normal_program = paddle.static.io.deserialize_program(new_program_bytes) + converted_back_program = paddle.static.io.deserialize_program( + legacy_program_bytes + ) + exe = paddle.static.Executor(paddle.CPUPlace()) + out = exe.run(normal_program, fetch_list=fetch_list) + np.testing.assert_allclose(out[0], expected_outputs[0]) + out = exe.run(converted_back_program, fetch_list=fetch_list) + np.testing.assert_allclose(out[0], expected_outputs[0]) + + def test_int32(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = np.array([[1, 1], [3, 4], [1, 3]]).astype(np.int32) + out = paddle.assign(x) + + normal_program_bytes = mp._get_desc().serialize_to_string() + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + self.assertNotEqual(normal_program_bytes, legacy_program_bytes) + self._test_for_new_program_format(normal_program_bytes) + self._test_for_legacy_program_format(legacy_program_bytes) + self._test_equivalence( + normal_program_bytes, + legacy_program_bytes, + fetch_list=[out.name], + expected_outputs=[x], + ) + + def test_int64(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = np.array([[1, 1], [3, 4], [1, 3]]).astype(np.int64) + out = paddle.assign(x) + + normal_program_bytes = mp._get_desc().serialize_to_string() + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + self.assertNotEqual(normal_program_bytes, legacy_program_bytes) + self._test_for_new_program_format(normal_program_bytes) + self._test_for_legacy_program_format(legacy_program_bytes) + self._test_equivalence( + normal_program_bytes, + legacy_program_bytes, + fetch_list=[out.name], + expected_outputs=[x], + ) + + def test_float32(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = np.random.random(size=(2, 5)).astype(np.float32) + out = paddle.assign(x) + + normal_program_bytes = mp._get_desc().serialize_to_string() + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + self.assertNotEqual(normal_program_bytes, legacy_program_bytes) + self._test_for_new_program_format(normal_program_bytes) + self._test_for_legacy_program_format(legacy_program_bytes) + self._test_equivalence( + normal_program_bytes, + legacy_program_bytes, + fetch_list=[out.name], + expected_outputs=[x], + ) + + def test_float64(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = np.random.random(size=(2, 5)).astype(np.float64) + out = paddle.assign(x) + + normal_program_bytes = mp._get_desc().serialize_to_string() + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + self.assertNotEqual(normal_program_bytes, legacy_program_bytes) + self._test_for_new_program_format(normal_program_bytes) + self._test_for_legacy_program_format(legacy_program_bytes) + self._test_equivalence( + normal_program_bytes, + legacy_program_bytes, + fetch_list=[out.name], + expected_outputs=[x], + ) + + def test_bool(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = np.random.choice(a=[False, True], size=(2, 5)).astype(np.bool_) + out = paddle.assign(x) + + normal_program_bytes = mp._get_desc().serialize_to_string() + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + self.assertNotEqual(normal_program_bytes, legacy_program_bytes) + self._test_for_new_program_format(normal_program_bytes) + self._test_for_legacy_program_format(legacy_program_bytes) + self._test_equivalence( + normal_program_bytes, + legacy_program_bytes, + fetch_list=[out.name], + expected_outputs=[x], + ) + + def test_complex64(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = ( + np.random.random(size=(2, 5)) + + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex64) + out = paddle.assign(x) + + with self.assertRaisesRegex(RuntimeError, "Invalid data type"): + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + def test_complex128(self): + mp = paddle.static.Program() + sp = paddle.static.Program() + with paddle.static.program_guard(mp, sp): + x = ( + np.random.random(size=(2, 5)) + + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex128) + out = paddle.assign(x) + + with self.assertRaisesRegex(RuntimeError, "Invalid data type"): + legacy_program_bytes = mp._get_desc().serialize_to_string( + legacy_format=True + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/xpu/test_assign_value_op_xpu.py b/test/xpu/test_assign_value_op_xpu.py index f6d2d2ec96ae3..e4414cdaafc05 100644 --- a/test/xpu/test_assign_value_op_xpu.py +++ b/test/xpu/test_assign_value_op_xpu.py @@ -53,7 +53,7 @@ def setUp(self): def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.float32) - self.attrs["fp32_values"] = [float(v) for v in self.value.flat] + self.attrs["values"] = [float(v) for v in self.value.flat] def test_forward(self): self.check_output_with_place(self.place) @@ -61,19 +61,40 @@ def test_forward(self): class TestAssignValueOp2(TestAssignValueOp): def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int32) - self.attrs["int32_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] class TestAssignValueOp3(TestAssignValueOp): def init_data(self): self.value = np.random.random(size=(2, 5)).astype(np.int64) - self.attrs["int64_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] class TestAssignValueOp4(TestAssignValueOp): def init_data(self): self.value = np.random.choice(a=[False, True], size=(2, 5)).astype( np.bool_ ) - self.attrs["bool_values"] = [int(v) for v in self.value.flat] + self.attrs["values"] = [int(v) for v in self.value.flat] + + class TestAssignValueOp5(TestAssignValueOp): + def init_data(self): + self.value = np.random.random(size=(2, 5)).astype(np.float64) + self.attrs["values"] = [float(v) for v in self.value.flat] + + class TestAssignValueOp6(TestAssignValueOp): + def init_data(self): + self.value = ( + np.random.random(size=(2, 5)) + + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex64) + self.attrs["values"] = list(self.value.flat) + + class TestAssignValueOp7(TestAssignValueOp): + def init_data(self): + self.value = ( + np.random.random(size=(2, 5)) + + 1j * np.random.random(size=(2, 5)) + ).astype(np.complex128) + self.attrs["values"] = list(self.value.flat) class TestAssignApi(unittest.TestCase): @@ -90,8 +111,7 @@ def init_dtype(self): def test_assign(self): main_program = base.Program() with base.program_guard(main_program): - x = paddle.tensor.create_tensor(dtype=self.dtype) - paddle.assign(self.value, output=x) + x = paddle.assign(self.value) exe = base.Executor(self.place) [fetched_x] = exe.run(main_program, feed={}, fetch_list=[x]) @@ -121,6 +141,35 @@ def init_dtype(self): self.dtype = "bool" +class TestAssignApi5(TestAssignApi): + def init_dtype(self): + self.dtype = "float64" + + +class TestAssignApi6(TestAssignApi): + def setUp(self): + self.init_dtype() + self.value = ( + np.random.random(size=(2, 5)) + 1j * (np.random.random(size=(2, 5))) + ).astype(np.complex64) + self.place = base.XPUPlace(0) + + def init_dtype(self): + self.dtype = "complex64" + + +class TestAssignApi7(TestAssignApi): + def setUp(self): + self.init_dtype() + self.value = ( + np.random.random(size=(2, 5)) + 1j * (np.random.random(size=(2, 5))) + ).astype(np.complex128) + self.place = base.XPUPlace(0) + + def init_dtype(self): + self.dtype = "complex128" + + support_types = get_xpu_op_support_types('assign_value') for stype in support_types: create_test_class(globals(), XPUTestAssignValueOp, stype) From d72ed8aeb0fcd0343e6fe15651af9c920d048964 Mon Sep 17 00:00:00 2001 From: xuxinyi389 <104957571+xuxinyi389@users.noreply.github.com> Date: Thu, 28 Dec 2023 18:52:14 +0800 Subject: [PATCH 122/146] =?UTF-8?q?Revert=20"=E3=80=90Hackathon=205th=20No?= =?UTF-8?q?.25=E3=80=91add=20`gammaln`=20api=20(#59311)"=20(#60450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit beba862cd2aa4dd2b14cdd0c6c4c08be33df62f2. --- paddle/phi/api/yaml/backward.yaml | 10 -- paddle/phi/api/yaml/ops.yaml | 10 -- paddle/phi/kernels/cpu/gammaln_grad_kernel.cc | 22 --- paddle/phi/kernels/cpu/gammaln_kernel.cc | 22 --- paddle/phi/kernels/gammaln_grad_kernel.h | 27 --- paddle/phi/kernels/gammaln_kernel.h | 26 --- paddle/phi/kernels/gpu/gammaln_grad_kernel.cu | 30 ---- paddle/phi/kernels/gpu/gammaln_kernel.cu | 29 ---- .../kernels/impl/gammaln_grad_kernel_impl.h | 92 ---------- paddle/phi/kernels/impl/gammaln_kernel_impl.h | 49 ------ python/paddle/__init__.py | 4 - python/paddle/tensor/__init__.py | 4 - python/paddle/tensor/math.py | 45 ----- test/legacy_test/test_gammaln_op.py | 160 ------------------ test/legacy_test/test_inplace.py | 8 - 15 files changed, 538 deletions(-) delete mode 100644 paddle/phi/kernels/cpu/gammaln_grad_kernel.cc delete mode 100644 paddle/phi/kernels/cpu/gammaln_kernel.cc delete mode 100644 paddle/phi/kernels/gammaln_grad_kernel.h delete mode 100644 paddle/phi/kernels/gammaln_kernel.h delete mode 100644 paddle/phi/kernels/gpu/gammaln_grad_kernel.cu delete mode 100644 paddle/phi/kernels/gpu/gammaln_kernel.cu delete mode 100644 paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h delete mode 100644 paddle/phi/kernels/impl/gammaln_kernel_impl.h delete mode 100644 test/legacy_test/test_gammaln_op.py diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index d5748145ffe49..938ea9d500046 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -922,16 +922,6 @@ kernel : func : frame_grad -- backward_op : gammaln_grad - forward : gammaln(Tensor x) -> Tensor(out) - args : (Tensor x, Tensor out_grad) - output : Tensor(x_grad) - infer_meta : - func : UnchangedInferMeta - param: [x] - kernel : - func : gammaln_grad - - backward_op : gather_grad forward : gather(Tensor x, Tensor index, Scalar axis=0) -> Tensor(out) args : (Tensor x, Tensor index, Tensor out_grad, Scalar axis=0) diff --git a/paddle/phi/api/yaml/ops.yaml b/paddle/phi/api/yaml/ops.yaml index dc545b7a2da54..de4d700cdf80e 100644 --- a/paddle/phi/api/yaml/ops.yaml +++ b/paddle/phi/api/yaml/ops.yaml @@ -1042,16 +1042,6 @@ data_type : dtype backend : place -- op : gammaln - args : (Tensor x) - output : Tensor(out) - infer_meta : - func : UnchangedInferMeta - kernel : - func : gammaln - inplace: (x -> out) - backward : gammaln_grad - - op : gather args : (Tensor x, Tensor index, Scalar axis=0) output : Tensor(out) diff --git a/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc b/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc deleted file mode 100644 index c52ee8b3848e9..0000000000000 --- a/paddle/phi/kernels/cpu/gammaln_grad_kernel.cc +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/phi/kernels/gammaln_grad_kernel.h" - -#include "paddle/phi/backends/cpu/cpu_context.h" -#include "paddle/phi/core/kernel_registry.h" -#include "paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h" - -PD_REGISTER_KERNEL( - gammaln_grad, CPU, ALL_LAYOUT, phi::GammalnGradKernel, float, double) {} diff --git a/paddle/phi/kernels/cpu/gammaln_kernel.cc b/paddle/phi/kernels/cpu/gammaln_kernel.cc deleted file mode 100644 index ff62f86d2522f..0000000000000 --- a/paddle/phi/kernels/cpu/gammaln_kernel.cc +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/phi/kernels/gammaln_kernel.h" - -#include "paddle/phi/backends/cpu/cpu_context.h" -#include "paddle/phi/core/kernel_registry.h" -#include "paddle/phi/kernels/impl/gammaln_kernel_impl.h" - -PD_REGISTER_KERNEL( - gammaln, CPU, ALL_LAYOUT, phi::GammalnKernel, float, double) {} diff --git a/paddle/phi/kernels/gammaln_grad_kernel.h b/paddle/phi/kernels/gammaln_grad_kernel.h deleted file mode 100644 index 440dca72a9d46..0000000000000 --- a/paddle/phi/kernels/gammaln_grad_kernel.h +++ /dev/null @@ -1,27 +0,0 @@ - -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/phi/core/dense_tensor.h" - -namespace phi { - -template -void GammalnGradKernel(const Context& dev_ctx, - const DenseTensor& x, - const DenseTensor& d_out, - DenseTensor* d_x); -} // namespace phi diff --git a/paddle/phi/kernels/gammaln_kernel.h b/paddle/phi/kernels/gammaln_kernel.h deleted file mode 100644 index db3015c4a747d..0000000000000 --- a/paddle/phi/kernels/gammaln_kernel.h +++ /dev/null @@ -1,26 +0,0 @@ - -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/phi/core/dense_tensor.h" - -namespace phi { - -template -void GammalnKernel(const Context& dev_ctx, - const DenseTensor& x, - DenseTensor* out); -} // namespace phi diff --git a/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu b/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu deleted file mode 100644 index b2513d9e3f25c..0000000000000 --- a/paddle/phi/kernels/gpu/gammaln_grad_kernel.cu +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/phi/kernels/gammaln_grad_kernel.h" - -#include "paddle/phi/backends/gpu/gpu_context.h" -#include "paddle/phi/common/amp_type_traits.h" -#include "paddle/phi/core/kernel_registry.h" - -#include "paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h" - -PD_REGISTER_KERNEL(gammaln_grad, - GPU, - ALL_LAYOUT, - phi::GammalnGradKernel, - float, - double, - phi::dtype::float16, - phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/gpu/gammaln_kernel.cu b/paddle/phi/kernels/gpu/gammaln_kernel.cu deleted file mode 100644 index 3d57be7b27733..0000000000000 --- a/paddle/phi/kernels/gpu/gammaln_kernel.cu +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/phi/kernels/gammaln_kernel.h" - -#include "paddle/phi/backends/gpu/gpu_context.h" -#include "paddle/phi/core/kernel_registry.h" - -#include "paddle/phi/kernels/impl/gammaln_kernel_impl.h" - -PD_REGISTER_KERNEL(gammaln, - GPU, - ALL_LAYOUT, - phi::GammalnKernel, - float, - double, - phi::dtype::float16, - phi::dtype::bfloat16) {} diff --git a/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h b/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h deleted file mode 100644 index 50c73cff27ce4..0000000000000 --- a/paddle/phi/kernels/impl/gammaln_grad_kernel_impl.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/phi/common/amp_type_traits.h" -#include "paddle/phi/kernels/funcs/for_range.h" - -namespace phi { -template -HOSTDEVICE T digamma(T x) { - static T c = T{8.5}; - static T euler_mascheroni = T{0.57721566490153286060}; - T r; - T value; - T x2; - - if (x <= T{0.0}) { - value = T{0.0}; - return value; - } - - if (x <= T{0.000001}) { - value = -euler_mascheroni - T{1.0} / x + T{1.6449340668482264365} * x; - return value; - } - - value = T{0.0}; - x2 = x; - while (x2 < c) { - value = value - T{1.0} / x2; - x2 = x2 + T{1.0}; - } - - r = T{1.0} / x2; - value = value + std::log(x2) - T{0.5} * r; - - r = r * r; - - value = value - - r * (T{1.0} / T{12.0} - - r * (T{1.0} / T{120.0} - - r * (T{1.0} / T{252.0} - - r * (T{1.0} / T{240.0} - r * (T{1.0} / T{132.0}))))); - - return value; -} - -template -struct GammalnGradFunctor { - GammalnGradFunctor(const T* dout, const T* x, T* output, int64_t numel) - : dout_(dout), x_(x), output_(output), numel_(numel) {} - - HOSTDEVICE void operator()(int64_t idx) const { - using MT = typename phi::dtype::MPTypeTrait::Type; - const MT mp_dout = static_cast(dout_[idx]); - const MT mp_x = static_cast(x_[idx]); - output_[idx] = static_cast(mp_dout * digamma(mp_x)); - } - - private: - const T* dout_; - const T* x_; - T* output_; - int64_t numel_; -}; -template -void GammalnGradKernel(const Context& dev_ctx, - const DenseTensor& x, - const DenseTensor& d_out, - DenseTensor* d_x) { - auto numel = d_out.numel(); - auto* dout_data = d_out.data(); - auto* x_data = x.data(); - auto* dx_data = - dev_ctx.template Alloc(d_x, static_cast(numel * sizeof(T))); - phi::funcs::ForRange for_range(dev_ctx, numel); - GammalnGradFunctor functor(dout_data, x_data, dx_data, numel); - for_range(functor); -} -} // namespace phi diff --git a/paddle/phi/kernels/impl/gammaln_kernel_impl.h b/paddle/phi/kernels/impl/gammaln_kernel_impl.h deleted file mode 100644 index 38385610de0de..0000000000000 --- a/paddle/phi/kernels/impl/gammaln_kernel_impl.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "paddle/phi/common/amp_type_traits.h" -#include "paddle/phi/kernels/funcs/for_range.h" - -namespace phi { -template -struct GammalnFunctor { - GammalnFunctor(const T* x, T* output, int64_t numel) - : x_(x), output_(output), numel_(numel) {} - - HOSTDEVICE void operator()(int64_t idx) const { - using MT = typename phi::dtype::MPTypeTrait::Type; - const MT mp_x = static_cast(x_[idx]); - output_[idx] = static_cast(std::lgamma(mp_x)); - } - - private: - const T* x_; - T* output_; - int64_t numel_; -}; - -template -void GammalnKernel(const Context& dev_ctx, - const DenseTensor& x, - DenseTensor* out) { - auto numel = x.numel(); - auto* x_data = x.data(); - auto* out_data = dev_ctx.template Alloc(out); - phi::funcs::ForRange for_range(dev_ctx, numel); - GammalnFunctor functor(x_data, out_data, numel); - for_range(functor); -} -} // namespace phi diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 1f0017562ebad..fc7b2a3533f89 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -398,8 +398,6 @@ frac, frac_, frexp, - gammaln, - gammaln_, gcd, gcd_, heaviside, @@ -775,8 +773,6 @@ 'square_', 'divide', 'divide_', - 'gammaln', - 'gammaln_', 'ceil', 'atan', 'atan_', diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index b718910348d8f..b26798892a2b2 100644 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -278,8 +278,6 @@ frac, frac_, frexp, - gammaln, - gammaln_, gcd, gcd_, heaviside, @@ -670,8 +668,6 @@ 'real', 'imag', 'is_floating_point', - 'gammaln', - 'gammaln_', 'digamma', 'digamma_', 'diagonal', diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index 6d75d41b4949c..acaa0905ce6f4 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -5003,51 +5003,6 @@ def conj(x, name=None): return out -def gammaln(x, name=None): - r""" - Calculates the logarithm of the absolute value of the gamma function elementwisely. - - Args: - x (Tensor): Input Tensor. Must be one of the following types: float16, float32, float64, bfloat16. - name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. - - Returns: - Tensor, The values of the logarithm of the absolute value of the gamma at the given tensor x. - - Examples: - .. code-block:: python - - >>> import paddle - - >>> x = paddle.arange(1.5, 4.5, 0.5) - >>> out = paddle.gammaln(x) - >>> print(out) - Tensor(shape=[6], dtype=float32, place=Place(cpu), stop_gradient=True, - [-0.12078224, 0. , 0.28468287, 0.69314718, 1.20097363, - 1.79175949]) - """ - if in_dynamic_or_pir_mode(): - return _C_ops.gammaln(x) - else: - check_variable_and_dtype( - x, 'x', ['float16', 'float32', 'float64', 'bfloat16'], 'gammaln' - ) - helper = LayerHelper('gammaln', **locals()) - out = helper.create_variable_for_type_inference(x.dtype) - helper.append_op(type='gammaln', inputs={'x': x}, outputs={'out': out}) - return out - - -@inplace_apis_in_dygraph_only -def gammaln_(x, name=None): - r""" - Inplace version of ``gammaln`` API, the output Tensor will be inplaced with input ``x``. - Please refer to :ref:`api_paddle_gammaln`. - """ - if in_dynamic_mode(): - return _C_ops.gammaln_(x) - - def digamma(x, name=None): r""" Calculates the digamma of the given input tensor, element-wise. diff --git a/test/legacy_test/test_gammaln_op.py b/test/legacy_test/test_gammaln_op.py deleted file mode 100644 index 50331af5c7a34..0000000000000 --- a/test/legacy_test/test_gammaln_op.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import numpy as np -from op_test import OpTest, convert_float_to_uint16 -from scipy import special - -import paddle -from paddle.base import core - - -def ref_gammaln(x): - return special.gammaln(x) - - -def ref_gammaln_grad(x, dout): - return dout * special.polygamma(0, x) - - -class TestGammalnOp(OpTest): - def setUp(self): - self.op_type = 'gammaln' - self.python_api = paddle.gammaln - self.init_dtype_type() - self.shape = (3, 40) - self.x = np.random.random(self.shape).astype(self.dtype) + 1 - self.inputs = {'x': self.x} - out = ref_gammaln(self.x) - self.outputs = {'out': out} - - def init_dtype_type(self): - self.dtype = np.float64 - - def test_check_output(self): - self.check_output(check_pir=True) - - def test_check_grad(self): - self.check_grad(['x'], 'out', check_pir=True) - - -class TestGammalnOpFp32(TestGammalnOp): - def init_dtype_type(self): - self.dtype = np.float32 - - -class TestGammalnFP16Op(TestGammalnOp): - def init_dtype_type(self): - self.dtype = np.float16 - - -class TestGammalnBigNumberOp(TestGammalnOp): - def setUp(self): - self.op_type = 'gammaln' - self.python_api = paddle.gammaln - self.init_dtype_type() - self.shape = (100, 1) - self.x = np.random.random(self.shape).astype(self.dtype) + 1 - self.x[:5, 0] = np.array([1e5, 1e10, 1e20, 1e40, 1e80]) - self.inputs = {'x': self.x} - out = ref_gammaln(self.x) - self.outputs = {'out': out} - - def init_dtype_type(self): - self.dtype = np.float64 - - def test_check_grad(self): - d_out = self.outputs['out'] - d_x = ref_gammaln_grad(self.x, d_out) - self.check_grad( - ['x'], - 'out', - user_defined_grads=[ - d_x, - ], - user_defined_grad_outputs=[ - d_out, - ], - check_pir=True, - ) - - -@unittest.skipIf( - not core.is_compiled_with_cuda() - or not core.is_bfloat16_supported(core.CUDAPlace(0)), - "core is not compiled with CUDA or not support bfloat16", -) -class TestGammalnBF16Op(OpTest): - def setUp(self): - self.op_type = 'gammaln' - self.python_api = paddle.gammaln - self.dtype = np.uint16 - self.shape = (5, 30) - x = np.random.random(self.shape).astype("float32") + 1 - self.inputs = {'x': convert_float_to_uint16(x)} - out = ref_gammaln(x) - self.outputs = {'out': convert_float_to_uint16(out)} - - def test_check_output(self): - self.check_output_with_place(core.CUDAPlace(0), check_pir=True) - - def test_check_grad(self): - self.check_grad_with_place( - core.CUDAPlace(0), ['x'], 'out', check_pir=True - ) - - -class TestGammalnOpApi(unittest.TestCase): - def setUp(self): - self.shape = [2, 3, 4, 5] - self.init_dtype_type() - self.x_np = np.random.random(self.shape).astype(self.dtype) + 1 - self.place = ( - paddle.CUDAPlace(0) - if core.is_compiled_with_cuda() - else paddle.CPUPlace() - ) - - def init_dtype_type(self): - self.dtype = "float64" - - def test_static_api(self): - paddle.enable_static() - with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data('x', self.x_np.shape, self.x_np.dtype) - out = paddle.gammaln(x) - exe = paddle.static.Executor(self.place) - (res,) = exe.run(feed={'x': self.x_np}, fetch_list=[out]) - out_ref = ref_gammaln(self.x_np) - np.testing.assert_allclose(out_ref, res, rtol=1e-5, atol=1e-5) - - def test_dygraph_api(self): - paddle.disable_static(self.place) - x = paddle.to_tensor(self.x_np) - out = paddle.gammaln(x) - out_ref = ref_gammaln(self.x_np) - np.testing.assert_allclose(out_ref, out.numpy(), rtol=1e-5, atol=1e-5) - paddle.enable_static() - - -class TestGammalnOpApiFp32(TestGammalnOpApi): - def init_dtype_type(self): - self.dtype = "float32" - - -if __name__ == "__main__": - paddle.enable_static() - unittest.main() diff --git a/test/legacy_test/test_inplace.py b/test/legacy_test/test_inplace.py index 38fbac0357d6d..42f9a46cfb910 100644 --- a/test/legacy_test/test_inplace.py +++ b/test/legacy_test/test_inplace.py @@ -869,14 +869,6 @@ def test_leaf_inplace_var_error(self): pass -class TestDygraphInplaceGammaln(TestDygraphInplaceWithContinuous): - def inplace_api_processing(self, var): - return paddle.gammaln_(var) - - def non_inplace_api_processing(self, var): - return paddle.gammaln(var) - - class TestDygraphInplaceNeg(TestDygraphInplaceWithContinuous): def inplace_api_processing(self, var): return paddle.neg_(var) From 51dc03178d3b90f0fa84eb6da336d7cb1aaf02e5 Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Thu, 28 Dec 2023 19:15:04 +0800 Subject: [PATCH 123/146] fix dead code elimination pass bug (#60430) --- paddle/fluid/pir/transforms/dead_code_elimination_pass.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc index 4c8fa32c6d635..bc2421cfe1a86 100644 --- a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc +++ b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc @@ -58,6 +58,10 @@ class DeadCodeEliminationPass : public pir::Pass { } } } + + if (!deleted_ops.empty()) { + EraseOp(block, num_erasers); + } } }; From fdc38b2ba32a5b4f27557c36de09bbcee3b9d816 Mon Sep 17 00:00:00 2001 From: Yuanle Liu Date: Thu, 28 Dec 2023 20:47:36 +0800 Subject: [PATCH 124/146] [DRR] change namespace pir::drr:: to paddle::drr:: (#60432) --- .../operator/transforms/pd_to_cinn_pass.cc | 40 +-- .../op_generator/op_creator_drr_gen.py | 8 +- paddle/fluid/pir/drr/README.md | 24 +- paddle/fluid/pir/drr/README_cn.md | 24 +- paddle/fluid/pir/drr/api/drr_pattern_base.h | 6 +- .../fluid/pir/drr/api/drr_pattern_context.cc | 5 +- .../fluid/pir/drr/api/drr_pattern_context.h | 4 +- paddle/fluid/pir/drr/api/match_context.cc | 4 +- paddle/fluid/pir/drr/api/match_context.h | 4 +- paddle/fluid/pir/drr/api/tensor_interface.cc | 4 +- paddle/fluid/pir/drr/api/tensor_interface.h | 4 +- paddle/fluid/pir/drr/attr_type_uilts.h | 20 +- paddle/fluid/pir/drr/drr_rewrite_pattern.cc | 42 +-- paddle/fluid/pir/drr/drr_rewrite_pattern.h | 11 +- paddle/fluid/pir/drr/ir_operation.h | 4 +- paddle/fluid/pir/drr/ir_operation_factory.cc | 24 +- paddle/fluid/pir/drr/ir_operation_factory.h | 8 +- paddle/fluid/pir/drr/ir_value.h | 8 +- paddle/fluid/pir/drr/match_context_impl.h | 4 +- paddle/fluid/pir/drr/pattern_graph.cc | 4 +- paddle/fluid/pir/drr/pattern_graph.h | 4 +- .../transforms/fusion/attention_fuse_pass.cc | 50 ++-- .../transforms/fusion/conv2d_add_fuse_pass.cc | 18 +- .../fc_elementwise_layernorm_fuse_pass.cc | 32 ++- .../pir/transforms/fusion/fc_fuse_pass.cc | 33 +-- .../fusion/fc_with_special_op_fuse_pass.cc | 68 +++-- .../fused_dot_product_attention_pass.cc | 250 ++++++++++-------- .../fusion/fused_dropout_add_pass.cc | 16 +- .../fusion/fused_gemm_epilogue_pass.cc | 75 +++--- .../fused_linear_param_grad_add_pass.cc | 132 +++++---- .../fusion/fused_weight_only_linear_pass.cc | 64 ++--- .../fusion/matmul_scale_fuse_pass.cc | 24 +- .../pir/transforms/identity_op_clean_pass.cc | 62 ++--- .../drr_same_type_binding_test.cc | 8 +- test/cpp/pir/pattern_rewrite/drr_test.cc | 38 +-- 35 files changed, 597 insertions(+), 529 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc index 295c50b0eae00..352fd9fdde322 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc @@ -31,11 +31,11 @@ namespace cinn { namespace dialect { namespace ir { -class SumOpPattern : public pir::drr::DrrPatternBase { +class SumOpPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pattern = ctx->SourcePattern(); + paddle::drr::SourcePattern pattern = ctx->SourcePattern(); const auto &full_int_array = pattern.Op(paddle::dialect::FullIntArrayOp::name(), {{"value", pattern.Attr("axis_info")}, @@ -48,7 +48,7 @@ class SumOpPattern : public pir::drr::DrrPatternBase { pattern.Tensor("ret") = sum(pattern.Tensor("arg0"), full_int_array()); // Result patterns - pir::drr::ResultPattern res = pattern.ResultPattern(); + paddle::drr::ResultPattern res = pattern.ResultPattern(); const auto &cinn_reduce_sum = res.Op(cinn::dialect::ReduceSumOp::name(), {{"dim", pattern.Attr("axis_info")}, @@ -57,11 +57,11 @@ class SumOpPattern : public pir::drr::DrrPatternBase { } }; -class MaxOpPattern : public pir::drr::DrrPatternBase { +class MaxOpPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pattern = ctx->SourcePattern(); + paddle::drr::SourcePattern pattern = ctx->SourcePattern(); const auto &full_int_array = pattern.Op(paddle::dialect::FullIntArrayOp::name(), {{"value", pattern.Attr("axis_info")}, @@ -73,7 +73,7 @@ class MaxOpPattern : public pir::drr::DrrPatternBase { pattern.Tensor("ret") = pd_max(pattern.Tensor("arg0"), full_int_array()); // Result patterns - pir::drr::ResultPattern res = pattern.ResultPattern(); + paddle::drr::ResultPattern res = pattern.ResultPattern(); const auto &cinn_reduce_max = res.Op(cinn::dialect::ReduceMaxOp::name(), {{"dim", pattern.Attr("axis_info")}, @@ -82,11 +82,11 @@ class MaxOpPattern : public pir::drr::DrrPatternBase { } }; -class MinOpPattern : public pir::drr::DrrPatternBase { +class MinOpPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pattern = ctx->SourcePattern(); + paddle::drr::SourcePattern pattern = ctx->SourcePattern(); const auto &full_int_array = pattern.Op(paddle::dialect::FullIntArrayOp::name(), {{"value", pattern.Attr("axis_info")}, @@ -98,7 +98,7 @@ class MinOpPattern : public pir::drr::DrrPatternBase { pattern.Tensor("ret") = pd_max(pattern.Tensor("arg0"), full_int_array()); // Result patterns - pir::drr::ResultPattern res = pattern.ResultPattern(); + paddle::drr::ResultPattern res = pattern.ResultPattern(); const auto &cinn_reduce_max = res.Op(cinn::dialect::ReduceMinOp::name(), {{"dim", pattern.Attr("axis_info")}, @@ -107,11 +107,11 @@ class MinOpPattern : public pir::drr::DrrPatternBase { } }; -class ProdOpPattern : public pir::drr::DrrPatternBase { +class ProdOpPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pattern = ctx->SourcePattern(); + paddle::drr::SourcePattern pattern = ctx->SourcePattern(); const auto &full_int_array = pattern.Op(paddle::dialect::FullIntArrayOp::name(), {{"value", pattern.Attr("axis_info")}, @@ -123,7 +123,7 @@ class ProdOpPattern : public pir::drr::DrrPatternBase { pattern.Tensor("ret") = pd_max(pattern.Tensor("arg0"), full_int_array()); // Result patterns - pir::drr::ResultPattern res = pattern.ResultPattern(); + paddle::drr::ResultPattern res = pattern.ResultPattern(); const auto &cinn_reduce_max = res.Op(cinn::dialect::ReduceProdOp::name(), {{"dim", pattern.Attr("axis_info")}, @@ -552,11 +552,11 @@ class SplitWithNumOpPattern } }; -class UniformOpPattern : public pir::drr::DrrPatternBase { +class UniformOpPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pattern = ctx->SourcePattern(); + paddle::drr::SourcePattern pattern = ctx->SourcePattern(); const auto &full_int_array = pattern.Op(paddle::dialect::FullIntArrayOp::name(), {{"value", pattern.Attr("axis_info")}, @@ -585,7 +585,7 @@ class UniformOpPattern : public pir::drr::DrrPatternBase { // int64_t[] shape, float min, float max, int seed, DataType dtype, int // diag_num, int diag_step, float diag_val) // Result patterns - pir::drr::ResultPattern res = pattern.ResultPattern(); + paddle::drr::ResultPattern res = pattern.ResultPattern(); const auto &cinn_uniform = res.Op(cinn::dialect::UniformRandomOp::name(), {{"shape", pattern.Attr("axis_info")}, diff --git a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py index 9a40f74429e52..18dc70f9fa7a7 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_creator_drr_gen.py @@ -27,7 +27,7 @@ {op_header} #include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" -namespace pir {{ +namespace paddle {{ namespace drr {{ void OperationFactory::Register{dialect}GeneratedOpCreator() {{ @@ -35,14 +35,14 @@ }} }} // namespace drr -}} // namespace pir +}} // namespace paddle """ NORMAL_FUNCTION_TEMPLATE = """ RegisterOperationCreator( "{op_name}", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) {{ return rewriter.Build<{namespace}::{op_class_name}>( @@ -53,7 +53,7 @@ MUTABLE_ATTR_FUNCTION_TEMPLATE = """ RegisterOperationCreator( "{op_name}", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) {{ // mutable_attr is tensor diff --git a/paddle/fluid/pir/drr/README.md b/paddle/fluid/pir/drr/README.md index 4abdbb1b64717..6fbac0756ae86 100644 --- a/paddle/fluid/pir/drr/README.md +++ b/paddle/fluid/pir/drr/README.md @@ -10,9 +10,9 @@ Taking PASS to eliminate redundant CastOp as an example, the code example develo ~~~ c++ // 1. Inherit specialized template class from DrPatternBase class RemoveRedundentCastPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { // 2. Overload operator() - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // 3. Define a SourcePattern containing two consecutive CastOps using Op, Tensor, and Attribute auto pat = ctx->SourcePattern(); @@ -55,7 +55,7 @@ Developers only need to define `SourcePattern`, `Constrains` and `ResultPattern` DrrPatternBase 
 virtual void operator()(
-        pir::drr::DrrPatternContext* ctx) const
+ paddle::drr::DrrPatternContext* ctx) const Implement the entry function of DRR PASS ctx: Context parameters required to create Patten @@ -165,11 +165,11 @@ Attribute Attr(const AttrComputeFunc& attr_compute_func) const ## 3 Example Example 1: Matmul + Add -> FusedGemmEpilogue ~~~ c++ -class FusedLinearPattern : public pir::drr::DrrPatternBase { +class FusedLinearPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Define SourcePattern - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -179,10 +179,10 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { pat.Tensor("out") = add(pat.Tensor("tmp"), pat.Tensor("bias")); // Define ResultPattern - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); // Define Constrain const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "none"; }); const auto &fused_gemm_epilogue = res.Op(paddle::dialect::FusedGemmEpilogueOp::name(), @@ -199,11 +199,11 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { Example 2: Full + Expand -> Full ~~~ c++ class FoldExpandToConstantPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Define SourcePattern - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &full1 = pat.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("shape_1")}, {"value", pat.Attr("value_1")}, @@ -218,7 +218,7 @@ class FoldExpandToConstantPattern pat.Tensor("ret") = expand(full1(), full_int_array1()); // Define ResultPattern - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &full2 = res.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("expand_shape_value")}, {"value", pat.Attr("value_1")}, diff --git a/paddle/fluid/pir/drr/README_cn.md b/paddle/fluid/pir/drr/README_cn.md index 456bf7921414b..1291bec2954c4 100644 --- a/paddle/fluid/pir/drr/README_cn.md +++ b/paddle/fluid/pir/drr/README_cn.md @@ -10,9 +10,9 @@ DRR ( Declarative Rewrite Rule ) 是来处理这种 DAG-to-DAG 类型的一套 P ~~~ c++ // 1. 继承 DrrPatternBase 的特化模板类 class RemoveRedundentCastPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { // 2. 重载 operator() - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // 3. 使用 Op、Tensor 和 Attribute 定义一个包含两个连续 CastOp 的 SourcePattern auto pat = ctx->SourcePattern(); @@ -56,7 +56,7 @@ DRR PASS 包含以下三个部分： DrrPatternBase 
 virtual void operator()(
-        pir::drr::DrrPatternContext* ctx) const
+ paddle::drr::DrrPatternContext* ctx) const 实现 DRR PASS 的入口函数 ctx: 创建 Patten 所需要的 Context 参数 @@ -168,11 +168,11 @@ Attribute Attr(const AttrComputeFunc& attr_compute_func) const ## 3 使用示例 Example 1: Matmul + Add -> FusedGemmEpilogue ~~~ c++ -class FusedLinearPattern : public pir::drr::DrrPatternBase { +class FusedLinearPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // 定义 Source Pattern - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -182,10 +182,10 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { pat.Tensor("out") = add(pat.Tensor("tmp"), pat.Tensor("bias")); // 定义 Result Pattern - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); // 定义 Constrain const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "none"; }); const auto &fused_gemm_epilogue = res.Op(paddle::dialect::FusedGemmEpilogueOp::name(), @@ -202,11 +202,11 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { Example 2: Full + Expand -> Full ~~~ c++ class FoldExpandToConstantPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // 定义 Source Pattern - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &full1 = pat.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("shape_1")}, {"value", pat.Attr("value_1")}, @@ -221,7 +221,7 @@ class FoldExpandToConstantPattern pat.Tensor("ret") = expand(full1(), full_int_array1()); // 定义 Result Pattern Constrains: 本 Pass 无额外约束规则 - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &full2 = res.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("expand_shape_value")}, {"value", pat.Attr("value_1")}, diff --git a/paddle/fluid/pir/drr/api/drr_pattern_base.h b/paddle/fluid/pir/drr/api/drr_pattern_base.h index 1a84c42800373..18252d536869f 100644 --- a/paddle/fluid/pir/drr/api/drr_pattern_base.h +++ b/paddle/fluid/pir/drr/api/drr_pattern_base.h @@ -17,7 +17,7 @@ #include "paddle/fluid/pir/drr/api/drr_pattern_context.h" #include "paddle/fluid/pir/drr/drr_rewrite_pattern.h" -namespace pir { +namespace paddle { namespace drr { template @@ -26,7 +26,7 @@ class DrrPatternBase { virtual ~DrrPatternBase() = default; // Define the Drr Pattern. - virtual void operator()(pir::drr::DrrPatternContext* ctx) const = 0; + virtual void operator()(paddle::drr::DrrPatternContext* ctx) const = 0; std::unique_ptr Build( pir::IrContext* ir_context, pir::PatternBenefit benefit = 1) const { @@ -39,4 +39,4 @@ class DrrPatternBase { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/drr_pattern_context.cc b/paddle/fluid/pir/drr/api/drr_pattern_context.cc index 50e94c3458265..7f98f0b34cbeb 100644 --- a/paddle/fluid/pir/drr/api/drr_pattern_context.cc +++ b/paddle/fluid/pir/drr/api/drr_pattern_context.cc @@ -17,7 +17,7 @@ #include "paddle/fluid/pir/drr/pattern_graph.h" #include "paddle/phi/core/enforce.h" -namespace pir { +namespace paddle { namespace drr { DrrPatternContext::DrrPatternContext() { @@ -28,6 +28,7 @@ DrrPatternContext::DrrPatternContext() { drr::SourcePattern DrrPatternContext::SourcePattern() { return drr::SourcePattern(this); } + const Op& DrrPatternContext::SourceOpPattern( const std::string& op_type, const std::unordered_map& attributes) { @@ -167,4 +168,4 @@ void Tensor::operator=(const Tensor& other) const { // NOLINT } } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/drr_pattern_context.h b/paddle/fluid/pir/drr/api/drr_pattern_context.h index 5c235215dd19b..feb0e988aa882 100644 --- a/paddle/fluid/pir/drr/api/drr_pattern_context.h +++ b/paddle/fluid/pir/drr/api/drr_pattern_context.h @@ -24,7 +24,7 @@ #include "paddle/fluid/pir/drr/api/match_context.h" -namespace pir { +namespace paddle { namespace drr { class Op; @@ -334,4 +334,4 @@ class SourcePattern { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/match_context.cc b/paddle/fluid/pir/drr/api/match_context.cc index 35b28db13254e..e5f15adf72e75 100644 --- a/paddle/fluid/pir/drr/api/match_context.cc +++ b/paddle/fluid/pir/drr/api/match_context.cc @@ -19,7 +19,7 @@ #include "paddle/fluid/pir/drr/ir_operation.h" #include "paddle/fluid/pir/drr/match_context_impl.h" -namespace pir { +namespace paddle { namespace drr { MatchContext::MatchContext(std::shared_ptr impl) @@ -46,4 +46,4 @@ template std::vector MatchContext::Attr>( const std::string&) const; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/match_context.h b/paddle/fluid/pir/drr/api/match_context.h index a1699ccb5bddf..762c86cf8a8e6 100644 --- a/paddle/fluid/pir/drr/api/match_context.h +++ b/paddle/fluid/pir/drr/api/match_context.h @@ -20,7 +20,7 @@ #include "paddle/fluid/pir/drr/api/tensor_interface.h" #include "paddle/fluid/pir/drr/ir_operation.h" -namespace pir { +namespace paddle { namespace drr { class TensorInterface; @@ -40,4 +40,4 @@ class MatchContext final { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/tensor_interface.cc b/paddle/fluid/pir/drr/api/tensor_interface.cc index 03a35031f0d91..335f95214887a 100644 --- a/paddle/fluid/pir/drr/api/tensor_interface.cc +++ b/paddle/fluid/pir/drr/api/tensor_interface.cc @@ -15,7 +15,7 @@ #include "paddle/fluid/pir/drr/api/tensor_interface.h" #include "paddle/fluid/pir/drr/ir_value.h" -namespace pir { +namespace paddle { namespace drr { bool ShapeInterface::operator==(const ShapeInterface& other) const { @@ -33,4 +33,4 @@ bool DtypeInterface::operator==(const DtypeInterface& other) const { IrDtype DtypeInterface::get() const { return *(this->dtype_); } } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/api/tensor_interface.h b/paddle/fluid/pir/drr/api/tensor_interface.h index 4684beba4ad84..24774f00d5a29 100644 --- a/paddle/fluid/pir/drr/api/tensor_interface.h +++ b/paddle/fluid/pir/drr/api/tensor_interface.h @@ -16,7 +16,7 @@ #include -namespace pir { +namespace paddle { namespace drr { class IrValue; @@ -60,4 +60,4 @@ class TensorInterface { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/attr_type_uilts.h b/paddle/fluid/pir/drr/attr_type_uilts.h index 4043aa3c64383..8904ed0e9ff6a 100644 --- a/paddle/fluid/pir/drr/attr_type_uilts.h +++ b/paddle/fluid/pir/drr/attr_type_uilts.h @@ -19,7 +19,7 @@ #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" #include "paddle/pir/core/builtin_attribute.h" -namespace pir { +namespace paddle { namespace drr { template @@ -32,11 +32,11 @@ struct CppTypeToIrAttribute; using type = ir_attr_type; \ }; -PD_SPECIALIZE_CppTypeToIrAttribute(bool, BoolAttribute); -PD_SPECIALIZE_CppTypeToIrAttribute(int32_t, Int32Attribute); -PD_SPECIALIZE_CppTypeToIrAttribute(int64_t, Int64Attribute); -PD_SPECIALIZE_CppTypeToIrAttribute(float, FloatAttribute); -PD_SPECIALIZE_CppTypeToIrAttribute(std::string, StrAttribute); +PD_SPECIALIZE_CppTypeToIrAttribute(bool, pir::BoolAttribute); +PD_SPECIALIZE_CppTypeToIrAttribute(int32_t, pir::Int32Attribute); +PD_SPECIALIZE_CppTypeToIrAttribute(int64_t, pir::Int64Attribute); +PD_SPECIALIZE_CppTypeToIrAttribute(float, pir::FloatAttribute); +PD_SPECIALIZE_CppTypeToIrAttribute(std::string, pir::StrAttribute); PD_SPECIALIZE_CppTypeToIrAttribute(phi::DataType, paddle::dialect::DataTypeAttribute); PD_SPECIALIZE_CppTypeToIrAttribute(phi::Place, paddle::dialect::PlaceAttribute); @@ -61,7 +61,8 @@ struct IrAttrbuteCreator> { std::vector attr_vec; attr_vec.reserve(obj.size()); for (int32_t x : obj) { - attr_vec.push_back(Int32Attribute::get(pir::IrContext::Instance(), x)); + attr_vec.push_back( + pir::Int32Attribute::get(pir::IrContext::Instance(), x)); } return pir::ArrayAttribute::get(pir::IrContext::Instance(), attr_vec); } @@ -73,7 +74,8 @@ struct IrAttrbuteCreator> { std::vector attr_vec; attr_vec.reserve(obj.size()); for (float x : obj) { - attr_vec.push_back(FloatAttribute::get(pir::IrContext::Instance(), x)); + attr_vec.push_back( + pir::FloatAttribute::get(pir::IrContext::Instance(), x)); } return pir::ArrayAttribute::get(pir::IrContext::Instance(), attr_vec); } @@ -140,4 +142,4 @@ struct IrAttrTypeCast> { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/drr_rewrite_pattern.cc b/paddle/fluid/pir/drr/drr_rewrite_pattern.cc index d0c0d71a3feaa..d408c1aab1349 100644 --- a/paddle/fluid/pir/drr/drr_rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/drr_rewrite_pattern.cc @@ -14,12 +14,12 @@ #include "paddle/fluid/pir/drr/drr_rewrite_pattern.h" -namespace pir { +namespace paddle { namespace drr { bool DrrRewritePattern::MatchAndRewrite( pir::Operation* op, - PatternRewriter& rewriter) const { // NOLINT + pir::PatternRewriter& rewriter) const { // NOLINT std::shared_ptr src_match_ctx = std::make_shared(); if (PatternGraphMatch(op, src_match_ctx.get())) { @@ -41,8 +41,8 @@ bool DrrRewritePattern::PatternGraphMatch( return false; } std::vector drr_output_sequence; - std::vector ir_output_sequence; - std::unordered_map output_op_map; + std::vector ir_output_sequence; + std::unordered_map output_op_map; for (const auto& pair : bind_map) { drr_output_sequence.push_back(pair.first); } @@ -50,8 +50,8 @@ bool DrrRewritePattern::PatternGraphMatch( auto permute = [&](auto&& permute, size_t index) -> bool { if (index == drr_output_sequence.size()) { // avoiding duplicate binding of ir op - std::unordered_set ir_output_set; - for (Operation* op : ir_output_sequence) { + std::unordered_set ir_output_set; + for (pir::Operation* op : ir_output_sequence) { auto pr = ir_output_set.insert(op); if (pr.second == false) { return false; @@ -64,7 +64,7 @@ bool DrrRewritePattern::PatternGraphMatch( drr_output_sequence.end(), ir_output_sequence.begin(), std::inserter(output_op_map, output_op_map.end()), - [](const OpCall* drr_op, Operation* ir_op) { + [](const OpCall* drr_op, pir::Operation* ir_op) { return std::make_pair(drr_op, ir_op); }); if (MatchFromOutputToInput( @@ -214,12 +214,12 @@ void DrrRewritePattern::DfsVisitor( } bool DrrRewritePattern::MatchFromOutputToInput( - std::unordered_map output_op_map, + std::unordered_map output_op_map, const SourcePatternGraph& source_pattern_graph, const std::shared_ptr& source_pattern_match_ctx) const { VLOG(6) << "MatchFromOutputToInput Start"; std::unordered_set drr_visited; - std::unordered_set ir_visited; + std::unordered_set ir_visited; std::queue drr_q; std::queue ir_q; bool matched = true; @@ -385,8 +385,8 @@ MatchContextImpl DrrRewritePattern::CreateOperations( } } - std::vector> temp_program; - std::unordered_map op_2_temp_program_index; + std::vector> temp_program; + std::unordered_map op_2_temp_program_index; for (auto& op : *rewriter.block()) { op_2_temp_program_index[&op] = temp_program.size(); temp_program.push_back({&op}); @@ -397,14 +397,14 @@ MatchContextImpl DrrRewritePattern::CreateOperations( graph_topo_visit.WalkGraphNodesTopoOrder([&](const OpCall& op_call) { // set insert point size_t max_input_op_index = 0; - Operation* max_index_op = nullptr; + pir::Operation* max_index_op = nullptr; for (const Tensor* input : op_call.inputs()) { if (input->is_none()) { continue; } auto ir_val = res_match_ctx.GetIrValue(input->name()); if (ir_val) { - Operation* ir_input_op = ir_val.dyn_cast().owner(); + pir::Operation* ir_input_op = ir_val.dyn_cast().owner(); if (op_2_temp_program_index.count(ir_input_op) == 0) { max_input_op_index = 0UL; } else if (max_input_op_index < @@ -431,7 +431,7 @@ MatchContextImpl DrrRewritePattern::CreateOperations( } if (max_input_op_index == 0UL) { VLOG(6) << "Not found producer op for (" << op_call.name() << ")"; - Operation* source_patter_first_op = + pir::Operation* source_patter_first_op = src_match_ctx.Operation(source_pattern_graph.owned_op_call()[0].get()) .get(); max_input_op_index = op_2_temp_program_index[source_patter_first_op]; @@ -440,7 +440,7 @@ MatchContextImpl DrrRewritePattern::CreateOperations( rewriter.SetInsertionPointAfter(max_index_op); } - Operation* new_op = + pir::Operation* new_op = CreateOperation(op_call, src_match_ctx, rewriter, &res_match_ctx); op_2_temp_program_index[new_op] = max_input_op_index + 1; if (max_input_op_index + 1 >= temp_program.size()) { @@ -487,11 +487,11 @@ void DrrRewritePattern::DeleteSourcePatternOp( const ResultPatternGraph& result_pattern_graph, const MatchContextImpl& src_match_ctx, pir::PatternRewriter& rewriter) const { // NOLINT - std::queue delete_ops_que; - std::unordered_set delete_ops_set; + std::queue delete_ops_que; + std::unordered_set delete_ops_set; GraphTopo graph_topo_visit(&source_pattern_graph); graph_topo_visit.WalkGraphNodesTopoOrder([&](const OpCall& op_call) { - Operation* op = src_match_ctx.Operation(&op_call).get(); + pir::Operation* op = src_match_ctx.Operation(&op_call).get(); VLOG(5) << "DRR delete op: " << op->name() << " pointer: " << op; if (delete_ops_set.count(op) == 0 && op->use_empty()) { delete_ops_que.push(op); @@ -500,9 +500,9 @@ void DrrRewritePattern::DeleteSourcePatternOp( }); while (!delete_ops_que.empty()) { - Operation* op = delete_ops_que.front(); + pir::Operation* op = delete_ops_que.front(); delete_ops_que.pop(); - std::vector inputs = op->operands_source(); + std::vector inputs = op->operands_source(); VLOG(5) << "Delete (" << op->name() << " @" << op << ") in source_pattern_graph."; rewriter.EraseOp(op); @@ -517,4 +517,4 @@ void DrrRewritePattern::DeleteSourcePatternOp( } } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/drr_rewrite_pattern.h b/paddle/fluid/pir/drr/drr_rewrite_pattern.h index 5d20a5947f13b..6163c6d9d0193 100644 --- a/paddle/fluid/pir/drr/drr_rewrite_pattern.h +++ b/paddle/fluid/pir/drr/drr_rewrite_pattern.h @@ -31,7 +31,7 @@ #include "paddle/pir/core/type_name.h" #include "paddle/pir/pattern_rewrite/pattern_match.h" -namespace pir { +namespace paddle { namespace drr { class DrrRewritePattern : public pir::RewritePattern { @@ -57,8 +57,9 @@ class DrrRewritePattern : public pir::RewritePattern { "source pattern definition code.")); } - bool MatchAndRewrite(pir::Operation* op, - PatternRewriter& rewriter) const override; // // NOLINT + bool MatchAndRewrite( + pir::Operation* op, + pir::PatternRewriter& rewriter) const override; // // NOLINT private: bool PatternGraphMatch(pir::Operation* op, @@ -78,7 +79,7 @@ class DrrRewritePattern : public pir::RewritePattern { output_op_bind_map) const; bool MatchFromOutputToInput( - std::unordered_map output_op_map, + std::unordered_map output_op_map, const SourcePatternGraph& source_pattern_graph, const std::shared_ptr& source_pattern_match_ctx) const; @@ -113,4 +114,4 @@ class DrrRewritePattern : public pir::RewritePattern { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/ir_operation.h b/paddle/fluid/pir/drr/ir_operation.h index 2764bc9245417..a88bb3bfff97c 100644 --- a/paddle/fluid/pir/drr/ir_operation.h +++ b/paddle/fluid/pir/drr/ir_operation.h @@ -16,7 +16,7 @@ #include "paddle/pir/core/operation.h" -namespace pir { +namespace paddle { namespace drr { class IrOperation { @@ -30,4 +30,4 @@ class IrOperation { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/ir_operation_factory.cc b/paddle/fluid/pir/drr/ir_operation_factory.cc index 6644026fabde0..bbc31e9df7c25 100644 --- a/paddle/fluid/pir/drr/ir_operation_factory.cc +++ b/paddle/fluid/pir/drr/ir_operation_factory.cc @@ -24,13 +24,13 @@ #include "paddle/pir/core/operation.h" #include "paddle/pir/core/value.h" -namespace pir { +namespace paddle { namespace drr { void OperationFactory::RegisterManualOpCreator() { RegisterOperationCreator( "pd_op.fused_gemm_epilogue", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) { return rewriter.Build( @@ -41,7 +41,7 @@ void OperationFactory::RegisterManualOpCreator() { }); RegisterOperationCreator( "pd_op.fused_gemm_epilogue_grad", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) { return rewriter.Build( @@ -52,14 +52,14 @@ void OperationFactory::RegisterManualOpCreator() { attrs); }); RegisterOperationCreator("builtin.combine", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) { return rewriter.Build(inputs); }); RegisterOperationCreator( "pd_op.scale", - [](const std::vector& inputs, + [](const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) { return rewriter.Build( @@ -130,18 +130,18 @@ pir::AttributeMap CreateAttributeMap(const OpCall& op_call, return attr_map; } -Value GetIrValueByDrrTensor(const Tensor& tensor, - const MatchContextImpl& res_match_ctx) { +pir::Value GetIrValueByDrrTensor(const Tensor& tensor, + const MatchContextImpl& res_match_ctx) { if (tensor.is_none()) { - return Value{}; + return pir::Value{}; } return res_match_ctx.GetIrValue(tensor.name()).get(); } -std::vector GetIrValuesByDrrTensors( +std::vector GetIrValuesByDrrTensors( const std::vector& tensors, const MatchContextImpl& res_match_ctx) { - std::vector ir_values; + std::vector ir_values; ir_values.reserve(tensors.size()); for (const auto* tensor : tensors) { ir_values.push_back(GetIrValueByDrrTensor(*tensor, res_match_ctx)); @@ -167,7 +167,7 @@ pir::Operation* CreateOperation(const OpCall& op_call, MatchContextImpl* res_match_ctx) { VLOG(6) << "Drr create [" << op_call.name() << "] op..."; const auto& inputs = op_call.inputs(); - std::vector ir_values = + std::vector ir_values = GetIrValuesByDrrTensors(inputs, *res_match_ctx); pir::Operation* op = OperationFactory::Instance().CreateOperation( op_call.name(), @@ -180,4 +180,4 @@ pir::Operation* CreateOperation(const OpCall& op_call, } } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/ir_operation_factory.h b/paddle/fluid/pir/drr/ir_operation_factory.h index adc76efb99b2d..40682904df62a 100644 --- a/paddle/fluid/pir/drr/ir_operation_factory.h +++ b/paddle/fluid/pir/drr/ir_operation_factory.h @@ -20,7 +20,7 @@ #include "paddle/fluid/pir/drr/match_context_impl.h" #include "paddle/pir/pattern_rewrite/pattern_match.h" -namespace pir { +namespace paddle { namespace drr { class OperationFactory { @@ -31,7 +31,7 @@ class OperationFactory { } using operation_create_fn = - std::function&, + std::function&, const pir::AttributeMap&, pir::PatternRewriter&)>; @@ -42,7 +42,7 @@ class OperationFactory { pir::Operation* CreateOperation( const std::string& op_name, - const std::vector& inputs, + const std::vector& inputs, const pir::AttributeMap& attrs, pir::PatternRewriter& rewriter) const { // NOLINT auto iter = op_creator_map.find(op_name); @@ -79,4 +79,4 @@ pir::Operation* CreateOperation(const OpCall& op_call, MatchContextImpl* res_match_ctx); } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/ir_value.h b/paddle/fluid/pir/drr/ir_value.h index 125f198dcc74c..ae99fd8c1964e 100644 --- a/paddle/fluid/pir/drr/ir_value.h +++ b/paddle/fluid/pir/drr/ir_value.h @@ -21,7 +21,7 @@ #include "paddle/pir/core/type.h" #include "paddle/pir/core/value.h" -namespace pir { +namespace paddle { namespace drr { class IrShape { @@ -101,10 +101,10 @@ class IrValue : public TensorInterface { } // Don't use it in drr pass! - const Value& get() const { return value_; } + const pir::Value& get() const { return value_; } private: - const Value value_; + const pir::Value value_; const IrShape shape_; const IrDtype dtype_; }; @@ -112,4 +112,4 @@ class IrValue : public TensorInterface { class IrAttr; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/match_context_impl.h b/paddle/fluid/pir/drr/match_context_impl.h index 37b06914cd2bd..b1234d8129936 100644 --- a/paddle/fluid/pir/drr/match_context_impl.h +++ b/paddle/fluid/pir/drr/match_context_impl.h @@ -25,7 +25,7 @@ #include "paddle/fluid/pir/drr/ir_value.h" #include "paddle/pir/core/builtin_attribute.h" -namespace pir { +namespace paddle { namespace drr { class MatchContextImpl final { @@ -131,4 +131,4 @@ class MatchContextImpl final { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/pattern_graph.cc b/paddle/fluid/pir/drr/pattern_graph.cc index 7d732b6576f68..58c79c65acf2f 100644 --- a/paddle/fluid/pir/drr/pattern_graph.cc +++ b/paddle/fluid/pir/drr/pattern_graph.cc @@ -19,7 +19,7 @@ #include "paddle/fluid/pir/drr/api/drr_pattern_context.h" #include "paddle/phi/core/enforce.h" -namespace pir { +namespace paddle { namespace drr { const drr::OpCall &PatternGraph::AddOpCall( @@ -238,4 +238,4 @@ std::ostream &operator<<(std::ostream &os, const PatternGraph &pattern_graph) { } } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/drr/pattern_graph.h b/paddle/fluid/pir/drr/pattern_graph.h index 63bd60eadf17f..e5cd74b2fa217 100644 --- a/paddle/fluid/pir/drr/pattern_graph.h +++ b/paddle/fluid/pir/drr/pattern_graph.h @@ -21,7 +21,7 @@ #include #include -namespace pir { +namespace paddle { namespace drr { class Constraint; @@ -105,4 +105,4 @@ class GraphTopo { }; } // namespace drr -} // namespace pir +} // namespace paddle diff --git a/paddle/fluid/pir/transforms/fusion/attention_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/attention_fuse_pass.cc index fbabf83539001..ab19247de4b26 100644 --- a/paddle/fluid/pir/transforms/fusion/attention_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/attention_fuse_pass.cc @@ -22,13 +22,13 @@ namespace { class MultiHeadMatmulFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // // Source Pattern. // - pir::drr::SourcePattern src = ctx->SourcePattern(); + paddle::drr::SourcePattern src = ctx->SourcePattern(); // The first path to matmul with scale (q). const auto &matmul_1 = src.Op("pd_op.matmul", @@ -115,7 +115,8 @@ class MultiHeadMatmulFusePattern // // Constraints. // - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { + src.RequireNativeCall([](const paddle::drr::MatchContext &match_ctx) + -> bool { const auto &softmax_axis = match_ctx.Attr("softmax_axis"); if (softmax_axis != -1 && softmax_axis != 3) return false; @@ -145,7 +146,7 @@ class MultiHeadMatmulFusePattern // // Result Pattern. // - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); // W combine. const auto &combine_1 = res.Op("builtin.combine"); combine_1({&res.Tensor("matmul_1_in_2"), @@ -153,11 +154,11 @@ class MultiHeadMatmulFusePattern &res.Tensor("matmul_3_in_2")}, {&res.Tensor("combine_1_out")}); const auto &concat_axis = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> int { return 0; }); + [](const paddle::drr::MatchContext &match_ctx) -> int { return 0; }); const auto &concat_1 = res.Op("pd_op.concat", {{"axis", concat_axis}}); res.Tensor("concat_1_out") = concat_1(res.Tensor("combine_1_out")); const auto &reshape_5_shape = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::vector { + [](const paddle::drr::MatchContext &match_ctx) -> std::vector { auto matmul_1_in_2 = match_ctx.Tensor("matmul_1_in_2").Shape(); return {-1, 3, matmul_1_in_2.at(1)}; }); @@ -175,7 +176,7 @@ class MultiHeadMatmulFusePattern const auto &concat_2 = res.Op("pd_op.concat", {{"axis", concat_axis}}); res.Tensor("concat_2_out") = concat_2(res.Tensor("combine_2_out")); const auto &reshape_6_shape = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::vector { + [](const paddle::drr::MatchContext &match_ctx) -> std::vector { return {3, -1}; }); const auto &reshape_6 = @@ -184,28 +185,31 @@ class MultiHeadMatmulFusePattern {&res.Tensor("reshape_6_out"), &res.NoneTensor()}); const auto &head_number = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> int { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> int { const auto &full_int_array_1_value = match_ctx.Attr>("full_int_array_1_value"); return full_int_array_1_value.at(2); }); const auto &alpha = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("full_1_value"); }); - const auto &multihead_matmul = res.Op( - "pd_op.multihead_matmul", - {{"transpose_q", res.Attr([](const pir::drr::MatchContext &match_ctx) { - return false; - })}, - {"transpose_k", res.Attr([](const pir::drr::MatchContext &match_ctx) { - return true; - })}, - {"transpose_v", res.Attr([](const pir::drr::MatchContext &match_ctx) { - return false; - })}, - {"head_number", head_number}, - {"alpha", alpha}}); + const auto &multihead_matmul = + res.Op("pd_op.multihead_matmul", + {{"transpose_q", + res.Attr([](const paddle::drr::MatchContext &match_ctx) { + return false; + })}, + {"transpose_k", + res.Attr([](const paddle::drr::MatchContext &match_ctx) { + return true; + })}, + {"transpose_v", + res.Attr([](const paddle::drr::MatchContext &match_ctx) { + return false; + })}, + {"head_number", head_number}, + {"alpha", alpha}}); multihead_matmul({&res.Tensor("matmul_1_in_1"), &res.Tensor("reshape_5_out"), &res.Tensor("reshape_6_out"), diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc index 86846508a519d..e86dc04037fa0 100644 --- a/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/conv2d_add_fuse_pass.cc @@ -29,10 +29,10 @@ namespace { class Conv2dAddFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &conv2d = pat.Op(paddle::dialect::Conv2dOp::name(), {{"strides", pat.Attr("strides")}, @@ -46,7 +46,7 @@ class Conv2dAddFusePattern {&pat.Tensor("conv2d_out")}); pat.Tensor("add_out") = add(pat.Tensor("conv2d_out"), pat.Tensor("bias")); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &fused_conv2d_add_act = res.Op( paddle::dialect::FusedConv2dAddActOp::name(), @@ -58,21 +58,21 @@ class Conv2dAddFusePattern {"groups", pat.Attr("groups")}, {"data_format", pat.Attr("data_format")}, {"activation", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return "identity"; })}, {"split_channels", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::vector { return {}; })}, {"exhaustive_search", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return false; })}, {"workspace_size_MB", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> int { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> int { return 32; })}, {"fuse_alpha", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return 0.0f; })}, }}); diff --git a/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc index fdb4621fb350b..7e5c4bbe8ea18 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_elementwise_layernorm_fuse_pass.cc @@ -22,10 +22,10 @@ namespace { class FcElementwiseLayerNormFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &fc = pat.Op(paddle::dialect::FcOp::name(), { @@ -47,7 +47,7 @@ class FcElementwiseLayerNormFusePattern &pat.Tensor("layernorm_mean"), &pat.Tensor("layernorm_variance")}); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { int64_t layer_norm_x = 1; for (int i = match_ctx.Attr("begin_norm_axis"); i < match_ctx.Tensor("fc_out").Shape().size(); @@ -60,12 +60,16 @@ class FcElementwiseLayerNormFusePattern return false; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &x_num_col_dims_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::any { return 1; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &x_num_col_dims_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { + return 1; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fused_fc_elementwise_op = res.Op(paddle::dialect::FusedFcElementwiseLayernormOp::name(), @@ -88,10 +92,10 @@ class FcElementwiseLayerNormFusePattern }; class FcElementwiseLayerNormFuse2Pattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &fc = pat.Op(paddle::dialect::FcOp::name(), { @@ -113,7 +117,7 @@ class FcElementwiseLayerNormFuse2Pattern &pat.Tensor("layernorm_mean"), &pat.Tensor("layernorm_variance")}); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { int64_t layer_norm_x = 1; for (int i = match_ctx.Attr("begin_norm_axis"); i < match_ctx.Tensor("fc_out").Shape().size(); @@ -126,7 +130,7 @@ class FcElementwiseLayerNormFuse2Pattern return false; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &fused_fc_elementwise_op = res.Op(paddle::dialect::FusedFcElementwiseLayernormOp::name(), diff --git a/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc index 2a320b75d6cc3..b49ab9ff4ac77 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_fuse_pass.cc @@ -21,10 +21,10 @@ namespace { -class MatmulAddPattern : public pir::drr::DrrPatternBase { +class MatmulAddPattern : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("transpose_x")}, {"transpose_y", pat.Attr("transpose_y")}}); @@ -32,7 +32,7 @@ class MatmulAddPattern : public pir::drr::DrrPatternBase { matmul({&pat.Tensor("x"), &pat.Tensor("w")}, {&pat.Tensor("matmul_out")}); pat.Tensor("add_out") = add(pat.Tensor("matmul_out"), pat.Tensor("y")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { if (match_ctx.Tensor("w").Shape().size() != 2 || match_ctx.Tensor("x").Shape().size() < 2) { return false; @@ -56,21 +56,23 @@ class MatmulAddPattern : public pir::drr::DrrPatternBase { return false; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &in_num_col_dims_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return match_ctx.Tensor("x").Shape().size() - 1; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fc = res.Op(paddle::dialect::FcOp::name(), {{ {"in_num_col_dims", in_num_col_dims_attr}, {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return ""; })}, {"padding_weights", false_attr}, }}); @@ -79,10 +81,11 @@ class MatmulAddPattern : public pir::drr::DrrPatternBase { } }; -class FcWithReluPattern : public pir::drr::DrrPatternBase { +class FcWithReluPattern + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &fc = pat.Op(paddle::dialect::FcOp::name(), {{ @@ -96,18 +99,18 @@ class FcWithReluPattern : public pir::drr::DrrPatternBase { relu({&pat.Tensor("fc_out")}, {&pat.Tensor("relu_out")}); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return match_ctx.Attr("activation_type").empty(); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &fc_with_relu = res.Op(paddle::dialect::FcOp::name(), {{ {"in_num_col_dims", pat.Attr("in_num_col_dims")}, {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return "relu"; })}, {"padding_weights", pat.Attr("padding_weights")}, }}); diff --git a/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc index 6bb2b3a6d512d..74dd21a0828fe 100644 --- a/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fc_with_special_op_fuse_pass.cc @@ -31,10 +31,10 @@ namespace { class SqueezeFcFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &squeeze_op = pat.Op(paddle::dialect::SqueezeOp::name()); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("transpose_x")}, @@ -46,7 +46,7 @@ class SqueezeFcFusePattern {&pat.Tensor("matmul_out")}); pat.Tensor("add_out") = add(pat.Tensor("matmul_out"), pat.Tensor("bias")); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { auto axis_type = match_ctx.Tensor("axis").Dtype().get(); if (axis_type.isa() && axis_type.dyn_cast().size() != 2) { @@ -87,19 +87,23 @@ class SqueezeFcFusePattern return false; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &in_num_col_dims_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::any { return 1; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &in_num_col_dims_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { + return 1; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fc = res.Op(paddle::dialect::FcOp::name(), {{ {"in_num_col_dims", in_num_col_dims_attr}, {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return ""; })}, {"padding_weights", false_attr}, }}); @@ -109,10 +113,10 @@ class SqueezeFcFusePattern }; class ReshapeFcFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &reshape_op = pat.Op(paddle::dialect::ReshapeOp::name()); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("transpose_x")}, @@ -124,7 +128,7 @@ class ReshapeFcFusePattern {&pat.Tensor("matmul_out")}); add({&pat.Tensor("matmul_out"), &pat.Tensor("bias")}, {&pat.Tensor("add_out")}); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { if (match_ctx.Tensor("w").Shape().size() != 2 || match_ctx.Attr("transpose_x") == true || match_ctx.Attr("transpose_y") == true) { @@ -212,10 +216,10 @@ class ReshapeFcFusePattern } return true; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &in_num_col_dims_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { int i = match_ctx.Tensor("x").Shape().size() - 1; int target = match_ctx.Tensor("reshape_out") @@ -228,15 +232,17 @@ class ReshapeFcFusePattern } return i; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fc = res.Op(paddle::dialect::FcOp::name(), {{ {"in_num_col_dims", in_num_col_dims_attr}, {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return ""; })}, {"padding_weights", false_attr}, }}); @@ -246,10 +252,10 @@ class ReshapeFcFusePattern }; class FlattenFcFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &flatten_op = pat.Op(paddle::dialect::FlattenOp::name(), {{"start_axis", pat.Attr("start_axis")}, {"stop_axis", pat.Attr("stop_axis")}}); @@ -263,7 +269,7 @@ class FlattenFcFusePattern {&pat.Tensor("matmul_out")}); pat.Tensor("add_out") = add(pat.Tensor("matmul_out"), pat.Tensor("bias")); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { bool flatten_flag = false; if (match_ctx.Tensor("x").Shape().size() == 4 && @@ -295,19 +301,23 @@ class FlattenFcFusePattern return false; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); - const auto &in_num_col_dims_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::any { return 1; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &in_num_col_dims_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { + return 1; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fc = res.Op(paddle::dialect::FcOp::name(), {{ {"in_num_col_dims", in_num_col_dims_attr}, {"activation_type", - res.Attr([](const pir::drr::MatchContext &match_ctx) + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::string { return ""; })}, {"padding_weights", false_attr}, }}); diff --git a/paddle/fluid/pir/transforms/fusion/fused_dot_product_attention_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_dot_product_attention_pass.cc index 639c0e0e4b414..9b2e7f2f3f2e7 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_dot_product_attention_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_dot_product_attention_pass.cc @@ -22,10 +22,10 @@ namespace { class FusedDotProductAttentionPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern src = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern src = ctx->SourcePattern(); // q[b, s, head, head_dim] -> transpose -> q[b, head, s, head_dim] -> scale const auto &q_transpose = src.Op("pd_op.transpose"); @@ -82,40 +82,45 @@ class FusedDotProductAttentionPattern src.Tensor("out") = o_transpose(src.Tensor("context_matmul_out")); // Constraints - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { - const auto &softmax_axis = match_ctx.Attr("softmax_axis"); - if (softmax_axis != -1 && softmax_axis != 3) return false; - - bool qk_matmul_transpose_x = - match_ctx.Attr("qk_matmul_transpose_x"); - bool qk_matmul_transpose_y = - match_ctx.Attr("qk_matmul_transpose_y"); - if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; - - bool context_matmul_transpose_x = - match_ctx.Attr("context_matmul_transpose_x"); - bool context_matmul_transpose_y = - match_ctx.Attr("context_matmul_transpose_y"); - if (context_matmul_transpose_x || context_matmul_transpose_y) - return false; - - return true; - }); + src.RequireNativeCall( + [](const paddle::drr::MatchContext &match_ctx) -> bool { + const auto &softmax_axis = match_ctx.Attr("softmax_axis"); + if (softmax_axis != -1 && softmax_axis != 3) return false; + + bool qk_matmul_transpose_x = + match_ctx.Attr("qk_matmul_transpose_x"); + bool qk_matmul_transpose_y = + match_ctx.Attr("qk_matmul_transpose_y"); + if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; + + bool context_matmul_transpose_x = + match_ctx.Attr("context_matmul_transpose_x"); + bool context_matmul_transpose_y = + match_ctx.Attr("context_matmul_transpose_y"); + if (context_matmul_transpose_x || context_matmul_transpose_y) + return false; + + return true; + }); // Result pattern - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); const auto &scaling_factor = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("q_scale_value"); }); const auto &dropout_prob = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return static_cast(0.0); }); - const auto &is_training = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &is_causal_masking = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &is_training = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &is_causal_masking = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &dot_product_attention = res.Op(paddle::dialect::FusedDotProductAttentionOp::name(), @@ -135,10 +140,10 @@ class FusedDotProductAttentionPattern }; class FusedDotProductAttentionGradPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern src = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern src = ctx->SourcePattern(); // q[b, s, head, head_dim] -> transpose -> q[b, head, s, head_dim] -> scale const auto &q_transpose = src.Op("pd_op.transpose"); @@ -239,40 +244,45 @@ class FusedDotProductAttentionGradPattern {&src.Tensor("k_grad")}); // Constraints - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { - const auto &softmax_axis = match_ctx.Attr("softmax_axis"); - if (softmax_axis != -1 && softmax_axis != 3) return false; - - bool qk_matmul_transpose_x = - match_ctx.Attr("qk_matmul_transpose_x"); - bool qk_matmul_transpose_y = - match_ctx.Attr("qk_matmul_transpose_y"); - if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; - - bool context_matmul_transpose_x = - match_ctx.Attr("context_matmul_transpose_x"); - bool context_matmul_transpose_y = - match_ctx.Attr("context_matmul_transpose_y"); - if (context_matmul_transpose_x || context_matmul_transpose_y) - return false; - - return true; - }); + src.RequireNativeCall( + [](const paddle::drr::MatchContext &match_ctx) -> bool { + const auto &softmax_axis = match_ctx.Attr("softmax_axis"); + if (softmax_axis != -1 && softmax_axis != 3) return false; + + bool qk_matmul_transpose_x = + match_ctx.Attr("qk_matmul_transpose_x"); + bool qk_matmul_transpose_y = + match_ctx.Attr("qk_matmul_transpose_y"); + if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; + + bool context_matmul_transpose_x = + match_ctx.Attr("context_matmul_transpose_x"); + bool context_matmul_transpose_y = + match_ctx.Attr("context_matmul_transpose_y"); + if (context_matmul_transpose_x || context_matmul_transpose_y) + return false; + + return true; + }); // Result pattern - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); const auto &scaling_factor = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("q_scale_value"); }); const auto &dropout_prob = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return static_cast(0.0); }); - const auto &is_training = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &is_causal_masking = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &is_training = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &is_causal_masking = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &dot_product_attention = res.Op(paddle::dialect::FusedDotProductAttentionOp::name(), @@ -307,11 +317,11 @@ class FusedDotProductAttentionGradPattern }; class FusedDotProductAttentionWithDropoutPattern - : public pir::drr::DrrPatternBase< + : public paddle::drr::DrrPatternBase< FusedDotProductAttentionWithDropoutPattern> { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern src = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern src = ctx->SourcePattern(); // q[b, s, head, head_dim] -> transpose -> q[b, head, s, head_dim] -> scale const auto &q_transpose = src.Op("pd_op.transpose"); @@ -376,40 +386,45 @@ class FusedDotProductAttentionWithDropoutPattern src.Tensor("out") = o_transpose(src.Tensor("context_matmul_out")); // Constraints - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { - const auto &softmax_axis = match_ctx.Attr("softmax_axis"); - if (softmax_axis != -1 && softmax_axis != 3) return false; - - bool qk_matmul_transpose_x = - match_ctx.Attr("qk_matmul_transpose_x"); - bool qk_matmul_transpose_y = - match_ctx.Attr("qk_matmul_transpose_y"); - if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; - - bool context_matmul_transpose_x = - match_ctx.Attr("context_matmul_transpose_x"); - bool context_matmul_transpose_y = - match_ctx.Attr("context_matmul_transpose_y"); - if (context_matmul_transpose_x || context_matmul_transpose_y) - return false; - - return true; - }); + src.RequireNativeCall( + [](const paddle::drr::MatchContext &match_ctx) -> bool { + const auto &softmax_axis = match_ctx.Attr("softmax_axis"); + if (softmax_axis != -1 && softmax_axis != 3) return false; + + bool qk_matmul_transpose_x = + match_ctx.Attr("qk_matmul_transpose_x"); + bool qk_matmul_transpose_y = + match_ctx.Attr("qk_matmul_transpose_y"); + if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; + + bool context_matmul_transpose_x = + match_ctx.Attr("context_matmul_transpose_x"); + bool context_matmul_transpose_y = + match_ctx.Attr("context_matmul_transpose_y"); + if (context_matmul_transpose_x || context_matmul_transpose_y) + return false; + + return true; + }); // Result pattern - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); const auto &scaling_factor = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("q_scale_value"); }); const auto &dropout_prob = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return static_cast(0.0); }); - const auto &is_training = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &is_causal_masking = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &is_training = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &is_causal_masking = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &dot_product_attention = res.Op(paddle::dialect::FusedDotProductAttentionOp::name(), @@ -429,11 +444,11 @@ class FusedDotProductAttentionWithDropoutPattern }; class FusedDotProductAttentionGradWithDropoutPattern - : public pir::drr::DrrPatternBase< + : public paddle::drr::DrrPatternBase< FusedDotProductAttentionGradWithDropoutPattern> { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern src = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern src = ctx->SourcePattern(); // q[b, s, head, head_dim] -> transpose -> q[b, head, s, head_dim] -> scale const auto &q_transpose = src.Op("pd_op.transpose"); @@ -548,36 +563,41 @@ class FusedDotProductAttentionGradWithDropoutPattern {&src.Tensor("k_grad")}); // Constraints - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { - const auto &softmax_axis = match_ctx.Attr("softmax_axis"); - if (softmax_axis != -1 && softmax_axis != 3) return false; - - bool qk_matmul_transpose_x = - match_ctx.Attr("qk_matmul_transpose_x"); - bool qk_matmul_transpose_y = - match_ctx.Attr("qk_matmul_transpose_y"); - if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; - - bool context_matmul_transpose_x = - match_ctx.Attr("context_matmul_transpose_x"); - bool context_matmul_transpose_y = - match_ctx.Attr("context_matmul_transpose_y"); - if (context_matmul_transpose_x || context_matmul_transpose_y) - return false; - - return true; - }); + src.RequireNativeCall( + [](const paddle::drr::MatchContext &match_ctx) -> bool { + const auto &softmax_axis = match_ctx.Attr("softmax_axis"); + if (softmax_axis != -1 && softmax_axis != 3) return false; + + bool qk_matmul_transpose_x = + match_ctx.Attr("qk_matmul_transpose_x"); + bool qk_matmul_transpose_y = + match_ctx.Attr("qk_matmul_transpose_y"); + if (qk_matmul_transpose_x || !qk_matmul_transpose_y) return false; + + bool context_matmul_transpose_x = + match_ctx.Attr("context_matmul_transpose_x"); + bool context_matmul_transpose_y = + match_ctx.Attr("context_matmul_transpose_y"); + if (context_matmul_transpose_x || context_matmul_transpose_y) + return false; + + return true; + }); // Result pattern - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); const auto &scaling_factor = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("q_scale_value"); }); - const auto &is_training = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &is_causal_masking = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &is_training = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &is_causal_masking = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &dot_product_attention = res.Op(paddle::dialect::FusedDotProductAttentionOp::name(), diff --git a/paddle/fluid/pir/transforms/fusion/fused_dropout_add_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_dropout_add_pass.cc index 35079d4f2cf1c..df8b39cfc8676 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_dropout_add_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_dropout_add_pass.cc @@ -22,10 +22,10 @@ namespace { class FusedDropoutAddPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &dropout = pat.Op(paddle::dialect::DropoutOp::name(), {{"p", pat.Attr("p")}, {"is_test", pat.Attr("is_test")}, @@ -38,7 +38,7 @@ class FusedDropoutAddPattern {&pat.Tensor("dropout_out"), &pat.Tensor("mask")}); pat.Tensor("add_out") = add(pat.Tensor("dropout_out"), pat.Tensor("y")); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &fused_dropout_add = res.Op(paddle::dialect::FusedDropoutAddOp::name(), {{{"p", pat.Attr("p")}, @@ -53,10 +53,10 @@ class FusedDropoutAddPattern }; class FusedDropoutGradAddGradPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &dropout = pat.Op(paddle::dialect::DropoutOp::name(), {{"p", pat.Attr("p")}, {"is_test", pat.Attr("is_test")}, @@ -81,7 +81,7 @@ class FusedDropoutGradAddGradPattern dropout_grad({&pat.Tensor("mask"), &pat.Tensor("dropout_out_grad")}, {&pat.Tensor("x_grad")}); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &fused_dropout_add = res.Op(paddle::dialect::FusedDropoutAddOp::name(), {{{"p", pat.Attr("p")}, diff --git a/paddle/fluid/pir/transforms/fusion/fused_gemm_epilogue_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_gemm_epilogue_pass.cc index 6bc15234efd31..02a6b4744cdcb 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_gemm_epilogue_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_gemm_epilogue_pass.cc @@ -21,10 +21,11 @@ namespace { -class FusedLinearPattern : public pir::drr::DrrPatternBase { +class FusedLinearPattern + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -33,15 +34,15 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { pat.Tensor("tmp") = matmul(pat.Tensor("x"), pat.Tensor("w")); pat.Tensor("out") = add(pat.Tensor("tmp"), pat.Tensor("bias")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("w").Shape().size() == 2 && match_ctx.Tensor("x").Shape().size() >= 2 && match_ctx.Tensor("bias").Shape().size() == 1); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "none"; }); const auto &fused_gemm_epilogue = @@ -56,10 +57,10 @@ class FusedLinearPattern : public pir::drr::DrrPatternBase { }; class FusedLinearGradPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -76,15 +77,15 @@ class FusedLinearGradPattern matmul_grad({&pat.Tensor("x"), &pat.Tensor("w"), &pat.Tensor("tmp_grad")}, {&pat.Tensor("x_grad"), &pat.Tensor("w_grad")}); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("w").Shape().size() == 2 && match_ctx.Tensor("x").Shape().size() >= 2 && match_ctx.Tensor("bias").Shape().size() == 1); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "none"; }); const auto &fused_gemm_epilogue = @@ -111,10 +112,10 @@ class FusedLinearGradPattern }; class FusedLinearGeluPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); // Source pattern const auto &fused_gemm_epilogue = pat.Op(paddle::dialect::FusedGemmEpilogueOp::name(), @@ -128,14 +129,14 @@ class FusedLinearGeluPattern pat.Tensor("out") = gelu(pat.Tensor("fuse_out")); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Attr("act") == "none"); }); // Result pattern - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "gelu"; }); const auto &fused_gemm_epilogue_gelu = @@ -149,10 +150,10 @@ class FusedLinearGeluPattern } }; class FusedLinearReluPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); // Source pattern const auto &fused_gemm_epilogue = pat.Op(paddle::dialect::FusedGemmEpilogueOp::name(), @@ -166,14 +167,14 @@ class FusedLinearReluPattern pat.Tensor("out") = relu(pat.Tensor("fuse_out")); // Constrains the activation is none - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Attr("act") == "none"); }); // Result pattern - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "relu"; }); const auto &fused_gemm_epilogue_relu = @@ -188,10 +189,10 @@ class FusedLinearReluPattern }; class FusedLinearGeluGradPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &fused_gemm_epilogue = pat.Op(paddle::dialect::FusedGemmEpilogueOp::name(), {{{"trans_x", pat.Attr("trans_x1")}, @@ -218,14 +219,14 @@ class FusedLinearGeluGradPattern pat.Tensor("gelu_dx") = pat.Op(paddle::dialect::GeluGradOp::name())( pat.Tensor("fuse_out"), pat.Tensor("x1_grad")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return match_ctx.Attr("act1") == "none" && match_ctx.Attr("act2") == "none"; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "gelu"; }); const auto &fused_gemm_epilogue_new = @@ -234,7 +235,7 @@ class FusedLinearGeluGradPattern {"trans_y", pat.Attr("trans_y1")}, {"activation", act_attr}}}); const auto &act_grad_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "gelu_grad"; }); const auto &fused_gemm_epilogue_grad_new = @@ -256,10 +257,10 @@ class FusedLinearGeluGradPattern }; class FusedLinearReluGradPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &fused_gemm_epilogue = pat.Op(paddle::dialect::FusedGemmEpilogueOp::name(), {{{"trans_x", pat.Attr("trans_x1")}, @@ -297,14 +298,14 @@ class FusedLinearReluGradPattern &pat.Tensor("w_grad"), &pat.Tensor("bias_grad")}); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return match_ctx.Attr("act1") == "relu" && match_ctx.Attr("act3") == "none"; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &act_grad_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "relu_grad"; }); const auto &res_fused_gemm_epilogue_grad1 = diff --git a/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc index 7a3afec65f33f..8c93ff9822675 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_linear_param_grad_add_pass.cc @@ -22,10 +22,10 @@ namespace { // add_grad + matmul_grad + add_ -> matmul + fused_liner_param_gard_add class FusedMatmulAddGradAddPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul0 = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -48,7 +48,7 @@ class FusedMatmulAddGradAddPattern pat.Tensor("add_out") = add_(pat.Tensor("dweight"), pat.Tensor("weight_grad")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { const auto &x_trans = match_ctx.Attr("trans_x"); const auto &y_trans = match_ctx.Attr("trans_y"); return (match_ctx.Tensor("weight_grad").Shape() == @@ -58,17 +58,21 @@ class FusedMatmulAddGradAddPattern x_trans == false && y_trans == false); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &matmul = res.Op(paddle::dialect::MatmulOp::name(), @@ -89,10 +93,10 @@ class FusedMatmulAddGradAddPattern // matmul_grad + add_ -> matmul + fused_liner_param_gard_add class FusedMatmulGradAddPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul_grad = pat.Op(paddle::dialect::MatmulGradOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -104,7 +108,7 @@ class FusedMatmulGradAddPattern pat.Tensor("add_out") = add_(pat.Tensor("dweight"), pat.Tensor("weight_grad")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { const auto &x_trans = match_ctx.Attr("trans_x"); const auto &y_trans = match_ctx.Attr("trans_y"); return (match_ctx.Tensor("weight_grad").Shape() == @@ -112,18 +116,22 @@ class FusedMatmulGradAddPattern x_trans == false && y_trans == false); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &matmul = res.Op(paddle::dialect::MatmulOp::name(), @@ -145,10 +153,10 @@ class FusedMatmulGradAddPattern // matmul + 0 = add_(0,1) -> fused_liner_param_gard_add class FusedMatmulAddaPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -159,22 +167,26 @@ class FusedMatmulAddaPattern pat.Tensor("add_out") = add_(pat.Tensor("dweight"), pat.Tensor("weight_grad")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("weight_grad").Shape() == match_ctx.Tensor("dweight").Shape()); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fused_linear_param_grad_add = res.Op( paddle::dialect::FusedLinearParamGradAddOp::name(), @@ -190,10 +202,10 @@ class FusedMatmulAddaPattern // matmul + 1 = add_(1,0) -> fused_liner_param_gard_add class FusedMatmulAddbPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -204,22 +216,26 @@ class FusedMatmulAddbPattern pat.Tensor("add_out") = add_(pat.Tensor("weight_grad"), pat.Tensor("dweight")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("weight_grad").Shape() == match_ctx.Tensor("dweight").Shape()); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); - const auto &false_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return false; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); + const auto &false_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return false; + }); const auto &fused_linear_param_grad_add = res.Op( paddle::dialect::FusedLinearParamGradAddOp::name(), @@ -235,10 +251,10 @@ class FusedMatmulAddbPattern // add_grad + matmul + 0 = add_(0,1) -> fused_liner_param_gard_add class FusedMatmulAddGradAddaPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -261,21 +277,23 @@ class FusedMatmulAddGradAddaPattern pat.Tensor("dweight_out") = add_(pat.Tensor("dweight"), pat.Tensor("weight_grad")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("weight_grad").Shape() == match_ctx.Tensor("dweight").Shape() && match_ctx.Tensor("out").Shape() == match_ctx.Tensor("dadd_out").Shape()); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); const auto &fused_linear_param_grad_add = res.Op( paddle::dialect::FusedLinearParamGradAddOp::name(), {{{"multi_precision", muti_precision_attr}, {"has_bias", true_attr}}}); @@ -290,10 +308,10 @@ class FusedMatmulAddGradAddaPattern // add_grad + matmul + 1 = add_(1,0) -> fused_liner_param_gard_add class FusedMatmulAddGradAddbPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("trans_x")}, {"transpose_y", pat.Attr("trans_y")}}); @@ -316,21 +334,23 @@ class FusedMatmulAddGradAddbPattern pat.Tensor("dweight_out") = add_(pat.Tensor("weight_grad"), pat.Tensor("dweight")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Tensor("weight_grad").Shape() == match_ctx.Tensor("dweight").Shape() && match_ctx.Tensor("out").Shape() == match_ctx.Tensor("dadd_out").Shape()); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &muti_precision_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { return !(match_ctx.Tensor("dweight").Dtype() == match_ctx.Tensor("weight_grad").Dtype()); }); - const auto &true_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> bool { return true; }); + const auto &true_attr = + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + }); const auto &fused_linear_param_grad_add = res.Op( paddle::dialect::FusedLinearParamGradAddOp::name(), {{{"multi_precision", muti_precision_attr}, {"has_bias", true_attr}}}); diff --git a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc index fa83418e562ba..82864f3d80e88 100644 --- a/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/fused_weight_only_linear_pass.cc @@ -36,13 +36,13 @@ int getSMVersion() { } class FusedWeightOnlyLinearPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // // Source Pattern. // - pir::drr::SourcePattern src = ctx->SourcePattern(); + paddle::drr::SourcePattern src = ctx->SourcePattern(); const auto &matmul = src.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", src.Attr("matmul_transpose_x")}, @@ -57,47 +57,49 @@ class FusedWeightOnlyLinearPattern // // Constraints. // - src.RequireNativeCall([](const pir::drr::MatchContext &match_ctx) -> bool { - bool matmul_trans_x = match_ctx.Attr("matmul_transpose_x"); - bool matmul_trans_y = match_ctx.Attr("matmul_transpose_y"); - if (matmul_trans_x || matmul_trans_y) return false; - - if (!(match_ctx.Tensor("w").Shape().size() == 2 && - match_ctx.Tensor("x").Shape().size() >= 2 && - match_ctx.Tensor("bias").Shape().size() == 1)) { - return false; - } - - auto w_dims = match_ctx.Tensor("w").Shape(); - if (w_dims.at(0) % 64 != 0 || w_dims.at(1) % 16 != 0) return false; - - auto w_dtype = match_ctx.Tensor("w").Dtype().get(); - if (!w_dtype.isa() && !w_dtype.isa()) - return false; - - auto x_dims = match_ctx.Tensor("x").Shape(); - if (x_dims.at(x_dims.size() - 1) != w_dims.at(1)) return false; - - return true; - }); + src.RequireNativeCall( + [](const paddle::drr::MatchContext &match_ctx) -> bool { + bool matmul_trans_x = match_ctx.Attr("matmul_transpose_x"); + bool matmul_trans_y = match_ctx.Attr("matmul_transpose_y"); + if (matmul_trans_x || matmul_trans_y) return false; + + if (!(match_ctx.Tensor("w").Shape().size() == 2 && + match_ctx.Tensor("x").Shape().size() >= 2 && + match_ctx.Tensor("bias").Shape().size() == 1)) { + return false; + } + + auto w_dims = match_ctx.Tensor("w").Shape(); + if (w_dims.at(0) % 64 != 0 || w_dims.at(1) % 16 != 0) return false; + + auto w_dtype = match_ctx.Tensor("w").Dtype().get(); + if (!w_dtype.isa() && + !w_dtype.isa()) + return false; + + auto x_dims = match_ctx.Tensor("x").Shape(); + if (x_dims.at(x_dims.size() - 1) != w_dims.at(1)) return false; + + return true; + }); // // Result Pattern. // - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); // quantize weight const auto &weight_only_int8_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "weight_only_int8"; }); const auto &arch_attr = - res.Attr([&](const pir::drr::MatchContext &match_ctx) -> int { + res.Attr([&](const paddle::drr::MatchContext &match_ctx) -> int { return getSMVersion(); }); const auto &group_size_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> int { return -1; }); + [](const paddle::drr::MatchContext &match_ctx) -> int { return -1; }); const auto &weight_quantize = res.Op(paddle::dialect::WeightQuantizeOp::name(), @@ -109,7 +111,7 @@ class FusedWeightOnlyLinearPattern &res.Tensor("weight_scale_tensor")}); const auto &weight_dtype_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> std::any { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> std::any { return "int8"; }); diff --git a/paddle/fluid/pir/transforms/fusion/matmul_scale_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/matmul_scale_fuse_pass.cc index 627c1cd516cc8..0bced0b8ec823 100644 --- a/paddle/fluid/pir/transforms/fusion/matmul_scale_fuse_pass.cc +++ b/paddle/fluid/pir/transforms/fusion/matmul_scale_fuse_pass.cc @@ -28,10 +28,10 @@ namespace { class MatmulScaleFusePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &matmul_op = pat.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("transpose_x")}, {"transpose_y", pat.Attr("transpose_y")}}); @@ -50,23 +50,23 @@ class MatmulScaleFusePattern scale_op({&pat.Tensor("matmul_out"), &full_op()}, {&pat.Tensor("scale_out")}); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return std::abs(match_ctx.Attr("bias")) <= 1e-6; }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &full_op_res = res.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("shape")}, {"value", pat.Attr("value")}, {"dtype", pat.Attr("dtype")}, {"place", pat.Attr("place")}}); - const auto &scale_op_res = - res.Op(paddle::dialect::ScaleOp::name(), - {{"bias", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { - return 0.0; - })}, - {"bias_after_scale", pat.Attr("bias_after_scale")}}); + const auto &scale_op_res = res.Op( + paddle::dialect::ScaleOp::name(), + {{"bias", + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { + return 0.0; + })}, + {"bias_after_scale", pat.Attr("bias_after_scale")}}); const auto &matmul_op_res = res.Op(paddle::dialect::MatmulOp::name(), {{"transpose_x", pat.Attr("transpose_x")}, diff --git a/paddle/fluid/pir/transforms/identity_op_clean_pass.cc b/paddle/fluid/pir/transforms/identity_op_clean_pass.cc index 377610196bf96..ac49d494d1c73 100644 --- a/paddle/fluid/pir/transforms/identity_op_clean_pass.cc +++ b/paddle/fluid/pir/transforms/identity_op_clean_pass.cc @@ -32,10 +32,10 @@ namespace { class RemoveUselessScalePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &full_op = pat.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("shape")}, {"value", pat.Attr("value")}, @@ -47,21 +47,21 @@ class RemoveUselessScalePattern {"bias_after_scale", pat.Attr("bias_after_scale")}}); scale_op({&pat.Tensor("x"), &full_op()}, {&pat.Tensor("scale_out")}); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { return (match_ctx.Attr("value") == 1.0 && match_ctx.Attr("bias") == 0.0); }); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); res.Tensor("scale_out").Assign(res.Tensor("x")); } }; class RemoveRedundentScalePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &full_op_1 = pat.Op(paddle::dialect::FullOp::name(), {{"shape", pat.Attr("shape_1")}, {"value", pat.Attr("value_1")}, @@ -84,10 +84,10 @@ class RemoveRedundentScalePattern scale_op_2({&pat.Tensor("scale_1_out"), &full_op_2()}, {&pat.Tensor("scale_2_out")}); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &bais_res = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { float res_bias_1 = 0.f; float res_bias_2 = 0.f; if (match_ctx.Attr("bias_after_scale_1")) { @@ -106,7 +106,7 @@ class RemoveRedundentScalePattern return res_bias_2; }); const auto &res_scale_input = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> float { + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> float { return match_ctx.Attr("value_1") * match_ctx.Attr("value_2"); }); @@ -116,22 +116,22 @@ class RemoveRedundentScalePattern {"value", res_scale_input}, {"dtype", pat.Attr("dtype_1")}, {"place", pat.Attr("place_1")}}); - const auto &scale_op_res = - res.Op("pd_op.scale", - {{"bias", bais_res}, - {"bias_after_scale", - res.Attr([](const pir::drr::MatchContext &match_ctx) -> bool { - return true; - })}}); + const auto &scale_op_res = res.Op( + "pd_op.scale", + {{"bias", bais_res}, + {"bias_after_scale", + res.Attr([](const paddle::drr::MatchContext &match_ctx) -> bool { + return true; + })}}); scale_op_res({&res.Tensor("x"), &full_op_res()}, {&res.Tensor("scale_2_out")}); } }; class RemoveUselessCastPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); pat.Tensor("ret") = pat.Op("pd_op.cast")(pat.Tensor("arg0")); pat.RequireEqual(pat.Tensor("ret").dtype(), pat.Tensor("arg0").dtype()); @@ -141,16 +141,16 @@ class RemoveUselessCastPattern }; class RemoveUselessConcatPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); const auto &combine = pat.Op(pir::CombineOp::name()); combine({&pat.Tensor("x")}, {&pat.Tensor("combine_out")}); pat.Tensor("out") = pat.Op(paddle::dialect::ConcatOp::name())( pat.Tensor("combine_out"), pat.Tensor("axis")); - pat.RequireNativeCall([&](const pir::drr::MatchContext &match_ctx) { - auto combine_out = dynamic_cast( + pat.RequireNativeCall([&](const paddle::drr::MatchContext &match_ctx) { + auto combine_out = dynamic_cast( match_ctx.Tensor("combine_out")); return combine_out.type_isa() && combine_out.type_dyn_cast().size() == 1; @@ -161,8 +161,8 @@ class RemoveUselessConcatPattern }; class RemoveRedundentCastPattern - : public pir::drr::DrrPatternBase { - void operator()(pir::drr::DrrPatternContext *ctx) const override { + : public paddle::drr::DrrPatternBase { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); pat.Tensor("tmp") = pat.Op( "pd_op.cast", {{"dtype", pat.Attr("dtype1")}})(pat.Tensor("arg0")); @@ -175,10 +175,10 @@ class RemoveRedundentCastPattern }; class RemoveRedundentTransposePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &transpose1 = pat.Op("pd_op.transpose", {{"perm", pat.Attr("perm_1")}}); const auto &transpose2 = @@ -186,9 +186,9 @@ class RemoveRedundentTransposePattern pat.Tensor("ret") = transpose2(transpose1(pat.Tensor("arg_transpose"))); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &new_perm_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::vector { + [](const paddle::drr::MatchContext &match_ctx) -> std::vector { const auto &perm1 = match_ctx.Attr>("perm_1"); const auto &perm2 = match_ctx.Attr>("perm_2"); std::vector new_perm; diff --git a/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc b/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc index b550212ad3654..1a938e7f600b7 100644 --- a/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc +++ b/test/cpp/pir/pattern_rewrite/drr_same_type_binding_test.cc @@ -53,10 +53,10 @@ class SameTypeBindingTestPattern // This class is for test cases of the same type of OP. // (without considering the computational logic between OPs, // only focusing on the process of matching and replacing) - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern src = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern src = ctx->SourcePattern(); // path 1 const auto &transpose_1 = @@ -141,7 +141,7 @@ class SameTypeBindingTestPattern const auto &relu_2 = src.Op("pd_op.relu"); src.Tensor("output6") = relu_2(src.Tensor("add_2_out")); - pir::drr::ResultPattern res = src.ResultPattern(); + paddle::drr::ResultPattern res = src.ResultPattern(); const auto &transpose_7 = res.Op("pd_op.transpose", {{"perm", src.Attr("perm_4")}}); res.Tensor("output0") = transpose_7(res.Tensor("input_1")); diff --git a/test/cpp/pir/pattern_rewrite/drr_test.cc b/test/cpp/pir/pattern_rewrite/drr_test.cc index fc0e7ae94f05f..54b5ff2025e49 100644 --- a/test/cpp/pir/pattern_rewrite/drr_test.cc +++ b/test/cpp/pir/pattern_rewrite/drr_test.cc @@ -24,11 +24,11 @@ #include "paddle/pir/pass/pass_manager.h" class RemoveRedundentReshapePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source patterns - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &reshape1 = pat.Op("pd_op.reshape"); const auto &reshape2 = pat.Op("pd_op.reshape"); @@ -38,18 +38,18 @@ class RemoveRedundentReshapePattern {&pat.Tensor("ret"), &pat.Tensor("xshape_1")}); // Result patterns - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); res.Op("pd_op.reshape")({&res.Tensor("arg0"), &res.Tensor("shape1")}, {&res.Tensor("ret"), &res.Tensor("xshape_1")}); } }; class FoldExpandToConstantPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { // Source Pattern - pir::drr::SourcePattern pat = ctx->SourcePattern(); + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &full1 = pat.Op("pd_op.full", {{"shape", pat.Attr("shape_1")}, {"value", pat.Attr("value_1")}, @@ -64,9 +64,9 @@ class FoldExpandToConstantPattern pat.Tensor("ret") = expand(full1(), full_int_array1()); // Result patterns - pir::drr::ResultPattern res = pat.ResultPattern(); - const auto &new_perm_attr = - res.Attr([](const pir::drr::MatchContext &match_ctx) -> phi::IntArray { + paddle::drr::ResultPattern res = pat.ResultPattern(); + const auto &new_perm_attr = res.Attr( + [](const paddle::drr::MatchContext &match_ctx) -> phi::IntArray { auto shape = match_ctx.Attr>("expand_shape_value"); @@ -82,10 +82,10 @@ class FoldExpandToConstantPattern }; class RemoveRedundentTransposePattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { - pir::drr::SourcePattern pat = ctx->SourcePattern(); + void operator()(paddle::drr::DrrPatternContext *ctx) const override { + paddle::drr::SourcePattern pat = ctx->SourcePattern(); const auto &transpose1 = pat.Op("pd_op.transpose", {{"perm", pat.Attr("perm_1")}}); const auto &transpose2 = @@ -93,9 +93,9 @@ class RemoveRedundentTransposePattern pat.Tensor("ret") = transpose2(transpose1(pat.Tensor("arg_transpose"))); - pir::drr::ResultPattern res = pat.ResultPattern(); + paddle::drr::ResultPattern res = pat.ResultPattern(); const auto &new_perm_attr = res.Attr( - [](const pir::drr::MatchContext &match_ctx) -> std::vector { + [](const paddle::drr::MatchContext &match_ctx) -> std::vector { const auto &perm1 = match_ctx.Attr>("perm_1"); const auto &perm2 = match_ctx.Attr>("perm_2"); std::vector new_perm; @@ -112,8 +112,8 @@ class RemoveRedundentTransposePattern }; class RemoveRedundentCastPattern - : public pir::drr::DrrPatternBase { - void operator()(pir::drr::DrrPatternContext *ctx) const override { + : public paddle::drr::DrrPatternBase { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); pat.Tensor("tmp") = pat.Op( "pd_op.cast", {{"dtype", pat.Attr("dtype1")}})(pat.Tensor("arg0")); @@ -126,9 +126,9 @@ class RemoveRedundentCastPattern }; class RemoveUselessCastPattern - : public pir::drr::DrrPatternBase { + : public paddle::drr::DrrPatternBase { public: - void operator()(pir::drr::DrrPatternContext *ctx) const override { + void operator()(paddle::drr::DrrPatternContext *ctx) const override { auto pat = ctx->SourcePattern(); pat.Tensor("ret") = pat.Op("pd_op.cast")(pat.Tensor("arg0")); pat.RequireEqual(pat.Tensor("ret").dtype(), pat.Tensor("arg0").dtype()); From 4cb084c3af2af42f567e5c24aa74fa08b2a2d21b Mon Sep 17 00:00:00 2001 From: Android zhang <53324261+zade23@users.noreply.github.com> Date: Fri, 29 Dec 2023 10:27:31 +0800 Subject: [PATCH 125/146] =?UTF-8?q?=E3=80=90CMake=20opt=20No.2=E3=80=91rm?= =?UTF-8?q?=20some=20DEPS=20of=20`test/cpp/auto=5Fparallel/CMakeLists.txt`?= =?UTF-8?q?=20(#60348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update CMakeLists.txt * fix conflict * add DEPS --- test/cpp/auto_parallel/CMakeLists.txt | 50 ++++++--------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/test/cpp/auto_parallel/CMakeLists.txt b/test/cpp/auto_parallel/CMakeLists.txt index 5911712dffdf2..311958d2e1031 100644 --- a/test/cpp/auto_parallel/CMakeLists.txt +++ b/test/cpp/auto_parallel/CMakeLists.txt @@ -15,49 +15,21 @@ if(WITH_DISTRIBUTE) SRCS dist_tensor_test.cc DEPS phi common) - paddle_test( - spmd_rule_test - SRCS - spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) - paddle_test( - softmax_grad_spmd_rule_test - SRCS - softmax_grad_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + paddle_test(spmd_rule_test SRCS spmd_rule_test.cc DEPS spmd_rule_test_util) - paddle_test( - tile_spmd_rule_test - SRCS - tile_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + paddle_test(softmax_grad_spmd_rule_test SRCS softmax_grad_spmd_rule_test.cc + DEPS spmd_rule_test_util) - paddle_test( - fused_linear_param_grad_add_spmd_rule_test - SRCS - fused_linear_param_grad_add_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + paddle_test(tile_spmd_rule_test SRCS tile_spmd_rule_test.cc DEPS + spmd_rule_test_util) paddle_test( - cross_entropy_softmax_spmd_rule_test - SRCS - cross_entropy_softmax_spmd_rule_test.cc - DEPS - spmd_rule_test_util - spmd_rules - phi) + fused_linear_param_grad_add_spmd_rule_test SRCS + fused_linear_param_grad_add_spmd_rule_test.cc DEPS spmd_rule_test_util) + + paddle_test(cross_entropy_softmax_spmd_rule_test SRCS + cross_entropy_softmax_spmd_rule_test.cc DEPS spmd_rule_test_util) + endif() cc_test( From 7909f768418b767708e14fa1c5bf6685b66e0ce4 Mon Sep 17 00:00:00 2001 From: chen2016013 <111894720+chen2016013@users.noreply.github.com> Date: Fri, 29 Dec 2023 10:31:06 +0800 Subject: [PATCH 126/146] [PIR] open test in test_ifelse for PIR (#60372) * open test * update * update * update as comment --- paddle/common/ddim.cc | 2 +- .../dialect/operator/ir/control_flow_op.cc | 18 +++++--- test/dygraph_to_static/ifelse_simple_func.py | 34 +++++++++++++-- test/dygraph_to_static/test_ifelse.py | 42 ++++++++++++++----- 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/paddle/common/ddim.cc b/paddle/common/ddim.cc index 7394dd03bfd8d..75eb1423cce8a 100644 --- a/paddle/common/ddim.cc +++ b/paddle/common/ddim.cc @@ -267,7 +267,7 @@ DDim DDim::transpose(const std::vector& axis) const { DDim ComputeCompatibleDim(const DDim& dim1, const DDim& dim2) { IR_ENFORCE(dim1.size() == dim2.size(), - "Does not support rank inconsistency: dim1=%d, dim2=%d", + "Does not support rank inconsistency: rank1=%d, rank2=%d", dim1.size(), dim2.size()); std::vector result; diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index 040fbb2837711..30d5ce5a1b685 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -102,12 +102,18 @@ void IfOp::Build(pir::Builder &builder, // NOLINT "The dtype in output[%d] of " "true_block&false_block must be equal.", i)); - PADDLE_ENFORCE_EQ(l_type.data_layout(), - r_type.data_layout(), - phi::errors::PreconditionNotMet( - "The date_layout in output[%d] of " - "true_block&false_block must be equal.", - i)); + if (l_type.data_layout() != phi::DataLayout::UNDEFINED && + r_type.data_layout() != phi::DataLayout::UNDEFINED) { + PADDLE_ENFORCE_EQ( + l_type.data_layout(), + r_type.data_layout(), + phi::errors::PreconditionNotMet( + "The data_layout in output[%d] of " + "true_block (%s) & false_block (%s) must be equal.", + i, + l_type.data_layout(), + r_type.data_layout())); + } PADDLE_ENFORCE_EQ(l_type.lod(), r_type.lod(), phi::errors::PreconditionNotMet( diff --git a/test/dygraph_to_static/ifelse_simple_func.py b/test/dygraph_to_static/ifelse_simple_func.py index d7767a3cfbefb..b011989fed709 100644 --- a/test/dygraph_to_static/ifelse_simple_func.py +++ b/test/dygraph_to_static/ifelse_simple_func.py @@ -60,9 +60,12 @@ def dyfunc_with_if_else2(x, col=100): # TODO: Don't support return non-Tensor in Tensor-dependent `if` statement currently. # `x` is Tensor, `col` is not Tensor, and `col` is the return value of `true_fn` after transformed. # col = -1 - col = paddle.tensor.fill_constant(shape=[1], value=-1, dtype="int64") + col = paddle.tensor.fill_constant(shape=[], value=-1, dtype="int64") + else: + col = paddle.tensor.fill_constant(shape=[], value=1, dtype="int64") if paddle.mean(x).numpy() > x.numpy()[row][col]: - y = paddle.nn.functional.relu(x) + x_pow = paddle.pow(x, 2) + y = paddle.nn.functional.relu(x_pow) else: x_pow = paddle.pow(x, 2) y = paddle.tanh(x_pow) @@ -100,9 +103,12 @@ def false_fn_0(q, x, y): x = x + 1 z = x + 2 q = x + 3 + m = x + 2 + n = x + 3 else: y = y + 1 z = x - 2 + q = x + 3 m = x + 2 n = x + 3 @@ -165,6 +171,22 @@ def nested_if_else(x_v): tmp = y * w y = paddle.nn.functional.relu(tmp) if paddle.mean(y).numpy() < batch_size: + tmp = paddle.tensor.fill_constant( + y.shape, dtype='float32', value=-1 + ) + y = paddle.abs(y) + else: + tmp = paddle.tensor.fill_constant( + y.shape, dtype='float32', value=-1 + ) + y = y - tmp + else: + tmp = y * w + y = paddle.nn.functional.relu(tmp) + if paddle.mean(y).numpy() < batch_size: + tmp = paddle.tensor.fill_constant( + y.shape, dtype='float32', value=-1 + ) y = paddle.abs(y) else: tmp = paddle.tensor.fill_constant( @@ -173,6 +195,11 @@ def nested_if_else(x_v): y = y - tmp else: y = x_v - bias + w = paddle.tensor.fill_constant([feat_size], dtype='float32', value=10) + tmp = y * w + y = paddle.nn.functional.relu(tmp) + tmp = paddle.tensor.fill_constant(y.shape, dtype='float32', value=-1) + y = paddle.abs(y) return y @@ -223,12 +250,14 @@ def nested_if_else_3(x): ) # `z` is created in above code block. z = y + 1 + out = x - 1 else: res = paddle.tensor.fill_constant( value=3, shape=x.shape, dtype="int32" ) # `out` is a new var. out = x + 1 + z = y - 1 return res @@ -378,7 +407,6 @@ def __init__(self): def if_tensor_case(x): x = base.dygraph.to_variable(x) - mean = paddle.mean(x) # It is equivalent to `if mean != 0` if mean: diff --git a/test/dygraph_to_static/test_ifelse.py b/test/dygraph_to_static/test_ifelse.py index 5f50780597e81..7f2262fca3ea6 100644 --- a/test/dygraph_to_static/test_ifelse.py +++ b/test/dygraph_to_static/test_ifelse.py @@ -22,6 +22,7 @@ disable_test_case, enable_to_static_guard, test_ast_only, + test_legacy_and_pt_and_pir, test_legacy_only, ) from ifelse_simple_func import ( @@ -69,6 +70,7 @@ def setUp(self): self.error = "Your if/else have different number of return value." @test_ast_only + @test_legacy_and_pt_and_pir def test_error(self): if self.dyfunc: with self.assertRaisesRegex(Dygraph2StaticException, self.error): @@ -76,6 +78,13 @@ def test_error(self): self.assertTrue(paddle.jit.to_static(self.dyfunc)(self.x)) +class TestDy2StIfElseRetInt2(TestDy2staticException): + def setUp(self): + self.x = np.random.random([5]).astype('float32') + self.error = "Your if/else have different number of return value." + self.dyfunc = dyfunc_ifelse_ret_int2 + + class TestDygraphIfElse(Dy2StTestBase): def setUp(self): self.x = np.random.random([10, 16]).astype('float32') @@ -104,7 +113,6 @@ def setUp(self): # TODO(dev): fix AST mode @disable_test_case((ToStaticMode.AST, IrMode.PT)) - @disable_test_case((ToStaticMode.AST, IrMode.LEGACY_IR)) def test_ast_to_func(self): self.assertTrue((self._run_dygraph() == self._run_static()).all()) @@ -143,6 +151,10 @@ def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = dyfunc_with_if_else_with_list_generator + @test_legacy_and_pt_and_pir + def test_ast_to_func(self): + self.assertTrue((self._run_dygraph() == self._run_static()).all()) + class TestDygraphNestedIfElse(Dy2StTestBase): def setUp(self): @@ -172,6 +184,10 @@ def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = nested_if_else_2 + @test_legacy_and_pt_and_pir + def test_ast_to_func(self): + self.assertTrue((self._run_dygraph() == self._run_static()).all()) + class TestDygraphNestedIfElse3(Dy2StTestBase): def setUp(self): @@ -269,12 +285,20 @@ def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_2 + @test_legacy_and_pt_and_pir + def test_ast_to_func(self): + self.assertTrue((self._run_dygraph() == self._run_static()).all()) + class TestDygraphIfElseWithAndOr3(TestDygraphIfElse): def setUp(self): self.x = np.random.random([10, 16]).astype('float32') self.dyfunc = if_with_and_or_3 + @test_legacy_and_pt_and_pir + def test_ast_to_func(self): + self.assertTrue((self._run_dygraph() == self._run_static()).all()) + class TestDygraphIfElseWithAndOr4(TestDygraphIfElse): def setUp(self): @@ -439,11 +463,14 @@ def init_net(self): def _run(self, mode, to_static): with enable_to_static_guard(to_static): - net = paddle.jit.to_static(self.Net(mode)) + if to_static: + net = paddle.jit.to_static(self.Net(mode)) + else: + net = self.Net(mode) ret = net(self.x, self.y) - return ret.numpy() + @test_legacy_and_pt_and_pir def test_train_mode(self): self.assertTrue( ( @@ -452,6 +479,7 @@ def test_train_mode(self): ).all() ) + @test_legacy_and_pt_and_pir def test_infer_mode(self): self.assertTrue( ( @@ -467,6 +495,7 @@ def init_net(self): class TestNewVarCreateInOneBranch(Dy2StTestBase): + @test_legacy_and_pt_and_pir def test_var_used_in_another_for(self): def case_func(training): # targets and targets_list is dynamically defined by training @@ -510,13 +539,6 @@ def test_ast_to_func(self): self.assertIsInstance(self.out[1], int) -class TestDy2StIfElseRetInt2(TestDy2staticException): - def setUp(self): - self.x = np.random.random([5]).astype('float32') - self.error = "Your if/else have different number of return value." - self.dyfunc = dyfunc_ifelse_ret_int2 - - class TestDy2StIfElseRetInt3(TestDy2StIfElseRetInt1): def setUp(self): self.x = np.random.random([5]).astype('float32') From 23bf65ac06647b3a92daaac1d2cefac4330d90f3 Mon Sep 17 00:00:00 2001 From: lanxianghit <47554610+lanxianghit@users.noreply.github.com> Date: Fri, 29 Dec 2023 10:33:50 +0800 Subject: [PATCH 127/146] add Get&Set APIs for value2sym_expr map (#60301) att, add Get&Set APIs for value2sym_expr map --- paddle/pir/dialect/shape/utils/shape_utils.cc | 12 ++++++++++++ paddle/pir/dialect/shape/utils/shape_utils.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/paddle/pir/dialect/shape/utils/shape_utils.cc b/paddle/pir/dialect/shape/utils/shape_utils.cc index faa5c498bb1f9..05bbb76db8937 100644 --- a/paddle/pir/dialect/shape/utils/shape_utils.cc +++ b/paddle/pir/dialect/shape/utils/shape_utils.cc @@ -175,4 +175,16 @@ std::string GetValueId(Value* val) { return "op_" + std::to_string(op_id) + "_rst_" + std::to_string(val_idx); } +const symbol::ShapeOrDataDimExprs& +ShapeConstraintIRAnalysis::GetShapeOrDataForValue(Value* val) { + auto val_id = GetValueId(val); + return value_id_to_shapeordata[val_id]; +} + +void ShapeConstraintIRAnalysis::SetShapeOrDataForValue( + Value* val, const symbol::ShapeOrDataDimExprs& shape_or_data) { + auto val_id = GetValueId(val); + value_id_to_shapeordata[val_id] = shape_or_data; +} + } // namespace pir diff --git a/paddle/pir/dialect/shape/utils/shape_utils.h b/paddle/pir/dialect/shape/utils/shape_utils.h index ac72c0bae88c7..8f383f3ad6e05 100644 --- a/paddle/pir/dialect/shape/utils/shape_utils.h +++ b/paddle/pir/dialect/shape/utils/shape_utils.h @@ -80,6 +80,11 @@ class IR_API ShapeConstraintIRAnalysis : public ShapeAnalysis { return "S" + std::to_string(next_sym_idx_++); } + const symbol::ShapeOrDataDimExprs& GetShapeOrDataForValue(Value* val); + + void SetShapeOrDataForValue(Value* val, + const symbol::ShapeOrDataDimExprs& shape_or_data); + // const symbol::ShapeOrData& GetShapeOrDataForValue() const; symbol::DimExprBuilder CreateDimExprBuilder() override; From f02f3d6c4f80aba59076b66ef7b87e572a157b8a Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Fri, 29 Dec 2023 10:53:58 +0800 Subject: [PATCH 128/146] adapt pir api (#60416) --- .../pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 3 + paddle/fluid/pybind/pir.cc | 12 ++ .../incubate/optimizer/functional/utils.py | 2 +- test/legacy_test/test_lr_scheduler.py | 32 ++++ test/legacy_test/test_lrn_op.py | 181 +++++++++--------- 6 files changed, 141 insertions(+), 90 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index d541f34a890dc..79cbad13c0f56 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -102,6 +102,7 @@ 'print', 'number_count', 'assign_value', + 'share_data', 'onednn_to_paddle_layout', ] diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 5bdcadc3cca03..b926b055daa6a 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -1153,6 +1153,9 @@ - op : share_data args : (Tensor x) output : Tensor(out) + infer_meta: + func: UnchangedInferMeta + param: [x] kernel: func: share_data param: [x] diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 9e87a3f39459d..e2471842c0729 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -777,6 +777,18 @@ void BindValue(py::module *m) { .def("apply", &apply) .def("is_same", &Value::operator==) .def("hash", [](Value self) { return std::hash{}(self); }) + .def("detach", + [](Value self) { + auto share_data_op = + ApiBuilder::Instance() + .GetBuilder() + ->Build(self); + auto out = share_data_op.out(); + out.set_attribute( + kAttrStopGradients, + BoolAttribute::get(pir::IrContext::Instance(), false)); + return out; + }) .def("__repr__", &Value2String); } diff --git a/python/paddle/incubate/optimizer/functional/utils.py b/python/paddle/incubate/optimizer/functional/utils.py index c6a6f1c6b405a..6fce7ef1703f5 100644 --- a/python/paddle/incubate/optimizer/functional/utils.py +++ b/python/paddle/incubate/optimizer/functional/utils.py @@ -23,7 +23,7 @@ def check_input_type(input, name, op_name): if not isinstance(input, paddle.Tensor): raise ValueError(f"The input: {input} must be tensor.") else: - check_type(input, name, Variable, op_name) + check_type(input, name, (Variable, paddle.pir.Value), op_name) def check_initial_inverse_hessian_estimate(H0): diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index 3db40ea291342..1109d29fa3214 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -1231,6 +1231,38 @@ def test_linear_warmp(self): natural_lr.step() natural_lr_warmup.step() + def test_pir_linear_warmup_lr(self): + params = { + 'learning_rate': 0.5, + 'warmup_steps': 10, + 'start_lr': 0, + 'end_lr': 0.5, + } + scheduler = paddle.optimizer.lr.LinearWarmup(**params) + adam = paddle.optimizer.Adam(learning_rate=scheduler) + with paddle.pir_utils.IrGuard(): + main_prog = paddle.static.Program() + start_prog = paddle.static.Program() + with paddle.static.program_guard(main_prog, start_prog): + x = paddle.static.data(name='x', shape=[3, 4, 5]) + loss = paddle.mean(x) + adam.minimize(loss) + lr_var = adam._global_learning_rate() + + exe = paddle.static.Executor() + exe.run(start_prog) + for epoch in range(5): + for batch_id in range(2): + out = exe.run( + main_prog, + feed={'x': np.random.randn(3, 4, 5).astype('float32')}, + fetch_list=[lr_var], + ) + self.assertEqual( + out, np.array(linear_warmup_lr(epoch, **params)) + ) + scheduler.step() + if __name__ == '__main__': paddle.enable_static() diff --git a/test/legacy_test/test_lrn_op.py b/test/legacy_test/test_lrn_op.py index 34ceff298ec3d..c97e8e7dd8536 100644 --- a/test/legacy_test/test_lrn_op.py +++ b/test/legacy_test/test_lrn_op.py @@ -19,7 +19,8 @@ import paddle from paddle import base -from paddle.base import Program, core, program_guard +from paddle.base import core +from paddle.pir_utils import test_with_pir_api class TestLRNOp(OpTest): @@ -115,98 +116,96 @@ def setUp(self): self.places.append(base.CUDAPlace(0)) def check_static_3d_input(self, place): - with paddle_static_guard(): - with base.program_guard(base.Program(), base.Program()): - in_np1 = np.random.random([3, 40, 40]).astype("float32") - in_np2 = np.transpose(in_np1, (0, 2, 1)) - - input1 = paddle.static.data( - name="input1", shape=[3, 40, 40], dtype="float32" - ) - input2 = paddle.static.data( - name="input2", shape=[3, 40, 40], dtype="float32" - ) - res1 = paddle.nn.functional.local_response_norm( - x=input1, size=5, data_format='NCL' - ) - res2 = paddle.nn.functional.local_response_norm( - x=input2, size=5, data_format='NLC' - ) - exe = base.Executor(place) - fetches = exe.run( - base.default_main_program(), - feed={"input1": in_np1, "input2": in_np2}, - fetch_list=[res1, res2], - ) - - fetches1_tran = np.transpose(fetches[1], (0, 2, 1)) - np.testing.assert_allclose( - fetches[0], fetches1_tran, rtol=1e-05 - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + in_np1 = np.random.random([3, 40, 40]).astype("float32") + in_np2 = np.transpose(in_np1, (0, 2, 1)) + + input1 = paddle.static.data( + name="input1", shape=[3, 40, 40], dtype="float32" + ) + input2 = paddle.static.data( + name="input2", shape=[3, 40, 40], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCL' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NLC' + ) + exe = base.Executor(place) + fetches = exe.run( + paddle.static.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) + + fetches1_tran = np.transpose(fetches[1], (0, 2, 1)) + np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_4d_input(self, place): - with paddle_static_guard(): - with base.program_guard(base.Program(), base.Program()): - input1 = paddle.static.data( - name="input1", shape=[3, 3, 40, 40], dtype="float32" - ) - input2 = paddle.static.data( - name="input2", shape=[3, 40, 40, 3], dtype="float32" - ) - - res1 = paddle.nn.functional.local_response_norm( - x=input1, size=5, data_format='NCHW' - ) - res2 = paddle.nn.functional.local_response_norm( - x=input2, size=5, data_format='NHWC' - ) - - in_np1 = np.random.random([3, 3, 40, 40]).astype("float32") - in_np2 = np.transpose(in_np1, (0, 2, 3, 1)) - - exe = base.Executor(place) - fetches = exe.run( - base.default_main_program(), - feed={"input1": in_np1, "input2": in_np2}, - fetch_list=[res1, res2], - ) - - fetches1_tran = np.transpose(fetches[1], (0, 3, 1, 2)) - np.testing.assert_allclose( - fetches[0], fetches1_tran, rtol=1e-05 - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input1 = paddle.static.data( + name="input1", shape=[3, 3, 40, 40], dtype="float32" + ) + input2 = paddle.static.data( + name="input2", shape=[3, 40, 40, 3], dtype="float32" + ) + + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NHWC' + ) + + in_np1 = np.random.random([3, 3, 40, 40]).astype("float32") + in_np2 = np.transpose(in_np1, (0, 2, 3, 1)) + + exe = base.Executor(place) + fetches = exe.run( + paddle.static.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) + + fetches1_tran = np.transpose(fetches[1], (0, 3, 1, 2)) + np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) def check_static_5d_input(self, place): - with paddle_static_guard(): - with base.program_guard(base.Program(), base.Program()): - input1 = paddle.static.data( - name="input1", shape=[3, 3, 3, 40, 40], dtype="float32" - ) - input2 = paddle.static.data( - name="input2", shape=[3, 3, 40, 40, 3], dtype="float32" - ) - res1 = paddle.nn.functional.local_response_norm( - x=input1, size=5, data_format='NCDHW' - ) - res2 = paddle.nn.functional.local_response_norm( - x=input2, size=5, data_format='NDHWC' - ) - - in_np1 = np.random.random([3, 3, 3, 40, 40]).astype("float32") - in_np2 = np.transpose(in_np1, (0, 2, 3, 4, 1)) - - exe = base.Executor(place) - fetches = exe.run( - base.default_main_program(), - feed={"input1": in_np1, "input2": in_np2}, - fetch_list=[res1, res2], - ) - - fetches1_tran = np.transpose(fetches[1], (0, 4, 1, 2, 3)) - np.testing.assert_allclose( - fetches[0], fetches1_tran, rtol=1e-05 - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + input1 = paddle.static.data( + name="input1", shape=[3, 3, 3, 40, 40], dtype="float32" + ) + input2 = paddle.static.data( + name="input2", shape=[3, 3, 40, 40, 3], dtype="float32" + ) + res1 = paddle.nn.functional.local_response_norm( + x=input1, size=5, data_format='NCDHW' + ) + res2 = paddle.nn.functional.local_response_norm( + x=input2, size=5, data_format='NDHWC' + ) + + in_np1 = np.random.random([3, 3, 3, 40, 40]).astype("float32") + in_np2 = np.transpose(in_np1, (0, 2, 3, 4, 1)) + + exe = base.Executor(place) + fetches = exe.run( + paddle.static.default_main_program(), + feed={"input1": in_np1, "input2": in_np2}, + fetch_list=[res1, res2], + ) + + fetches1_tran = np.transpose(fetches[1], (0, 4, 1, 2, 3)) + np.testing.assert_allclose(fetches[0], fetches1_tran, rtol=1e-05) + @test_with_pir_api def test_static(self): with paddle_static_guard(): for place in self.places: @@ -276,9 +275,12 @@ def test_dygraph(self): class TestLocalResponseNormFAPIError(unittest.TestCase): + @test_with_pir_api def test_errors(self): with paddle_static_guard(): - with program_guard(Program(), Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): def test_Variable(): # the input of lrn must be Variable. @@ -346,6 +348,7 @@ def test_dygraph(self): res2_tran = np.transpose(res2.numpy(), (0, 3, 1, 2)) np.testing.assert_allclose(res1.numpy(), res2_tran, rtol=1e-05) + @test_with_pir_api def test_static_fp16_gpu(self): if paddle.base.core.is_compiled_with_cuda(): place = paddle.CUDAPlace(0) From 4ac428075bfa169a35a42a4b2b4e10dc78f913c5 Mon Sep 17 00:00:00 2001 From: ming1753 <61511741+ming1753@users.noreply.github.com> Date: Fri, 29 Dec 2023 11:34:44 +0800 Subject: [PATCH 129/146] support optimized update shape_range_info_path (#60457) --- .../inference/analysis/ir_passes/tensorrt_subgraph_pass.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc index 2cabfe567b5d9..851e7863d7af2 100644 --- a/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/tensorrt_subgraph_pass.cc @@ -193,6 +193,12 @@ void analysis::TensorRtSubgraphPass::ApplyImpl( std::vector repetitive_params; std::vector engine_names; for (auto *node : graph->Nodes()) { + // load optimized model may update shape_range_info_path + auto shape_range_info_path = Get("trt_shape_range_info_path"); + if (node->IsOp() && node->Op()->Type() == "tensorrt_engine" && + !shape_range_info_path.empty()) { + node->Op()->SetAttr("shape_range_info_path", shape_range_info_path); + } if (node->IsOp() && !framework::ir::Agent(node).subgraph()->empty()) { engine_names.push_back(CreateTensorRTOp( node, graph, graph_param_names, &repetitive_params, use_cuda_graph)); From 567cd55d9b303d2c83a4c2b409f4d63515200c02 Mon Sep 17 00:00:00 2001 From: wuhuachaocoding <77733235+wuhuachaocoding@users.noreply.github.com> Date: Fri, 29 Dec 2023 11:50:53 +0800 Subject: [PATCH 130/146] remove assert for sharding and mp hybrid parallel. (#60455) --- .../dygraph_optimizer/dygraph_sharding_optimizer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py index b6b4c3c01842f..605c08039d534 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/dygraph_sharding_optimizer.py @@ -537,9 +537,6 @@ def __init__(self, optimizer, hcg): self.pp_overlap = pp_config.sharding_comm_overlap self.pp_release_grads = pp_config.release_gradients - # TODO(liuzhenhai):support it latter - assert not self.comm_overlap, "not supported yet" - self._build_comm_buffers(acc_steps) # NOTE(shenliang03): Sort the comm_buffers by dst rank, # it will improve the performance in reduce communicate. Default From 21ee5780d519d1c21c8bf0bc3d12b34fb3c70d1c Mon Sep 17 00:00:00 2001 From: Bo Zhang <105368690+zhangbopd@users.noreply.github.com> Date: Fri, 29 Dec 2023 12:37:17 +0800 Subject: [PATCH 131/146] bug fix (#60461) --- paddle/pir/dialect/shape/utils/shape_utils.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/pir/dialect/shape/utils/shape_utils.h b/paddle/pir/dialect/shape/utils/shape_utils.h index 8f383f3ad6e05..7e4eafa672276 100644 --- a/paddle/pir/dialect/shape/utils/shape_utils.h +++ b/paddle/pir/dialect/shape/utils/shape_utils.h @@ -105,6 +105,9 @@ class IR_API ShapeConstraintIRAnalysis : public ShapeAnalysis { int64_t next_sym_idx_ = 0; std::vector constraints_; + std::unordered_map + value_id_to_shapeordata; + public: explicit ShapeConstraintIRAnalysis(std::shared_ptr&& program) : ShapeConstraintIRAnalysis(program->module_op()) { From c15a6d3d44fbdda97b6e113f3a33797370c8d720 Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Fri, 29 Dec 2023 13:59:47 +0800 Subject: [PATCH 132/146] [XPU] avoid pre-allocating gm buffer (#60387) --- paddle/fluid/distributed/collective/process_group_bkcl.cc | 2 ++ paddle/phi/backends/xpu/xpu_context.cc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/paddle/fluid/distributed/collective/process_group_bkcl.cc b/paddle/fluid/distributed/collective/process_group_bkcl.cc index 8b306e29f52b3..cdc31cf9a6489 100644 --- a/paddle/fluid/distributed/collective/process_group_bkcl.cc +++ b/paddle/fluid/distributed/collective/process_group_bkcl.cc @@ -207,6 +207,8 @@ void ProcessGroupBKCL::CreateBKCLEnvCache(const Place& place, platform::DeviceContextPool::Instance().Get(place)); // must use XPUDeviceContext here to make sure XPUContext::Init() is called auto comm_ctx = std::make_unique(place); + // comm_ctx does not require a pre-allocated GM buffer + comm_ctx->x_context()->set_option("XPUAPI_DEFAULT_SIZE", "1"); auto bkcl_comm_ctx = this->GetCommContext(); comm_ctx->SetBkclContext(bkcl_comm_ctx->GetBKCLComm()); diff --git a/paddle/phi/backends/xpu/xpu_context.cc b/paddle/phi/backends/xpu/xpu_context.cc index ad0047b4e9ad6..9de9744393d4a 100644 --- a/paddle/phi/backends/xpu/xpu_context.cc +++ b/paddle/phi/backends/xpu/xpu_context.cc @@ -200,6 +200,9 @@ struct XPUContext::Impl { << tname << " currently " << context_map_.size() << " contexts existing"; xpu::Context* ctx_t = xpu::create_context(); + // DataLoader does not require a pre-allocated GM buffer + // to avoid xpu_wait calls + ctx_t->set_option("XPUAPI_DEFAULT_SIZE", "1"); context_map_[tname] = ctx_t; } } From 16710f72e6696a2c45afac52360e4e21f05b047b Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Fri, 29 Dec 2023 14:08:21 +0800 Subject: [PATCH 133/146] [PIR] Fix some pir interpreter bug and refine some code (#60420) * fix * fix * fix * fix * fix * fix * fix --- .../{ => control_flow}/assert_instruction.cc | 2 +- .../{ => control_flow}/assert_instruction.h | 0 .../has_elements_instruction.cc | 2 +- .../has_elements_instruction.h | 0 .../{ => control_flow}/if_instruction.cc | 2 +- .../{ => control_flow}/if_instruction.h | 0 .../select_input_instruction.cc | 2 +- .../select_input_instruction.h | 0 .../tuple_pop_instruction.cc | 2 +- .../tuple_pop_instruction.h | 0 .../tuple_push_instruction.cc | 2 +- .../tuple_push_instruction.h | 0 .../{ => control_flow}/while_instruction.cc | 2 +- .../{ => control_flow}/while_instruction.h | 0 .../framework/new_executor/pir_interpreter.cc | 14 ++++---- .../translator/program_translator.cc | 32 ++++++++++++++++++- .../translator/program_translator.h | 2 ++ test/dygraph_to_static/test_for_enumerate.py | 3 -- test/legacy_test/test_cond.py | 7 +++- 19 files changed, 53 insertions(+), 19 deletions(-) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/assert_instruction.cc (97%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/assert_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/has_elements_instruction.cc (96%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/has_elements_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/if_instruction.cc (99%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/if_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/select_input_instruction.cc (98%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/select_input_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/tuple_pop_instruction.cc (98%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/tuple_pop_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/tuple_push_instruction.cc (97%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/tuple_push_instruction.h (100%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/while_instruction.cc (99%) rename paddle/fluid/framework/new_executor/instruction/{ => control_flow}/while_instruction.h (100%) diff --git a/paddle/fluid/framework/new_executor/instruction/assert_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.cc similarity index 97% rename from paddle/fluid/framework/new_executor/instruction/assert_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.cc index 96d1fcc57b943..d2835dd65ccad 100644 --- a/paddle/fluid/framework/new_executor/instruction/assert_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/assert_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" #include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" diff --git a/paddle/fluid/framework/new_executor/instruction/assert_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/assert_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.cc similarity index 96% rename from paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.cc index 958daf2239eaf..900667071091b 100644 --- a/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/has_elements_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" #include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" diff --git a/paddle/fluid/framework/new_executor/instruction/has_elements_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/has_elements_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/if_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.cc similarity index 99% rename from paddle/fluid/framework/new_executor/instruction/if_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.cc index 57146acdfb5df..ef856c7fc0162 100644 --- a/paddle/fluid/framework/new_executor/instruction/if_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/if_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.h" #include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h" #include "paddle/fluid/framework/new_executor/interpreter/stream_analyzer.h" diff --git a/paddle/fluid/framework/new_executor/instruction/if_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/if_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/select_input_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.cc similarity index 98% rename from paddle/fluid/framework/new_executor/instruction/select_input_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.cc index 893915f841d7f..987edeb97eda0 100644 --- a/paddle/fluid/framework/new_executor/instruction/select_input_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/select_input_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" #include "paddle/fluid/framework/new_executor/new_executor_defs.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" diff --git a/paddle/fluid/framework/new_executor/instruction/select_input_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/select_input_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.cc similarity index 98% rename from paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.cc index a3a8f4461865e..1cb27abb3e2d9 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.cc @@ -14,7 +14,7 @@ #include -#include "paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.cc similarity index 97% rename from paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.cc index bb01125bf3eca..3f0082a4af5c8 100644 --- a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/instruction_util.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" diff --git a/paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.h diff --git a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc b/paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.cc similarity index 99% rename from paddle/fluid/framework/new_executor/instruction/while_instruction.cc rename to paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.cc index b281e2b8a6cbe..a9f23fd60e176 100644 --- a/paddle/fluid/framework/new_executor/instruction/while_instruction.cc +++ b/paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/framework/new_executor/instruction/while_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.h" #include "paddle/fluid/framework/new_executor/interpreter/interpreter_util.h" #include "paddle/fluid/framework/new_executor/interpreter/stream_analyzer.h" diff --git a/paddle/fluid/framework/new_executor/instruction/while_instruction.h b/paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.h similarity index 100% rename from paddle/fluid/framework/new_executor/instruction/while_instruction.h rename to paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.h diff --git a/paddle/fluid/framework/new_executor/pir_interpreter.cc b/paddle/fluid/framework/new_executor/pir_interpreter.cc index 82bf2973345ad..2afdfb5e9717a 100644 --- a/paddle/fluid/framework/new_executor/pir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/pir_interpreter.cc @@ -48,16 +48,16 @@ #include "paddle/fluid/framework/new_executor/instruction/cinn_jit_instruction.h" #endif -#include "paddle/fluid/framework/new_executor/instruction/assert_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/builtin_combine_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/has_elements_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/if_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/assert_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/has_elements_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/if_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/select_input_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/tuple_pop_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/tuple_push_instruction.h" +#include "paddle/fluid/framework/new_executor/instruction/control_flow/while_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/legacy_kernel_instruction.h" #include "paddle/fluid/framework/new_executor/instruction/phi_kernel_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/select_input_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/tuple_pop_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/tuple_push_instruction.h" -#include "paddle/fluid/framework/new_executor/instruction/while_instruction.h" #include "paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_attribute.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_dialect.h" diff --git a/paddle/fluid/ir_adaptor/translator/program_translator.cc b/paddle/fluid/ir_adaptor/translator/program_translator.cc index 5eaaa5052f457..7eca5767750b9 100644 --- a/paddle/fluid/ir_adaptor/translator/program_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/program_translator.cc @@ -131,6 +131,8 @@ ProgramTranslator::ProgramTranslator(const ProgramDesc* legacy_program, void ProgramTranslator::Translate() { GetParameterForSingleBlock(legacy_program_->Block(0)); + InsertDataOpForSingleBlock(legacy_program_->Block(0)); + TranslateBlock(legacy_program_->Block(0), 0, legacy_program_->Block(0).OpSize(), @@ -155,7 +157,7 @@ void ProgramTranslator::TranslateBlock(const BlockDesc& src_block, uint64_t end_id, TranslationContext* translation_ctx, pir::Block* dst_block) { - VLOG(8) << "=============>start to translate a block"; + VLOG(8) << "=============>start to translate a block: " << &src_block; PADDLE_ENFORCE( (src_block.OpSize() >= end_id) && (start_id <= end_id), platform::errors::NotFound( @@ -419,6 +421,34 @@ inline pir::Operation* InsertSetParamaterOp(pir::IrContext* ctx, return operation; } +void ProgramTranslator::InsertDataOpForSingleBlock(const BlockDesc& block) { + std::unordered_set all_var_names; + for (auto& var : block.AllVars()) { + all_var_names.insert(var->Name()); + } + + std::unordered_set inner_outputs; + for (auto op_desc : block.AllOps()) { + for (const auto& n : op_desc->Inputs()) { + const auto& input_var_names = n.second; + for (const auto& var_name : input_var_names) { + if (param_map_.count(var_name) != 0) continue; + if (no_cast_var_names.count(var_name) != 0) continue; + if (all_var_names.count(var_name) == 0) continue; + if (inner_outputs.count(var_name) == 0) { + CreateUndefinedVariable(var_name, block); + } + } + } + for (const auto& n : op_desc->Outputs()) { + const auto& output_var_names = n.second; + for (const auto& var_name : output_var_names) { + inner_outputs.insert(var_name); + } + } + } +} + void ProgramTranslator::GetParameterForSingleBlock(const BlockDesc& block) { for (auto& var : block.AllVars()) { if (!var->Persistable()) continue; diff --git a/paddle/fluid/ir_adaptor/translator/program_translator.h b/paddle/fluid/ir_adaptor/translator/program_translator.h index 5fce6b08c2648..cff7684226c52 100644 --- a/paddle/fluid/ir_adaptor/translator/program_translator.h +++ b/paddle/fluid/ir_adaptor/translator/program_translator.h @@ -120,6 +120,8 @@ class ProgramTranslator { void TranslateGeneralOperation(const OpDesc* src_op, TranslationContext* translation_ctx, pir::Block* dst_block); + + void InsertDataOpForSingleBlock(const BlockDesc& block); void GetParameterForSingleBlock(const BlockDesc& block); void SetParameterFromSingleBlock(const BlockDesc& block); void SetStopGradientAttributeForAllValue(const BlockDesc& block); diff --git a/test/dygraph_to_static/test_for_enumerate.py b/test/dygraph_to_static/test_for_enumerate.py index a540cef2e387b..7b754fb1343ea 100644 --- a/test/dygraph_to_static/test_for_enumerate.py +++ b/test/dygraph_to_static/test_for_enumerate.py @@ -19,7 +19,6 @@ import numpy as np from dygraph_to_static_utils import ( Dy2StTestBase, - compare_legacy_with_pt, enable_to_static_guard, test_legacy_and_pt_and_pir, ) @@ -495,7 +494,6 @@ class TestForIterVarList(TestForInRangeConfig): def set_test_func(self): self.dygraph_func = for_iter_var_list - @compare_legacy_with_pt def test_transformed_result_compare(self): self.set_test_func() self.transformed_result_compare() @@ -505,7 +503,6 @@ class TestForEnumerateVarList(TestForInRangeConfig): def set_test_func(self): self.dygraph_func = for_enumerate_var_list - @compare_legacy_with_pt def test_transformed_result_compare(self): self.set_test_func() self.transformed_result_compare() diff --git a/test/legacy_test/test_cond.py b/test/legacy_test/test_cond.py index 1323d7caa6eae..3dcd127e51c4b 100644 --- a/test/legacy_test/test_cond.py +++ b/test/legacy_test/test_cond.py @@ -32,6 +32,7 @@ class TestCondInputOutput(unittest.TestCase): @compare_legacy_with_pt + @test_with_pir_api def test_return_single_var(self): """ pseudocode: @@ -73,7 +74,11 @@ def false_func(): else base.CPUPlace() ) exe = base.Executor(place) - (ret,) = exe.run(main_program, fetch_list=[out.name]) + if paddle.framework.in_pir_mode(): + (ret,) = exe.run(main_program, fetch_list=[out]) + else: + (ret,) = exe.run(main_program, fetch_list=[out.name]) + np.testing.assert_allclose( np.asarray(ret), np.full((3, 2), -1, np.int32), rtol=1e-05 ) From 5828223b6f13ced50b36439ec26248f29bb58d99 Mon Sep 17 00:00:00 2001 From: enzodechine Date: Fri, 29 Dec 2023 14:47:12 +0800 Subject: [PATCH 134/146] correct the unittest for bf16 op (#60415) --- .../legacy/xpu/elementwise_divide_kernel.cc | 1 + .../legacy/xpu/elementwise_multiply_kernel.cc | 1 + .../legacy/xpu/elementwise_subtract_kernel.cc | 1 + test/xpu/test_elementwise_div_op_xpu.py | 354 ++++-------------- test/xpu/test_elementwise_mul_op_xpu.py | 205 +++++----- test/xpu/test_elementwise_sub_op_xpu.py | 175 ++++----- test/xpu/test_reduce_sum_op_xpu.py | 39 +- 7 files changed, 278 insertions(+), 498 deletions(-) diff --git a/paddle/phi/kernels/legacy/xpu/elementwise_divide_kernel.cc b/paddle/phi/kernels/legacy/xpu/elementwise_divide_kernel.cc index 5318cb464001f..ccdfcd750f091 100644 --- a/paddle/phi/kernels/legacy/xpu/elementwise_divide_kernel.cc +++ b/paddle/phi/kernels/legacy/xpu/elementwise_divide_kernel.cc @@ -50,4 +50,5 @@ PD_REGISTER_KERNEL(divide_raw, ALL_LAYOUT, phi::DivideRawKernel, phi::dtype::float16, + phi::dtype::bfloat16, float) {} diff --git a/paddle/phi/kernels/legacy/xpu/elementwise_multiply_kernel.cc b/paddle/phi/kernels/legacy/xpu/elementwise_multiply_kernel.cc index 790bd72b24091..2986e555cda70 100644 --- a/paddle/phi/kernels/legacy/xpu/elementwise_multiply_kernel.cc +++ b/paddle/phi/kernels/legacy/xpu/elementwise_multiply_kernel.cc @@ -50,6 +50,7 @@ PD_REGISTER_KERNEL(multiply_raw, ALL_LAYOUT, phi::MultiplyRawKernel, phi::dtype::float16, + phi::dtype::bfloat16, float, int, int64_t) {} diff --git a/paddle/phi/kernels/legacy/xpu/elementwise_subtract_kernel.cc b/paddle/phi/kernels/legacy/xpu/elementwise_subtract_kernel.cc index 421a30a240a43..7fb4144d7705b 100644 --- a/paddle/phi/kernels/legacy/xpu/elementwise_subtract_kernel.cc +++ b/paddle/phi/kernels/legacy/xpu/elementwise_subtract_kernel.cc @@ -45,4 +45,5 @@ PD_REGISTER_KERNEL(subtract_raw, phi::SubtractRawKernel, float, phi::dtype::float16, + phi::dtype::bfloat16, int64_t) {} diff --git a/test/xpu/test_elementwise_div_op_xpu.py b/test/xpu/test_elementwise_div_op_xpu.py index 52e2e62e067d2..ca190b7eb1230 100644 --- a/test/xpu/test_elementwise_div_op_xpu.py +++ b/test/xpu/test_elementwise_div_op_xpu.py @@ -20,7 +20,10 @@ create_test_class, get_xpu_op_support_types, ) -from op_test import skip_check_grad_ci +from op_test import ( + convert_float_to_uint16, + skip_check_grad_ci, +) from op_test_xpu import XPUOpTest import paddle @@ -28,6 +31,8 @@ paddle.enable_static() +INT_GROUP = [np.int32, np.int64] + class XPUTestElementwiseDivOp(XPUOpTestWrapper): def __init__(self): @@ -40,6 +45,7 @@ def setUp(self): self.dtype = self.in_type self.init_dtype() self.use_xpu = True + self.init_shape() self.init_input_output() """ Warning CPU gradient check error! @@ -47,20 +53,40 @@ def setUp(self): 'Y': np.random.random((32,84)).astype("float32") """ + def gen_data_depend_on_dtype(self, shape): + if self.dtype in INT_GROUP: + return np.random.randint(1, 100, size=shape) + else: + return np.random.uniform(-1, 1, size=shape) + + def reshape_y_depend_on_x(self): + if len(self.x_shape) <= len(self.y_shape) or self.y_shape == (): + return self.y + reshape_dims = [ + 1 if i not in self.y_shape else i for i in self.x_shape + ] + return np.reshape(self.y, reshape_dims) + def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: + self.x = self.gen_data_depend_on_dtype(self.x_shape) + self.y = self.gen_data_depend_on_dtype(self.y_shape) + reshaped_y = self.reshape_y_depend_on_x() + if self.dtype == np.uint16: + self.outputs = {'Out': np.divide(self.x, reshaped_y)} self.inputs = { - 'X': np.random.randint(1, 100, [13, 17]).astype(self.dtype), - 'Y': np.random.randint(1, 100, [13, 17]).astype(self.dtype), + 'X': convert_float_to_uint16(self.x), + 'Y': convert_float_to_uint16(self.y), } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} else: self.inputs = { - 'X': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype), + 'X': self.x.astype(self.dtype), + 'Y': self.y.astype(self.dtype), } + reshaped_y.astype(self.dtype) self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) + 'Out': self.inputs['X'] // reshaped_y + if self.dtype in INT_GROUP + else np.divide(self.inputs['X'], reshaped_y) } def test_check_output(self): @@ -100,306 +126,80 @@ def test_check_grad_ingore_y(self): def init_dtype(self): pass + def init_shape(self): + self.x_shape = [13, 17] + self.y_shape = [13, 17] + class TestElementwiseDivOp_ZeroDim1(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, []).astype(self.dtype), - 'Y': np.random.randint(1, 100, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(-1, 1, []).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} + def init_shape(self): + self.x_shape = [] + self.y_shape = [] class TestElementwiseDivOp_ZeroDim2(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [13, 17]).astype(self.dtype), - 'Y': np.random.randint(1, 100, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(-1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} + def init_shape(self): + self.x_shape = [13, 17] + self.y_shape = [] @skip_check_grad_ci( reason="[skip shape check] Use y_shape(1) to test broadcast." ) class TestElementwiseDivOp_scalar(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [20, 3, 4]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [1]).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [1]).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} + def init_shape(self): + self.x_shape = [20, 3, 4] + self.y_shape = [1] class TestElementwiseDivOp_Vector(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [100]).astype(self.dtype), - 'Y': np.random.randint(1, 100, [100]).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } + def init_shape(self): + self.x_shape = [100] + self.y_shape = [100] class TestElementwiseDivOp_broadcast_0(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [100, 3, 4]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': self.inputs['X'] - // self.inputs['Y'].reshape(100, 1, 1) - } - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': np.divide( - self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1) - ) - } - + def init_shape(self): + self.x_shape = [100, 3, 4] + self.y_shape = [100] self.attrs = {'axis': 0} class TestElementwiseDivOp_broadcast_1(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 100, 4]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': self.inputs['X'] - // self.inputs['Y'].reshape(1, 100, 1) - } - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': np.divide( - self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1) - ) - } - + def init_shape(self): + self.x_shape = [2, 100, 4] + self.y_shape = [100] self.attrs = {'axis': 1} class TestElementwiseDivOp_broadcast_2(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 3, 100]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': self.inputs['X'] - // self.inputs['Y'].reshape(1, 1, 100) - } - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [100]).astype(self.dtype), - } - self.outputs = { - 'Out': np.divide( - self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100) - ) - } + def init_shape(self): + self.x_shape = [2, 3, 100] + self.y_shape = [100] class TestElementwiseDivOp_broadcast_3(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 10, 12, 5]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [10, 12]).astype(self.dtype), - } - self.outputs = { - 'Out': self.inputs['X'] - // self.inputs['Y'].reshape(1, 10, 12, 1) - } - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), - } - self.outputs = { - 'Out': np.divide( - self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1) - ) - } - + def init_shape(self): + self.x_shape = [2, 10, 12, 5] + self.y_shape = [10, 12] self.attrs = {'axis': 1} class TestElementwiseDivOp_broadcast_4(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 3, 50]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [2, 1, 50]).astype( - self.dtype - ), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype( - self.dtype - ), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } + def init_shape(self): + self.x_shape = [2, 3, 50] + self.y_shape = [2, 1, 50] class TestElementwiseDivOp_broadcast_5(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 3, 4, 20]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [2, 3, 1, 20]).astype( - self.dtype - ), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype( - self.dtype - ), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } + def init_shape(self): + self.x_shape = [2, 3, 4, 20] + self.y_shape = [2, 3, 1, 20] class TestElementwiseDivOp_commonuse_1(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [2, 3, 100]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [1, 1, 100]).astype( - self.dtype - ), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [1, 1, 100]).astype( - self.dtype - ), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } + def init_shape(self): + self.x_shape = [2, 3, 100] + self.y_shape = [1, 1, 100] class TestElementwiseDivOp_commonuse_2(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [30, 3, 1, 5]).astype( - self.dtype - ), - 'Y': np.random.randint(1, 100, [30, 1, 4, 1]).astype( - self.dtype - ), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [30, 3, 1, 5]).astype( - self.dtype - ), - 'Y': np.random.uniform(0.1, 1, [30, 1, 4, 1]).astype( - self.dtype - ), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } + def init_shape(self): + self.x_shape = [30, 3, 1, 5] + self.y_shape = [30, 1, 4, 1] class TestElementwiseDivOp_xsize_lessthan_ysize(ElementwiseDivOp): - def init_input_output(self): - if self.dtype == np.int32 or self.dtype == np.int64: - self.inputs = { - 'X': np.random.randint(1, 100, [10, 12]).astype(self.dtype), - 'Y': np.random.randint(1, 100, [2, 3, 10, 12]).astype( - self.dtype - ), - } - self.outputs = {'Out': self.inputs['X'] // self.inputs['Y']} - else: - self.inputs = { - 'X': np.random.uniform(0.1, 1, [10, 12]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 10, 12]).astype( - self.dtype - ), - } - self.outputs = { - 'Out': np.divide(self.inputs['X'], self.inputs['Y']) - } - + def init_shape(self): + self.x_shape = [10, 12] + self.y_shape = [2, 3, 10, 12] self.attrs = {'axis': 2} class TestElementwiseDivBroadcast(unittest.TestCase): diff --git a/test/xpu/test_elementwise_mul_op_xpu.py b/test/xpu/test_elementwise_mul_op_xpu.py index 6bd604df07e40..b8fda9a5b6217 100644 --- a/test/xpu/test_elementwise_mul_op_xpu.py +++ b/test/xpu/test_elementwise_mul_op_xpu.py @@ -20,7 +20,10 @@ create_test_class, get_xpu_op_support_types, ) -from op_test import OpTest, skip_check_grad_ci +from op_test import ( + convert_float_to_uint16, + skip_check_grad_ci, +) from op_test_xpu import XPUOpTest import paddle @@ -40,13 +43,34 @@ def init_kernel_type(self): def setUp(self): self.op_type = 'elementwise_mul' self.use_xpu = True + self.cal_x = None + self.cal_y = None self.dtype = self.in_type self.axis = -1 - self.init_dtype() + self.init_data() + self.gen_output() self.init_input_output() self.init_kernel_type() self.init_axis() + def gen_output(self): + if self.cal_x is None: + self.cal_x = self.x + if self.cal_y is None: + self.cal_y = self.y + if self.dtype == np.uint16: + self.out = np.multiply(self.cal_x, self.cal_y) + else: + self.out = np.multiply( + self.cal_x.astype(self.dtype), self.cal_y.astype(self.dtype) + ) + + def gen_data_depend_on_dtype(self, shape): + if self.dtype == np.int32 or self.dtype == np.int64: + return np.random.randint(1, 100, size=shape) + else: + return np.random.uniform(0.1, 1, size=shape) + def test_check_output(self): if paddle.is_compiled_with_xpu(): place = paddle.XPUPlace(0) @@ -84,158 +108,109 @@ def test_check_grad_ingore_y(self): check_dygraph=False, ) + def init_data(self): + self.x = self.gen_data_depend_on_dtype([13, 17]) + self.y = self.gen_data_depend_on_dtype([13, 17]) + def init_input_output(self): - self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) - self.y = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) - self.out = np.multiply(self.x, self.y) + if self.dtype == np.uint16: + self.x = convert_float_to_uint16(self.x) + self.y = convert_float_to_uint16(self.y) + else: + self.x = self.x.astype(self.dtype) + self.y = self.y.astype(self.dtype) + self.inputs = { - 'X': OpTest.np_dtype_to_base_dtype(self.x), - 'Y': OpTest.np_dtype_to_base_dtype(self.y), + 'X': self.x, + 'Y': self.y, } self.outputs = {'Out': self.out} self.attrs = {'axis': self.axis, 'use_mkldnn': self.use_mkldnn} - def init_dtype(self): - pass - def init_axis(self): pass class TestElementwiseMulOp_ZeroDim1(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, []).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([]) + self.y = self.gen_data_depend_on_dtype([]) class TestElementwiseMulOp_ZeroDim2(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([13, 17]) + self.y = self.gen_data_depend_on_dtype([]) class TestElementwiseMulOp_ZeroDim3(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, []).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, [13, 17]).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([]) + self.y = self.gen_data_depend_on_dtype([13, 17]) @skip_check_grad_ci( reason="[skip shape check] Use y_shape(1) to test broadcast." ) class TestElementwiseMulOp_scalar(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([10, 3, 4]) + self.y = self.gen_data_depend_on_dtype([1]) class TestElementwiseMulOp_Vector(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.random((100,)).astype(self.dtype), - 'Y': np.random.random((100,)).astype(self.dtype), - } - self.outputs = { - 'Out': np.multiply(self.inputs['X'], self.inputs['Y']) - } + def init_data(self): + self.x = self.gen_data_depend_on_dtype([100]) + self.y = self.gen_data_depend_on_dtype([100]) class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(100, 2, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - self.outputs = { - 'Out': self.inputs['X'] * self.inputs['Y'].reshape(100, 1, 1) - } - self.attrs = {'axis': 0} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([100, 2, 3]) + self.y = self.gen_data_depend_on_dtype([100]) + self.cal_y = self.y.reshape(100, 1, 1) + self.axis = 0 class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - - self.attrs = {'axis': 1} - self.outputs = { - 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 100, 1) - } + def init_data(self): + self.x = self.gen_data_depend_on_dtype([2, 100, 3]) + self.y = self.gen_data_depend_on_dtype([100]) + self.cal_y = self.y.reshape(1, 100, 1) + self.axis = 1 class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - - self.outputs = { - 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 1, 100) - } + def init_data(self): + self.x = self.gen_data_depend_on_dtype([2, 3, 100]) + self.y = self.gen_data_depend_on_dtype([100]) + self.cal_y = self.y.reshape(1, 1, 100) class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype), - } - - self.attrs = {'axis': 1} - self.outputs = { - 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 10, 12, 1) - } + def init_data(self): + self.x = self.gen_data_depend_on_dtype([2, 10, 12, 3]) + self.y = self.gen_data_depend_on_dtype([10, 12]) + self.cal_y = self.y.reshape(1, 10, 12, 1) + self.axis = 1 class TestElementwiseMulOp_broadcast_4(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 2, 11).astype(self.dtype), - 'Y': np.random.rand(10, 1, 11).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([10, 2, 11]) + self.y = self.gen_data_depend_on_dtype([10, 1, 11]) class TestElementwiseMulOp_broadcast_5(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 4, 2, 3).astype(self.dtype), - 'Y': np.random.rand(10, 4, 1, 3).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([10, 4, 2, 3]) + self.y = self.gen_data_depend_on_dtype([10, 4, 1, 3]) class TestElementwiseMulOp_commonuse_1(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([2, 3, 100]) + self.y = self.gen_data_depend_on_dtype([1, 1, 100]) class TestElementwiseMulOp_commonuse_2(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(30, 3, 1, 5).astype(self.dtype), - 'Y': np.random.rand(30, 1, 4, 1).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] * self.inputs['Y']} + def init_data(self): + self.x = self.gen_data_depend_on_dtype([30, 3, 1, 5]) + self.y = self.gen_data_depend_on_dtype([30, 1, 4, 1]) class TestElementwiseMulOp_xsize_lessthan_ysize(ElementwiseMulOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 10).astype(self.dtype), - 'Y': np.random.rand(2, 2, 10, 10).astype(self.dtype), - } - - self.attrs = {'axis': 2} - - self.outputs = { - 'Out': self.inputs['X'].reshape(1, 1, 10, 10) * self.inputs['Y'] - } + def init_data(self): + self.x = self.gen_data_depend_on_dtype([10, 10]) + self.y = self.gen_data_depend_on_dtype([2, 2, 10, 10]) + self.cal_x = self.x.reshape(1, 1, 10, 10) + self.axis = 2 support_types = get_xpu_op_support_types('elementwise_mul') diff --git a/test/xpu/test_elementwise_sub_op_xpu.py b/test/xpu/test_elementwise_sub_op_xpu.py index 8e595932eae29..3cb440f05de06 100644 --- a/test/xpu/test_elementwise_sub_op_xpu.py +++ b/test/xpu/test_elementwise_sub_op_xpu.py @@ -21,13 +21,18 @@ create_test_class, get_xpu_op_support_types, ) -from op_test import skip_check_grad_ci +from op_test import ( + convert_float_to_uint16, + skip_check_grad_ci, +) from op_test_xpu import XPUOpTest import paddle paddle.enable_static() +INT_GROUP = [np.int32, np.int64] + class XPUTestElementwiseSubOp(XPUOpTestWrapper): def __init__(self): @@ -39,14 +44,43 @@ def setUp(self): self.op_type = "elementwise_sub" self.use_xpu = True self.dtype = self.in_type + self.init_shape() self.init_input_output() + def reshape_data(self, x, y): + if len(x.shape) < len(y.shape): + reshape_dims = [1 if i not in x.shape else i for i in y.shape] + return np.reshape(x, reshape_dims) + else: + return x + + def gen_data_depend_on_dtype(self, shape): + if self.dtype in INT_GROUP: + return np.random.randint(1, 100, size=shape) + else: + return np.random.uniform(-1, 1, size=shape) + def init_input_output(self): + self.x = self.gen_data_depend_on_dtype(self.x_shape) + self.y = self.gen_data_depend_on_dtype(self.y_shape) + if self.dtype == np.uint16: + tmp_x = self.reshape_data(self.x, self.y) + tmp_y = self.reshape_data(self.y, self.x) + self.outputs = {'Out': tmp_x - tmp_y} + self.x = convert_float_to_uint16(self.x) + self.y = convert_float_to_uint16(self.y) + else: + tmp_x = self.reshape_data(self.x, self.y).astype(self.dtype) + tmp_y = self.reshape_data(self.y, self.x).astype(self.dtype) + self.outputs = {'Out': tmp_x - tmp_y} self.inputs = { - 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), - 'Y': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype(self.dtype), + 'X': self.x, + 'Y': self.y, } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + + def init_shape(self): + self.x_shape = [2, 3, 4, 5] + self.y_shape = [2, 3, 4, 5] def test_check_output(self): if paddle.is_compiled_with_xpu(): @@ -81,132 +115,77 @@ def test_check_grad_ingore_y(self): ) class TestElementwiseSubOp_ZeroDim1(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, []).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [] + self.y_shape = [] class TestElementwiseSubOp_ZeroDim2(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, [13, 17]).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, []).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [13, 17] + self.y_shape = [] class TestElementwiseSubOp_ZeroDim3(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.uniform(-1, 1, []).astype(self.dtype), - 'Y': np.random.uniform(-1, 1, [13, 17]).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [] + self.y_shape = [13, 17] @skip_check_grad_ci( reason="[skip shape check] Use y_shape(1) to test broadcast." ) class TestElementwiseSubOp_scalar(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 3, 4).astype(self.dtype), - 'Y': np.random.rand(1).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [10, 3, 4] + self.y_shape = [1] class TestElementwiseSubOp_Vector(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.random((100,)).astype(self.dtype), - 'Y': np.random.random((100,)).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [100] + self.y_shape = [100] class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(100, 3, 2).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - + def init_shape(self): + self.x_shape = [100, 3, 2] + self.y_shape = [100] self.attrs = {'axis': 0} - self.outputs = { - 'Out': self.inputs['X'] - self.inputs['Y'].reshape(100, 1, 1) - } class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 100, 3).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - + def init_shape(self): + self.x_shape = [2, 100, 3] + self.y_shape = [100] self.attrs = {'axis': 1} - self.outputs = { - 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 100, 1) - } class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(100).astype(self.dtype), - } - - self.outputs = { - 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 1, 100) - } + def init_shape(self): + self.x_shape = [2, 3, 100] + self.y_shape = [100] class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 10, 12, 3).astype(self.dtype), - 'Y': np.random.rand(10, 12).astype(self.dtype), - } - + def init_shape(self): + self.x_shape = [2, 10, 12, 3] + self.y_shape = [10, 12] self.attrs = {'axis': 1} - self.outputs = { - 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 10, 12, 1) - } class TestElementwiseSubOp_broadcast_4(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 5, 3, 12).astype(self.dtype), - 'Y': np.random.rand(2, 5, 1, 12).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [2, 5, 3, 12] + self.y_shape = [2, 5, 1, 12] class TestElementwiseSubOp_commonuse_1(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(2, 3, 100).astype(self.dtype), - 'Y': np.random.rand(1, 1, 100).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [2, 3, 100] + self.y_shape = [1, 1, 100] class TestElementwiseSubOp_commonuse_2(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 3, 1, 4).astype(self.dtype), - 'Y': np.random.rand(10, 1, 12, 1).astype(self.dtype), - } - self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + def init_shape(self): + self.x_shape = [10, 3, 1, 4] + self.y_shape = [10, 1, 12, 1] class TestElementwiseSubOp_xsize_lessthan_ysize(TestElementwiseOp): - def init_input_output(self): - self.inputs = { - 'X': np.random.rand(10, 12).astype(self.dtype), - 'Y': np.random.rand(2, 3, 10, 12).astype(self.dtype), - } - + def init_shape(self): + self.x_shape = [10, 12] + self.y_shape = [2, 3, 10, 12] self.attrs = {'axis': 2} - self.outputs = { - 'Out': self.inputs['X'].reshape(1, 1, 10, 12) - self.inputs['Y'] - } - support_types = get_xpu_op_support_types('elementwise_sub') for stype in support_types: diff --git a/test/xpu/test_reduce_sum_op_xpu.py b/test/xpu/test_reduce_sum_op_xpu.py index 06c62d29fb263..cbf144c923bcb 100644 --- a/test/xpu/test_reduce_sum_op_xpu.py +++ b/test/xpu/test_reduce_sum_op_xpu.py @@ -20,6 +20,7 @@ create_test_class, get_xpu_op_support_types, ) +from op_test import convert_float_to_uint16 from op_test_xpu import XPUOpTest import paddle @@ -38,6 +39,16 @@ def setUp(self): self.init_case() self.set_case() + def gen_data_depend_on_dtype(self, shape): + if ( + self.dtype == np.int32 + or self.dtype == np.int64 + or self.dtype == np.uint8 + ): + return np.random.randint(1, 100, size=shape) + else: + return np.random.uniform(-1, 1, size=shape) + def set_case(self): self.op_type = 'reduce_sum' self.attrs = { @@ -46,17 +57,29 @@ def set_case(self): 'keep_dim': self.keep_dim, 'dim': self.axis, } - self.inputs = {'X': np.random.random(self.shape).astype(self.dtype)} - if self.attrs['reduce_all']: - self.outputs = {'Out': self.inputs['X'].sum()} + tmp_x = self.gen_data_depend_on_dtype(self.shape) + if self.dtype == np.uint16: + tmp_out = ( + tmp_x.sum() + if self.attrs['reduce_all'] + else tmp_x.sum( + axis=self.axis, keepdims=self.attrs['keep_dim'] + ) + ) + self.outputs = {'Out': tmp_out} + tmp_x = convert_float_to_uint16(tmp_x) + self.inputs = {'X': tmp_x} else: - self.outputs = { - 'Out': self.inputs['X'].sum( + tmp_x = tmp_x.astype(self.dtype) + self.inputs = {'X': tmp_x} + tmp_out = ( + tmp_x.sum() + if self.attrs['reduce_all'] + else tmp_x.sum( axis=self.axis, keepdims=self.attrs['keep_dim'] ) - } - if self.dtype == np.uint16: - self.outputs['Out'] = self.outputs['Out'].astype(np.uint16) + ) + self.outputs = {'Out': tmp_out} def init_case(self): self.shape = (5, 6, 10) From b7c36ed6aff14810920ce7fdaa7f1cd98340b53b Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Fri, 29 Dec 2023 14:53:09 +0800 Subject: [PATCH 135/146] [XPU][Phi Kernel] xpu::nonzero support simulator XPUSIM_SKIP_RUN mode (#60388) --- paddle/phi/kernels/xpu/masked_select_kernel.cc | 10 +++++++++- paddle/phi/kernels/xpu/nonzero_kernel.cc | 5 ++--- ...gmoid_cross_entropy_with_logits_grad_kernel.cc | 10 ++++++++++ .../sigmoid_cross_entropy_with_logits_kernel.cc | 11 ++++++++++- test/xpu/test_masked_select_op_xpu.py | 15 +++++++++++++++ 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/paddle/phi/kernels/xpu/masked_select_kernel.cc b/paddle/phi/kernels/xpu/masked_select_kernel.cc index 62803fde27aa5..85687c19f6c06 100644 --- a/paddle/phi/kernels/xpu/masked_select_kernel.cc +++ b/paddle/phi/kernels/xpu/masked_select_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/masked_select_kernel.h" +#include "glog/logging.h" + #include "paddle/phi/backends/xpu/enforce_xpu.h" #include "paddle/phi/core/kernel_registry.h" @@ -54,7 +56,13 @@ void MaskedSelectKernel(const Context& dev_ctx, mask.place(), static_cast(out_size), sizeof(int32_t)); - + if (std::getenv("XPUSIM_SKIP_RUN") && + std::strcmp(std::getenv("XPUSIM_SKIP_RUN"), "1") == 0) { + VLOG(3) << "WARNING: In the simulator mode, the variable out_size_cpu " + "stores an uninitialized value. To avoid allocating a memory of " + "random size, we assign numel to out_size_cpu"; + out_size_cpu = mask.numel(); + } DDim out_dim{out_size_cpu}; out->Resize(out_dim); auto out_data = reinterpret_cast(dev_ctx.template Alloc(out)); diff --git a/paddle/phi/kernels/xpu/nonzero_kernel.cc b/paddle/phi/kernels/xpu/nonzero_kernel.cc index e2a1339504bae..8dfd7734cff52 100644 --- a/paddle/phi/kernels/xpu/nonzero_kernel.cc +++ b/paddle/phi/kernels/xpu/nonzero_kernel.cc @@ -46,9 +46,8 @@ void NonZeroKernel(const Context& dev_ctx, std::strcmp(std::getenv("XPUSIM_SKIP_RUN"), "1") == 0) { VLOG(3) << "WARNING: In the simulator mode, the variable true_num_cpu " "stores an uninitialized value. To avoid allocating a memory of " - "random size, we limit the value of true_num_cpu to the range 0 " - "<= true_num_cpu < numel"; - true_num_cpu = std::min(std::max(true_num_cpu, 0), static_cast(numel)); + "random size, we assign numel to true_num_cpu"; + true_num_cpu = numel; } out->Resize(common::make_ddim({static_cast(true_num_cpu), rank})); diff --git a/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_grad_kernel.cc b/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_grad_kernel.cc index cf383439a77e9..9ee967b5e5725 100644 --- a/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_grad_kernel.cc +++ b/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_grad_kernel.cc @@ -16,6 +16,8 @@ #include "paddle/phi/kernels/sigmoid_cross_entropy_with_logits_grad_kernel.h" +#include "glog/logging.h" + #include "paddle/phi/backends/xpu/enforce_xpu.h" #include "paddle/phi/backends/xpu/xpu_context.h" #include "paddle/phi/core/kernel_registry.h" @@ -79,6 +81,14 @@ void SigmoidCrossEntropyWithLogitsGradKernel( dev_ctx.GetPlace(), static_cast(non_zero), sizeof(int)); + if (std::getenv("XPUSIM_SKIP_RUN") && + std::strcmp(std::getenv("XPUSIM_SKIP_RUN"), "1") == 0) { + VLOG(3) + << "WARNING: In the simulator mode, the variable non_zero_cpu " + "stores an uninitialized value. To avoid allocating a memory of " + "random size, we assign numel to true_num_cpu"; + non_zero_cpu = x.numel(); + } r = xpu::scale(dev_ctx.x_context(), reinterpret_cast(in_grad->data()), reinterpret_cast(in_grad->data()), diff --git a/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_kernel.cc b/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_kernel.cc index c189c143adb74..fa2b6f24c173a 100644 --- a/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_kernel.cc +++ b/paddle/phi/kernels/xpu/sigmoid_cross_entropy_with_logits_kernel.cc @@ -16,6 +16,8 @@ #include "paddle/phi/kernels/sigmoid_cross_entropy_with_logits_kernel.h" +#include "glog/logging.h" + #include "paddle/phi/backends/xpu/enforce_xpu.h" #include "paddle/phi/backends/xpu/xpu_context.h" #include "paddle/phi/core/kernel_registry.h" @@ -75,7 +77,14 @@ void SigmoidCrossEntropyWithLogitsKernel( dev_ctx.GetPlace(), static_cast(non_zero), sizeof(int)); - + if (std::getenv("XPUSIM_SKIP_RUN") && + std::strcmp(std::getenv("XPUSIM_SKIP_RUN"), "1") == 0) { + VLOG(3) + << "WARNING: In the simulator mode, the variable non_zero_cpu " + "stores an uninitialized value. To avoid allocating a memory of " + "random size, we assign numel to non_zero_cpu"; + non_zero_cpu = x.numel(); + } r = xpu::scale(dev_ctx.x_context(), reinterpret_cast(out->data()), reinterpret_cast(out->data()), diff --git a/test/xpu/test_masked_select_op_xpu.py b/test/xpu/test_masked_select_op_xpu.py index f2ed82cd1e8d7..30b91f38b66d6 100644 --- a/test/xpu/test_masked_select_op_xpu.py +++ b/test/xpu/test_masked_select_op_xpu.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest import numpy as np @@ -108,6 +109,20 @@ def test_static_mode(self): ) self.assertEqual(np.allclose(res, np_out), True) + def test_simulator_skip_run_mode(self): + os.environ['XPUSIM_SKIP_RUN'] = '1' + paddle.disable_static(paddle.XPUPlace(0)) + shape = (88, 6, 8) + np_x = np.random.random(shape).astype('float32') + np_mask = np.array(np.random.randint(2, size=shape, dtype=bool)) + x = paddle.to_tensor(np_x) + mask = paddle.to_tensor(np_mask) + out = paddle.masked_select(x, mask) + # only check the numel of output + np.testing.assert_equal(out.numpy().size, np_x.size) + paddle.enable_static() + del os.environ['XPUSIM_SKIP_RUN'] + class TestMaskedSelectError(unittest.TestCase): def test_error(self): From ebc859a69b7732a59ced3d68e8d7788f45cfaf50 Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Fri, 29 Dec 2023 14:54:31 +0800 Subject: [PATCH 136/146] [DimExpr] Add utils for DimExpr, convert between DimExpr and pir::Attribute (#60390) * convert between DimExpr and pir::Attribute * add two helper functions: SubstituteDimExpr and MakeGetterDimExpr4SymbolName * Fix compile bug and add unittest * Fix bug * Add IR_API * Fix windows compile error --- .../pir/dialect/shape/utils/dim_expr_util.cc | 362 ++++++++++++++++++ .../pir/dialect/shape/utils/dim_expr_util.h | 42 ++ test/cpp/pir/shape_dialect/CMakeLists.txt | 3 + .../symbol_dim_expr_util_test.cc | 99 +++++ 4 files changed, 506 insertions(+) create mode 100644 paddle/pir/dialect/shape/utils/dim_expr_util.cc create mode 100644 paddle/pir/dialect/shape/utils/dim_expr_util.h create mode 100644 test/cpp/pir/shape_dialect/symbol_dim_expr_util_test.cc diff --git a/paddle/pir/dialect/shape/utils/dim_expr_util.cc b/paddle/pir/dialect/shape/utils/dim_expr_util.cc new file mode 100644 index 0000000000000..8421f500c23da --- /dev/null +++ b/paddle/pir/dialect/shape/utils/dim_expr_util.cc @@ -0,0 +1,362 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/pir/dialect/shape/utils/dim_expr_util.h" +#include "paddle/pir/core/builder.h" +#include "paddle/pir/core/builtin_attribute.h" + +namespace symbol { + +namespace { + +template +std::string GetSerializedTag(); + +template <> +std::string GetSerializedTag>() { + return "Negative"; +} + +template <> +std::string GetSerializedTag>() { + return "Reciprocal"; +} + +template <> +std::string GetSerializedTag>() { + return "Add"; +} + +template <> +std::string GetSerializedTag>() { + return "Mul"; +} + +template <> +std::string GetSerializedTag>() { + return "Max"; +} + +template <> +std::string GetSerializedTag>() { + return "Min"; +} + +template <> +std::string GetSerializedTag>() { + return "Broadcast"; +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const std::int64_t& dim_expr) { + return builder->int64_attr(dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const std::string& dim_expr) { + return builder->str_attr(dim_expr); +} + +template +::pir::Attribute ConvertUnaryDimExprToAttributeImpl(::pir::Builder* builder, + const T& dim_expr) { + std::vector<::pir::Attribute> attr_vecs{}; + attr_vecs.push_back(builder->str_attr(GetSerializedTag())); + const auto& operand = dim_expr->data; + attr_vecs.push_back(ConvertDimExprToAttribute(builder, operand)); + return builder->array_attr(attr_vecs); +} + +::pir::Attribute ConvertDimExprToAttributeImpl( + ::pir::Builder* builder, const Negative& dim_expr) { + return ConvertUnaryDimExprToAttributeImpl(builder, dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl( + ::pir::Builder* builder, const Reciprocal& dim_expr) { + return ConvertUnaryDimExprToAttributeImpl(builder, dim_expr); +} + +template +::pir::Attribute ConvertVariadicDimExprToAttribute(::pir::Builder* builder, + const T& dim_expr) { + std::vector<::pir::Attribute> attr_vecs{}; + attr_vecs.push_back(builder->str_attr(GetSerializedTag())); + const auto& operands = *(dim_expr.operands); + for (const auto& operand : operands) { + attr_vecs.push_back(ConvertDimExprToAttribute(builder, operand)); + } + return builder->array_attr(attr_vecs); +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const Add& dim_expr) { + return ConvertVariadicDimExprToAttribute(builder, dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const Mul& dim_expr) { + return ConvertVariadicDimExprToAttribute(builder, dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const Max& dim_expr) { + return ConvertVariadicDimExprToAttribute(builder, dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl(::pir::Builder* builder, + const Min& dim_expr) { + return ConvertVariadicDimExprToAttribute(builder, dim_expr); +} + +::pir::Attribute ConvertDimExprToAttributeImpl( + ::pir::Builder* builder, const Broadcast& dim_expr) { + return ConvertVariadicDimExprToAttribute(builder, dim_expr); +} + +std::optional ConvertInt64AttributeToDimExpr( + const ::pir::Int64Attribute& attribute) { + return DimExpr{attribute.data()}; +} + +std::optional ConvertStrAttributeToDimExpr( + const ::pir::StrAttribute& attribute) { + return DimExpr{attribute.AsString()}; +} + +template +std::optional ConvertArrayAttributeToUnaryDimExpr( + const ::pir::ArrayAttribute& attribute) { + if (attribute.size() != 2) { + return std::nullopt; + } + std::optional operand = ConvertAttributeToDimExpr(attribute.at(1)); + if (!operand.has_value()) { + return std::nullopt; + } + return T{operand.value()}; +} + +template +std::optional ConvertArrayAttributeToVariadicDimExpr( + const ::pir::ArrayAttribute& attribute) { + if (attribute.size() < 2) { + return std::nullopt; + } + List operands{}; + for (std::size_t i = 1; i < attribute.size(); ++i) { + std::optional operand = ConvertAttributeToDimExpr(attribute.at(i)); + if (!operand.has_value()) { + return std::nullopt; + } + operands->push_back(operand.value()); + } + return T{operands}; +} + +typedef std::optional (*ArrayAttributeConverterT)( + const ::pir::ArrayAttribute& attribute); + +std::optional GetArrayAttributeConverter( + const std::string& tag) { + static std::unordered_map map{ + {GetSerializedTag>(), + &ConvertArrayAttributeToUnaryDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToUnaryDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToVariadicDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToVariadicDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToVariadicDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToVariadicDimExpr>}, + {GetSerializedTag>(), + &ConvertArrayAttributeToVariadicDimExpr>}, + }; + const auto& iter = map.find(tag); + if (iter == map.end()) { + return std::nullopt; + } + return iter->second; +} + +std::optional ConvertArrayAttributeToDimExpr( + const ::pir::ArrayAttribute& attribute) { + if (attribute.empty()) { + return std::nullopt; + } + if (!attribute.at(0).isa<::pir::StrAttribute>()) { + return std::nullopt; + } + const auto& tag = attribute.at(0).dyn_cast<::pir::StrAttribute>().AsString(); + auto opt_func = GetArrayAttributeConverter(tag); + if (!opt_func.has_value()) { + return std::nullopt; + } + return opt_func.value()(attribute); +} + +} // namespace + +::pir::Attribute ConvertDimExprToAttribute(::pir::Builder* builder, + const DimExpr& dim_expr) { + return std::visit( + [&](const auto& impl) { + return ConvertDimExprToAttributeImpl(builder, impl); + }, + dim_expr.variant()); +} + +std::optional ConvertAttributeToDimExpr(::pir::Attribute attribute) { + if (attribute.isa<::pir::Int64Attribute>()) { + return ConvertInt64AttributeToDimExpr( + attribute.dyn_cast<::pir::Int64Attribute>()); + } + if (attribute.isa<::pir::StrAttribute>()) { + return ConvertStrAttributeToDimExpr( + attribute.dyn_cast<::pir::StrAttribute>()); + } + if (attribute.isa<::pir::ArrayAttribute>()) { + return ConvertArrayAttributeToDimExpr( + attribute.dyn_cast<::pir::ArrayAttribute>()); + } + return std::nullopt; +} + +class SubstituteDimExprHelper final { + public: + using DimExpr4SymbolNameT = + std::function(const std::string& symbol_name)>; + + explicit SubstituteDimExprHelper( + const DimExpr4SymbolNameT& DimExpr4SymbolName) + : DimExpr4SymbolName_(DimExpr4SymbolName) {} + + std::optional Substitute(const DimExpr& dim_expr) { + return std::visit([&](const auto& impl) { return SubstituteImpl(impl); }, + dim_expr.variant()); + } + + private: + std::optional SubstituteImpl(const std::int64_t& dim_expr) { + return dim_expr; + } + std::optional SubstituteImpl(const std::string& dim_expr) { + return DimExpr4SymbolName_(dim_expr); + } + + std::optional SubstituteImpl(const Negative& dim_expr) { + return SubstituteUnary(dim_expr); + } + std::optional SubstituteImpl(const Reciprocal& dim_expr) { + return SubstituteUnary(dim_expr); + } + + template + std::optional SubstituteUnary(const T& dim_expr) { + const auto& operand = dim_expr->data; + const auto& substituted_operand = Substitute(operand); + if (!substituted_operand.has_value()) { + return std::nullopt; + } + return T{substituted_operand.value()}; + } + + std::optional SubstituteImpl(const Add& dim_expr) { + return SubstituteVariadic(dim_expr); + } + + std::optional SubstituteImpl(const Mul& dim_expr) { + return SubstituteVariadic(dim_expr); + } + + std::optional SubstituteImpl(const Max& dim_expr) { + return SubstituteVariadic(dim_expr); + } + + std::optional SubstituteImpl(const Min& dim_expr) { + return SubstituteVariadic(dim_expr); + } + + std::optional SubstituteImpl(const Broadcast& dim_expr) { + return SubstituteVariadic(dim_expr); + } + + template + std::optional SubstituteVariadic(const T& dim_expr) { + const auto& operands = *(dim_expr.operands); + List substituted_operands{}; + for (const auto& operand : operands) { + const auto& substituted_operand = Substitute(operand); + if (!substituted_operand.has_value()) { + return std::nullopt; + } + substituted_operands->push_back(substituted_operand.value()); + } + return T{substituted_operands}; + } + + DimExpr4SymbolNameT DimExpr4SymbolName_; +}; + +std::optional SubstituteDimExpr( + const DimExpr& dim_expr, + const std::function(const std::string& symbol_name)>& + DimExpr4SymbolName) { + return SubstituteDimExprHelper(DimExpr4SymbolName).Substitute(dim_expr); +} + +std::function(const std::string& symbol_name)> +MakeGetterDimExpr4SymbolName( + const std::vector>& symbol_bindings, + const std::function( + int in_tensor_idx, int in_tensor_dim_idx)>& DimExpr4InputDim) { + std::unordered_map>> + symbol_name2in_tensor_dim_pos; + for (const auto& tuple : symbol_bindings) { + const auto& [symbol_name, in_tensor_idx, in_tensor_dim_idx] = tuple; + symbol_name2in_tensor_dim_pos[symbol_name].emplace_back( + std::pair{in_tensor_idx, in_tensor_dim_idx}); + } + return [map = std::move(symbol_name2in_tensor_dim_pos), DimExpr4InputDim]( + const std::string& symbol_name) -> std::optional { + const auto& iter = map.find(symbol_name); + if (iter == map.end()) { + return std::nullopt; + } + const auto& positions = iter->second; + std::optional ret = std::nullopt; + for (const auto& [in_tensor_idx, in_tensor_dim_idx] : positions) { + const auto& current = DimExpr4InputDim(in_tensor_idx, in_tensor_dim_idx); + if (!current.has_value()) { + return std::nullopt; + } + if (ret.has_value()) { + // Same names, same DimExprs. + if (ret.value() != current.value()) { + return std::nullopt; + } + } else { + ret = current; + } + } + return ret; + }; +} + +} // namespace symbol diff --git a/paddle/pir/dialect/shape/utils/dim_expr_util.h b/paddle/pir/dialect/shape/utils/dim_expr_util.h new file mode 100644 index 0000000000000..3ed4550c2248d --- /dev/null +++ b/paddle/pir/dialect/shape/utils/dim_expr_util.h @@ -0,0 +1,42 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/pir/core/builder.h" +#include "paddle/pir/core/dll_decl.h" +#include "paddle/pir/dialect/shape/utils/dim_expr.h" + +namespace symbol { + +IR_API ::pir::Attribute ConvertDimExprToAttribute(::pir::Builder* builder, + const DimExpr& dim_expr); +IR_API std::optional ConvertAttributeToDimExpr( + ::pir::Attribute attribute); + +IR_API std::optional SubstituteDimExpr( + const DimExpr& dim_expr, + const std::function(const std::string& symbol_name)>& + DimExpr4SymbolName); + +IR_API std::function(const std::string& symbol_name)> +MakeGetterDimExpr4SymbolName( + const std::vector>& symbol_bindings, + const std::function( + int in_tensor_idx, int in_tensor_dim_idx)>& DimExpr4InputDim); + +} // namespace symbol diff --git a/test/cpp/pir/shape_dialect/CMakeLists.txt b/test/cpp/pir/shape_dialect/CMakeLists.txt index f508efb56947e..5c3aa2b9f4344 100644 --- a/test/cpp/pir/shape_dialect/CMakeLists.txt +++ b/test/cpp/pir/shape_dialect/CMakeLists.txt @@ -4,6 +4,9 @@ paddle_test(shape_struct_test SRCS shape_struct_test.cc DEPS gtest) paddle_test(symbol_dim_expr_test SRCS symbol_dim_expr_test.cc DEPS gtest) +paddle_test(symbol_dim_expr_util_test SRCS symbol_dim_expr_util_test.cc DEPS + gtest) + if(WITH_CINN) paddle_test( shape_optimization_test diff --git a/test/cpp/pir/shape_dialect/symbol_dim_expr_util_test.cc b/test/cpp/pir/shape_dialect/symbol_dim_expr_util_test.cc new file mode 100644 index 0000000000000..0893a6d502705 --- /dev/null +++ b/test/cpp/pir/shape_dialect/symbol_dim_expr_util_test.cc @@ -0,0 +1,99 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" + +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/pir/dialect/shape/utils/dim_expr_builder.h" +#include "paddle/pir/dialect/shape/utils/dim_expr_util.h" + +#include "test/cpp/pir/tools/test_pir_utils.h" + +namespace symbol { + +namespace { +DimExpr CreateExampleDimExpr() { + DimExprBuilder dim_expr_builder{nullptr}; + DimExpr sym0 = DimExpr("S0"); + DimExpr sym1 = DimExpr("S1"); + DimExpr constant = DimExpr(2); + DimExpr expr1 = (sym0 - sym1) * constant / sym0; + DimExpr expr2 = dim_expr_builder.Max(expr1, sym0); + DimExpr output = dim_expr_builder.Min(expr2, sym1); + return output; +} +} // namespace + +TEST(DimExprUtil, Convert) { + pir::IrContext* ctx = pir::IrContext::Instance(); + pir::Program program(ctx); + pir::Builder builder = pir::Builder(ctx, program.block()); + + DimExpr dim_expr = CreateExampleDimExpr(); + ::pir::Attribute attr = ConvertDimExprToAttribute(&builder, dim_expr); + std::optional opt_expr = ConvertAttributeToDimExpr(attr); + ASSERT_TRUE(opt_expr.has_value()); + ASSERT_EQ(opt_expr.value(), dim_expr); +} + +TEST(DimExprUtil, Substitute) { + DimExpr dim_expr = CreateExampleDimExpr(); + const auto& opt_expr = SubstituteDimExpr( + dim_expr, [](const std::string& str) -> std::optional { + if (str == "S0") { + return DimExpr("symbol0"); + } else if (str == "S1") { + return DimExpr("symbol1"); + } else { + return std::nullopt; + } + }); + ASSERT_TRUE(opt_expr.has_value()); + const auto& ret_expr = SubstituteDimExpr( + opt_expr.value(), [](const std::string& str) -> std::optional { + if (str == "symbol0") { + return DimExpr("S0"); + } else if (str == "symbol1") { + return DimExpr("S1"); + } else { + return std::nullopt; + } + }); + ASSERT_TRUE(ret_expr.has_value()); + ASSERT_EQ(ret_expr.value(), dim_expr); +} + +TEST(DimExprUtil, MakeGetterDimExpr4SymbolName) { + std::vector> + symbol_bindings{}; + symbol_bindings.push_back(std::make_tuple("Symbol", 0, 0)); + const auto& dim_expr = CreateExampleDimExpr(); + const auto& DimExpr4SymbolName = MakeGetterDimExpr4SymbolName( + symbol_bindings, + [dim_expr](int in_tensor_idx, + int in_tensor_dim_idx) -> std::optional { + if (in_tensor_idx == 0 && in_tensor_dim_idx == 0) { + return dim_expr; + } else { + return std::nullopt; + } + }); + const auto& opt_dim_expr = DimExpr4SymbolName("Symbol"); + ASSERT_TRUE(opt_dim_expr.has_value()); + ASSERT_EQ(opt_dim_expr.value(), dim_expr); +} + +} // namespace symbol From e4b39bb56a4e55213383e96daf262f4f72c1811d Mon Sep 17 00:00:00 2001 From: lijin23 <41257772+lj970926@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:05:25 +0800 Subject: [PATCH 137/146] [XPU][PHI Kernels] refine bf16 test for fused_rope (#60439) * refine fuesd_rope bf16 test * format code --- ..._fused_rotary_position_embedding_op_xpu.py | 94 ++++++++++++++----- 1 file changed, 71 insertions(+), 23 deletions(-) diff --git a/test/xpu/test_fused_rotary_position_embedding_op_xpu.py b/test/xpu/test_fused_rotary_position_embedding_op_xpu.py index 0fe25194c1633..6aac9d828cc03 100644 --- a/test/xpu/test_fused_rotary_position_embedding_op_xpu.py +++ b/test/xpu/test_fused_rotary_position_embedding_op_xpu.py @@ -196,6 +196,7 @@ def get_forward_backward( fw.append(out_q) fw.append(out_k) fw.append(out_v) + paddle.seed(seed + 1) out_gq = paddle.randn(out_q.shape, self.dtype) out_gk = paddle.randn(out_q.shape, self.dtype) out_gv = paddle.randn(out_q.shape, self.dtype) @@ -203,9 +204,9 @@ def get_forward_backward( paddle.autograd.backward( [out_q, out_k, out_v], [out_gq, out_gk, out_gv], True ) - bw.append(tensor_q) - bw.append(tensor_k) - bw.append(tensor_v) + bw.append(tensor_q.grad) + bw.append(tensor_k.grad) + bw.append(tensor_v.grad) return fw, bw @@ -368,28 +369,28 @@ def setUp(self): self.shape = [2, 8, 2, 16] def test_api(self): - q_fp32 = paddle.rand(self.shape, dtype="float32") - k_fp32 = paddle.rand(self.shape, dtype="float32") - v_fp32 = paddle.rand(self.shape, dtype="float32") - sin_fp32 = paddle.rand( - [1, self.shape[1], 1, self.shape[3]], dtype="float32" + paddle.disable_static() + q_bf16 = paddle.randn(self.shape, dtype="bfloat16") + k_bf16 = paddle.randn(self.shape, dtype="bfloat16") + v_bf16 = paddle.randn(self.shape, dtype="bfloat16") + sin_bf16 = paddle.randn( + [1, self.shape[1], 1, self.shape[3]], dtype="bfloat16" ) - cos_fp32 = paddle.rand( - [1, self.shape[1], 1, self.shape[3]], dtype="float32" + cos_bf16 = paddle.randn( + [1, self.shape[1], 1, self.shape[3]], dtype="bfloat16" ) - q_bf16 = paddle.to_tensor(q_fp32, dtype="bfloat16") - k_bf16 = paddle.to_tensor(k_fp32, dtype="bfloat16") - v_bf16 = paddle.to_tensor(v_fp32, dtype="bfloat16") - sin_bf16 = paddle.to_tensor(sin_fp32, dtype="bfloat16") - cos_bf16 = paddle.to_tensor(cos_fp32, dtype="bfloat16") - - out_fp32 = fused_rotary_position_embedding( - q_fp32, - k_fp32, - v_fp32, - sin_fp32, - cos_fp32, - use_neox_rotary_style=False, + q_bf16.stop_gradient = False + k_bf16.stop_gradient = False + v_bf16.stop_gradient = False + q_fp32 = paddle.to_tensor(q_bf16, dtype="float32", stop_gradient=False) + k_fp32 = paddle.to_tensor(k_bf16, dtype="float32", stop_gradient=False) + v_fp32 = paddle.to_tensor(v_bf16, dtype="float32", stop_gradient=False) + sin_fp32 = paddle.to_tensor(sin_bf16, dtype="float32") + cos_fp32 = paddle.to_tensor(cos_bf16, dtype="float32") + + position_ids = paddle.arange(0, self.shape[1], dtype="int64") + position_ids = paddle.stack( + [position_ids for _ in range(self.shape[0])], axis=0 ) out_bf16 = fused_rotary_position_embedding( q_bf16, @@ -397,13 +398,60 @@ def test_api(self): v_bf16, sin_bf16, cos_bf16, + position_ids=position_ids, + use_neox_rotary_style=False, + ) + + grad_out_q_bf16 = paddle.randn(self.shape, dtype="bfloat16") + grad_out_k_bf16 = paddle.randn(self.shape, dtype="bfloat16") + grad_out_v_bf16 = paddle.randn(self.shape, dtype="bfloat16") + + paddle.autograd.backward( + out_bf16, [grad_out_q_bf16, grad_out_k_bf16, grad_out_v_bf16], True + ) + grad_bf16 = [q_bf16.grad, k_bf16.grad, v_bf16.grad] + + out_fp32 = paddle_fused_rotary_position_embedding( + q_fp32, + k_fp32, + v_fp32, + sin_fp32, + cos_fp32, + position_ids=position_ids, use_neox_rotary_style=False, ) + + grad_out_q_fp32 = paddle.to_tensor(grad_out_q_bf16, dtype="float32") + grad_out_k_fp32 = paddle.to_tensor(grad_out_k_bf16, dtype="float32") + grad_out_v_fp32 = paddle.to_tensor(grad_out_v_bf16, dtype="float32") + paddle.autograd.backward( + out_fp32, [grad_out_q_fp32, grad_out_k_fp32, grad_out_v_fp32], True + ) + grad_fp32 = [q_fp32.grad, k_fp32.grad, v_fp32.grad] + for fp32_val, bf16_val in zip(out_fp32, out_bf16): bf16_val = convert_uint16_to_float(bf16_val.numpy()) np.testing.assert_allclose( fp32_val.numpy(), bf16_val, rtol=1e-2, atol=1e-2 ) + for grad_fp32_val, grad_bf16_val in zip(grad_fp32, grad_bf16): + grad_bf16_val = convert_uint16_to_float(grad_bf16_val.numpy()) + np.testing.assert_allclose( + grad_fp32_val.numpy(), grad_bf16_val, rtol=1e-2, atol=1e-2 + ) + + +class XPUTestFusedRotaryPositionEmbeddingBf16_2( + XPUTestFusedRotaryPositionEmbeddingBf16_1 +): + def setUp(self): + self.shape = [2, 2048, 16, 128] + + +# too long for CI +# class XPUTestFusedRotaryPositionEmbeddingBf16_3(XPUTestFusedRotaryPositionEmbeddingBf16_1): +# def setUp(self): +# self.shape = [2, 8192, 8, 128] if __name__ == '__main__': From 63776cfae91119b8f169536691f5f3aa1b23f1b8 Mon Sep 17 00:00:00 2001 From: xysheng-baidu <121540080+xysheng-baidu@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:40:44 +0800 Subject: [PATCH 138/146] [auto config] Resume from history csv file (#60417) --- python/paddle/distributed/auto_tuner/tuner.py | 77 +++++++++++++++++++ python/paddle/distributed/launch/main.py | 71 +++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/python/paddle/distributed/auto_tuner/tuner.py b/python/paddle/distributed/auto_tuner/tuner.py index b3b6cbf3cdc52..6a6a0ba4e082f 100644 --- a/python/paddle/distributed/auto_tuner/tuner.py +++ b/python/paddle/distributed/auto_tuner/tuner.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import csv +import os from .utils import default_candidates, gbs_default_candidates @@ -54,6 +56,8 @@ def __init__(self, tuner_cfg): raise NotImplementedError() self.history_cfgs = [] + self.resume_cfgs = [] + self.tuner_cfg = tuner_cfg def search_once(self): """Return a new task config.""" @@ -67,3 +71,76 @@ def search_once(self): def add_cfg(self, cfg): """Add cfg into history cfgs""" self.history_cfgs.append(cfg) + + def resume_form_history(self, history_csv_path="./history.csv"): + """Resume form history csv file""" + # The breakpoint resume function does not start when the resume csv file does not exist. + if not os.path.exists(history_csv_path): + return + resume_csv_path = os.path.join( + os.path.dirname(history_csv_path), + f'{os.path.basename(history_csv_path).split(".")[0]}_copy.csv', + ) + with open(history_csv_path, "r") as fread: + reader = csv.reader(fread) + data_list = list(reader) + with open(resume_csv_path, "w") as fwrite: + writer = csv.writer(fwrite) + for row in data_list: + writer.writerow(row) + # chang str type to real type + for row in data_list: + for i, value in enumerate(row): + try: + row[i] = int(value) + except ValueError: + try: + row[i] = float(value) + except ValueError: + pass + + data_dict = [] + keys = data_list[0] + values = data_list[1:] + for val in values: + val = [x if x != '' else None for x in val] + val = [True if x == 'True' else x for x in val] + val = [False if x == 'False' else x for x in val] + dictionary = dict(zip(keys, val)) + time_val = -1 + target_key = self.tuner_cfg["metric_cfg"]["name"] + if dictionary[target_key]: + time_val = dictionary[target_key] + dictionary["time"] = time_val + data_dict.append(dictionary) + self.resume_cfgs = data_dict + + def get_cfg_from_resume(self, cur_cfg): + """Get cfg from resume cfgs""" + keys_to_compare = [ + 'mp_degree', + 'sharding_degree', + 'pp_degree', + 'dp_degree', + 'sharding_stage', + 'micro_batch_size', + 'vpp_degree', + 'use_recompute', + 'recompute_granularity', + 'num_gpus', + 'nodes', + 'global_batch_size', + 'sharding_overlap', + 'acc_steps', + ] + for cfg in self.resume_cfgs: + ret_is_same = True + for key in keys_to_compare: + if not cfg.get(key) and not cur_cfg.get(key): + continue + else: + is_same = str(cfg.get(key)) == str(cur_cfg.get(key)) + ret_is_same = ret_is_same and is_same + if ret_is_same: + return cfg + return None diff --git a/python/paddle/distributed/launch/main.py b/python/paddle/distributed/launch/main.py index 0869ac7bbfcd9..40caf7f223677 100644 --- a/python/paddle/distributed/launch/main.py +++ b/python/paddle/distributed/launch/main.py @@ -587,6 +587,10 @@ def launch(): logger.info( f"Launch {len(auto_tuner.algo.all_tasks)} tasks by auto tuner: " ) + resume_csv_file_path = tuner_cfg.get( + "resume_csv_file_path", history_file_path + ) + auto_tuner.resume_form_history(resume_csv_file_path) cur_cfg = auto_tuner.search_once() auto_tuner.add_cfg(cur_cfg) assert cur_cfg is not None, "No config can run." @@ -658,6 +662,73 @@ def launch(): ) logger.info(f"Launch task: job_id {task_job_id}, log_dir {log_dir}") + cur_resume_cfg = auto_tuner.get_cfg_from_resume(cur_cfg) + if cur_resume_cfg: + cur_cfg = cur_resume_cfg + cur_cfg['job_id'] = job_id + auto_tuner.history_cfgs.pop(-1) + auto_tuner.add_cfg(cur_cfg) + recorder.add_cfg(**cur_cfg) + cur_best_cfgs, err = recorder.get_best( + metric=tuner_cfg['metric_cfg']['name'], + direction=tuner_cfg['metric_cfg']['OptimizationDirection'], + ) + if not err: + ctx.logger.info(f"Current best config: {cur_best_cfgs}") + logger.info(f"Current best config: {cur_best_cfgs}") + else: + ctx.logger.info( + "Get best config failed. Currently no config can be run." + ) + logger.info( + "Get best config failed. Currently no config can be run." + ) + if ( + "sharding_overlap" in cur_cfg + and cur_cfg["sharding_overlap"] + ): + add_overlap_performance( + cur_cfg, tuner_cfg, recorder.history + ) + + if cur_cfg["error_info"]: + error_task_nums += 1 + error_info = cur_cfg["error_info"] + task_nums = len(auto_tuner.algo.all_tasks) + cur_task_id = auto_tuner.algo.idx + ctx.logger.info( + "Auto Tuner Schedule: [{}/{}], Pruned nums {}, Error nums {}, Error info {}, Remaining time {} min".format( + cur_task_id, + task_nums, + cur_task_id - job_id, + error_task_nums, + error_info, + round( + (task_nums - cur_task_id) * max_time_per_task / 60, + 2, + ), + ) + ) + logger.info( + "Auto Tuner Schedule: [{}/{}], Pruned nums {}, Error nums {}, Error info {}, Remaining time {} min".format( + cur_task_id, + task_nums, + cur_task_id - job_id, + error_task_nums, + error_info, + round( + (task_nums - cur_task_id) * max_time_per_task / 60, + 2, + ), + ) + ) + recorder.store_history(history_file_path) + # generate a new config + new_cfg = auto_tuner.search_once() + cur_cfg = copy.deepcopy(new_cfg) + auto_tuner.add_cfg(cur_cfg) + continue + # in single dp estimation scene, just some nodes not all nodes run ctx = gen_new_ctx(ctx, cur_cfg, tuner_cfg) actual_nnodes = int(ctx.args.nnodes.split(":")[0]) From 10b352e32dbaa804ffa54830f953a43225b2e0c8 Mon Sep 17 00:00:00 2001 From: Jianbang Yang Date: Fri, 29 Dec 2023 17:08:28 +0800 Subject: [PATCH 139/146] [XPU] update XHPC to 20231229 (#60421) --- cmake/external/xpu.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index 64e9154f9f8e3..c0aea59730832 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE) set(XPU_BASE_DATE "20231203") endif() if(NOT DEFINED XPU_XHPC_BASE_DATE) - set(XPU_XHPC_BASE_DATE "20231226") + set(XPU_XHPC_BASE_DATE "20231229") endif() set(XPU_XCCL_BASE_VERSION "1.1.8.1") if(NOT DEFINED XPU_XFT_BASE_VERSION) From 044dec73f552136757e23f67b73e58fa1dcf305b Mon Sep 17 00:00:00 2001 From: Yichen Zhang <32740647+pkuzyc@users.noreply.github.com> Date: Fri, 29 Dec 2023 19:59:00 +0800 Subject: [PATCH 140/146] refine shard_layer api (#60468) --- .../paddle/distributed/auto_parallel/api.py | 3 +- test/auto_parallel/test_shard_layer_api.py | 118 +++++++++++++++--- 2 files changed, 105 insertions(+), 16 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/api.py b/python/paddle/distributed/auto_parallel/api.py index d3f19baded5e6..c012d7a59d1c6 100644 --- a/python/paddle/distributed/auto_parallel/api.py +++ b/python/paddle/distributed/auto_parallel/api.py @@ -525,8 +525,7 @@ def replicate_layer_params_and_buffers( else: # TODO(chenweihang): Support static mode branch later. raise NotImplementedError( - "`paddle.distributed.shard_layer` only supports dynamic graph mode " - "now. It will be supported for static graph mode later." + "`paddle.distributed.shard_layer` only supports dynamic graph mode." ) diff --git a/test/auto_parallel/test_shard_layer_api.py b/test/auto_parallel/test_shard_layer_api.py index fb0476303cd6e..20e3d13946056 100644 --- a/test/auto_parallel/test_shard_layer_api.py +++ b/test/auto_parallel/test_shard_layer_api.py @@ -14,6 +14,8 @@ import unittest +import numpy as np + import paddle import paddle.distributed as dist from paddle import nn @@ -43,6 +45,33 @@ def forward(self, x): return self.seq(x) +def shard_fn(layer_name, layer, process_mesh): + if isinstance(layer, nn.Linear): + for name, param in layer.named_parameters(): + if 'weight' in name: + dist_param = dist.shard_tensor( + param, process_mesh, [dist.Replicate()] + ) + else: + dist_param = dist.shard_tensor( + param, process_mesh, [dist.Replicate()] + ) + layer.add_parameter(name, dist_param) + + +class RandomDataset(paddle.io.Dataset): + def __init__(self, images, labels, num_samples): + self.images = images + self.labels = labels + self.num_samples = num_samples + + def __getitem__(self, idx): + return self.images[idx], self.labels[idx] + + def __len__(self): + return self.num_samples + + class TestShardLayer(unittest.TestCase): def setUp(self): self.mesh = dist.ProcessMesh([0, 1], dim_names=["x"]) @@ -52,19 +81,6 @@ def setUp(self): def test_shard_layer_base(self): layer = MyLayer(self.num_features, self.num_layers) - def shard_fn(layer_name, layer, process_mesh): - if isinstance(layer, nn.Linear): - for name, param in layer.named_parameters(): - if 'weight' in name: - dist_param = dist.shard_tensor( - param, process_mesh, [dist.Replicate()] - ) - else: - dist_param = dist.shard_tensor( - param, process_mesh, [dist.Replicate()] - ) - layer.add_parameter(name, dist_param) - # test shard parameters sharded_params_layer = dist.shard_layer(layer, self.mesh, shard_fn) @@ -155,11 +171,85 @@ def test_shard_layer_static_mode(self): dist.shard_layer(layer, self.mesh) except NotImplementedError as ex: self.assertIn( - "`paddle.distributed.shard_layer` only supports dynamic graph mode now", + "`paddle.distributed.shard_layer` only supports dynamic graph mode.", str(ex), ) exception = ex self.assertIsNotNone(exception) + paddle.disable_static() + + def create_data_loader(self): + batch_size = 4 + hidden_size = self.num_features + images = np.random.rand(batch_size, hidden_size).astype('float32') + labels = np.random.rand(batch_size, hidden_size).astype('float32') + dataset = RandomDataset(images, labels, batch_size) + loader = paddle.io.DataLoader(dataset, batch_size=batch_size) + return loader + + def test_shard_layer_to_static(self): + def input_fn(inputs, process_mesh): + return dist.shard_tensor( + inputs[0], process_mesh, [dist.Replicate()] + ) + + def output_fn(outputs, process_mesh): + return dist.shard_tensor(outputs, process_mesh, [dist.Shard(0)]) + + layer = MyLayer(self.num_features, self.num_layers) + + sharded_layer = dist.shard_layer( + layer, self.mesh, shard_fn, input_fn=input_fn, output_fn=output_fn + ) + + loader = self.create_data_loader() + + dist_model, dist_loader = dist.to_static(sharded_layer, loader) + + serial_main_program = dist_model.serial_main_program() + for param in serial_main_program.all_parameters(): + self.assertTrue(param.dist_attr.is_annotated("dims_mapping")) + self.assertEqual(param.dist_attr.dims_mapping, [-1, -1]) + + input_var = serial_main_program.global_block().var("input0") + output_var = serial_main_program.global_block().var( + "matmul_v2_19.tmp_0" + ) + self.assertListEqual(input_var.dist_attr.dims_mapping, [-1, -1]) + self.assertListEqual(output_var.dist_attr.dims_mapping, [0, -1]) + + paddle.disable_static() + + def test_shard_layer_to_static_with_buffer(self): + layer = MyLayer(self.num_features, self.num_layers) + test_buffer0 = paddle.randn([3]) + layer.register_buffer("test_buffer0", test_buffer0, persistable=True) + test_buffer1 = paddle.randn([10]) + layer.register_buffer("test_buffer1", test_buffer1, persistable=True) + layer.test_buffer1 = dist.shard_tensor( + layer.test_buffer1, self.mesh, [dist.Shard(0)] + ) + sharded_buffers_layer = dist.shard_layer(layer, self.mesh, shard_fn) + + loader = self.create_data_loader() + dist_model, dist_loader = dist.to_static(sharded_buffers_layer, loader) + + serial_main_program = dist_model.serial_main_program() + for param in serial_main_program.all_parameters(): + self.assertTrue(param.dist_attr.is_annotated("dims_mapping")) + self.assertEqual(param.dist_attr.dims_mapping, [-1, -1]) + + buffer_vars = [ + var + for var in serial_main_program.list_vars() + if var.name.startswith("generated") + ] + buffer0_var = buffer_vars[1] + buffer1_var = buffer_vars[0] + self.assertTrue(buffer0_var.dist_attr.is_annotated("dims_mapping")) + self.assertEqual(buffer0_var.dist_attr.dims_mapping, [-1]) + self.assertTrue(buffer1_var.dist_attr.is_annotated("dims_mapping")) + self.assertEqual(buffer1_var.dist_attr.dims_mapping, [0]) if __name__ == '__main__': From c4bc9e15e9293f4d38afa2231e1106a36205bacf Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Fri, 29 Dec 2023 20:24:47 +0800 Subject: [PATCH 141/146] [PirPass]Refine ApplyPirPass logic in @to_static (#59702) * [PirPass]Refine ApplyPirPass logic in @to_static * fix shared_ptr * fix code * del usless code * fix codestyle * fix UT --- .../group_merge/cinn_group_lowering_pass.cc | 4 +- .../hlir/framework/pir/op_lowering_impl.cc | 2 + paddle/fluid/pybind/pir.cc | 27 +++--- .../jit/dy2static/pir_partial_program.py | 95 +++++-------------- test/ir/pir/cinn/test_cinn_sub_graph.py | 7 +- 5 files changed, 45 insertions(+), 90 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/cinn_group_lowering_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/cinn_group_lowering_pass.cc index f11613ead1bfc..f4aa34bbc7263 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/group_merge/cinn_group_lowering_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/group_merge/cinn_group_lowering_pass.cc @@ -211,7 +211,7 @@ class GroupOpPattern : public pir::OpRewritePattern { } private: - std::shared_ptr shape_analysis_; + std::shared_ptr shape_analysis_{nullptr}; }; class CinnGroupLoweringPass : public pir::PatternRewritePass { @@ -237,7 +237,7 @@ class CinnGroupLoweringPass : public pir::PatternRewritePass { } private: - const std::shared_ptr& shape_analysis_; + std::shared_ptr shape_analysis_{nullptr}; }; } // namespace diff --git a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc index 1255a05825bab..643e4ed294b4c 100644 --- a/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc +++ b/paddle/cinn/hlir/framework/pir/op_lowering_impl.cc @@ -520,6 +520,7 @@ std::vector OpLowererImpl::LowerOps( auto& strategy = Operator::GetAttrs("CINNStrategy"); std::vector func_bodies; for (auto* op : ops) { + VLOG(4) << "start lowering op:" << op->name(); // 1.Select Op impl std::vector op_func_arg_tensors = CollectInputTensor(group, op, group_func_arg_tensors, tensor_map); @@ -891,6 +892,7 @@ ir::Tensor OpLowererImpl::GetTensor(const GroupPtr& group, auto in_shape = ::common::vectorize(type_info.dims()); auto dtype = type_info.dtype(); std::string input_id = ValueName(value); + VLOG(3) << "group->shape_analysis:" << group->shape_analysis; if (group->shape_analysis != nullptr) { auto sym_vec = group->shape_analysis->GetOrCreateSymbolicDimsForRankedValue(value); diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index e2471842c0729..8813ff59de53e 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1607,37 +1607,36 @@ static bool HasDynamicShape(const Program &program) { return false; } -void ApplyPirPass(Program &forward_program) { // NOLINT +void AddCinnPass(std::shared_ptr &pass_manager, // NOLINT + Program &program) { // NOLINT #ifdef PADDLE_WITH_CINN pir::IrContext *ctx = pir::IrContext::Instance(); ctx->GetOrRegisterDialect(); ctx->GetOrRegisterDialect(); ctx->GetOrRegisterDialect(); - bool has_dynamic_shape = HasDynamicShape(forward_program); + bool has_dynamic_shape = HasDynamicShape(program); auto shape_analysis = has_dynamic_shape ? std::make_shared(ctx) : nullptr; - pir::PassManager pass_manager(ctx); - pass_manager.AddPass(pir::CreateShapeOptimizationPass()); - cinn::dialect::ir::PdOp2CinnOpConverter(&forward_program); + pass_manager->AddPass(pir::CreateShapeOptimizationPass()); + cinn::dialect::ir::PdOp2CinnOpConverter(&program); - pass_manager.AddPass( + pass_manager->AddPass( std::make_unique()); - pass_manager.AddPass(pir::CreateDeadCodeEliminationPass()); - pass_manager.AddPass(pir::CreateBuildCinnPass()); + pass_manager->AddPass(pir::CreateDeadCodeEliminationPass()); + pass_manager->AddPass(pir::CreateBuildCinnPass()); if (has_dynamic_shape) { - pass_manager.AddPass(pir::CreateInferSymbolicShapePass(shape_analysis)); + pass_manager->AddPass(pir::CreateInferSymbolicShapePass(shape_analysis)); } - pass_manager.AddPass( + pass_manager->AddPass( cinn::dialect::ir::CreateCinnGroupLoweringPass(shape_analysis)); - - pass_manager.Run(&forward_program); - VLOG(3) << "after BuildCinnPass, forward_program:\n" << forward_program; + VLOG(4) << "has_dynamic_shape :" << has_dynamic_shape + << ", shape_analysis: " << shape_analysis; #else PADDLE_THROW(platform::errors::Unimplemented( "Currently we only support CINN Pass for Pir under @to_static, please " @@ -1645,7 +1644,7 @@ void ApplyPirPass(Program &forward_program) { // NOLINT #endif } void BindIrPass(pybind11::module *m) { - m->def("apply_pir_pass", ApplyPirPass); + m->def("add_cinn_pass", AddCinnPass); py::class_> pass(*m, "Pass", diff --git a/python/paddle/jit/dy2static/pir_partial_program.py b/python/paddle/jit/dy2static/pir_partial_program.py index a5858df1886e8..88b51f827581c 100644 --- a/python/paddle/jit/dy2static/pir_partial_program.py +++ b/python/paddle/jit/dy2static/pir_partial_program.py @@ -280,11 +280,12 @@ def apply_pir_program_pass(self, pass_fn): def pass_fn(forward_program, backward_program): return forward_program, backward_program """ - program_name_attr = self.program_name_attr origin_fwd = self.forward_program origin_bwd = self.backward_program + # NOTE(dev): Add this line to trigger program_name_attr logic + program_name_attr = self.program_name_attr self.forward_program, self.backward_program = pass_fn( - self.forward_program, self.backward_program, program_name_attr + origin_fwd, origin_bwd ) # cached property can ensure program is splited only once. @@ -382,55 +383,6 @@ def backward_program(self): return self._forward_backward_program[0][1] -class PirPassContext: - """ - PirPassContext is a class that only has staticmethod currently. - It will create a new RunableProgram after calling apply method. - """ - - INPUT_OP_NAME = "pd_op.data" - PARAM_OP_NAME = "builtin.parameter" - OUTPUT_OP_NAME = "builtin.shadow_output" - - @classmethod - def apply(cls, runable_program, build_strategy): - # TODO(Aurelius84): Currently only support infer mode, - # and we just use forward_program because backward_program - # is empty. - if not build_strategy.build_cinn_pass: - return runable_program - elif not paddle.is_compiled_with_cinn(): - raise RuntimeError( - "Please install PaddlePaddle compiled with CINN while setting build_strategy.build_cinn_pass = True." - ) - fwd_program, _ = paddle.base.libpaddle.pir.clone_program( - runable_program.forward_program - ) - paddle.base.libpaddle.pir.apply_pir_pass(fwd_program) - in_out_values = cls._prepare_attr(fwd_program) - return RunableProgram(fwd_program, in_out_values) - - @classmethod - def _prepare_attr(cls, program): - """ - After applying Pass, we need to update the Input/Parameter/Output Value - that refer to the new program. - - NOTE: We assume that Inputs come from INPUT_OP, Params come from - PARM_OP and Output come from OUTPUT_OP. - """ - inputs, params, outputs = [], [], [] - for op in program.global_block().ops: - op_name = op.name() - if op_name == cls.INPUT_OP_NAME: - inputs.append(op.result(0)) - elif op_name == cls.PARAM_OP_NAME: - params.append(op.result(0)) - elif op_name == cls.OUTPUT_OP_NAME: - outputs.append(op.operand(0).source()) - return inputs, params, outputs - - class PartialProgramLayerHook: def before_append_backward(self, forward_program, src_vars): ... @@ -596,13 +548,19 @@ def _get_scope(self, program_id=None, use_scope_cache=False): @switch_to_static_graph def _create_program(self, is_infer_mode=False): if is_infer_mode: + + def pass_fn(forward_program, backward_program): + pm = paddle.base.libpaddle.pir.PassManager() + if self._build_strategy.build_cinn_pass: + paddle.base.libpaddle.pir.add_cinn_pass(pm, forward_program) + pm.run(forward_program) + return forward_program, backward_program + # TODO(xiongkun) who to transfer the pruning program? infer_program = self.origin_runable_program.clone() if self._hooker: self._hooker.after_infer(infer_program) - infer_program = PirPassContext.apply( - infer_program, self._build_strategy - ) + infer_program.apply_pir_program_pass(pass_fn) return infer_program else: train_program: RunableProgram = self.origin_runable_program.clone() @@ -610,23 +568,20 @@ def _create_program(self, is_infer_mode=False): # Note: Only set grad type once after initializing train program. So we put it here. self._set_grad_type(self._params, train_program) - # (NOTE:@xiongkun) HOW TO APPLY PASS: this is a example for forward/backward clone pass, just replace with your cases. - def pass_fn(forward_program, backward_program, name_attr): - fwd, _ = paddle.base.libpaddle.pir.clone_program( - forward_program - ) - - if self._build_strategy.build_cinn_pass: - paddle.base.libpaddle.pir.apply_pir_pass(fwd) - - bwd, _ = paddle.base.libpaddle.pir.clone_program( - backward_program - ) + def pass_fn(forward_program, backward_program): + fwd_pm = paddle.base.libpaddle.pir.PassManager() + bwd_pm = paddle.base.libpaddle.pir.PassManager() if self._build_strategy.build_cinn_pass: - paddle.base.libpaddle.pir.apply_pir_pass(bwd) - - return fwd, bwd + paddle.base.libpaddle.pir.add_cinn_pass( + fwd_pm, forward_program + ) + paddle.base.libpaddle.pir.add_cinn_pass( + bwd_pm, backward_program + ) + fwd_pm.run(forward_program) + bwd_pm.run(backward_program) + return forward_program, backward_program train_program.apply_pir_program_pass(pass_fn) return train_program @@ -748,7 +703,7 @@ def _insert_aggregation_ops_for_var(target_program, var): shape=var.shape, ) # step2: rename the var.name@GRAD to var.name@GRAD@dy2static - for idx, op in finded_ops: + for _, op in finded_ops: op._rename_input(var_grad_name, new_grad_name) op._rename_output(var_grad_name, new_grad_name) # step3: insert sum op to aggregate the gradient. diff --git a/test/ir/pir/cinn/test_cinn_sub_graph.py b/test/ir/pir/cinn/test_cinn_sub_graph.py index 32b0bd5779dd9..ad4c65d3d3541 100644 --- a/test/ir/pir/cinn/test_cinn_sub_graph.py +++ b/test/ir/pir/cinn/test_cinn_sub_graph.py @@ -203,10 +203,9 @@ def test_forward(self): cinn_out = self.train(use_cinn=True) dy_out = self.train(use_cinn=False) - # TODO(zhangliujie) fix precision error - # np.testing.assert_allclose( - # cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 - # ) + np.testing.assert_allclose( + cinn_out.numpy(), dy_out.numpy(), atol=1e-8, rtol=1e-4 + ) class TestCinnDropout(TestCinnSubGraphBase): From 5bc7a5926308d2c1a22d1c696d98a199bf60ff3c Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Fri, 29 Dec 2023 23:17:01 +0800 Subject: [PATCH 142/146] [CodeStyle][ruff] clean I001 ignore - Part 2 (#60466) --- pyproject.toml | 1 - python/paddle/distribution/__init__.py | 60 ++++++++++++++++---------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64727e39f1d64..eaf239288bd39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,6 +135,5 @@ known-first-party = ["paddle"] "python/paddle/distributed/launch/controllers/__init__.py" = ["I001"] "python/paddle/distributed/passes/__init__.py" = ["I001"] "python/paddle/distributed/rpc/__init__.py" = ["I001"] -"python/paddle/distribution/__init__.py" = ["I001"] "python/paddle/incubate/distributed/fleet/__init__.py" = ["I001"] "python/paddle/incubate/distributed/fleet/parameter_server/pslib/__init__.py" = ["I001"] diff --git a/python/paddle/distribution/__init__.py b/python/paddle/distribution/__init__.py index c56da5805ad66..446c75aeaea70 100644 --- a/python/paddle/distribution/__init__.py +++ b/python/paddle/distribution/__init__.py @@ -12,29 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle.distribution import transform -from paddle.distribution.bernoulli import Bernoulli -from paddle.distribution.beta import Beta -from paddle.distribution.categorical import Categorical -from paddle.distribution.cauchy import Cauchy -from paddle.distribution.continuous_bernoulli import ContinuousBernoulli -from paddle.distribution.dirichlet import Dirichlet -from paddle.distribution.distribution import Distribution -from paddle.distribution.gumbel import Gumbel -from paddle.distribution.exponential_family import ExponentialFamily -from paddle.distribution.independent import Independent -from paddle.distribution.kl import kl_divergence, register_kl -from paddle.distribution.lognormal import LogNormal -from paddle.distribution.multinomial import Multinomial -from paddle.distribution.multivariate_normal import MultivariateNormal -from paddle.distribution.normal import Normal -from paddle.distribution.transform import * # noqa: F403 -from paddle.distribution.transformed_distribution import TransformedDistribution -from paddle.distribution.uniform import Uniform -from paddle.distribution.laplace import Laplace -from paddle.distribution.geometric import Geometric -from paddle.distribution.binomial import Binomial -from paddle.distribution.poisson import Poisson +from . import transform +from .bernoulli import Bernoulli +from .beta import Beta +from .binomial import Binomial +from .categorical import Categorical +from .cauchy import Cauchy +from .continuous_bernoulli import ContinuousBernoulli +from .dirichlet import Dirichlet +from .distribution import Distribution +from .exponential_family import ExponentialFamily +from .geometric import Geometric +from .gumbel import Gumbel +from .independent import Independent +from .kl import kl_divergence, register_kl +from .laplace import Laplace +from .lognormal import LogNormal +from .multinomial import Multinomial +from .multivariate_normal import MultivariateNormal +from .normal import Normal +from .poisson import Poisson +from .transform import ( # noqa:F401 + AbsTransform, + AffineTransform, + ChainTransform, + ExpTransform, + IndependentTransform, + PowerTransform, + ReshapeTransform, + SigmoidTransform, + SoftmaxTransform, + StackTransform, + StickBreakingTransform, + TanhTransform, + Transform, +) +from .transformed_distribution import TransformedDistribution +from .uniform import Uniform __all__ = [ 'Bernoulli', From 3177d59b2915ce345964449a5189e16a8e0ca544 Mon Sep 17 00:00:00 2001 From: ronnywang Date: Sat, 30 Dec 2023 20:58:49 +0800 Subject: [PATCH 143/146] [CustomDevice] release all xccl_comm in DeviceManager::Release (#60465) --- paddle/phi/backends/device_manager.cc | 4 +++ .../phi/core/distributed/xccl_comm_context.cc | 27 +++++++++++++++++++ .../phi/core/distributed/xccl_comm_context.h | 3 +++ 3 files changed, 34 insertions(+) diff --git a/paddle/phi/backends/device_manager.cc b/paddle/phi/backends/device_manager.cc index 1e57fb736b7c2..87a163b2cb4fa 100644 --- a/paddle/phi/backends/device_manager.cc +++ b/paddle/phi/backends/device_manager.cc @@ -14,6 +14,7 @@ #include "paddle/phi/backends/device_manager.h" #include "paddle/phi/common/complex.h" +#include "paddle/phi/core/distributed/xccl_comm_context.h" #if !defined(_WIN32) #include @@ -699,6 +700,9 @@ DeviceManager& DeviceManager::Instance() { void DeviceManager::Release() { event::Event::ReleaseAll(); stream::Stream::ReleaseAll(); +#ifdef PADDLE_WITH_CUSTOM_DEVICE + phi::distributed::XCCLCommContext::ReleaseAll(); +#endif Instance().device_map_.clear(); Instance().device_impl_map_.clear(); } diff --git a/paddle/phi/core/distributed/xccl_comm_context.cc b/paddle/phi/core/distributed/xccl_comm_context.cc index ba7e24ab06b9e..3e3608e4d88a5 100644 --- a/paddle/phi/core/distributed/xccl_comm_context.cc +++ b/paddle/phi/core/distributed/xccl_comm_context.cc @@ -14,6 +14,8 @@ #include "paddle/phi/core/distributed/xccl_comm_context.h" +#include + #include "glog/logging.h" #include "paddle/phi/core/dense_tensor.h" @@ -25,6 +27,29 @@ namespace phi { namespace distributed { +std::list g_xccl_comm_contexts; +std::mutex g_xccl_comm_contexts_mutex; + +void XCCLCommContext::ReleaseAll() { + std::unique_lock lock(g_xccl_comm_contexts_mutex); + for (auto xccl_comm_ctx : g_xccl_comm_contexts) { + phi::DeviceManager::CCLDestroyComm(xccl_comm_ctx->GetDeviceType(), + xccl_comm_ctx->GetXcclComm()); + xccl_comm_ctx->xccl_comm_ = nullptr; + } + g_xccl_comm_contexts.clear(); +} + +XCCLCommContext::~XCCLCommContext() { + std::unique_lock lock(g_xccl_comm_contexts_mutex); + if (phi::DeviceManager::HasDeviceType(this->GetDeviceType()) && + xccl_comm_ != nullptr) { + phi::DeviceManager::CCLDestroyComm(this->GetDeviceType(), xccl_comm_); + xccl_comm_ = nullptr; + } + g_xccl_comm_contexts.remove(this); +} + XCCLCommContext::XCCLCommContext(const phi::Place& place, int rank, int size, @@ -38,6 +63,8 @@ XCCLCommContext::XCCLCommContext(const phi::Place& place, &xccl_comm_); stream_ = std::make_shared(); stream_->Init(place_); + std::unique_lock lock(g_xccl_comm_contexts_mutex); + g_xccl_comm_contexts.push_back(this); } void XCCLCommContext::Broadcast(phi::DenseTensor* out_tensor, diff --git a/paddle/phi/core/distributed/xccl_comm_context.h b/paddle/phi/core/distributed/xccl_comm_context.h index 0c253eb925bb4..8cdc7e4153d76 100644 --- a/paddle/phi/core/distributed/xccl_comm_context.h +++ b/paddle/phi/core/distributed/xccl_comm_context.h @@ -28,6 +28,9 @@ class XCCLCommContext final : public CommContext { int rank, int size, const ccl::CCLRootId& xccl_id); + ~XCCLCommContext(); + + static void ReleaseAll(); ccl::CCLComm GetXcclComm() const { return xccl_comm_; } From 77d7638a75bf527c1db3b7df3688102b4210d74f Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Tue, 2 Jan 2024 10:27:53 +0800 Subject: [PATCH 144/146] [DimExpr] DimExpr support hash (#60471) --- paddle/pir/dialect/shape/utils/dim_expr.cc | 52 +++++++++++++++++++ paddle/pir/dialect/shape/utils/dim_expr.h | 13 +++++ .../pir/shape_dialect/symbol_dim_expr_test.cc | 34 +++++++++--- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/paddle/pir/dialect/shape/utils/dim_expr.cc b/paddle/pir/dialect/shape/utils/dim_expr.cc index 0d9b6ece23245..61f7a582cb5a5 100644 --- a/paddle/pir/dialect/shape/utils/dim_expr.cc +++ b/paddle/pir/dialect/shape/utils/dim_expr.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/pir/dialect/shape/utils/dim_expr.h" +#include "paddle/pir/core/utils.h" namespace symbol { @@ -184,4 +185,55 @@ std::ostream& operator<<(std::ostream& stream, const DimExpr& dim_expr) { return stream; } +namespace { + +std::size_t GetHashValueImpl(const std::int64_t& dim_expr) { return dim_expr; } + +std::size_t GetHashValueImpl(const std::string& dim_expr) { + return std::hash()(dim_expr); +} + +std::size_t GetHashValueImpl(const Negative& dim_expr) { + return -GetHashValue(dim_expr->data); +} + +std::size_t GetHashValueImpl(const Reciprocal& dim_expr) { + return pir::hash_combine(1, -GetHashValue(dim_expr->data)); +} + +std::size_t GetHashValueImpl(const List& exprs) { + std::size_t ret = 0; + for (const auto& expr : *exprs) { + ret = pir::hash_combine(ret, GetHashValue(expr)); + } + return ret; +} + +std::size_t GetHashValueImpl(const Add& dim_expr) { + return pir::hash_combine(1, GetHashValueImpl(dim_expr.operands)); +} + +std::size_t GetHashValueImpl(const Mul& dim_expr) { + return pir::hash_combine(2, GetHashValueImpl(dim_expr.operands)); +} + +std::size_t GetHashValueImpl(const Max& dim_expr) { + return pir::hash_combine(3, GetHashValueImpl(dim_expr.operands)); +} + +std::size_t GetHashValueImpl(const Min& dim_expr) { + return pir::hash_combine(4, GetHashValueImpl(dim_expr.operands)); +} + +std::size_t GetHashValueImpl(const Broadcast& dim_expr) { + return pir::hash_combine(5, GetHashValueImpl(dim_expr.operands)); +} + +} // namespace + +std::size_t GetHashValue(const DimExpr& dim_expr) { + return std::visit([](const auto& impl) { return GetHashValueImpl(impl); }, + dim_expr.variant()); +} + } // namespace symbol diff --git a/paddle/pir/dialect/shape/utils/dim_expr.h b/paddle/pir/dialect/shape/utils/dim_expr.h index 277a6febe66ed..a65390200cd06 100644 --- a/paddle/pir/dialect/shape/utils/dim_expr.h +++ b/paddle/pir/dialect/shape/utils/dim_expr.h @@ -253,4 +253,17 @@ IR_API std::string ToString(const DimExpr& dim_expr); IR_API std::ostream& operator<<(std::ostream&, const DimExpr& dim_expr); +IR_API std::size_t GetHashValue(const DimExpr& dim_expr); + } // namespace symbol + +namespace std { + +template <> +struct hash { + std::size_t operator()(const symbol::DimExpr& dim_expr) const { + return symbol::GetHashValue(dim_expr); + } +}; + +} // namespace std diff --git a/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc b/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc index 6157850e3842c..3aebb367d1a27 100644 --- a/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc +++ b/test/cpp/pir/shape_dialect/symbol_dim_expr_test.cc @@ -22,7 +22,7 @@ namespace symbol::test { // Construct DimExpr by overloaded operator(+, - , *, /) -TEST(DimExpr, dim_expr_naive) { +TEST(DimExpr, DimExprNaive) { DimExpr sym0 = DimExpr("S0"); DimExpr sym1 = DimExpr("S1"); DimExpr constant1 = DimExpr(1); @@ -30,7 +30,7 @@ TEST(DimExpr, dim_expr_naive) { } // Construct DimExpr by DimExprBuilder -TEST(DimExpr, dim_expr_builder) { +TEST(DimExpr, DimExprBuilder) { DimExprBuilder builder{nullptr}; DimExpr sym0 = DimExpr("S0"); DimExpr sym1 = DimExpr("S1"); @@ -40,7 +40,7 @@ TEST(DimExpr, dim_expr_builder) { } // Add constraints by DimExprBuilder -TEST(DimExpr, constraint) { +TEST(DimExpr, Constraint) { std::vector constraints{}; DimExprBuilder builder(&constraints); DimExpr sym0 = DimExpr("S0"); @@ -55,7 +55,7 @@ TEST(DimExpr, constraint) { extend_x = x.shape out = pd.reshape(y, extend_x) */ -TEST(DimExpr, data_shape_expr) { +TEST(DimExpr, DataShapeExpr) { // Show ideal ShapeOrDataDimExprs of each pir::Value std::vector x_shapes{DimExpr("S0"), DimExpr(2)}; std::vector y_shapes{DimExpr(1), DimExpr("S1"), DimExpr(2)}; @@ -80,7 +80,7 @@ TEST(Simplify, NumberArithmetic) { ASSERT_EQ((mul_div.Get()), 1); } -TEST(DimExpr, equal) { +TEST(DimExpr, Equal) { DimExprBuilder builder{nullptr}; DimExpr sym0 = DimExpr("S0"); DimExpr sym1 = DimExpr("S1"); @@ -111,7 +111,7 @@ TEST(DimExpr, equal) { builder.Broadcast(DimExpr("S0"), constant1)); } -TEST(DimExpr, print) { +TEST(DimExpr, Print) { DimExprBuilder builder{nullptr}; DimExpr sym0 = DimExpr("S0"); DimExpr sym1 = DimExpr("S1"); @@ -124,4 +124,26 @@ TEST(DimExpr, print) { ASSERT_EQ((ToString(builder.Broadcast(sym0, sym1))), "Broadcast(S0, S1)"); } +TEST(DimExpr, Hash) { + DimExprBuilder builder{nullptr}; + DimExpr sym0 = DimExpr("S0"); + DimExpr sym1 = DimExpr("S1"); + ASSERT_EQ((std::hash()(sym0 + sym1)), + (std::hash()(sym0 + sym1))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(sym1 + sym0))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(sym0 - sym1))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(sym0 * sym1))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(sym0 / sym1))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(builder.Max(sym0, sym1)))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(builder.Min(sym0, sym1)))); + ASSERT_NE((std::hash()(sym0 + sym1)), + (std::hash()(builder.Broadcast(sym0, sym1)))); +} + } // namespace symbol::test From 8fcf35b724cddc2fd327f0656850e70efbb74ddb Mon Sep 17 00:00:00 2001 From: ooo oo <106524776+ooooo-create@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:38:25 +0800 Subject: [PATCH 145/146] open warning with `paddle.utils.deprecated` (#60458) * open_warning * update unittest * update * fix typos * fix warning in test runner * uncomment * cleanup todo * using VisibleDeprecationWarning * update comment * fix typo * fix indentation * fix * fix * fix indent level and test * update --------- Co-authored-by: SigureMo --- python/paddle/utils/deprecated.py | 23 ++++++-- test/legacy_test/CMakeLists.txt | 2 - test/legacy_test/test_deprecated_decorator.py | 59 ++++++++----------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/python/paddle/utils/deprecated.py b/python/paddle/utils/deprecated.py index 873c6b3a6a9fc..39b1f73748098 100755 --- a/python/paddle/utils/deprecated.py +++ b/python/paddle/utils/deprecated.py @@ -16,6 +16,7 @@ """ import functools +import inspect import sys import warnings @@ -24,6 +25,18 @@ __all__ = [] +class VisibleDeprecationWarning(UserWarning): + """Visible deprecation warning. + + Since Python 3.7, Python only show the DeprecationWarning if the module + is __main__. So we use this warning to make the deprecation warning visible. + + See more details from https://peps.python.org/pep-0565/ + """ + + ... + + def deprecated(update_to="", since="", reason="", level=0): """Decorate a function to signify its deprecation. @@ -47,8 +60,6 @@ def deprecated(update_to="", since="", reason="", level=0): """ def decorator(func): - # TODO(zhiqiu): temporally disable the warnings - return func """construct warning message, and return a decorated function or class.""" assert isinstance(update_to, str), 'type of "update_to" must be str.' assert isinstance(since, str), 'type of "since" must be str.' @@ -75,9 +86,11 @@ def decorator(func): ) msg += f' Please use "{_update_to}" instead.' if len(_reason) > 0: - msg += f"\nreason: {_reason}" + msg += f"\n Reason: {_reason}" if func.__doc__: - func.__doc__ = ('\n\nWarning: ' + msg + '\n') + func.__doc__ + func.__doc__ = ( + '\n\nWarning:\n ' + msg + '\n\n' + ) + inspect.cleandoc(func.__doc__) if level == 0: return func @@ -110,7 +123,7 @@ def wrapper(*args, **kwargs): or v_current >= v_since ): warnings.warn( - warningmsg, category=DeprecationWarning, stacklevel=2 + warningmsg, category=VisibleDeprecationWarning, stacklevel=2 ) return func(*args, **kwargs) diff --git a/test/legacy_test/CMakeLists.txt b/test/legacy_test/CMakeLists.txt index 824d50d8a6aaf..ed0f40f982d23 100644 --- a/test/legacy_test/CMakeLists.txt +++ b/test/legacy_test/CMakeLists.txt @@ -118,8 +118,6 @@ if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) list(REMOVE_ITEM TEST_OPS test_fleet_executor_cond_interceptor) endif() -list(REMOVE_ITEM TEST_OPS test_deprecated_decorator) - if(WIN32) list(REMOVE_ITEM TEST_OPS test_multiprocess_reader_exception) list(REMOVE_ITEM TEST_OPS test_trainer_desc) diff --git a/test/legacy_test/test_deprecated_decorator.py b/test/legacy_test/test_deprecated_decorator.py index 81ae80a1f9bf7..0c7ce0f062590 100755 --- a/test/legacy_test/test_deprecated_decorator.py +++ b/test/legacy_test/test_deprecated_decorator.py @@ -19,23 +19,20 @@ import numpy as np import paddle -from paddle import _legacy_C_ops from paddle.utils import deprecated LOWEST_WARNING_POSTION = 3 ERROR_WARNING_POSTION = sys.maxsize # custom paddle version -paddle.version.major = '1' -paddle.version.minor = '8' +paddle.version.major = '0' +paddle.version.minor = '0' paddle.version.patch = '0' paddle.version.rc = '0' -paddle.__version__ = '1.8.0' -paddle.version.full_version = '1.8.0' +paddle.__version__ = '0.0.0' +paddle.version.full_version = '0.0.0' print("current paddle version: ", paddle.__version__) -paddle.disable_static() - def get_warning_index(api): """ @@ -49,22 +46,25 @@ def get_warning_index(api): index (int): the index of the Warinng information in its doc string if exists. """ - doc_lst = api.__doc__.splitlines() - for idx, val in enumerate(doc_lst): + doc_list = api.__doc__.splitlines() + if len(doc_list) < 2: + return ERROR_WARNING_POSTION + for idx, (current_line, next_line) in enumerate( + zip(doc_list[:-1], doc_list[1:]) + ): if ( - val.startswith("Warning: ") - and val.endswith(" instead.") - and "and will be removed in future versions." in val + current_line == "Warning:" + and next_line.endswith(" instead.") + and "and will be removed in future versions." in next_line ): return idx return ERROR_WARNING_POSTION -class TestDeprecatedDocorator(unittest.TestCase): +class TestDeprecatedDecorator(unittest.TestCase): """ - tests for paddle's Deprecated Docorator. + tests for paddle's deprecated decorator. test_new_multiply: test for new api, which should not insert warning information. - test_ops_elementwise_mul: test for C++ elementwise_mul op, which should not insert warning information. """ def test_new_multiply(self): @@ -87,26 +87,15 @@ def test_new_multiply(self): # testting self.assertLess(expected, captured) - def test_ops_elementwise_mul(self): - """ - Test for new C++ elementwise_op, expected result should be True, - because not matter what base.layers.elementwise_mul is deprecated. - """ - - a = np.random.uniform(0.1, 1, [51, 76]).astype(np.float32) - b = np.random.uniform(0.1, 1, [51, 76]).astype(np.float32) - x = paddle.to_tensor(a) - y = paddle.to_tensor(b) - res = _legacy_C_ops.elementwise_mul(x, y) - - # expected - expected = LOWEST_WARNING_POSTION - - # captured - captured = get_warning_index(paddle.multiply) - - # testting - self.assertGreater(expected, captured) + def test_indent_level(self): + # test for different indent_level + dataset = paddle.base.DatasetFactory().create_dataset("InMemoryDataset") + with warnings.catch_warnings(record=True): + dataset.set_merge_by_lineid() + assert ( + '\nSet merge by' + in paddle.base.InMemoryDataset.set_merge_by_lineid.__doc__ + ) def test_tensor_gradient(self): paddle.__version__ = '2.1.0' From a08580e25c6db807e7ba7318550f566db55ac1f8 Mon Sep 17 00:00:00 2001 From: zhaoyingli <86812880+zhaoyinglia@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:45:08 +0800 Subject: [PATCH 146/146] [AutoParallel] Auto Trans PP to VPP (#60467) * [AutoParallel] Auto Trans PP to VPP * add comment --- .../auto_parallel/static/completion.py | 189 ++++++++++++++---- .../distributed/auto_parallel/static/utils.py | 2 +- .../pipeline_scheduler_vpp_unittest.py | 80 ++++++-- 3 files changed, 207 insertions(+), 64 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/completion.py b/python/paddle/distributed/auto_parallel/static/completion.py index 76c6a9d181766..692d02b7563c6 100644 --- a/python/paddle/distributed/auto_parallel/static/completion.py +++ b/python/paddle/distributed/auto_parallel/static/completion.py @@ -1057,22 +1057,43 @@ def set_chunk_id(block, op, chunk_id, var_to_chunk_id): dist_op = self._dist_context.get_dist_op_for_program(op) dist_op.dist_attr.chunk_id = chunk_id for name in op.input_arg_names + op.output_arg_names: - var = block._find_var_recursive(name) if "lod_tensor_blocking_queue" in name: continue if name not in var_to_chunk_id: - op_dist_attr = ( - self._dist_context.get_op_dist_attr_for_program(op) + var = block._find_var_recursive(name) + dist_tensor = ( + self._dist_context.get_dist_tensor_for_program(var) ) - tensor_dist_attr = ( - self._dist_context.get_tensor_dist_attr_for_program(var) + if ( + dist_op.dist_attr.process_mesh + == dist_tensor.dist_attr.process_mesh + ): + dist_tensor.dist_attr.chunk_id = chunk_id + var_to_chunk_id[var.name] = chunk_id + + def set_process_mesh(block, op, process_mesh, var_to_process_mesh): + dist_op = self._dist_context.get_dist_op_for_program(op) + for name in op.input_arg_names: + if name not in var_to_process_mesh: + var = block._find_var_recursive(name) + dist_tensor = ( + self._dist_context.get_dist_tensor_for_program(var) ) if ( - op_dist_attr.process_mesh - == tensor_dist_attr.process_mesh + dist_op.dist_attr.process_mesh + == dist_tensor.dist_attr.process_mesh ): - tensor_dist_attr.chunk_id = op_dist_attr.chunk_id - var_to_chunk_id[var.name] = op_dist_attr.chunk_id + dist_tensor.dist_attr.process_mesh = process_mesh + var_to_process_mesh[var.name] = process_mesh + for name in op.output_arg_names: + if name not in var_to_process_mesh: + var = block._find_var_recursive(name) + dist_tensor = ( + self._dist_context.get_dist_tensor_for_program(var) + ) + dist_tensor.dist_attr.process_mesh = process_mesh + var_to_process_mesh[var.name] = process_mesh + dist_op.dist_attr.process_mesh = process_mesh if ( not self._dist_context.strategy @@ -1080,7 +1101,7 @@ def set_chunk_id(block, op, chunk_id, var_to_chunk_id): ): return - pp_degree = get_pp_degree(self._dist_context) + pp_degree, sub_process_meshes = get_pp_degree(self._dist_context) vpp_degree = self._dist_context.strategy.pipeline.vpp_degree seg_method = self._dist_context.strategy.pipeline.vpp_seg_method schedule_mode = self._dist_context.strategy.pipeline.schedule_mode @@ -1099,8 +1120,11 @@ def set_chunk_id(block, op, chunk_id, var_to_chunk_id): block = serial_main_program.global_block() ops = block.ops - # 1. search seg_method in op's struct_name, and get all ops of segments - seg_op_deps = collections.OrderedDict() + # Step1: search seg_method in op's struct_name + # 1. get op_idx of each segment + # 2. get process_mesh or each segment + seg_op_deps = collections.OrderedDict() # struct_name -> [idx] + seg_op_mesh = collections.OrderedDict() # struct_name -> process_mesh regex = re.compile(seg_method, re.IGNORECASE) for i, op in enumerate(ops): struct_name = op.struct_name @@ -1109,59 +1133,93 @@ def set_chunk_id(block, op, chunk_id, var_to_chunk_id): continue struct_name = struct_name[m.start(0) :].split("/")[0] + dist_op = self._dist_context.get_dist_op_for_program(op) if struct_name not in seg_op_deps: seg_op_deps[struct_name] = [i] + seg_op_mesh[struct_name] = dist_op.dist_attr.process_mesh else: assert ( seg_op_deps[struct_name][-1] + 1 == i ), "The segment's ops should be continuous." - pre_op = ops[seg_op_deps[struct_name][-1]] - pre_dist_op = self._dist_context.get_dist_op_for_program(pre_op) - dist_op = self._dist_context.get_dist_op_for_program(op) + pre_mesh = seg_op_mesh[struct_name] assert ( - pre_dist_op.dist_attr.process_mesh - == dist_op.dist_attr.process_mesh + pre_mesh == dist_op.dist_attr.process_mesh ), "The segment's ops should have same process_mesh." seg_op_deps[struct_name].extend([i]) - # the num of chunk is equal to vpp_degree - num_parts = pp_degree * vpp_degree + num_chunks = pp_degree * vpp_degree assert ( - len(seg_op_deps.keys()) % num_parts == 0 - ), "number of layers[{}] ({}) should be devided by part number ({}).".format( - seg_method, len(seg_op_deps.keys()), num_parts + len(seg_op_deps) % num_chunks == 0 + ), "The number of layers[{}] ({}) should be devided by part number ({}).".format( + seg_method, len(seg_op_deps), num_chunks ) - part_size = len(seg_op_deps.keys()) // vpp_degree + # Step2: analysis whether the pp_stage is non-decreasing among segments + # 1. if non_decreasing is True, the ops' process_mesh will be changed by vpp strategy + # 2. if non_decreasing is False, the ops's process_mesh will not be changed. + non_decreasing = True + seg_pp_stages = [-1] + for seg_pm in seg_op_mesh.values(): + assert seg_pm in sub_process_meshes + pp_stage = sub_process_meshes.index(seg_pm) + if seg_pp_stages[-1] > pp_stage: + non_decreasing = False + break + seg_pp_stages.append(pp_stage) - # 2. get boundary index of each chunk - results = [0] * (vpp_degree + 1) - memory_counter = 0 - result_idx = 1 - for struct_name, idxs in seg_op_deps.items(): + if not non_decreasing: + _logger.info("Cannot Use Auto VPP") + else: + _logger.info("Using Auto VPP") + + # Step3: Get op index boundary, pp_stage, chunk_id, struct_names of each segment + seg_pp_stages = [i % pp_degree for i in range(num_chunks)] + seg_chunk_ids = [i // pp_degree for i in range(num_chunks)] + part_size = len(seg_op_deps) // num_chunks + segment_struct_names = [] + segment_parts = [0] * (num_chunks + 1) + memory_counter, seg_idx = 0, 1 + struct_name = [] + for name, idxs in seg_op_deps.items(): + struct_name.append(name) memory_counter += 1 if memory_counter == part_size: - results[result_idx] = idxs[-1] + 1 - result_idx += 1 - memory_counter = 0 - results[vpp_degree] = len(ops) + segment_parts[seg_idx] = idxs[-1] + 1 + memory_counter, seg_idx = 0, seg_idx + 1 + segment_struct_names.append(struct_name) + struct_name = [] + segment_parts[num_chunks] = len(ops) - # 3. set right chunk_id for each op + # Step4: set right chunk_id and process_mesh for each op and var var_to_chunk_id = {} - for chunk_id in range(len(results) - 1): - start_idx = results[chunk_id] - end_idx = results[chunk_id + 1] + var_to_process_mesh = {} + for seg_id in range(len(segment_parts) - 1): + start_idx = segment_parts[seg_id] + end_idx = segment_parts[seg_id + 1] + pp_stage = seg_pp_stages[seg_id] + chunk_id = seg_chunk_ids[seg_id] + process_mesh = sub_process_meshes[pp_stage] + struct_names = segment_struct_names[seg_id] + seg_op_idx = [] + for name in struct_names: + seg_op_idx.extend(seg_op_deps[name]) + _logger.info( - "[chunk_{}] start op: [{}]: [{}] [{}]".format( + "stage=[{}], chunk_id=[{}], layer_name=[{}]".format( + pp_stage, chunk_id, + struct_names, + ) + ) + _logger.info( + "start op: [{}]: [{}] [{}]".format( ops[start_idx].type, ops[start_idx].input_arg_names, ops[start_idx].output_arg_names, ) ) _logger.info( - "[chunk_{}] end op: [{}]: [{}] [{}]".format( - chunk_id, + "end op: [{}]: [{}] [{}]".format( ops[end_idx - 1].type, ops[end_idx - 1].input_arg_names, ops[end_idx - 1].output_arg_names, @@ -1173,9 +1231,28 @@ def set_chunk_id(block, op, chunk_id, var_to_chunk_id): if op.has_attr("sub_block"): block_id = op.attr('sub_block').id sub_block = serial_main_program.blocks[block_id] - for op in sub_block.ops: - set_chunk_id(sub_block, op, chunk_id, var_to_chunk_id) + if non_decreasing and idx in seg_op_idx: + set_process_mesh( + block, op, process_mesh, var_to_process_mesh + ) + set_chunk_id(block, op, chunk_id, var_to_chunk_id) + + for sub_op in sub_block.ops: + if non_decreasing and idx in seg_op_idx: + set_process_mesh( + sub_block, + sub_op, + process_mesh, + var_to_process_mesh, + ) + set_chunk_id( + sub_block, sub_op, chunk_id, var_to_chunk_id + ) else: + if non_decreasing and idx in seg_op_idx: + set_process_mesh( + block, op, process_mesh, var_to_process_mesh + ) set_chunk_id(block, op, chunk_id, var_to_chunk_id) def _update_dist_attr_for_dp(self): @@ -1915,8 +1992,34 @@ def infer_backward_op_partial_status( grad_op_dist_attr.set_output_dims_mapping( output_name, ref_fwd_dims_mapping ) - grad_op_dist_attr.process_mesh = ref_fwd_process_mesh - grad_op_dist_attr.chunk_id = ref_fwd_chunk_id + # NOTE(zhaoyingli): + # The sum op is used to accmulate the grads' value of the same forward var, + # sum op's chunk_id is same with the last op which generate the grad. + ref_chunk_id = None + ref_process_mesh = None + for pre_idx in range( + idx - 1, first_backward_op_idx + 1, -1 + ): + pre_grad_op = ops[pre_idx] + inter_arg_name = list( + set(pre_grad_op.output_arg_names) + & set(grad_op.input_arg_names) + ) + if len(inter_arg_name) > 0: + pre_op_dist_attr = ( + self._dist_context.get_op_dist_attr_for_program( + pre_grad_op + ) + ) + ref_chunk_id = pre_op_dist_attr.chunk_id + ref_process_mesh = pre_op_dist_attr.process_mesh + break + assert ( + ref_chunk_id is not None + and ref_process_mesh is not None + ) + grad_op_dist_attr.process_mesh = ref_process_mesh + grad_op_dist_attr.chunk_id = ref_chunk_id self._dist_context.set_op_dist_attr_for_program( grad_op, grad_op_dist_attr ) diff --git a/python/paddle/distributed/auto_parallel/static/utils.py b/python/paddle/distributed/auto_parallel/static/utils.py index 296196230d086..359767c7345e8 100644 --- a/python/paddle/distributed/auto_parallel/static/utils.py +++ b/python/paddle/distributed/auto_parallel/static/utils.py @@ -2335,7 +2335,7 @@ def get_pp_degree(dist_context): for idx in reversed(global_pm_idx): process_meshes.pop(idx) - return len(process_meshes) + return len(process_meshes), process_meshes def get_pp_stage(dist_context, rank): diff --git a/test/auto_parallel/pipeline_scheduler_vpp_unittest.py b/test/auto_parallel/pipeline_scheduler_vpp_unittest.py index 431e782cb073e..bed72232a05ca 100644 --- a/test/auto_parallel/pipeline_scheduler_vpp_unittest.py +++ b/test/auto_parallel/pipeline_scheduler_vpp_unittest.py @@ -37,8 +37,8 @@ class MyLinear(nn.Layer): def __init__( self, - hidden_size=1024, - intermediate_size=4 * 1024, + hidden_size=784, + intermediate_size=4 * 784, dropout_ratio=0.1, weight_attr=None, ): @@ -64,10 +64,11 @@ def forward(self, input): class MLPLayer(nn.Layer): def __init__( self, - hidden_size=1024, - intermediate_size=4 * 1024, + hidden_size=784, + intermediate_size=4 * 784, dropout_ratio=0.1, initializer_range=0.02, + manual=True, ): super().__init__() @@ -86,7 +87,10 @@ def __init__( self.linear = nn.Linear(hidden_size, 1, weight_attr, bias_attr=None) self.norm = nn.LayerNorm(hidden_size, epsilon=1e-5) - self.layer_to_mesh = [PP_MESH_0, PP_MESH_1, PP_MESH_0, PP_MESH_1] + if manual: + self.layer_to_mesh = [PP_MESH_0, PP_MESH_1, PP_MESH_0, PP_MESH_1] + else: + self.layer_to_mesh = [PP_MESH_0, PP_MESH_0, PP_MESH_1, PP_MESH_1] def forward(self, input): out = self.norm(input) @@ -99,6 +103,11 @@ def forward(self, input): return out +def loss_fn(pred, label): + loss = F.l1_loss(pred, label) + return loss + + def apply_pass(schedule_mode, acc_step): strategy = auto.Strategy() strategy.auto_mode = "semi" @@ -126,8 +135,8 @@ def __init__(self, num_samples): self.num_samples = num_samples def __getitem__(self, index): - input = np.random.uniform(size=1024).astype("float32") - label = np.random.randint(0, 9, dtype="int64") + input = np.random.uniform(size=784).astype("float32") + label = np.random.uniform(size=1).astype("float32") return input, label def __len__(self): @@ -136,8 +145,6 @@ def __len__(self): class TestVPPPass(unittest.TestCase): def setUp(self): - self.rtol = 1e-5 - self.atol = 1e-8 self.batch_size = 4 self.batch_num = 10 self.clip_norm = 0.2 @@ -151,23 +158,50 @@ def init(self, engine): place = paddle.base.CUDAPlace(ParallelEnv().dev_id) engine._executor = paddle.static.Executor(place) - def get_engine(self, schedule_mode, acc_step): + def get_engine(self, schedule_mode, acc_step, manual=True): reset_prog() strategy = apply_pass(schedule_mode, acc_step) clip = paddle.nn.ClipGradByGlobalNorm(self.clip_norm) opt = paddle.optimizer.AdamW(learning_rate=0.00001, grad_clip=clip) - model = MLPLayer() - loss = paddle.nn.CrossEntropyLoss() + model = MLPLayer(manual=manual) - engine = auto.Engine(model, loss, opt, strategy=strategy) + engine = auto.Engine(model, loss_fn, opt, strategy=strategy) self.init(engine) return engine def test_pp_pass(self): - # pp2-vpp - engine = self.get_engine(schedule_mode="VPP", acc_step=4) - engine.fit(self.dataset, batch_size=self.batch_size, log_freq=1) + # pp2-vpp-manual + engine = self.get_engine(schedule_mode="VPP", acc_step=4, manual=True) + out_manual = engine.fit( + self.dataset, batch_size=self.batch_size, log_freq=1 + ) + assert engine._strategy.pipeline.schedule_mode == "VPP" + + fw_chunk_ids = [] + bw_chunk_ids = [] + for op in engine.main_program.global_block().ops: + if is_optimize_op(op): + break + + dist_op = engine.dist_context.get_dist_op_for_program(op) + if is_forward_op(op): + fw_chunk_ids.append(dist_op.dist_attr.chunk_id) + if is_backward_op(op): + bw_chunk_ids.append(dist_op.dist_attr.chunk_id) + + if paddle.distributed.get_rank() == 0: + self.assertEqual(sum(fw_chunk_ids), 8) + self.assertEqual(sum(bw_chunk_ids), 13) + else: + self.assertEqual(sum(fw_chunk_ids), 12) + self.assertEqual(sum(bw_chunk_ids), 19) + + # pp2-vpp-auto + engine = self.get_engine(schedule_mode="VPP", acc_step=4, manual=False) + out_auto = engine.fit( + self.dataset, batch_size=self.batch_size, log_freq=1 + ) assert engine._strategy.pipeline.schedule_mode == "VPP" fw_chunk_ids = [] @@ -183,11 +217,17 @@ def test_pp_pass(self): bw_chunk_ids.append(dist_op.dist_attr.chunk_id) if paddle.distributed.get_rank() == 0: - assert sum(fw_chunk_ids) == 8 - assert sum(bw_chunk_ids) == 13 + self.assertEqual(sum(fw_chunk_ids), 9) + self.assertEqual(sum(bw_chunk_ids), 13) else: - assert sum(fw_chunk_ids) == 12 - assert sum(bw_chunk_ids) == 18 + self.assertEqual(sum(fw_chunk_ids), 13) + self.assertEqual(sum(bw_chunk_ids), 19) + + if paddle.distributed.get_rank() == 1: + self.assertEqual( + np.mean(out_manual.history["loss"][0]), + np.mean(out_auto.history["loss"][0]), + ) if __name__ == "__main__":