Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[PHI decoupling] simplify "convert_utils.h" in fluid #48168

Merged
merged 13 commits into from
Nov 24, 2022
Merged

[PHI decoupling] simplify "convert_utils.h" in fluid #48168

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
320623a
rm dependence to "convert_utils.h" in some files
huangjiyi Nov 19, 2022
dca7c14
fix bugs
huangjiyi Nov 19, 2022
e5fc728
replace DataType2String with DataTypeToString
huangjiyi Nov 19, 2022
d4050c7
replace framework::DataTypeSize with phi::SizeOf
huangjiyi Nov 19, 2022
60745c1
mv convert_function from fluid to phi and rm old map
huangjiyi Nov 19, 2022
60b4a92
recommit with pre-commit
huangjiyi Nov 19, 2022
0756811
repalce ProtoVarType with ProtoDataType and update comment.
huangjiyi Nov 21, 2022
6277881
Merge remote-tracking branch 'upstream/develop' into clean_convert_utils
huangjiyi Nov 21, 2022
f9d0dd9
fix error about include "dnnl.hpp"
huangjiyi Nov 22, 2022
bbc51d2
Merge remote-tracking branch 'upstream/develop' into clean_convert_utils
huangjiyi Nov 22, 2022
9347cb2
revert add dep mkldnn to convert_utils in phi
huangjiyi Nov 22, 2022
9cd30a8
add mkldnn deps in convert_utils.h in phi
huangjiyi Nov 22, 2022
81064c6
move deps to convert_utils.h in phi
huangjiyi Nov 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 14 additions & 18 deletions paddle/fluid/distributed/ps/service/brpc_utils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,13 @@ void SerializeLodTensor(framework::Variable* var,
}
// IO Buffer
if (platform::is_cpu_place(tensor->place())) {
auto data_len = tensor->numel() * framework::DataTypeSize(tensor->dtype());
auto data_len = tensor->numel() * phi::SizeOf(tensor->dtype());
iobuf->append(reinterpret_cast<const char*>(&data_len), 8);
iobuf->append(reinterpret_cast<const char*>(tensor->data()), data_len);
} else {
#ifdef PADDLE_WITH_CUDA
char* temp_ptr =
new char[tensor->numel() *
framework::DataTypeSize(tensor->dtype())]; // NOLINT
new char[tensor->numel() * phi::SizeOf(tensor->dtype())]; // NOLINT
auto stream = reinterpret_cast<const phi::GPUContext&>(ctx).stream();
memory::Copy(
platform::CPUPlace(),
Expand All @@ -128,7 +127,7 @@ void SerializeLodTensor(framework::Variable* var,
tensor->numel() * framework::SizeOfType(
framework::TransToProtoVarType(tensor->dtype())),
stream);
auto data_len = tensor->numel() * framework::DataTypeSize(tensor->dtype());
auto data_len = tensor->numel() * phi::SizeOf(tensor->dtype());
iobuf->append(reinterpret_cast<const char*>(&data_len), 8);
iobuf->append(reinterpret_cast<const char*>(temp_ptr), data_len);
delete[] temp_ptr;
Expand Down Expand Up @@ -159,14 +158,13 @@ void SerializeSelectedRows(framework::Variable* var,
}
// IO Buffer
if (platform::is_cpu_place(tensor->place())) {
auto data_len = tensor->numel() * framework::DataTypeSize(tensor->dtype());
auto data_len = tensor->numel() * phi::SizeOf(tensor->dtype());
iobuf->append(reinterpret_cast<const char*>(&data_len), 8);
iobuf->append(reinterpret_cast<const char*>(tensor->data()), data_len);
} else {
#ifdef PADDLE_WITH_CUDA
char* temp_ptr =
new char[tensor->numel() *
framework::DataTypeSize(tensor->dtype())]; // NOLINT
new char[tensor->numel() * phi::SizeOf(tensor->dtype())]; // NOLINT
auto stream = reinterpret_cast<const phi::GPUContext&>(ctx).stream();
memory::Copy(
platform::CPUPlace(),
Expand All @@ -176,7 +174,7 @@ void SerializeSelectedRows(framework::Variable* var,
tensor->numel() * framework::SizeOfType(
framework::TransToProtoVarType(tensor->dtype())),
stream);
auto data_len = tensor->numel() * framework::DataTypeSize(tensor->dtype());
auto data_len = tensor->numel() * phi::SizeOf(tensor->dtype());
iobuf->append(reinterpret_cast<const char*>(&data_len), 8);
iobuf->append(reinterpret_cast<const char*>(temp_ptr), data_len);
delete[] temp_ptr;
Expand Down Expand Up @@ -259,16 +257,15 @@ void DeserializeLodTensor(framework::Variable* var,
#ifdef PADDLE_WITH_CUDA
unsigned long data_len; // NOLINT
char* temp_ptr =
new char[tensor->numel() *
framework::DataTypeSize(tensor->dtype())]; // NOLINT
io_buffer_itr.copy_and_forward((void*)(&data_len), 8); // NOLINT
io_buffer_itr.copy_and_forward((void*)temp_ptr, data_len); // NOLINT
new char[tensor->numel() * phi::SizeOf(tensor->dtype())]; // NOLINT
io_buffer_itr.copy_and_forward((void*)(&data_len), 8); // NOLINT
io_buffer_itr.copy_and_forward((void*)temp_ptr, data_len); // NOLINT
auto stream = reinterpret_cast<const phi::GPUContext&>(ctx).stream();
memory::Copy(place,
tensor_data,
platform::CPUPlace(),
(void*)temp_ptr, // NOLINT
tensor->numel() * framework::DataTypeSize(tensor->dtype()),
tensor->numel() * phi::SizeOf(tensor->dtype()),
stream);
delete[] temp_ptr;
#endif
Expand Down Expand Up @@ -303,17 +300,16 @@ void DeserializeSelectedRows(
} else if (platform::is_gpu_place(place)) {
#ifdef PADDLE_WITH_CUDA
char* temp_ptr =
new char[tensor->numel() *
framework::DataTypeSize(tensor->dtype())]; // NOLINT
unsigned long data_len; // NOLINT
io_buffer_itr.copy_and_forward((void*)(&data_len), 8); // NOLINT
new char[tensor->numel() * phi::SizeOf(tensor->dtype())]; // NOLINT
unsigned long data_len; // NOLINT
io_buffer_itr.copy_and_forward((void*)(&data_len), 8); // NOLINT
io_buffer_itr.copy_and_forward(temp_ptr, data_len);
auto stream = reinterpret_cast<const phi::GPUContext&>(ctx).stream();
memory::Copy(place,
tensor_data,
platform::CPUPlace(),
temp_ptr,
tensor->numel() * framework::DataTypeSize(tensor->dtype()),
tensor->numel() * phi::SizeOf(tensor->dtype()),
stream);
delete[] temp_ptr;
#endif
Expand Down
4 changes: 2 additions & 2 deletions paddle/fluid/distributed/ps/service/heter_client.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ int GetMicroId(const platform::DeviceContext& ctx,
} else {
#ifdef PADDLE_WITH_CUDA
std::vector<char> temp;
temp.resize(tensor->numel() * framework::DataTypeSize(tensor->dtype()));
temp.resize(tensor->numel() * phi::SizeOf(tensor->dtype()));
char* temp_ptr = temp.data();
auto stream = reinterpret_cast<const phi::GPUContext&>(ctx).stream();
memory::Copy(platform::CPUPlace(),
temp_ptr,
tensor->place(),
tensor->data(),
tensor->numel() * framework::DataTypeSize(tensor->dtype()),
tensor->numel() * phi::SizeOf(tensor->dtype()),
stream);
float* temp_ptr_float = reinterpret_cast<float*>(temp_ptr);
micro_id = static_cast<int>(temp_ptr_float[0]);
Expand Down
4 changes: 2 additions & 2 deletions paddle/fluid/eager/amp_auto_cast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ inline std::vector<paddle::experimental::Tensor> AmpAutoCasts(
std::string op_name) {
VLOG(6) << "AMP AmpAutoCasts:"
<< " inputs(" << inputs_name << ") dst_dtype("
<< paddle::framework::DataType2String(dst_dtype) << ").";
<< phi::DataTypeToString(dst_dtype) << ").";
std::vector<paddle::experimental::Tensor> inputs_casted;
for (auto& input : inputs) {
if (NeedCast(input, dst_dtype)) {
Expand All @@ -71,7 +71,7 @@ inline paddle::experimental::Tensor AmpAutoCast(
std::string op_name) {
VLOG(6) << "AMP AmpAutoCasts:"
<< " input(" << input_name << ") dst_dtype("
<< paddle::framework::DataType2String(dst_dtype) << ").";
<< phi::DataTypeToString(dst_dtype) << ").";
if (dst_dtype == paddle::experimental::DataType::FLOAT16) {
if (op_name == "run_program") {
return input;
Expand Down
4 changes: 2 additions & 2 deletions paddle/fluid/eager/eager_amp_auto_cast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ inline std::vector<paddle::experimental::Tensor> EagerAmpAutoCasts(
bool trace_backward = true) {
VLOG(6) << "AMP AmpAutoCasts:"
<< " inputs(" << inputs_name << ") dst_dtype("
<< paddle::framework::DataType2String(dst_dtype) << ").";
<< phi::DataTypeToString(dst_dtype) << ").";
std::vector<paddle::experimental::Tensor> inputs_casted;
for (auto& input : inputs) {
if (NeedCast(input, dst_dtype)) {
Expand All @@ -88,7 +88,7 @@ inline paddle::experimental::Tensor EagerAmpAutoCast(
bool trace_backward = true) {
VLOG(6) << "AMP AmpAutoCasts:"
<< " input(" << egr::EagerUtils::TensorStr(input) << " to dst_dtype("
<< paddle::framework::DataType2String(dst_dtype) << ").";
<< phi::DataTypeToString(dst_dtype) << ").";
if (dst_dtype == paddle::experimental::DataType::FLOAT16) {
if (op_name == "run_program") {
return input;
Expand Down
8 changes: 4 additions & 4 deletions paddle/fluid/eager/grad_node_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,16 @@ static void CheckTensor(const paddle::experimental::Tensor& pre,
"The tensor in before and after hook are not consistent"));
}
if (pre.initialized() && post.initialized()) {
VLOG(7) << paddle::framework::DataType2String(pre.dtype()) << " "
<< paddle::framework::DataType2String(post.dtype());
VLOG(7) << phi::DataTypeToString(pre.dtype()) << " "
<< phi::DataTypeToString(post.dtype());
PADDLE_ENFORCE_EQ(
pre.dtype(),
post.dtype(),
paddle::platform::errors::PermissionDenied(
"The dtype of tensor before(%s) and after(%s) hook are not "
"consistent",
paddle::framework::DataType2String(pre.dtype()),
paddle::framework::DataType2String(post.dtype())));
phi::DataTypeToString(pre.dtype()),
phi::DataTypeToString(post.dtype())));
PADDLE_ENFORCE_EQ(pre.place(),
post.place(),
paddle::platform::errors::PermissionDenied(
Expand Down
18 changes: 1 addition & 17 deletions paddle/fluid/framework/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,23 +1246,7 @@ cc_test(
SRCS phi_utils_test.cc
DEPS phi_utils)

if(WITH_GPU OR WITH_ROCM)
cc_library(
fluid_convert_utils
SRCS convert_utils.cc
DEPS data_type place gpu_info)
else()
cc_library(
fluid_convert_utils
SRCS convert_utils.cc
DEPS data_type place)
endif()

# every source file that includes "dnnl.h" must depends on mkldnn
# or, the first one should depends on mkldnn
if(WITH_MKLDNN)
add_dependencies(fluid_convert_utils mkldnn)
endif()
cc_library(fluid_convert_utils DEPS data_type)

cc_test(
convert_utils_test
Expand Down
166 changes: 0 additions & 166 deletions paddle/fluid/framework/convert_utils.cc
View file
Open in desktop

This file was deleted.

Loading