Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[xpu] support paddle compile with XFT #51082

Merged
merged 5 commits into from
Mar 6, 2023
Merged

[xpu] support paddle compile with XFT #51082

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3f907e0
[xpu] support paddle compile with XFT
mayang002 Mar 1, 2023
4157799
[xpu] support paddle compile with XFT
mayang002 Mar 1, 2023
a4730c5
[xpu] support paddle compile with XFT
mayang002 Mar 1, 2023
27ba73a
Update cmake/external/xpu.cmake
mayang002 Mar 2, 2023
2162fac
[xpu] support paddle compile with XFT: add WITH_XPU_XFT into env_dict…
mayang002 Mar 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ option(WITH_MPI "Compile PaddlePaddle with MPI" OFF)
option(WITH_TENSORRT "Compile PaddlePaddle with NVIDIA TensorRT" OFF)
option(WITH_XPU "Compile PaddlePaddle with BAIDU KUNLUN XPU" OFF)
option(WITH_XPU_KP "Compile PaddlePaddle with BAIDU XPU compiler " OFF)
option(WITH_XPU_XFT "Compile PaddlePaddle with BAIDU XPU-XFT" OFF)
option(WITH_MLU "Compile PaddlePaddle with CAMBRICON MLU" OFF)
option(WITH_WIN_DUMP_DBG "Compile with windows core dump debug mode" OFF)
option(WITH_ASCEND "Compile PaddlePaddle with ASCEND" OFF)
Expand All @@ -74,6 +75,9 @@ endif()
if(WITH_GPU AND WITH_XPU_KP)
message(FATAL_ERROR "Error when compile GPU and XPU2 at the same time")
endif()
if(WITH_GPU AND WITH_XPU_XFT)
message(FATAL_ERROR "Error when compile GPU and XPU-XFT at the same time")
endif()
if(WITH_GPU AND WITH_ASCEND)
message(FATAL_ERROR "Error when compile GPU and ASCEND at the same time")
endif()
Expand Down Expand Up @@ -419,6 +423,15 @@ if(WITH_XPU_KP AND NOT WITH_XPU)
CACHE STRING "Enable WITH_XPU when compiling with WITH_XPU_KP" FORCE)
endif()

if(NOT WITH_XPU AND WITH_XPU_XFT)
message(
WARNING
"Enable WITH_XPU when compiling with WITH_XPU_XFT. Force WITH_XPU=ON.")
set(WITH_XPU
ON
CACHE STRING "Enable WITH_XPU when compiling with WITH_XPU_XFT" FORCE)
endif()

if(NOT WITH_XPU AND WITH_XPU_BKCL)
message(
WARNING "Disable BKCL when compiling without XPU. Force WITH_XPU_BKCL=OFF.")
Expand Down
38 changes: 37 additions & 1 deletion cmake/external/xpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@ include(ExternalProject)
set(XPU_PROJECT "extern_xpu")
set(XPU_API_LIB_NAME "libxpuapi.so")
set(XPU_RT_LIB_NAME "libxpurt.so")
set(XPU_XFT_LIB_NAME "libxft.so")

set(XPU_BASE_DATE "20230227")
set(XPU_XCCL_BASE_VERSION "1.0.10")
set(XPU_XFT_BASE_VERSION "latest")

if(NOT DEFINED XPU_BASE_URL)
set(XPU_BASE_URL_WITHOUT_DATE
Expand All @@ -22,30 +24,42 @@ set(XPU_XCCL_BASE_URL
"https://klx-sdk-release-public.su.bcebos.com/xccl/release/${XPU_XCCL_BASE_VERSION}"
)

if(NOT XPU_XFT_BASE_URL)
set(XPU_XFT_BASE_URL
"https://klx-sdk-release-public.su.bcebos.com/xft/dev/${XPU_XFT_BASE_VERSION}"
)
endif()

if(WITH_AARCH64)
set(XPU_XRE_DIR_NAME "xre-kylin_aarch64")
set(XPU_XDNN_DIR_NAME "xdnn-kylin_aarch64")
set(XPU_XCCL_DIR_NAME "xccl-kylin_aarch64")
set(XPU_XFT_DIR_NAME "") # TODO: xft has no kylin output at now.
elseif(WITH_SUNWAY)
set(XPU_XRE_DIR_NAME "xre-deepin_sw6_64")
set(XPU_XDNN_DIR_NAME "xdnn-deepin_sw6_64")
set(XPU_XCCL_DIR_NAME "xccl-deepin_sw6_64")
set(XPU_XFT_DIR_NAME "") # TODO: xft has no deepin output at now.
elseif(WITH_BDCENTOS)
set(XPU_XRE_DIR_NAME "xre-bdcentos_x86_64")
set(XPU_XDNN_DIR_NAME "xdnn-bdcentos_x86_64")
set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
elseif(WITH_UBUNTU)
set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
elseif(WITH_CENTOS)
set(XPU_XRE_DIR_NAME "xre-centos7_x86_64")
set(XPU_XDNN_DIR_NAME "xdnn-centos7_x86_64")
set(XPU_XCCL_DIR_NAME "xccl-bdcentos_x86_64")
set(XPU_XFT_DIR_NAME "xft_bdcentos6u3_x86_64_gcc82")
else()
set(XPU_XRE_DIR_NAME "xre-ubuntu_x86_64")
set(XPU_XDNN_DIR_NAME "xdnn-ubuntu_x86_64")
set(XPU_XCCL_DIR_NAME "xccl-ubuntu_x86_64")
set(XPU_XFT_DIR_NAME "xft_ubuntu1604_x86_64")
endif()

set(XPU_XRE_URL
Expand All @@ -57,12 +71,16 @@ set(XPU_XDNN_URL
set(XPU_XCCL_URL
"${XPU_XCCL_BASE_URL}/${XPU_XCCL_DIR_NAME}.tar.gz"
CACHE STRING "" FORCE)
set(XPU_XFT_URL "${XPU_XFT_BASE_URL}/${XPU_XFT_DIR_NAME}.tar.gz")
set(XPU_PACK_DEPENCE_URL
"https://baidu-kunlun-public.su.bcebos.com/paddle_depence/pack_paddle_depence.sh"
CACHE STRING "" FORCE)
set(XPU_CHECK_DEPENCE_URL
"https://baidu-kunlun-public.su.bcebos.com/paddle_depence/check_xpu_dependence.sh"
CACHE STRING "" FORCE)
set(XPU_XFT_GET_DEPENCE_URL
"https://baidu-kunlun-public.su.bcebos.com/paddle_depence/get_xft_dependence.sh"
CACHE STRING "" FORCE)

set(SNAPPY_PREFIX_DIR "${THIRD_PARTY_PATH}/xpu")
set(XPU_DOWNLOAD_DIR "${SNAPPY_PREFIX_DIR}/src/${XPU_PROJECT}")
Expand Down Expand Up @@ -90,7 +108,9 @@ ExternalProject_Add(
wget ${XPU_CHECK_DEPENCE_URL} && bash check_xpu_dependence.sh
${XPU_BASE_URL} ${XPU_XCCL_BASE_URL} && wget ${XPU_PACK_DEPENCE_URL} && bash
pack_paddle_depence.sh ${XPU_XRE_URL} ${XPU_XRE_DIR_NAME} ${XPU_XDNN_URL}
${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME}
${XPU_XDNN_DIR_NAME} ${XPU_XCCL_URL} ${XPU_XCCL_DIR_NAME} && wget
${XPU_XFT_GET_DEPENCE_URL} && bash get_xft_dependence.sh ${XPU_XFT_URL}
${XPU_XFT_DIR_NAME}
DOWNLOAD_NO_PROGRESS 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT}
Expand All @@ -116,7 +136,23 @@ if(WITH_XPU_BKCL)
set(XPU_BKCL_LIB "${XPU_LIB_DIR}/${XPU_BKCL_LIB_NAME}")
set(XPU_BKCL_INC_DIR "${THIRD_PARTY_PATH}/install/xpu/include")
include_directories(${XPU_BKCL_INC_DIR})
endif()

if(WITH_XPU_XFT)
message(STATUS "Compile with XPU XFT!")
add_definitions(-DPADDLE_WITH_XPU_XFT)

set(XPU_XFT_LIB_DIR "${XPU_INSTALL_DIR}/lib")
set(XPU_XFT_LIB "${XPU_XFT_LIB_DIR}/${XPU_XFT_LIB_NAME}")
mayang002 marked this conversation as resolved.
Show resolved Hide resolved
endif()

if(WITH_XPU_BKCL AND WITH_XPU_XFT)
target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_BKCL_LIB}
${XPU_XFT_LIB})
elseif(WITH_XPU_BKCL)
target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_BKCL_LIB})
elseif(WITH_XPU_XFT)
target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB} ${XPU_XFT_LIB})
else()
target_link_libraries(xpulib ${XPU_API_LIB} ${XPU_RT_LIB})
endif()
Expand Down
4 changes: 4 additions & 0 deletions python/setup.py.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,10 @@ if '${WITH_XPU_BKCL}' == 'ON':
shutil.copy('${XPU_BKCL_LIB}', libs_path)
package_data['paddle.libs']+=['${XPU_BKCL_LIB_NAME}']

if '${WITH_XPU_XFT}' == 'ON':
shutil.copy('${XPU_XFT_LIB}', libs_path)
package_data['paddle.libs']+=['${XPU_XFT_LIB_NAME}']

# remove unused paddle/libs/__init__.py
if os.path.isfile(libs_path+'/__init__.py'):
os.remove(libs_path+'/__init__.py')
Expand Down