From a283ec201f37ef42d12e8c640bbc551104108aa0 Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Mon, 6 Dec 2021 19:58:09 +0800 Subject: [PATCH 01/11] feat: zookeeper client support kerberos --- scripts/linux/clear_zk.sh | 2 +- scripts/linux/start_zk.sh | 25 +++++++++++------- scripts/linux/stop_zk.sh | 2 +- src/meta/test/CMakeLists.txt | 2 ++ src/runtime/security/init.cpp | 17 ++++++++++++ src/runtime/security/init.h | 3 +++ src/runtime/security/kinit_context.cpp | 6 +++-- src/runtime/security/negotiation.cpp | 1 + src/runtime/service_api_c.cpp | 9 +++++++ src/zookeeper/CMakeLists.txt | 2 ++ src/zookeeper/test/CMakeLists.txt | 2 ++ src/zookeeper/zookeeper_session.cpp | 36 +++++++++++++++++++++----- thirdparty/CMakeLists.txt | 13 +++++----- 13 files changed, 94 insertions(+), 26 deletions(-) diff --git a/scripts/linux/clear_zk.sh b/scripts/linux/clear_zk.sh index 1d4823946a..50de0d75be 100755 --- a/scripts/linux/clear_zk.sh +++ b/scripts/linux/clear_zk.sh @@ -11,7 +11,7 @@ fi cd $INSTALL_DIR -ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10 +ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin if [ -d "$ZOOKEEPER_HOME" ] then diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh index f26fdc558f..e442c6d5d5 100755 --- a/scripts/linux/start_zk.sh +++ b/scripts/linux/start_zk.sh @@ -27,29 +27,34 @@ fi cd $INSTALL_DIR -ZOOKEEPER_PKG=${PROJECT_DIR}/thirdparty/build/Download/zookeeper/zookeeper-3.4.10.tar.gz -if [ ! -f ${ZOOKEEPER_PKG} ]; then - echo "no such file \"${ZOOKEEPER_PKG}\"" - echo "please install third-parties first" - exit 1 +ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin +ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz + +if [ ! -f $ZOOKEEPER_TAR_NAME ]; then + echo "Downloading zookeeper..." + download_url="https://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz" + if ! wget -T 5 -t 1 $download_url; then + echo "ERROR: download zookeeper failed" + exit 1 + fi fi -if [ ! -d zookeeper-3.4.10 ]; then +if [ ! -d $ZOOKEEPER_ROOT ]; then echo "Decompressing zookeeper..." - cp ${ZOOKEEPER_PKG} . - tar xf zookeeper-3.4.10.tar.gz - if [ $? -ne 0 ]; then + if ! tar xf $ZOOKEEPER_TAR_NAME; then echo "ERROR: decompress zookeeper failed" exit 1 fi fi -ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10 +ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT ZOOKEEPER_PORT=$PORT cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@" $ZOOKEEPER_HOME/conf/zoo.cfg sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@" $ZOOKEEPER_HOME/conf/zoo.cfg +echo "admin.enableServer=false" >> $ZOOKEEPER_HOME/conf/zoo.cfg +echo "4lw.commands.whitelist=ruok" >> $ZOOKEEPER_HOME/conf/zoo.cfg mkdir -p $ZOOKEEPER_HOME/data $ZOOKEEPER_HOME/bin/zkServer.sh start diff --git a/scripts/linux/stop_zk.sh b/scripts/linux/stop_zk.sh index be38eea380..8bb43be622 100755 --- a/scripts/linux/stop_zk.sh +++ b/scripts/linux/stop_zk.sh @@ -11,7 +11,7 @@ fi cd $INSTALL_DIR -ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10 +ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin if [ -d "$ZOOKEEPER_HOME" ] then diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index a429f08df4..bf9190bfb0 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -8,6 +8,8 @@ file(GLOB MY_PROJ_SRC ) set(MY_PROJ_SRC ${MY_PROJ_SRC} misc/misc.cpp) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lssl -lcrypto") + # Search mode for source files under CURRENT project directory? # "GLOB_RECURSE" for recursive search # "GLOB" for non-recursive search diff --git a/src/runtime/security/init.cpp b/src/runtime/security/init.cpp index 99e515f6a2..1b42e313ef 100644 --- a/src/runtime/security/init.cpp +++ b/src/runtime/security/init.cpp @@ -68,5 +68,22 @@ bool init(bool is_server) return true; } +bool init_for_zookeeper_client() +{ + error_s err = run_kinit(); + if (!err.is_ok()) { + derror_f("initialize kerberos failed, with err = {}", err.description()); + return false; + } + ddebug("initialize kerberos for zookeeper client succeed"); + + err = init_sasl(false); + if (!err.is_ok()) { + derror_f("initialize sasl failed, with err = {}", err.description()); + return false; + } + ddebug("initialize sasl for zookeeper client succeed"); + return true; +} } // namespace security } // namespace dsn diff --git a/src/runtime/security/init.h b/src/runtime/security/init.h index aa4abadd14..f17c53f47f 100644 --- a/src/runtime/security/init.h +++ b/src/runtime/security/init.h @@ -23,5 +23,8 @@ namespace dsn { namespace security { // init security(kerberos and sasl) bool init(bool is_server); + +// init security only for zookeeper client(kerberos and sasl) +bool init_for_zookeeper_client(); } // namespace security } // namespace dsn diff --git a/src/runtime/security/kinit_context.cpp b/src/runtime/security/kinit_context.cpp index 4767783ce4..e7bd41e3f7 100644 --- a/src/runtime/security/kinit_context.cpp +++ b/src/runtime/security/kinit_context.cpp @@ -33,6 +33,7 @@ namespace dsn { namespace security { DSN_DECLARE_bool(enable_auth); +DSN_DECLARE_bool(enable_zookeeper_kerberos); #define KRB5_RETURN_NOT_OK(err, msg) \ do { \ @@ -53,8 +54,9 @@ DSN_DEFINE_string("security", service_name, "", "service name"); // will not pass. error_s check_configuration() { - dassert(FLAGS_enable_auth, - "There is no need to check configuration if FLAGS_enable_auth is not true"); + dassert(FLAGS_enable_auth || FLAGS_enable_zookeeper_kerberos, + "There is no need to check configuration if FLAGS_enable_auth" + " and FLAGS_enable_zookeeper_kerberos both are not true"); if (0 == strlen(FLAGS_krb5_keytab) || !utils::filesystem::file_exists(FLAGS_krb5_keytab)) { return error_s::make(ERR_INVALID_PARAMETERS, diff --git a/src/runtime/security/negotiation.cpp b/src/runtime/security/negotiation.cpp index e5934a6829..cb2d000728 100644 --- a/src/runtime/security/negotiation.cpp +++ b/src/runtime/security/negotiation.cpp @@ -32,6 +32,7 @@ const std::set supported_mechanisms{"GSSAPI"}; DSN_DEFINE_bool("security", enable_auth, false, "whether open auth or not"); DSN_DEFINE_bool("security", mandatory_auth, false, "wheter to do authertication mandatorily"); +DSN_DEFINE_bool("security", enable_zookeeper_kerberos, false, "whether to enable kerberos for zookeeper client"); DSN_TAG_VARIABLE(mandatory_auth, FT_MUTABLE); negotiation::~negotiation() {} diff --git a/src/runtime/service_api_c.cpp b/src/runtime/service_api_c.cpp index 649af2611b..19f3f8ee9b 100644 --- a/src/runtime/service_api_c.cpp +++ b/src/runtime/service_api_c.cpp @@ -56,6 +56,7 @@ namespace dsn { namespace security { DSN_DECLARE_bool(enable_auth); +DSN_DECLARE_bool(enable_zookeeper_kerberos); } // namespace security } // namespace dsn // @@ -470,6 +471,14 @@ bool run(const char *config_file, if (!dsn::security::init(is_server)) { return false; } + // if FLAGS_enable_auth is false but FLAGS_enable_zookeeper_kerberos, we should init kerberos for it separately + // include two steps: + // 1) apply kerberos ticket and keep it valid + // 2) complete sasl init for client(use FLAGS_sasl_plugin_path) + } else if (dsn::security::FLAGS_enable_zookeeper_kerberos && app_list == "meta") { + if (!dsn::security::init_for_zookeeper_client()) { + return false; + } } // split app_name and app_index diff --git a/src/zookeeper/CMakeLists.txt b/src/zookeeper/CMakeLists.txt index 3a226b4ad6..e42b2b6c58 100644 --- a/src/zookeeper/CMakeLists.txt +++ b/src/zookeeper/CMakeLists.txt @@ -17,4 +17,6 @@ set(MY_BINPLACES "") dsn_add_static_library() +add_definitions(-DHAVE_CYRUS_SASL_H) + add_subdirectory(test) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index cb64c4795e..adb856a792 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -9,6 +9,8 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lssl -lcrypto") + set(MY_PROJ_LIBS dsn.replication.zookeeper_provider dsn_runtime diff --git a/src/zookeeper/zookeeper_session.cpp b/src/zookeeper/zookeeper_session.cpp index ad8fd5c07c..8bac2e5738 100644 --- a/src/zookeeper/zookeeper_session.cpp +++ b/src/zookeeper/zookeeper_session.cpp @@ -33,10 +33,20 @@ */ #include +#include #include "zookeeper_session.h" #include "zookeeper_session_mgr.h" +#include + +namespace dsn { +namespace security { +DSN_DECLARE_bool(enable_zookeeper_kerberos); +DSN_DEFINE_string("security", zookeeper_kerberos_service_name, "zookeeper", "zookeeper kerberos service name"); +} // namespace security +} // namespace dsn + namespace dsn { namespace dist { @@ -141,12 +151,26 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb) { utils::auto_write_lock l(_watcher_lock); if (nullptr == _handle) { - _handle = zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(), - global_watcher, - zookeeper_session_mgr::instance().timeout(), - nullptr, - this, - 0); + if (dsn::security::FLAGS_enable_zookeeper_kerberos) { + zoo_sasl_params_t sasl_params = { 0 }; + sasl_params.service = dsn::security::FLAGS_zookeeper_kerberos_service_name; + sasl_params.mechlist = "GSSAPI"; + _handle = zookeeper_init_sasl(zookeeper_session_mgr::instance().zoo_hosts(), + global_watcher, + zookeeper_session_mgr::instance().timeout(), + nullptr, + this, + 0, + NULL, + &sasl_params); + } else { + _handle = zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(), + global_watcher, + zookeeper_session_mgr::instance().timeout(), + nullptr, + this, + 0); + } dassert(_handle != nullptr, "zookeeper session init failed"); } diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index c7b27be74d..3936655df1 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -153,14 +153,15 @@ if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () ExternalProject_Add(zookeeper - URL ${OSS_URL_PREFIX}/zookeeper-3.4.10.tar.gz - http://ftp.jaist.ac.jp/pub/apache/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz - URL_MD5 e4cf1b1593ca870bf1c7a75188f09678 - CONFIGURE_COMMAND cd src/c && CFLAGS=${ZOOKEEPER_CFLAGS} ./configure --enable-static=yes --enable-shared=no --prefix=${TP_OUTPUT} --with-pic=yes - BUILD_COMMAND cd src/c && make - INSTALL_COMMAND cd src/c && make install + URL https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz + https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz + URL_MD5 44c2a33e01931aed94ef7f3d39d0963e + CONFIGURE_COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && autoreconf -if && CFLAGS=${ZOOKEEPER_CFLAGS} ./configure --disable-shared --prefix=${TP_OUTPUT} --enable-debug --with-pic=yes --with-sasl=${TP_OUTPUT} + BUILD_COMMAND cd zookeeper-client/zookeeper-client-c && make + INSTALL_COMMAND cd zookeeper-client/zookeeper-client-c && make install BUILD_IN_SOURCE 1 ) +add_dependencies(zookeeper cyrus-sasl) ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz From 41ae98e8987efae93028b4c737794fbd4d773d1d Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Tue, 7 Dec 2021 20:02:26 +0800 Subject: [PATCH 02/11] format code --- src/runtime/security/negotiation.cpp | 5 ++++- src/runtime/service_api_c.cpp | 9 +++++---- src/zookeeper/zookeeper_session.cpp | 7 +++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/runtime/security/negotiation.cpp b/src/runtime/security/negotiation.cpp index cb2d000728..bb109bc838 100644 --- a/src/runtime/security/negotiation.cpp +++ b/src/runtime/security/negotiation.cpp @@ -32,7 +32,10 @@ const std::set supported_mechanisms{"GSSAPI"}; DSN_DEFINE_bool("security", enable_auth, false, "whether open auth or not"); DSN_DEFINE_bool("security", mandatory_auth, false, "wheter to do authertication mandatorily"); -DSN_DEFINE_bool("security", enable_zookeeper_kerberos, false, "whether to enable kerberos for zookeeper client"); +DSN_DEFINE_bool("security", + enable_zookeeper_kerberos, + false, + "whether to enable kerberos for zookeeper client"); DSN_TAG_VARIABLE(mandatory_auth, FT_MUTABLE); negotiation::~negotiation() {} diff --git a/src/runtime/service_api_c.cpp b/src/runtime/service_api_c.cpp index 19f3f8ee9b..f36730afab 100644 --- a/src/runtime/service_api_c.cpp +++ b/src/runtime/service_api_c.cpp @@ -471,10 +471,11 @@ bool run(const char *config_file, if (!dsn::security::init(is_server)) { return false; } - // if FLAGS_enable_auth is false but FLAGS_enable_zookeeper_kerberos, we should init kerberos for it separately - // include two steps: - // 1) apply kerberos ticket and keep it valid - // 2) complete sasl init for client(use FLAGS_sasl_plugin_path) + // if FLAGS_enable_auth is false but FLAGS_enable_zookeeper_kerberos, we should init + // kerberos for it separately + // include two steps: + // 1) apply kerberos ticket and keep it valid + // 2) complete sasl init for client(use FLAGS_sasl_plugin_path) } else if (dsn::security::FLAGS_enable_zookeeper_kerberos && app_list == "meta") { if (!dsn::security::init_for_zookeeper_client()) { return false; diff --git a/src/zookeeper/zookeeper_session.cpp b/src/zookeeper/zookeeper_session.cpp index 8bac2e5738..4c9bb5b9fa 100644 --- a/src/zookeeper/zookeeper_session.cpp +++ b/src/zookeeper/zookeeper_session.cpp @@ -43,7 +43,10 @@ namespace dsn { namespace security { DSN_DECLARE_bool(enable_zookeeper_kerberos); -DSN_DEFINE_string("security", zookeeper_kerberos_service_name, "zookeeper", "zookeeper kerberos service name"); +DSN_DEFINE_string("security", + zookeeper_kerberos_service_name, + "zookeeper", + "zookeeper kerberos service name"); } // namespace security } // namespace dsn @@ -152,7 +155,7 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb) utils::auto_write_lock l(_watcher_lock); if (nullptr == _handle) { if (dsn::security::FLAGS_enable_zookeeper_kerberos) { - zoo_sasl_params_t sasl_params = { 0 }; + zoo_sasl_params_t sasl_params = {0}; sasl_params.service = dsn::security::FLAGS_zookeeper_kerberos_service_name; sasl_params.mechlist = "GSSAPI"; _handle = zookeeper_init_sasl(zookeeper_session_mgr::instance().zoo_hosts(), From a11921d19d950da7540eeea4124de174a49e66eb Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Fri, 17 Dec 2021 19:02:23 +0800 Subject: [PATCH 03/11] cr 12.17 --- scripts/linux/start_zk.sh | 5 +++++ src/runtime/security/negotiation.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh index e442c6d5d5..309f7b3850 100755 --- a/scripts/linux/start_zk.sh +++ b/scripts/linux/start_zk.sh @@ -29,6 +29,7 @@ cd $INSTALL_DIR ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz +ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a" if [ ! -f $ZOOKEEPER_TAR_NAME ]; then echo "Downloading zookeeper..." @@ -37,6 +38,10 @@ if [ ! -f $ZOOKEEPER_TAR_NAME ]; then echo "ERROR: download zookeeper failed" exit 1 fi + if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` != $ZOOKEEPER_TAR_MD5_VALUE ]; then + echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!" + exit 1 + fi fi if [ ! -d $ZOOKEEPER_ROOT ]; then diff --git a/src/runtime/security/negotiation.cpp b/src/runtime/security/negotiation.cpp index bb109bc838..f2819e87fc 100644 --- a/src/runtime/security/negotiation.cpp +++ b/src/runtime/security/negotiation.cpp @@ -31,11 +31,11 @@ namespace security { const std::set supported_mechanisms{"GSSAPI"}; DSN_DEFINE_bool("security", enable_auth, false, "whether open auth or not"); -DSN_DEFINE_bool("security", mandatory_auth, false, "wheter to do authertication mandatorily"); DSN_DEFINE_bool("security", enable_zookeeper_kerberos, false, "whether to enable kerberos for zookeeper client"); +DSN_DEFINE_bool("security", mandatory_auth, false, "wheter to do authertication mandatorily"); DSN_TAG_VARIABLE(mandatory_auth, FT_MUTABLE); negotiation::~negotiation() {} From a92c4ced03640169ea65f0d7e4592e57c0d0d2d4 Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Thu, 24 Mar 2022 19:13:52 +0800 Subject: [PATCH 04/11] tmp --- thirdparty/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 3936655df1..be24b826eb 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -153,8 +153,8 @@ if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () ExternalProject_Add(zookeeper - URL https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz - https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz + URL http://jfrog.sensorsdata.cn:80/artifactory/dragon-internal/inf/skv/apache-zookeeper-3.7.0.tar.gz + https://archive.apache.org/dist/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz URL_MD5 44c2a33e01931aed94ef7f3d39d0963e CONFIGURE_COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && autoreconf -if && CFLAGS=${ZOOKEEPER_CFLAGS} ./configure --disable-shared --prefix=${TP_OUTPUT} --enable-debug --with-pic=yes --with-sasl=${TP_OUTPUT} BUILD_COMMAND cd zookeeper-client/zookeeper-client-c && make From 596db3ac37c8af09c1cdbdb66caf365d1f6c626a Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Tue, 29 Mar 2022 21:22:18 +0800 Subject: [PATCH 05/11] change use lib style --- src/meta/test/CMakeLists.txt | 3 +-- src/zookeeper/test/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index bf9190bfb0..20dbc9706a 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -8,8 +8,6 @@ file(GLOB MY_PROJ_SRC ) set(MY_PROJ_SRC ${MY_PROJ_SRC} misc/misc.cpp) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lssl -lcrypto") - # Search mode for source files under CURRENT project directory? # "GLOB_RECURSE" for recursive search # "GLOB" for non-recursive search @@ -35,6 +33,7 @@ set(MY_PROJ_LIBS PocoJSON crypto gtest + ssl ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index adb856a792..a128f30575 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -9,13 +9,13 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lssl -lcrypto") - set(MY_PROJ_LIBS dsn.replication.zookeeper_provider dsn_runtime zookeeper_mt gtest + ssl + crypto ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) From 93c9fb483d9e904f8c166bdab0291b19f331ea50 Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Wed, 30 Mar 2022 23:00:54 +0800 Subject: [PATCH 06/11] change download zk url --- thirdparty/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index be24b826eb..61efad7d53 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -153,9 +153,9 @@ if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () ExternalProject_Add(zookeeper - URL http://jfrog.sensorsdata.cn:80/artifactory/dragon-internal/inf/skv/apache-zookeeper-3.7.0.tar.gz - https://archive.apache.org/dist/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz - URL_MD5 44c2a33e01931aed94ef7f3d39d0963e + URL https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz + https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz + URL_MD5 c564fbf2877f611890522e4aa6e4eb24 CONFIGURE_COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && autoreconf -if && CFLAGS=${ZOOKEEPER_CFLAGS} ./configure --disable-shared --prefix=${TP_OUTPUT} --enable-debug --with-pic=yes --with-sasl=${TP_OUTPUT} BUILD_COMMAND cd zookeeper-client/zookeeper-client-c && make INSTALL_COMMAND cd zookeeper-client/zookeeper-client-c && make install From 947bbd84dd00c9e3f1e4e99ba542f16df79a822f Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Thu, 7 Apr 2022 14:23:54 +0800 Subject: [PATCH 07/11] from laiyingchun --- src/failure_detector/test/CMakeLists.txt | 1 + src/meta/CMakeLists.txt | 2 +- src/meta/test/CMakeLists.txt | 3 +- .../test/balancer_simulator/CMakeLists.txt | 1 + src/meta/test/meta_state/CMakeLists.txt | 1 + src/replica/backup/test/CMakeLists.txt | 1 + src/replica/bulk_load/test/CMakeLists.txt | 1 + src/replica/duplication/test/CMakeLists.txt | 3 +- src/replica/split/test/CMakeLists.txt | 1 + src/replica/storage/simple_kv/CMakeLists.txt | 2 +- .../storage/simple_kv/test/CMakeLists.txt | 3 +- src/replica/test/CMakeLists.txt | 3 +- src/zookeeper/CMakeLists.txt | 2 +- src/zookeeper/test/CMakeLists.txt | 3 +- thirdparty/CMakeLists.txt | 41 ++++--------------- 15 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/failure_detector/test/CMakeLists.txt b/src/failure_detector/test/CMakeLists.txt index d739a5cd27..a0d22102fb 100644 --- a/src/failure_detector/test/CMakeLists.txt +++ b/src/failure_detector/test/CMakeLists.txt @@ -16,6 +16,7 @@ set(MY_PROJ_LIBS dsn_replication_common dsn.failure_detector gtest + hashtable ) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/meta/CMakeLists.txt b/src/meta/CMakeLists.txt index 2e1e582ca7..9f9197ac53 100644 --- a/src/meta/CMakeLists.txt +++ b/src/meta/CMakeLists.txt @@ -21,7 +21,7 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_aio - zookeeper_mt + zookeeper galaxy-fds-sdk-cpp PocoNet PocoFoundation diff --git a/src/meta/test/CMakeLists.txt b/src/meta/test/CMakeLists.txt index b0cb7ff711..bd3bd8d841 100644 --- a/src/meta/test/CMakeLists.txt +++ b/src/meta/test/CMakeLists.txt @@ -25,7 +25,8 @@ set(MY_PROJ_LIBS dsn_http dsn_runtime dsn_aio - zookeeper_mt + zookeeper + hashtable galaxy-fds-sdk-cpp PocoNet PocoFoundation diff --git a/src/meta/test/balancer_simulator/CMakeLists.txt b/src/meta/test/balancer_simulator/CMakeLists.txt index 534e3e3bf0..3719d9f80b 100644 --- a/src/meta/test/balancer_simulator/CMakeLists.txt +++ b/src/meta/test/balancer_simulator/CMakeLists.txt @@ -13,6 +13,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replication_common dsn_runtime + hashtable gtest) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/meta/test/meta_state/CMakeLists.txt b/src/meta/test/meta_state/CMakeLists.txt index 9fdf3512fb..e4ec1a6b8a 100644 --- a/src/meta/test/meta_state/CMakeLists.txt +++ b/src/meta/test/meta_state/CMakeLists.txt @@ -14,6 +14,7 @@ set(MY_PROJ_LIBS dsn_replica_server dsn_replication_common dsn_runtime + hashtable gtest ) diff --git a/src/replica/backup/test/CMakeLists.txt b/src/replica/backup/test/CMakeLists.txt index eeaa56d71e..37ddb2101f 100644 --- a/src/replica/backup/test/CMakeLists.txt +++ b/src/replica/backup/test/CMakeLists.txt @@ -11,6 +11,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn.block_service.local dsn.block_service.fds dsn_utils + hashtable gtest ) diff --git a/src/replica/bulk_load/test/CMakeLists.txt b/src/replica/bulk_load/test/CMakeLists.txt index 9140a759b1..fdbf418065 100644 --- a/src/replica/bulk_load/test/CMakeLists.txt +++ b/src/replica/bulk_load/test/CMakeLists.txt @@ -8,6 +8,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replica_server dsn_replication_common dsn_runtime + hashtable gtest ) diff --git a/src/replica/duplication/test/CMakeLists.txt b/src/replica/duplication/test/CMakeLists.txt index 0eb03e9199..6bac15ea07 100644 --- a/src/replica/duplication/test/CMakeLists.txt +++ b/src/replica/duplication/test/CMakeLists.txt @@ -10,7 +10,8 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replication_common dsn.failure_detector dsn_utils - zookeeper_mt + zookeeper + hashtable gtest ) diff --git a/src/replica/split/test/CMakeLists.txt b/src/replica/split/test/CMakeLists.txt index ee7ca688b7..890348c821 100644 --- a/src/replica/split/test/CMakeLists.txt +++ b/src/replica/split/test/CMakeLists.txt @@ -8,6 +8,7 @@ set(MY_PROJ_LIBS dsn_meta_server dsn_replica_server dsn_replication_common dsn_runtime + hashtable gtest ) diff --git a/src/replica/storage/simple_kv/CMakeLists.txt b/src/replica/storage/simple_kv/CMakeLists.txt index aaf7029fe8..390dab46d5 100644 --- a/src/replica/storage/simple_kv/CMakeLists.txt +++ b/src/replica/storage/simple_kv/CMakeLists.txt @@ -13,7 +13,7 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS}) # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime) +set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/replica/storage/simple_kv/test/CMakeLists.txt b/src/replica/storage/simple_kv/test/CMakeLists.txt index 17eba92636..af8cda0f6c 100644 --- a/src/replica/storage/simple_kv/test/CMakeLists.txt +++ b/src/replica/storage/simple_kv/test/CMakeLists.txt @@ -12,7 +12,8 @@ set(MY_PROJ_LIBS dsn_replica_server dsn.failure_detector dsn.replication.zookeeper_provider dsn_runtime - zookeeper_mt + zookeeper + hashtable gtest ) diff --git a/src/replica/test/CMakeLists.txt b/src/replica/test/CMakeLists.txt index 8e4dbfe519..e4900ef849 100644 --- a/src/replica/test/CMakeLists.txt +++ b/src/replica/test/CMakeLists.txt @@ -20,7 +20,8 @@ set(MY_PROJ_LIBS dsn_meta_server dsn.failure_detector dsn_http dsn_runtime - zookeeper_mt + zookeeper + hashtable gtest) set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) diff --git a/src/zookeeper/CMakeLists.txt b/src/zookeeper/CMakeLists.txt index e42b2b6c58..0d244249fd 100644 --- a/src/zookeeper/CMakeLists.txt +++ b/src/zookeeper/CMakeLists.txt @@ -10,7 +10,7 @@ set(MY_PROJ_SRC "") # "GLOB" for non-recursive search set(MY_SRC_SEARCH_MODE "GLOB") -set(MY_PROJ_LIBS "") +set(MY_PROJ_LIBS zookeeper hashtable ssl crypto) # Extra files that will be installed set(MY_BINPLACES "") diff --git a/src/zookeeper/test/CMakeLists.txt b/src/zookeeper/test/CMakeLists.txt index a128f30575..a7d850c3e2 100644 --- a/src/zookeeper/test/CMakeLists.txt +++ b/src/zookeeper/test/CMakeLists.txt @@ -12,7 +12,8 @@ set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS dsn.replication.zookeeper_provider dsn_runtime - zookeeper_mt + zookeeper + hashtable gtest ssl crypto diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 1b395b4b84..170b8c3897 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -163,12 +163,15 @@ ExternalProject_Add(zookeeper URL https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz URL_MD5 c564fbf2877f611890522e4aa6e4eb24 - CONFIGURE_COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && autoreconf -if && CFLAGS=${ZOOKEEPER_CFLAGS} ./configure --disable-shared --prefix=${TP_OUTPUT} --enable-debug --with-pic=yes --with-sasl=${TP_OUTPUT} - BUILD_COMMAND cd zookeeper-client/zookeeper-client-c && make - INSTALL_COMMAND cd zookeeper-client/zookeeper-client-c && make install + PATCH_COMMAND "" + COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -WITH_OPENSSL=OFF -DWITH_CYRUS_SASL=${TP_OUTPUT} -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + COMMAND cd zookeeper-client/zookeeper-client-c && make + COMMAND cp -R zookeeper-client/zookeeper-client-c/include/. ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/generated/zookeeper.jute.h ${TP_OUTPUT}/include/zookeeper && cp zookeeper-client/zookeeper-client-c/libzookeeper.a ${TP_OUTPUT}/lib && cp zookeeper-client/zookeeper-client-c/libhashtable.a ${TP_OUTPUT}/lib + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" BUILD_IN_SOURCE 1 ) -add_dependencies(zookeeper cyrus-sasl) ExternalProject_Add(libevent URL ${OSS_URL_PREFIX}/libevent-release-2.1.8-stable.tar.gz @@ -361,36 +364,6 @@ ExternalProject_Add(rocksdb -DPORTABLE=${ROCKSDB_PORTABLE} ) -if (NOT APPLE) -# kerberos -ExternalProject_Add(krb5 - URL ${OSS_URL_PREFIX}/krb5-1.16.1.tar.gz - http://web.mit.edu/kerberos/dist/krb5/1.16/krb5-1.16.1.tar.gz - URL_MD5 848e9b80d6aaaa798e3f3df24b83c407 - CONFIGURE_COMMAND cd src && ./configure --prefix=${TP_OUTPUT} - BUILD_COMMAND cd src && make - INSTALL_COMMAND cd src && make install - BUILD_IN_SOURCE 1 - ) - -# cyrus-sasl -ExternalProject_Add(cyrus-sasl - URL ${OSS_URL_PREFIX}/cyrus-sasl-2.1.27.tar.gz - http://www.cyrusimap.org/releases/cyrus-sasl-2.1.27.tar.gz - URL_MD5 a33820c66e0622222c5aefafa1581083 - CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} - --enable-gssapi=${TP_OUTPUT} - --enable-scram=no - --enable-digest=no - --enable-cram=no - --enable-otp=no - BUILD_COMMAND make - INSTALL_COMMAND make install - BUILD_IN_SOURCE 1 - ) -add_dependencies(cyrus-sasl krb5) -endif() - ExternalProject_Add(http-parser URL ${OSS_URL_PREFIX}/http-parser-2.9.4.zip https://github.com/nodejs/http-parser/archive/v2.9.4.zip From e3d0f53c8e2d8149678a4712556755afd3805b0c Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Fri, 8 Apr 2022 18:37:20 +0800 Subject: [PATCH 08/11] change docker source --- .github/workflows/pull_request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index ddcb9c9ee6..fa7ee2c5ab 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -45,7 +45,7 @@ jobs: needs: lint runs-on: self-hosted container: - image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804 + image: apachepegasus/thirdparties-bin:ubuntu1804 env: CCACHE_DIR: /tmp/ccache/pegasus CCACHE_MAXSIZE: 10G From 3d5633d415cbbd00f2e7fef85d58150461b06c71 Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Fri, 8 Apr 2022 19:30:21 +0800 Subject: [PATCH 09/11] change download url --- scripts/linux/start_zk.sh | 2 +- thirdparty/CMakeLists.txt | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh index 309f7b3850..cd68fa2872 100755 --- a/scripts/linux/start_zk.sh +++ b/scripts/linux/start_zk.sh @@ -33,7 +33,7 @@ ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a" if [ ! -f $ZOOKEEPER_TAR_NAME ]; then echo "Downloading zookeeper..." - download_url="https://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz" + download_url="https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz" if ! wget -T 5 -t 1 $download_url; then echo "ERROR: download zookeeper failed" exit 1 diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 170b8c3897..f6ea9e719c 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -108,6 +108,7 @@ ExternalProject_Add(gperftools set(HDFS_CLIENT_DIR "hadoop-hdfs-project/hadoop-hdfs-native-client") ExternalProject_Add(hadoop URL https://github.com/apache/hadoop/archive/refs/tags/rel/release-2.8.4.tar.gz + https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/hadoop-release-2.8.4.tar.gz URL_MD5 a1be737d4bff14923689619ab6545a96 PATCH_COMMAND "" COMMAND cd ${HDFS_CLIENT_DIR} && mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip=true -Dtar @@ -160,9 +161,9 @@ if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () ExternalProject_Add(zookeeper - URL https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz - https://github.com/apache/zookeeper/archive/refs/tags/release-3.7.0.tar.gz - URL_MD5 c564fbf2877f611890522e4aa6e4eb24 + URL http://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz + https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0.tar.gz + URL_MD5 44c2a33e01931aed94ef7f3d39d0963e PATCH_COMMAND "" COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -WITH_OPENSSL=OFF -DWITH_CYRUS_SASL=${TP_OUTPUT} -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} COMMAND cd zookeeper-client/zookeeper-client-c && make @@ -231,6 +232,7 @@ ExternalProject_Add(fds # fmtlib >=6.x requires c++14 support, do not update this library for now ExternalProject_Add(fmt URL https://github.com/fmtlib/fmt/archive/refs/tags/5.3.0.tar.gz + https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/fmt-5.3.0.tar.gz URL_MD5 1015bf3ff2a140dfe03de50ee2469401 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_BUILD_TYPE=release @@ -332,6 +334,7 @@ message(STATUS "USE_JEMALLOC = ${USE_JEMALLOC}") ExternalProject_Add(jemalloc URL https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 + https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/jemalloc-5.2.1.tar.bz2 URL_MD5 3d41fbf006e6ebffd489bdb304d009ae CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} --enable-cxx --enable-stats --enable-prof BUILD_COMMAND make From 63431bc4adc9e1e95694eba21fd7d583d6ea8843 Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Mon, 11 Apr 2022 16:07:52 +0800 Subject: [PATCH 10/11] change docker for Test Sanitizer --- .github/workflows/pull_request.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index fa7ee2c5ab..0b8cf1ee30 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -79,7 +79,7 @@ jobs: needs: lint runs-on: self-hosted container: - image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804 + image: apachepegasus/thirdparties-bin:ubuntu1804 env: CCACHE_DIR: /tmp/ccache/pegasus CCACHE_MAXSIZE: 10G @@ -113,7 +113,7 @@ jobs: needs: lint runs-on: self-hosted container: - image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804 + image: apachepegasus/thirdparties-bin:ubuntu1804 env: CCACHE_DIR: /tmp/ccache/pegasus CCACHE_MAXSIZE: 10G @@ -147,7 +147,7 @@ jobs: needs: lint runs-on: self-hosted container: - image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804 + image: apachepegasus/thirdparties-bin:ubuntu1804 env: CCACHE_DIR: /tmp/ccache/pegasus CCACHE_MAXSIZE: 10G From 8585f4b0355a3f40369953e60073f939fdbd15ba Mon Sep 17 00:00:00 2001 From: liguohao <948193394@qq.com> Date: Tue, 12 Apr 2022 20:26:15 +0800 Subject: [PATCH 11/11] add oss url prefix --- scripts/linux/start_zk.sh | 2 +- thirdparty/CMakeLists.txt | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh index cd68fa2872..45236cc7bf 100755 --- a/scripts/linux/start_zk.sh +++ b/scripts/linux/start_zk.sh @@ -33,7 +33,7 @@ ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a" if [ ! -f $ZOOKEEPER_TAR_NAME ]; then echo "Downloading zookeeper..." - download_url="https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz" + download_url="http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz" if ! wget -T 5 -t 1 $download_url; then echo "ERROR: download zookeeper failed" exit 1 diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index f6ea9e719c..8d62d1b96e 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -48,7 +48,7 @@ set(TP_OUTPUT ${PROJECT_SOURCE_DIR}/output) # LOG_DIR = set_property(DIRECTORY PROPERTY EP_BASE ${TP_DIR}/build) -set(OSS_URL_PREFIX "http://pegasus-thirdparties.oss-cn-beijing.aliyuncs.com") +set(OSS_URL_PREFIX "http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com") message(STATUS "Setting up third-parties...") @@ -107,8 +107,8 @@ ExternalProject_Add(gperftools set(HDFS_CLIENT_DIR "hadoop-hdfs-project/hadoop-hdfs-native-client") ExternalProject_Add(hadoop - URL https://github.com/apache/hadoop/archive/refs/tags/rel/release-2.8.4.tar.gz - https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/hadoop-release-2.8.4.tar.gz + URL ${OSS_URL_PREFIX}/hadoop-release-2.8.4.tar.gz + https://github.com/apache/hadoop/archive/refs/tags/rel/release-2.8.4.tar.gz URL_MD5 a1be737d4bff14923689619ab6545a96 PATCH_COMMAND "" COMMAND cd ${HDFS_CLIENT_DIR} && mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip=true -Dtar @@ -161,8 +161,8 @@ if (COMPILER_SUPPORTS_FORMAT_OVERFLOW) set(ZOOKEEPER_CFLAGS -Wno-error=format-overflow) endif () ExternalProject_Add(zookeeper - URL http://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz - https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0.tar.gz + URL ${OSS_URL_PREFIX}/apache-zookeeper-3.7.0.tar.gz + http://downloads.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0.tar.gz URL_MD5 44c2a33e01931aed94ef7f3d39d0963e PATCH_COMMAND "" COMMAND cd zookeeper-jute && mvn compile && cd ../zookeeper-client/zookeeper-client-c && cmake -DCMAKE_BUILD_TYPE=release -DWANT_CPPUNIT=OFF -WITH_OPENSSL=OFF -DWITH_CYRUS_SASL=${TP_OUTPUT} -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} @@ -211,7 +211,8 @@ ExternalProject_Add(poco ) ExternalProject_Add(fds - URL https://github.com/XiaoMi/galaxy-fds-sdk-cpp/archive/refs/tags/v1.0.0.tar.gz + URL ${OSS_URL_PREFIX}/fds-1.0.0.tar.gz + https://github.com/XiaoMi/galaxy-fds-sdk-cpp/archive/refs/tags/v1.0.0.tar.gz URL_MD5 f7e0f86534f7b15c2a9b349f5cb45503 PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_fds_for_macos.patch CMAKE_ARGS -DPOCO_INCLUDE=${TP_OUTPUT}/include @@ -231,8 +232,8 @@ ExternalProject_Add(fds # fmtlib >=6.x requires c++14 support, do not update this library for now ExternalProject_Add(fmt - URL https://github.com/fmtlib/fmt/archive/refs/tags/5.3.0.tar.gz - https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/fmt-5.3.0.tar.gz + URL ${OSS_URL_PREFIX}/fmt-5.3.0.tar.gz + https://github.com/fmtlib/fmt/archive/refs/tags/5.3.0.tar.gz URL_MD5 1015bf3ff2a140dfe03de50ee2469401 CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_BUILD_TYPE=release @@ -333,8 +334,8 @@ option(USE_JEMALLOC "use jemalloc" OFF) message(STATUS "USE_JEMALLOC = ${USE_JEMALLOC}") ExternalProject_Add(jemalloc - URL https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 - https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/jemalloc-5.2.1.tar.bz2 + URL ${OSS_URL_PREFIX}/jemalloc-5.2.1.tar.bz2 + https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 URL_MD5 3d41fbf006e6ebffd489bdb304d009ae CONFIGURE_COMMAND ./configure --prefix=${TP_OUTPUT} --enable-cxx --enable-stats --enable-prof BUILD_COMMAND make