feat: zookeeper client support kerberos

.github/workflows/pull_request.yaml

@@ -45,7 +45,7 @@ jobs:
     needs: lint
     runs-on: self-hosted
     container:
-      image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804
+      image: apachepegasus/thirdparties-bin:ubuntu1804
       env:
         CCACHE_DIR: /tmp/ccache/pegasus
         CCACHE_MAXSIZE: 10G
@@ -79,7 +79,7 @@ jobs:
     needs: lint
     runs-on: self-hosted
     container:
-      image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804
+      image: apachepegasus/thirdparties-bin:ubuntu1804
       env:
         CCACHE_DIR: /tmp/ccache/pegasus
         CCACHE_MAXSIZE: 10G
@@ -113,7 +113,7 @@ jobs:
     needs: lint
     runs-on: self-hosted
     container:
-      image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804
+      image: apachepegasus/thirdparties-bin:ubuntu1804
       env:
         CCACHE_DIR: /tmp/ccache/pegasus
         CCACHE_MAXSIZE: 10G
@@ -147,7 +147,7 @@ jobs:
     needs: lint
     runs-on: self-hosted
     container:
-      image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804
+      image: apachepegasus/thirdparties-bin:ubuntu1804
       env:
         CCACHE_DIR: /tmp/ccache/pegasus
         CCACHE_MAXSIZE: 10G

scripts/linux/clear_zk.sh

@@ -11,7 +11,7 @@ fi
 
 cd $INSTALL_DIR
 
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
 
 if [ -d "$ZOOKEEPER_HOME" ]
 then

scripts/linux/start_zk.sh

@@ -27,29 +27,39 @@ fi
 
 cd $INSTALL_DIR
 
-ZOOKEEPER_PKG=${PROJECT_DIR}/thirdparty/build/Download/zookeeper/zookeeper-3.4.10.tar.gz
-if [ ! -f ${ZOOKEEPER_PKG} ]; then
-    echo "no such file \"${ZOOKEEPER_PKG}\""
-    echo "please install third-parties first"
-    exit 1
+ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin
+ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz
+ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a"
+
+if [ ! -f $ZOOKEEPER_TAR_NAME ]; then
+    echo "Downloading zookeeper..."
+    download_url="https://pegasus-resource-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz"
+    if ! wget -T 5 -t 1 $download_url; then
+        echo "ERROR: download zookeeper failed"
+        exit 1
+    fi
+    if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` != $ZOOKEEPER_TAR_MD5_VALUE ]; then
+        echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!"
+        exit 1
+    fi
 fi
 
-if [ ! -d zookeeper-3.4.10 ]; then
+if [ ! -d $ZOOKEEPER_ROOT ]; then
     echo "Decompressing zookeeper..."
-    cp ${ZOOKEEPER_PKG} .
-    tar xf zookeeper-3.4.10.tar.gz
-    if [ $? -ne 0 ]; then
+    if ! tar xf $ZOOKEEPER_TAR_NAME; then
         echo "ERROR: decompress zookeeper failed"
         exit 1
     fi
 fi
 
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10
+ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT
 ZOOKEEPER_PORT=$PORT
 
 cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg
 sed -i "s@dataDir=/tmp/zookeeper@dataDir=$ZOOKEEPER_HOME/data@" $ZOOKEEPER_HOME/conf/zoo.cfg
 sed -i "s@clientPort=2181@clientPort=$ZOOKEEPER_PORT@" $ZOOKEEPER_HOME/conf/zoo.cfg
+echo "admin.enableServer=false" >> $ZOOKEEPER_HOME/conf/zoo.cfg
+echo "4lw.commands.whitelist=ruok" >> $ZOOKEEPER_HOME/conf/zoo.cfg
 
 mkdir -p $ZOOKEEPER_HOME/data
 $ZOOKEEPER_HOME/bin/zkServer.sh start

scripts/linux/stop_zk.sh

@@ -11,7 +11,7 @@ fi
 
 cd $INSTALL_DIR
 
-ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
 
 if [ -d "$ZOOKEEPER_HOME" ]
 then

src/failure_detector/test/CMakeLists.txt

@@ -16,6 +16,7 @@ set(MY_PROJ_LIBS
     dsn_replication_common
     dsn.failure_detector
     gtest
+    hashtable
     )
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

src/meta/CMakeLists.txt

@@ -21,7 +21,7 @@ set(MY_PROJ_LIBS
     dsn_http
     dsn_runtime
     dsn_aio
-    zookeeper_mt
+    zookeeper
     galaxy-fds-sdk-cpp
     PocoNet
     PocoFoundation

src/meta/test/CMakeLists.txt

@@ -25,14 +25,16 @@ set(MY_PROJ_LIBS
         dsn_http
         dsn_runtime
         dsn_aio
-        zookeeper_mt
+        zookeeper
+        hashtable
         galaxy-fds-sdk-cpp
         PocoNet
         PocoFoundation
         PocoNetSSL
         PocoJSON
         crypto
         gtest
+        ssl
         hdfs)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

src/meta/test/balancer_simulator/CMakeLists.txt

@@ -13,6 +13,7 @@ set(MY_PROJ_LIBS
     dsn_meta_server
     dsn_replication_common
     dsn_runtime
+    hashtable
     gtest)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

src/meta/test/meta_state/CMakeLists.txt

@@ -14,6 +14,7 @@ set(MY_PROJ_LIBS
     dsn_replica_server
     dsn_replication_common
     dsn_runtime
+    hashtable
     gtest
     )
 

src/replica/backup/test/CMakeLists.txt

@@ -11,6 +11,7 @@ set(MY_PROJ_LIBS dsn_meta_server
         dsn.block_service.local
         dsn.block_service.fds
         dsn_utils
+        hashtable
         gtest
 )
 

src/replica/bulk_load/test/CMakeLists.txt

@@ -8,6 +8,7 @@ set(MY_PROJ_LIBS dsn_meta_server
         dsn_replica_server
         dsn_replication_common
         dsn_runtime
+        hashtable
         gtest
 )
 

src/replica/duplication/test/CMakeLists.txt

@@ -10,7 +10,8 @@ set(MY_PROJ_LIBS dsn_meta_server
         dsn_replication_common
         dsn.failure_detector
         dsn_utils
-        zookeeper_mt
+        zookeeper
+        hashtable
         gtest
 )
 

src/replica/split/test/CMakeLists.txt

@@ -8,6 +8,7 @@ set(MY_PROJ_LIBS dsn_meta_server
         dsn_replica_server
         dsn_replication_common
         dsn_runtime
+        hashtable
         gtest
 )
 

src/replica/storage/simple_kv/CMakeLists.txt

@@ -13,7 +13,7 @@ set(MY_PROJ_SRC ${SIMPLE_KV_THRIFT_SRCS})
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime)
+set(MY_PROJ_LIBS dsn_replica_server dsn_meta_server dsn_client dsn_runtime hashtable)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
 

src/replica/storage/simple_kv/test/CMakeLists.txt

@@ -12,7 +12,8 @@ set(MY_PROJ_LIBS dsn_replica_server
                  dsn.failure_detector
                  dsn.replication.zookeeper_provider
                  dsn_runtime
-                 zookeeper_mt
+                 zookeeper
+                 hashtable
                  gtest
                  )
 

src/replica/test/CMakeLists.txt

@@ -20,7 +20,8 @@ set(MY_PROJ_LIBS dsn_meta_server
                  dsn.failure_detector
                  dsn_http
                  dsn_runtime
-                 zookeeper_mt
+                 zookeeper
+                 hashtable
                  gtest)
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

src/runtime/security/init.cpp

@@ -68,5 +68,22 @@ bool init(bool is_server)
     return true;
 }
 
+bool init_for_zookeeper_client()
+{
+    error_s err = run_kinit();
+    if (!err.is_ok()) {
+        derror_f("initialize kerberos failed, with err = {}", err.description());
+        return false;
+    }
+    ddebug("initialize kerberos for zookeeper client succeed");
+
+    err = init_sasl(false);
+    if (!err.is_ok()) {
+        derror_f("initialize sasl failed, with err = {}", err.description());
+        return false;
+    }
+    ddebug("initialize sasl for zookeeper client succeed");
+    return true;
+}
 } // namespace security
 } // namespace dsn

src/runtime/security/init.h

@@ -23,5 +23,8 @@ namespace dsn {
 namespace security {
 // init security(kerberos and sasl)
 bool init(bool is_server);
+
+// init security only for zookeeper client(kerberos and sasl)
+bool init_for_zookeeper_client();
 } // namespace security
 } // namespace dsn

src/runtime/security/kinit_context.cpp

@@ -33,6 +33,7 @@
 namespace dsn {
 namespace security {
 DSN_DECLARE_bool(enable_auth);
+DSN_DECLARE_bool(enable_zookeeper_kerberos);
 
 #define KRB5_RETURN_NOT_OK(err, msg)                                                               \
     do {                                                                                           \
@@ -53,8 +54,9 @@ DSN_DEFINE_string("security", service_name, "", "service name");
 // will not pass.
 error_s check_configuration()
 {
-    dassert(FLAGS_enable_auth,
-            "There is no need to check configuration if FLAGS_enable_auth is not true");
+    dassert(FLAGS_enable_auth || FLAGS_enable_zookeeper_kerberos,
+            "There is no need to check configuration if FLAGS_enable_auth"
+            " and FLAGS_enable_zookeeper_kerberos both are not true");
 
     if (0 == strlen(FLAGS_krb5_keytab) || !utils::filesystem::file_exists(FLAGS_krb5_keytab)) {
         return error_s::make(ERR_INVALID_PARAMETERS,

src/runtime/security/negotiation.cpp

@@ -31,6 +31,10 @@ namespace security {
 const std::set<std::string> supported_mechanisms{"GSSAPI"};
 
 DSN_DEFINE_bool("security", enable_auth, false, "whether open auth or not");
+DSN_DEFINE_bool("security",
+                enable_zookeeper_kerberos,
+                false,
+                "whether to enable kerberos for zookeeper client");
 DSN_DEFINE_bool("security", mandatory_auth, false, "wheter to do authertication mandatorily");
 DSN_TAG_VARIABLE(mandatory_auth, FT_MUTABLE);
 

src/runtime/service_api_c.cpp

@@ -56,6 +56,7 @@
 namespace dsn {
 namespace security {
 DSN_DECLARE_bool(enable_auth);
+DSN_DECLARE_bool(enable_zookeeper_kerberos);
 } // namespace security
 } // namespace dsn
 //
@@ -467,6 +468,15 @@ bool run(const char *config_file,
         if (!dsn::security::init(is_server)) {
             return false;
         }
+        // if FLAGS_enable_auth is false but FLAGS_enable_zookeeper_kerberos, we should init
+        // kerberos for it separately
+        // include two steps:
+        // 1) apply kerberos ticket and keep it valid
+        // 2) complete sasl init for client(use FLAGS_sasl_plugin_path)
+    } else if (dsn::security::FLAGS_enable_zookeeper_kerberos && app_list == "meta") {
+        if (!dsn::security::init_for_zookeeper_client()) {
+            return false;
+        }
     }
 
     // split app_name and app_index

src/zookeeper/CMakeLists.txt

@@ -10,11 +10,13 @@ set(MY_PROJ_SRC "")
 # "GLOB" for non-recursive search
 set(MY_SRC_SEARCH_MODE "GLOB")
 
-set(MY_PROJ_LIBS "")
+set(MY_PROJ_LIBS zookeeper hashtable ssl crypto)
 
 # Extra files that will be installed
 set(MY_BINPLACES "")
 
 dsn_add_static_library()
 
+add_definitions(-DHAVE_CYRUS_SASL_H)
+
 add_subdirectory(test)

src/zookeeper/test/CMakeLists.txt

@@ -12,8 +12,11 @@ set(MY_SRC_SEARCH_MODE "GLOB")
 set(MY_PROJ_LIBS 
     dsn.replication.zookeeper_provider
     dsn_runtime
-    zookeeper_mt
+    zookeeper
+    hashtable
     gtest
+    ssl
+    crypto
     )
 
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)

src/zookeeper/zookeeper_session.cpp

@@ -33,10 +33,23 @@
  */
 
 #include <zookeeper/zookeeper.h>
+#include <sasl/sasl.h>
 
 #include "zookeeper_session.h"
 #include "zookeeper_session_mgr.h"
 
+#include <dsn/utility/flags.h>
+
+namespace dsn {
+namespace security {
+DSN_DECLARE_bool(enable_zookeeper_kerberos);
+DSN_DEFINE_string("security",
+                  zookeeper_kerberos_service_name,
+                  "zookeeper",
+                  "zookeeper kerberos service name");
+} // namespace security
+} // namespace dsn
+
 namespace dsn {
 namespace dist {
 
@@ -141,12 +154,26 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb)
 {
     utils::auto_write_lock l(_watcher_lock);
     if (nullptr == _handle) {
-        _handle = zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(),
-                                 global_watcher,
-                                 zookeeper_session_mgr::instance().timeout(),
-                                 nullptr,
-                                 this,
-                                 0);
+        if (dsn::security::FLAGS_enable_zookeeper_kerberos) {
+            zoo_sasl_params_t sasl_params = {0};
+            sasl_params.service = dsn::security::FLAGS_zookeeper_kerberos_service_name;
+            sasl_params.mechlist = "GSSAPI";
+            _handle = zookeeper_init_sasl(zookeeper_session_mgr::instance().zoo_hosts(),
+                                          global_watcher,
+                                          zookeeper_session_mgr::instance().timeout(),
+                                          nullptr,
+                                          this,
+                                          0,
+                                          NULL,
+                                          &sasl_params);
+        } else {
+            _handle = zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(),
+                                     global_watcher,
+                                     zookeeper_session_mgr::instance().timeout(),
+                                     nullptr,
+                                     this,
+                                     0);
+        }
         dassert(_handle != nullptr, "zookeeper session init failed");
     }