ZOOKEEPER-1112: Add (Cyrus) SASL authentication support to C client l…

…ibrary

This changeset allows C clients to use SASL to authenticate with the
ZooKeeper server.  It is loosely based on patches apache#1 and apache#2 by Tom
Klonikowski, at https://reviews.apache.org/r/2252/, but the result has
been extensively reworked to follow the semantics of the Java client:

  * No SASL operations are exposed through the API;

  * The configuration is provided, and stored, at "handle init time";

  * SASL authentication is automatically performed after each
    (re)connect.

It introduces an optional dependency on the Cyrus SASL library, which
can either be autodetected (default) or configured using the
--without-sasl/--with-sasl[=DIR] flags.

TestServerRequireClientSASLAuth.cc has been renamed to TestSASLAuth.cc,
and a test has been added which successfully (re)authenticates using
the DIGEST-MD5 mechanism.  An earlier version of this code has been
used to successfully authenticate clients via Kerberos.

While cli.c is not modified by this commit, we are planning to submit
a subsequent contribution which enables SASL support in that client
using the ZOOKEEPER-3599 (use getopt if available) mechanism.

Co-authored-by: Tom Klonikowski <[email protected]>

tools/cmake/Modules/FindCyrusSASL.cmake

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# - Find Cyrus SASL (sasl.h, libsasl2.so)
+#
+# This module defines
+#  CYRUS_SASL_INCLUDE_DIR, directory containing headers
+#  CYRUS_SASL_SHARED_LIB, path to Cyrus SASL's shared library
+#  CYRUS_SASL_FOUND, whether Cyrus SASL and its plugins have been found
+#
+# It also defines the following IMPORTED targets:
+#  CyrusSASL
+#
+# Hints:
+#  Set CYRUS_SASL_ROOT_DIR to the root directory of a Cyrus SASL installation.
+#
+# The initial version of this file was extracted from
+# https://github.com/cloudera/kudu, at the following commit:
+#
+#  commit 9806863e78107505a622b44112a897189d9b3c24
+#  Author: Dan Burkert <[email protected]>
+#  Date:   Mon Nov 30 12:15:36 2015 -0800
+#
+#      Enable C++11
+
+find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h HINTS "${CYRUS_SASL_ROOT_DIR}/include")
+find_library(CYRUS_SASL_SHARED_LIB sasl2 HINTS "${CYRUS_SASL_ROOT_DIR}/lib")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CYRUS_SASL REQUIRED_VARS
+  CYRUS_SASL_SHARED_LIB CYRUS_SASL_INCLUDE_DIR)
+
+if(CYRUS_SASL_FOUND)
+  if(NOT TARGET CyrusSASL)
+    add_library(CyrusSASL UNKNOWN IMPORTED)
+    set_target_properties(CyrusSASL PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${CYRUS_SASL_INCLUDE_DIR}"
+      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+      IMPORTED_LOCATION "${CYRUS_SASL_SHARED_LIB}")
+  endif()
+endif()

zookeeper-client/zookeeper-client-c/CMakeLists.txt

@@ -20,6 +20,8 @@ project(zookeeper VERSION 3.6.0)
 set(email [email protected])
 set(description "zookeeper C client")
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../../tools/cmake/Modules")
+
 # general options
 if(UNIX)
   add_compile_options(-Wall -fPIC)
@@ -61,6 +63,20 @@ if(WANT_SOCK_CLOEXEC AND HAVE_SOCK_CLOEXEC)
   set(SOCK_CLOEXEC_ENABLED 1)
 endif()
 
+# Cyrus SASL 2.x
+option(WITH_CYRUS_SASL "turn ON/OFF Cyrus SASL 2.x support, or define SASL library location (default: ON)" ON)
+message("-- using WITH_CYRUS_SASL=${WITH_CYRUS_SASL}")
+if(NOT WITH_CYRUS_SASL STREQUAL "OFF")
+  if(NOT WITH_CYRUS_SASL STREQUAL "ON")
+    set(CYRUS_SASL_ROOT_DIR "${WITH_CYRUS_SASL}")
+  endif()
+  find_package(CyrusSASL)
+  if(CYRUS_SASL_FOUND)
+    message("-- Cyrus SASL 2.x found! will build with SASL support.")
+  else()
+    message("-- WARNING: unable to find Cyrus SASL 2.x! will build without SASL support.")
+  endif()
+endif()
 
 # The function `to_have(in out)` converts a header name like `arpa/inet.h`
 # into an Autotools style preprocessor definition `HAVE_ARPA_INET_H`.
@@ -171,6 +187,10 @@ else()
   list(APPEND zookeeper_sources src/st_adaptor.c)
 endif()
 
+if(CYRUS_SASL_FOUND)
+  list(APPEND zookeeper_sources src/zk_sasl.c)
+endif()
+
 if(WIN32)
   list(APPEND zookeeper_sources src/winport.c)
 endif()
@@ -203,6 +223,11 @@ if(WANT_SYNCAPI AND NOT WIN32)
   target_link_libraries(zookeeper PUBLIC Threads::Threads)
 endif()
 
+if(CYRUS_SASL_FOUND)
+  target_compile_definitions(zookeeper PUBLIC HAVE_CYRUS_SASL_H)
+  target_link_libraries(zookeeper PUBLIC CyrusSASL)
+endif()
+
 # cli executable
 add_executable(cli src/cli.c)
 target_link_libraries(cli zookeeper)

zookeeper-client/zookeeper-client-c/Makefile.am

@@ -13,10 +13,16 @@ if WANT_OPENSSL
   OPENSSL_LIB_LDFLAGS = -lssl -lcrypto
 endif
 
-AM_CPPFLAGS = -I${srcdir}/include -I${srcdir}/tests -I${srcdir}/generated $(SOLARIS_CPPFLAGS) $(OPENSSL_CPPFLAGS)
+if WANT_SASL
+  SASL_CPPFLAGS = -DHAVE_CYRUS_SASL_H
+  SASL_LIB_LDFLAGS = -lsasl2
+  SASL_SRC = src/zk_sasl.c
+endif
+
+AM_CPPFLAGS = -I${srcdir}/include -I${srcdir}/tests -I${srcdir}/generated $(SOLARIS_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SASL_CPPFLAGS)
 AM_CFLAGS = -Wall -Werror -Wdeclaration-after-statement
 AM_CXXFLAGS = -Wall $(USEIPV6)
-LIB_LDFLAGS = -no-undefined -version-info 2 $(SOLARIS_LIB_LDFLAGS) $(OPENSSL_LIB_LDFLAGS)
+LIB_LDFLAGS = -no-undefined -version-info 2 $(SOLARIS_LIB_LDFLAGS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS)
 
 # Additional flags for coverage testing (if enabled)
 if ENABLEGCOV
@@ -37,7 +43,7 @@ COMMON_SRC = src/zookeeper.c include/zookeeper.h include/zookeeper_version.h inc
     src/recordio.c include/recordio.h include/proto.h \
     src/zk_adaptor.h generated/zookeeper.jute.c \
     src/zk_log.c src/zk_hashtable.h src/zk_hashtable.c \
-	src/addrvec.h src/addrvec.c
+    src/addrvec.h src/addrvec.c $(SASL_SRC)
 
 # These are the symbols (classes, mostly) we want to export from our library.
 EXPORT_SYMBOLS = '(zoo_|zookeeper_|zhandle|Z|format_log_message|log_message|logLevel|deallocate_|allocate_|zerror|is_unrecoverable)'
@@ -59,6 +65,7 @@ libzkmt_la_LIBADD = -lm $(CLOCK_GETTIME_LIBS)
 
 lib_LTLIBRARIES += libzookeeper_mt.la
 libzookeeper_mt_la_SOURCES =
+libzookeeper_mt_la_CFLAGS = -DTHREADED
 libzookeeper_mt_la_LIBADD=libzkmt.la libhashtable.la -lpthread
 libzookeeper_mt_la_DEPENDENCIES=libzkmt.la libhashtable.la
 libzookeeper_mt_la_LDFLAGS = $(LIB_LDFLAGS) -export-symbols-regex $(EXPORT_SYMBOLS)
@@ -113,7 +120,7 @@ TEST_SOURCES = \
 	tests/ZooKeeperQuorumServer.h \
 	tests/TestReadOnlyClient.cc \
 	tests/TestLogClientEnv.cc \
-    tests/TestServerRequireClientSASLAuth.cc \
+        tests/TestSASLAuth.cc \
 	$(NULL)
 
 if SOLARIS
@@ -127,14 +134,14 @@ check_PROGRAMS = zktest-st
 TESTS_ENVIRONMENT = ZKROOT=${srcdir}/../.. \
                     CLASSPATH=$$CLASSPATH:$$CLOVER_HOME/lib/clover*.jar
 nodist_zktest_st_SOURCES = $(TEST_SOURCES)
-zktest_st_LDADD = libzkst.la libhashtable.la $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) -ldl
+zktest_st_LDADD = libzkst.la libhashtable.la $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS) -ldl
 zktest_st_CXXFLAGS = -DUSE_STATIC_LIB $(CPPUNIT_CFLAGS) $(USEIPV6) $(SOLARIS_CPPFLAGS)
 zktest_st_LDFLAGS = -shared $(SYMBOL_WRAPPERS) $(SOLARIS_LIB_LDFLAGS)
 
 if WANT_SYNCAPI
   check_PROGRAMS += zktest-mt
   nodist_zktest_mt_SOURCES = $(TEST_SOURCES) tests/PthreadMocks.cc
-  zktest_mt_LDADD = libzkmt.la libhashtable.la -lpthread $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) -ldl
+  zktest_mt_LDADD = libzkmt.la libhashtable.la -lpthread $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS) -ldl
   zktest_mt_CXXFLAGS = -DUSE_STATIC_LIB -DTHREADED $(CPPUNIT_CFLAGS) $(USEIPV6)
 if SOLARIS
   SHELL_SYMBOL_WRAPPERS_MT = cat ${srcdir}/tests/wrappers-mt.opt

zookeeper-client/zookeeper-client-c/configure.ac

@@ -143,6 +143,36 @@ fi
 
 AM_CONDITIONAL([WANT_SYNCAPI],[test "x$with_syncapi" != xno])
 
+dnl Cyrus SASL 2.x
+AC_ARG_WITH(sasl,
+ [AC_HELP_STRING([--with-sasl[=DIR]], [build with SASL support via Cyrus SASL 2.x (default=auto)])],
+ [], [with_sasl=yes])
+if test "x$with_sasl" != "xno"; then
+    saved_CPPFLAGS="$CPPFLAGS"
+    saved_LDFLAGS="$LDFLAGS"
+    if test "x$with_sasl" != "xyes" ; then
+        CPPFLAGS="$CPPFLAGS -I$with_sasl/include"
+        LDFLAGS="$LDFLAGS -L$with_sasl/lib"
+    fi
+    have_sasl=no
+    AC_CHECK_HEADER(sasl/sasl.h, [
+     AC_CHECK_LIB(sasl2, sasl_client_init, [have_sasl=yes])])
+    if test "x$have_sasl" != "xyes"; then
+        CPPFLAGS="$saved_CPPFLAGS"
+        LDFLAGS="$saved_LDFLAGS"
+    fi
+fi
+if test "x$with_sasl" != xno && test "x$have_sasl" = xno; then
+    AC_MSG_WARN([cannot build SASL support -- sasl2 not found])
+    with_sasl=no
+fi
+if test "x$with_sasl" != xno; then
+    AC_MSG_NOTICE([building with SASL support])
+else
+    AC_MSG_NOTICE([building without SASL support])
+fi
+AM_CONDITIONAL([WANT_SASL],[test "x$with_sasl" != xno])
+
 # Checks for header files.
 AC_HEADER_STDC
 AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h sys/time.h unistd.h sys/utsname.h])

zookeeper-client/zookeeper-client-c/include/proto.h

@@ -46,6 +46,7 @@ extern "C" {
 #define ZOO_CLOSE_OP -11
 #define ZOO_SETAUTH_OP 100
 #define ZOO_SETWATCHES_OP 101
+#define ZOO_SASL_OP 102
 
 #ifdef __cplusplus
 }

zookeeper-client/zookeeper-client-c/include/zookeeper.h

@@ -39,6 +39,10 @@
 #include <stdio.h>
 #include <ctype.h>
 
+#ifdef HAVE_CYRUS_SASL_H
+#include <sasl/sasl.h>
+#endif /* HAVE_CYRUS_SASL_H */
+
 #include "proto.h"
 #include "zookeeper_version.h"
 #include "recordio.h"
@@ -575,6 +579,93 @@ ZOOAPI zhandle_t *zookeeper_init2(const char *host, watcher_fn fn,
   int recv_timeout, const clientid_t *clientid, void *context, int flags,
   log_callback_fn log_callback);
 
+#ifdef HAVE_CYRUS_SASL_H
+
+/**
+ * \brief zoo_sasl_params structure.
+ *
+ * This structure holds the SASL parameters for the connection.
+ *
+ * Its \c service, \c host and \c callbacks fields are used with Cyrus
+ * SASL's \c sasl_client_new; its \c mechlist field with \c
+ * sasl_client_start.  Please refer to these functions for precise
+ * semantics.
+ *
+ * Note while "string" parameters are copied into the ZooKeeper
+ * client, the callbacks array is simply referenced: its lifetime must
+ * therefore cover that of the handle.
+ */
+typedef struct zoo_sasl_params {
+  const char *service;          /*!< The service name, usually "zookeeper" */
+  const char *host;             /*!< The server name, e.g. "zk-sasl-md5" */
+  const char *mechlist;         /*!< Mechanisms to try, e.g. "DIGEST-MD5" */
+  const sasl_callback_t *callbacks;  /*!< List of callbacks */
+} zoo_sasl_params_t;
+
+/**
+ * \brief create a handle to communicate with zookeeper.
+ *
+ * This function is identical to \ref zookeeper_init2 except that it
+ * allows specifying optional SASL connection parameters.  It is only
+ * available if the client library was configured to link against the
+ * Cyrus SASL library, and only visible when \c HAVE_CYRUS_SASL_H is defined.
+ *
+ * This method creates a new handle and a zookeeper session that corresponds
+ * to that handle. Session establishment is asynchronous, meaning that the
+ * session should not be considered established until (and unless) an
+ * event of state ZOO_CONNECTED_STATE is received.
+ * \param host comma separated host:port pairs, each corresponding to a zk
+ *   server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
+ * \param fn the global watcher callback function. When notifications are
+ *   triggered this function will be invoked.
+ * \param clientid the id of a previously established session that this
+ *   client will be reconnecting to. Pass 0 if not reconnecting to a previous
+ *   session. Clients can access the session id of an established, valid,
+ *   connection by calling \ref zoo_client_id. If the session corresponding to
+ *   the specified clientid has expired, or if the clientid is invalid for
+ *   any reason, the returned zhandle_t will be invalid -- the zhandle_t
+ *   state will indicate the reason for failure (typically
+ *   ZOO_EXPIRED_SESSION_STATE).
+ * \param context the handback object that will be associated with this instance
+ *   of zhandle_t. Application can access it (for example, in the watcher
+ *   callback) using \ref zoo_get_context. The object is not used by zookeeper
+ *   internally and can be null.
+ * \param flags reserved for future use. Should be set to zero.
+ * \param log_callback All log messages will be passed to this callback function.
+ *   For more details see \ref zoo_get_log_callback and \ref zoo_set_log_callback.
+ * \param sasl_params a pointer to a \ref zoo_sasl_params_t structure
+ *   specifying SASL connection parameters, or NULL to skip SASL
+ *   authentication
+ * \return a pointer to the opaque zhandle structure. If it fails to create
+ * a new zhandle the function returns NULL and the errno variable
+ * indicates the reason.
+ */
+ZOOAPI zhandle_t *zookeeper_init_sasl(const char *host, watcher_fn fn,
+  int recv_timeout, const clientid_t *clientid, void *context, int flags,
+  log_callback_fn log_callback, zoo_sasl_params_t *sasl_params);
+
+/**
+ * \brief allocates and initializes a basic array of Cyrus SASL callbacks.
+ *
+ * This small helper function makes it easy to pass "static"
+ * parameters to Cyrus SASL's underlying callback-based API.  Its use
+ * is not mandatory; you can still implement interactive dialogs by
+ * defining your own callbacks.
+ *
+ * \param user the "canned" response to \c SASL_CB_USER and \c SASL_CB_AUTHNAME,
+ *   or NULL for none
+ * \param realm the "canned" response to \c SASL_CB_GETREALM, or NULL for none
+ * \param password_file the name of a file whose first line is read in
+ *   response to \c SASL_CB_PASS, or NULL for none
+ * \return the freshly-malloc()ed callbacks array, or NULL if allocation
+ *   failed.  Deallocate with free(), but only after the corresponding
+ *   ZooKeeper handle is closed.
+ */
+ZOOAPI sasl_callback_t *zoo_sasl_make_basic_callbacks(const char *user,
+  const char *realm, const char* password_file);
+
+#endif /* HAVE_CYRUS_SASL_H */
+
 /**
  * \brief update the list of servers this client will connect to.
  *

zookeeper-client/zookeeper-client-c/src/zk_adaptor.h

@@ -18,6 +18,7 @@
 
 #ifndef ZK_ADAPTOR_H_
 #define ZK_ADAPTOR_H_
+
 #include <zookeeper.jute.h>
 #ifdef THREADED
 #ifndef WIN32
@@ -182,6 +183,8 @@ typedef struct _auth_list_head {
 #endif
 } auth_list_head_t;
 
+typedef struct _zoo_sasl_client zoo_sasl_client_t;
+
 /**
  * This structure represents the connection to zookeeper.
  */
@@ -261,6 +264,10 @@ struct _zhandle {
     /** used for chroot path at the client side **/
     char *chroot;
 
+#ifdef HAVE_CYRUS_SASL_H
+    zoo_sasl_client_t *sasl_client;
+#endif /* HAVE_CYRUS_SASL_H */
+
     /** Indicates if this client is allowed to go to r/o mode */
     char allow_read_only;
     /** Indicates if we connected to a majority server before */