[CELEBORN-1741][CIP-14] Add processBase utils to cppClient

cpp/CMakeLists.txt

@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+cmake_minimum_required(VERSION 3.24)
+
+if (NOT DEFINED PACKAGE_VERSION)
+    set(PACKAGE_VERSION "0.0.1")
+endif ()
+
+project("celeborn" VERSION ${PACKAGE_VERSION} LANGUAGES CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+message("Appending CMAKE_CXX_FLAGS with ${SCRIPT_CXX_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}")
+if ("${TREAT_WARNINGS_AS_ERRORS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+endif ()
+
+# Avoid folly::f14::detail::F14LinkCheck problem on x86-64 platform.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
+
+# Set CMAKE_BUILD_TYPE to 'Release' if it is not specified.
+if (NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release)
+endif ()
+message(STATUS "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
+
+# Known warnings that are benign can be disabled.
+set(DISABLED_WARNINGS
+        "-Wno-nullability-completeness -Wno-deprecated-declarations")
+
+# Important warnings that must be explicitly enabled.
+set(ENABLE_WARNINGS "-Wreorder")
+
+# The CMAKE_PREFIX_PATH should be set to the thirdparty's install path
+# (thirdparty/installed by default), to find all the dependencies.
+message(STATUS "Using CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}")
+# Currently, we use the "-lgtest" to link the gtest library, and gtest library
+# is in the "thirdparty/installed/lib64" directory in the linux environment, so
+# the lib search path needs to be specified here.
+if (EXISTS ${CMAKE_PREFIX_PATH}/lib64)
+    link_directories(${CMAKE_PREFIX_PATH}/lib64)
+    # thrift.a is installed in the directory thirdparty/installed/lib by
+    # default in the Linux environment.
+    link_directories(${CMAKE_PREFIX_PATH}/lib)
+elseif (CMAKE_SYSTEM_NAME MATCHES "Linux")
+    message(
+            WARNING
+            "You did not use the recommended way "
+            "(using 'thirdparty/build-thirdparty.sh') to build & install "
+            "thirdparty libraries.")
+endif ()
+
+#if (NOT APPLE)
+#    set(Boost_USE_STATIC_RUNTIME ON)
+#    set(Boost_USE_STATIC_LIBS ON)
+#endif ()
+set(Boost_USE_MULTITHREADED TRUE)
+find_package(
+        Boost
+	#1.75.0
+        1.84.0
+        REQUIRED
+        program_options
+        context
+        filesystem
+        regex
+        thread
+        system
+        date_time
+        atomic)
+include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
+
+# GFlags
+#
+# NOTE: The name here must be exactly "gflags", that is, use all lowercase.
+# Otherwise, e.g. "GFLAGS" or "GFlags", the generated `GFLAGS_LIBRARIES` will
+# point to the shared library instead of the static library, even if we
+# explicitly specify to link against the static library (via "COMPONENTS
+# static"). This may be a problem that the cmake script of GFlags does not
+# consider comprehensively (for the case of words).
+#
+# See [1] for the recommended `find_package` commands to use to find GFlags, in
+# which the "@PACKAGE_NAME@" will be replaced with "gflags" when installed.
+#
+# [1] https://github.com/gflags/gflags/blob/v2.2.2/cmake/config.cmake.in#L50-L56
+if (APPLE)
+    # Use the shared library of gflags on MacOS because it is installed via
+    # Homebrew and only shared library is installed.
+    find_package(gflags REQUIRED COMPONENTS shared)
+else ()
+    find_package(gflags REQUIRED COMPONENTS static)
+endif ()
+
+find_package(glog REQUIRED)
+find_library(FMT fmt)
+
+find_package(folly CONFIG REQUIRED)
+set(FOLLY_WITH_DEPENDENCIES
+        ${FOLLY_LIBRARIES}
+        Boost::context
+        dl
+)
+
+# Include third party header files
+find_path(OPT_OPENSSL_DIR NAMES opt/[email protected])
+set(OPENSSL_ROOT_DIR "${OPT_OPENSSL_DIR}/opt/[email protected]")
+find_package(OpenSSL REQUIRED)
+
+find_package(Protobuf REQUIRED)
+
+set(CMAKE_MODULE_PATH
+        "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+        ${CMAKE_MODULE_PATH})
+find_package(Sodium REQUIRED)
+find_library(FIZZ fizz REQUIRED)
+find_library(WANGLE wangle REQUIRED)
+
+find_library(RE2 re2)
+
+find_package(fizz CONFIG REQUIRED)
+find_package(wangle CONFIG REQUIRED)
+
+set(WANGLE_LIBRARIES ${WANGLE} ${FIZZ})
+
+include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR})
+
+include_directories(SYSTEM celeborn)
+include_directories(.)
+
+add_subdirectory(celeborn)

cpp/README.md

@@ -37,3 +37,14 @@ cd scripts
 bash setup-ubuntu.sh
 ```
 Other platforms are not supported yet, and you could use the container above as your dev environment.
+
+## Compile
+Currently, the modules are under development. 
+You could compile the code within the dev container by
+```
+cd celeborn/cpp
+mkdir -p build && cd build
+cmake ..
+make
+```
+

cpp/celeborn/CMakeLists.txt

@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+add_subdirectory(utils)

cpp/celeborn/utils/CMakeLists.txt

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+add_library(utils ProcessBase.cpp)
+
+target_link_libraries(
+        utils
+        ${WANGLE}
+        ${FIZZ}
+        ${LIBSODIUM_LIBRARY}
+        ${FOLLY_WITH_DEPENDENCIES}
+        ${GLOG}
+        ${GFLAGS_LIBRARIES}
+)

cpp/celeborn/utils/ProcessBase.cpp

@@ -0,0 +1,129 @@
+/*
+ * Based on ProcessBase.cpp from Facebook Velox
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "celeborn/utils/ProcessBase.h"
+
+#include <limits.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <folly/CpuId.h>
+#include <folly/FileUtil.h>
+#include <folly/String.h>
+#include <gflags/gflags.h>
+
+constexpr const char* kProcSelfCmdline = "/proc/self/cmdline";
+
+namespace celeborn {
+namespace utils {
+
+DECLARE_bool(celeborn_avx2); // Enables use of AVX2 when available NOLINT
+
+DECLARE_bool(celeborn_bmi2); // Enables use of BMI2 when available NOLINT
+
+/**
+ * Current executable's name.
+ */
+std::string getAppName() {
+  const char* result = getenv("_");
+  if (result) {
+    return result;
+  }
+
+  // if we're running under gtest, getenv will return null
+  std::string appName;
+  if (folly::readFile(kProcSelfCmdline, appName)) {
+    auto pos = appName.find('\0');
+    if (pos != std::string::npos) {
+      appName = appName.substr(0, pos);
+    }
+
+    return appName;
+  }
+
+  return "";
+}
+
+/**
+ * This machine's name.
+ */
+std::string getHostName() {
+  char hostbuf[_POSIX_HOST_NAME_MAX + 1];
+  if (gethostname(hostbuf, _POSIX_HOST_NAME_MAX + 1) < 0) {
+    return "";
+  } else {
+    // When the host name is precisely HOST_NAME_MAX bytes long, gethostname
+    // returns 0 even though the result is not NUL-terminated. Manually NUL-
+    // terminate to handle that case.
+    hostbuf[_POSIX_HOST_NAME_MAX] = '\0';
+    return hostbuf;
+  }
+}
+
+/**
+ * Process identifier.
+ */
+pid_t getProcessId() {
+  return getpid();
+}
+
+/**
+ * Current thread's identifier.
+ */
+pthread_t getThreadId() {
+  return pthread_self();
+}
+
+/**
+ * Get current working directory.
+ */
+std::string getCurrentDirectory() {
+  char buf[PATH_MAX];
+  return getcwd(buf, PATH_MAX);
+}
+
+uint64_t threadCpuNanos() {
+  timespec ts;
+  clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
+  return ts.tv_sec * 1'000'000'000 + ts.tv_nsec;
+}
+
+namespace {
+bool bmi2CpuFlag = folly::CpuId().bmi2();
+bool avx2CpuFlag = folly::CpuId().avx2();
+} // namespace
+
+bool hasAvx2() {
+#ifdef __AVX2__
+  return avx2CpuFlag && FLAGS_celeborn_avx2;
+#else
+  return false;
+#endif
+}
+
+bool hasBmi2() {
+#ifdef __BMI2__
+  return bmi2CpuFlag && FLAGS_celeborn_bmi2;
+#else
+  return false;
+#endif
+}
+
+} // namespace utils
+} // namespace celeborn

cpp/celeborn/utils/ProcessBase.h

@@ -0,0 +1,68 @@
+/*
+ * Based on ProcessBase.h from Facebook Velox
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <pthread.h>
+#include <sys/types.h>
+#include <string>
+#include <vector>
+
+namespace celeborn {
+namespace utils {
+
+/**
+ * Current executable's name.
+ */
+std::string getAppName();
+
+/**
+ * This machine'a name.
+ */
+std::string getHostName();
+
+/**
+ * Process identifier.
+ */
+pid_t getProcessId();
+
+/**
+ * Current thread's identifier.
+ */
+pthread_t getThreadId();
+
+/**
+ * Get current working directory.
+ */
+std::string getCurrentDirectory();
+
+/**
+ * Returns elapsed CPU nanoseconds on the calling thread
+ */
+uint64_t threadCpuNanos();
+
+// True if the machine has Intel AVX2 instructions and these are not disabled by
+// flag.
+bool hasAvx2();
+
+// True if the machine has Intel BMI2 instructions and these are not disabled by
+// flag.
+bool hasBmi2();
+
+} // namespace utils
+} // namespace celeborn