From 5e397f1d146041c1f66db7c66676d39c113479f6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 9 Nov 2023 17:58:24 +0800 Subject: [PATCH] refactor(s2): Bump s2geometry to 0.10.0 --- .licenserc.yaml | 2 +- cmake_modules/BaseFunctions.cmake | 3 ++ src/geo/bench/CMakeLists.txt | 2 + src/geo/bench/bench.cpp | 1 - src/geo/lib/geo_client.h | 1 - src/geo/lib/latlng_codec.h | 1 - src/geo/test/CMakeLists.txt | 2 + src/geo/test/geo_test.cpp | 1 - src/geo/test/latlng_codec_test.cpp | 1 - src/redis_protocol/proxy/CMakeLists.txt | 2 + src/redis_protocol/proxy/main.cpp | 1 - src/redis_protocol/proxy_lib/redis_parser.h | 1 - src/redis_protocol/proxy_ut/CMakeLists.txt | 2 + .../proxy_ut/redis_proxy_test.cpp | 1 - src/server/hotspot_partition_calculator.cpp | 4 +- src/server/hotspot_partition_calculator.h | 1 - src/server/info_collector.h | 3 +- src/server/info_collector_app.h | 1 - src/server/main.cpp | 1 - src/shell/CMakeLists.txt | 2 + src/shell/commands/bulk_load.cpp | 1 - src/shell/commands/cold_backup.cpp | 1 - src/shell/commands/data_operations.cpp | 1 - src/shell/commands/debugger.cpp | 1 - src/shell/commands/detect_hotkey.cpp | 1 - src/shell/commands/duplication.cpp | 1 - src/shell/commands/global_properties.cpp | 1 - src/shell/commands/node_management.cpp | 1 - src/shell/commands/rebalance.cpp | 1 - src/shell/commands/recovery.cpp | 1 - src/shell/commands/table_management.cpp | 1 - src/shell/main.cpp | 1 - src/test/function_test/base_api/test_copy.cpp | 1 - thirdparty/CMakeLists.txt | 42 +++++++++---------- .../fix_s2_build_with_absl_and_gtest.patch | 42 +++++++++++++++++++ thirdparty/fix_s2_for_aarch64.patch | 10 ----- 36 files changed, 78 insertions(+), 62 deletions(-) create mode 100644 thirdparty/fix_s2_build_with_absl_and_gtest.patch delete mode 100644 thirdparty/fix_s2_for_aarch64.patch diff --git a/.licenserc.yaml b/.licenserc.yaml index 2cea94f325..4d5c1a0494 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -69,7 +69,7 @@ header: - 'thirdparty/fix_jemalloc_for_m1_on_macos.patch' - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' - - 'thirdparty/fix_s2_for_aarch64.patch' + - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch' - 'thirdparty/fix_thrift_for_cpp11.patch' - 'thirdparty/rocksdb_fix_atomic_flush_0879c240.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index ad4133e0bc..1c50d630a4 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -353,6 +353,9 @@ function(dsn_setup_thirdparty_libs) link_libraries(${OPENSSL_CRYPTO_LIBRARY}) link_libraries(${OPENSSL_SSL_LIBRARY}) + # abseil + find_package(absl REQUIRED) + link_directories(${THIRDPARTY_INSTALL_DIR}/lib) if (NOT APPLE) link_directories(${THIRDPARTY_INSTALL_DIR}/lib64) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 1c4ea97a51..c55feb512c 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -28,6 +28,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + absl::flat_hash_set + absl::strings pegasus_geo_lib s2testing s2 diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index 18324e7673..47687b0b89 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/lib/geo_client.h b/src/geo/lib/geo_client.h index ae3672c214..1d00dc7fe5 100644 --- a/src/geo/lib/geo_client.h +++ b/src/geo/lib/geo_client.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include #include #include diff --git a/src/geo/lib/latlng_codec.h b/src/geo/lib/latlng_codec.h index b8aa5caa33..91d986d1f3 100644 --- a/src/geo/lib/latlng_codec.h +++ b/src/geo/lib/latlng_codec.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include #include diff --git a/src/geo/test/CMakeLists.txt b/src/geo/test/CMakeLists.txt index b59934773b..69435f085d 100644 --- a/src/geo/test/CMakeLists.txt +++ b/src/geo/test/CMakeLists.txt @@ -28,6 +28,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + absl::flat_hash_set + absl::strings pegasus_geo_lib s2testing s2 diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp index e9df7a422a..806b1691ad 100644 --- a/src/geo/test/geo_test.cpp +++ b/src/geo/test/geo_test.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/test/latlng_codec_test.cpp b/src/geo/test/latlng_codec_test.cpp index 22f0243cbe..08c257dbc3 100644 --- a/src/geo/test/latlng_codec_test.cpp +++ b/src/geo/test/latlng_codec_test.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "geo/lib/latlng_codec.h" diff --git a/src/redis_protocol/proxy/CMakeLists.txt b/src/redis_protocol/proxy/CMakeLists.txt index f0584fc1a8..f93bc506aa 100644 --- a/src/redis_protocol/proxy/CMakeLists.txt +++ b/src/redis_protocol/proxy/CMakeLists.txt @@ -27,6 +27,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus.rproxylib + absl::flat_hash_set + absl::strings pegasus_geo_lib pegasus_reporter event diff --git a/src/redis_protocol/proxy/main.cpp b/src/redis_protocol/proxy/main.cpp index 7ac27e805a..21de0aa583 100644 --- a/src/redis_protocol/proxy/main.cpp +++ b/src/redis_protocol/proxy/main.cpp @@ -18,7 +18,6 @@ */ #include -#include #include #include #include diff --git a/src/redis_protocol/proxy_lib/redis_parser.h b/src/redis_protocol/proxy_lib/redis_parser.h index 92ab3af2c7..f66bce0002 100644 --- a/src/redis_protocol/proxy_lib/redis_parser.h +++ b/src/redis_protocol/proxy_lib/redis_parser.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include #include diff --git a/src/redis_protocol/proxy_ut/CMakeLists.txt b/src/redis_protocol/proxy_ut/CMakeLists.txt index 73cf0a74d0..521abddc20 100644 --- a/src/redis_protocol/proxy_ut/CMakeLists.txt +++ b/src/redis_protocol/proxy_ut/CMakeLists.txt @@ -31,6 +31,8 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) set(MY_PROJ_LIBS pegasus.rproxylib pegasus_base + absl::flat_hash_set + absl::strings pegasus_geo_lib s2 pegasus_client_static diff --git a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp index 400c41b4ce..7b54af9f68 100644 --- a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp +++ b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp @@ -35,7 +35,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 7f8920ff5d..59ffce7b63 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -84,7 +84,7 @@ void hotspot_partition_calculator::init_perf_counter(int partition_count) { for (int data_type = 0; data_type <= 1; data_type++) { for (int i = 0; i < partition_count; i++) { - string partition_desc = + std::string partition_desc = _app_name + '.' + (data_type == partition_qps_type::WRITE_HOTSPOT_DATA ? "write." : "read.") + std::to_string(i); @@ -95,7 +95,7 @@ void hotspot_partition_calculator::init_perf_counter(int partition_count) "app.pegasus", counter_name.c_str(), COUNTER_TYPE_NUMBER, counter_desc.c_str()); } - string total_desc = + std::string total_desc = _app_name + '.' + (data_type == partition_qps_type::WRITE_HOTSPOT_DATA ? "write.total" : "read.total"); std::string counter_name = fmt::format("app.stat.hotspots.{}", total_desc); diff --git a/src/server/hotspot_partition_calculator.h b/src/server/hotspot_partition_calculator.h index b497934e98..d117bdc717 100644 --- a/src/server/hotspot_partition_calculator.h +++ b/src/server/hotspot_partition_calculator.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 258a3d6318..649d0dee09 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include // IWYU pragma: no_include @@ -233,7 +232,7 @@ class info_collector ::dsn::task_ptr _storage_size_stat_timer_task; ::dsn::utils::ex_lock_nr _capacity_unit_update_info_lock; // mapping 'node address' --> 'last updated timestamp' - std::map _capacity_unit_update_info; + std::map _capacity_unit_update_info; // _hotspot_calculator_store is to save hotspot_partition_calculator for each table, a // hotspot_partition_calculator saves historical hotspot data and alert perf_counters of // corresponding table diff --git a/src/server/info_collector_app.h b/src/server/info_collector_app.h index 3a0d552721..e21d9ca148 100644 --- a/src/server/info_collector_app.h +++ b/src/server/info_collector_app.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include diff --git a/src/server/main.cpp b/src/server/main.cpp index 39ed8a7a21..7eb9253fab 100644 --- a/src/server/main.cpp +++ b/src/server/main.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 70447ed766..109df61658 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -51,6 +51,8 @@ set(MY_PROJ_LIBS PocoXML pegasus_geo_lib RocksDB::rocksdb + absl::flat_hash_set + absl::strings s2 hdfs ) diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp index 6f6f16d73b..17153dcfbe 100644 --- a/src/shell/commands/bulk_load.cpp +++ b/src/shell/commands/bulk_load.cpp @@ -23,7 +23,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/commands/cold_backup.cpp b/src/shell/commands/cold_backup.cpp index f6dbefefc7..ba159c4cea 100644 --- a/src/shell/commands/cold_backup.cpp +++ b/src/shell/commands/cold_backup.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/data_operations.cpp b/src/shell/commands/data_operations.cpp index f13b2fec99..7343e95cdb 100644 --- a/src/shell/commands/data_operations.cpp +++ b/src/shell/commands/data_operations.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 621fe2025a..4febb46401 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index b413c63b77..6d300d31d3 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -16,7 +16,6 @@ // under the License. #include -#include #include #include #include diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index caf25c9023..97bff425b0 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -18,7 +18,6 @@ */ #include -#include #include #include #include diff --git a/src/shell/commands/global_properties.cpp b/src/shell/commands/global_properties.cpp index 485570dab5..fd51bed611 100644 --- a/src/shell/commands/global_properties.cpp +++ b/src/shell/commands/global_properties.cpp @@ -17,7 +17,6 @@ * under the License. */ -#include #include #include diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index 7e51d67c79..fd9369b5c0 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -19,7 +19,6 @@ // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/shell/commands/rebalance.cpp b/src/shell/commands/rebalance.cpp index 37f0acb5f2..43a591cfc4 100644 --- a/src/shell/commands/rebalance.cpp +++ b/src/shell/commands/rebalance.cpp @@ -19,7 +19,6 @@ // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/shell/commands/recovery.cpp b/src/shell/commands/recovery.cpp index 340b1ab7f3..8df98616c6 100644 --- a/src/shell/commands/recovery.cpp +++ b/src/shell/commands/recovery.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/commands/table_management.cpp b/src/shell/commands/table_management.cpp index 818ca8a715..806a7a13dc 100644 --- a/src/shell/commands/table_management.cpp +++ b/src/shell/commands/table_management.cpp @@ -21,7 +21,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 99ba64eb34..50c1614c10 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include diff --git a/src/test/function_test/base_api/test_copy.cpp b/src/test/function_test/base_api/test_copy.cpp index d2155f12f0..92cd79c19b 100644 --- a/src/test/function_test/base_api/test_copy.cpp +++ b/src/test/function_test/base_api/test_copy.cpp @@ -21,7 +21,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 8645ee1a1c..98a4d4178a 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -86,9 +86,9 @@ ExternalProject_Add(readerwriterqueue ) ExternalProject_Add(googletest - URL ${OSS_URL_PREFIX}/googletest-release-1.8.0.tar.gz - https://codeload.github.com/google/googletest/tar.gz/release-1.8.0 - URL_MD5 16877098823401d1bf2ed7891d7dce36 + URL ${OSS_URL_PREFIX}/googletest-1.14.0.tar.gz + https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz + URL_MD5 c8340a482851ef6a3fe618a082304cfc CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} @@ -336,21 +336,26 @@ ExternalProject_Add(nlohmann_json BUILD_IN_SOURCE 1 ) +ExternalProject_Add(abseil + URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip + https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip + URL_MD5 5c6193dbc82834f8e762c6a28c9cc615 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DABSL_FIND_GOOGLETEST=OFF + -DCMAKE_CXX_STANDARD=17) + ExternalProject_Add(s2geometry - URL ${OSS_URL_PREFIX}/s2geometry-e8d16637a467d9f096a92a6d81a50a9b747ca828.zip - https://github.com/neverchanje/s2geometry/archive/e8d16637a467d9f096a92a6d81a50a9b747ca828.zip - URL_MD5 75cc44c9c31382083d8a2d0e42590788 - PATCH_COMMAND patch -p0 < ${TP_DIR}/fix_s2_for_aarch64.patch + URL ${OSS_URL_PREFIX}/s2geometry-0.10.0.tar.gz + https://github.com/google/s2geometry/archive/refs/tags/v0.10.0.tar.gz + URL_MD5 c68f3c5d326dde9255681b9201393a9f + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_s2_build_with_absl_and_gtest.patch CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DGTEST_ROOT=${googletest_SRC}/googletest -DBUILD_SHARED_LIBS=OFF - -DBUILD_PYTHON=OFF - -DBUILD_TESTING=OFF + -DGTEST_ROOT=${googletest_SRC}/googletest + -DWITH_PYTHON=OFF -DBUILD_EXAMPLES=OFF - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - DEPENDS googletest - ) + DEPENDS googletest abseil) option(USE_JEMALLOC "use jemalloc" OFF) message(STATUS "USE_JEMALLOC = ${USE_JEMALLOC}") @@ -412,12 +417,3 @@ ExternalProject_Add(http-parser INSTALL_COMMAND cp -R http_parser.h ${TP_OUTPUT}/include/nodejs/http_parser.h BUILD_IN_SOURCE 1 ) - -ExternalProject_Add(abseil - URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip - https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip - URL_MD5 5c6193dbc82834f8e762c6a28c9cc615 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DABSL_FIND_GOOGLETEST=OFF - -DCMAKE_CXX_STANDARD=14 - ) diff --git a/thirdparty/fix_s2_build_with_absl_and_gtest.patch b/thirdparty/fix_s2_build_with_absl_and_gtest.patch new file mode 100644 index 0000000000..43c534a137 --- /dev/null +++ b/thirdparty/fix_s2_build_with_absl_and_gtest.patch @@ -0,0 +1,42 @@ +From 3e3d469e81b5b8934abc4b15cd9e33065b5a489f Mon Sep 17 00:00:00 2001 +From: Yingchun Lai +Date: Tue, 21 Nov 2023 23:43:25 +0800 +Subject: [PATCH] Build as thirdparty of Apache Pegasus + +--- + CMakeLists.txt | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f080b4b..8e3471a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -19,7 +19,7 @@ endif() + # undefined symbol errors since ABSL_HAVE_STD_STRING_VIEW etc will + # end up defined differently. There is probably a better way to achieve + # this than assuming what absl used. +-set(CMAKE_CXX_STANDARD 11) ++set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + # No compiler-specific extensions, i.e. -std=c++11, not -std=gnu++11. + set(CMAKE_CXX_EXTENSIONS OFF) +@@ -40,6 +40,7 @@ add_feature_info(SHARED_LIBS BUILD_SHARED_LIBS + "builds shared libraries instead of static.") + + option(BUILD_EXAMPLES "Build s2 documentation examples." ON) ++option(BUILD_TESTS "Build s2 tests." OFF) + + option(WITH_PYTHON "Add python interface" OFF) + add_feature_info(PYTHON WITH_PYTHON "provides python interface to S2") +@@ -429,7 +430,7 @@ install(TARGETS ${S2_TARGETS} + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + message("GTEST_ROOT: ${GTEST_ROOT}") +-if (GTEST_ROOT) ++if (GTEST_ROOT AND BUILD_TESTS) + add_subdirectory(${GTEST_ROOT} build_gtest) + include_directories(${GTEST_ROOT}/include) + +-- +2.41.0 + diff --git a/thirdparty/fix_s2_for_aarch64.patch b/thirdparty/fix_s2_for_aarch64.patch deleted file mode 100644 index c218233acc..0000000000 --- a/thirdparty/fix_s2_for_aarch64.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- CMakeLists.txt 2022-04-24 06:33:29.829438987 +0000 -+++ CMakeLists.aarch64.txt 2022-04-24 06:34:09.079226044 +0000 -@@ -355,6 +355,7 @@ install(FILES src/s2/third_party/absl/ba - src/s2/third_party/absl/base/port.h - src/s2/third_party/absl/base/thread_annotations.h - DESTINATION include/s2/third_party/absl/base) -+execute_process(COMMAND sed -i "/attributes\.h/a\#include \"s2\/base\/integral_types\.h\"" ${PROJECT_SOURCE_DIR}/src/s2/third_party/absl/base/internal/unaligned_access.h) - install(FILES src/s2/third_party/absl/base/internal/identity.h - src/s2/third_party/absl/base/internal/inline_variable.h - src/s2/third_party/absl/base/internal/invoke.h