From c89bddedfb9ff9fd03a12f568df6762897ffeb33 Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Thu, 9 Nov 2023 17:58:24 +0800 Subject: [PATCH] refactor(s2): Bump s2geometry to 0.10.0 --- .licenserc.yaml | 2 +- cmake_modules/BaseFunctions.cmake | 3 ++ src/geo/bench/CMakeLists.txt | 2 + src/geo/bench/bench.cpp | 1 - src/geo/lib/geo_client.h | 1 - src/geo/lib/latlng_codec.h | 1 - src/geo/test/CMakeLists.txt | 2 + src/geo/test/geo_test.cpp | 1 - src/geo/test/latlng_codec_test.cpp | 1 - src/redis_protocol/proxy/CMakeLists.txt | 2 + src/redis_protocol/proxy/main.cpp | 1 - src/redis_protocol/proxy_lib/redis_parser.h | 1 - src/redis_protocol/proxy_ut/CMakeLists.txt | 2 + .../proxy_ut/redis_proxy_test.cpp | 1 - src/server/hotspot_partition_calculator.cpp | 4 +- src/server/hotspot_partition_calculator.h | 1 - src/server/info_collector.h | 3 +- src/server/info_collector_app.h | 1 - src/server/main.cpp | 1 - src/shell/CMakeLists.txt | 2 + src/shell/commands/bulk_load.cpp | 1 - src/shell/commands/cold_backup.cpp | 1 - src/shell/commands/data_operations.cpp | 1 - src/shell/commands/debugger.cpp | 1 - src/shell/commands/detect_hotkey.cpp | 1 - src/shell/commands/duplication.cpp | 1 - src/shell/commands/global_properties.cpp | 1 - src/shell/commands/node_management.cpp | 1 - src/shell/commands/rebalance.cpp | 1 - src/shell/commands/recovery.cpp | 1 - src/shell/commands/table_management.cpp | 1 - src/shell/main.cpp | 1 - src/test/function_test/base_api/test_copy.cpp | 1 - thirdparty/CMakeLists.txt | 41 +++++++++--------- .../fix_s2_build_with_absl_and_gtest.patch | 42 +++++++++++++++++++ 35 files changed, 78 insertions(+), 51 deletions(-) create mode 100644 thirdparty/fix_s2_build_with_absl_and_gtest.patch diff --git a/.licenserc.yaml b/.licenserc.yaml index d6b5360827..15a2e4b78b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -70,7 +70,7 @@ header: - 'thirdparty/fix_libevent_for_macos.patch' - 'thirdparty/fix_prometheus-cpp_limits.patch' - 'thirdparty/fix_rocksdb-cmake-PORTABLE-option.patch' - - 'thirdparty/fix_s2_build_with_cxx17.patch' + - 'thirdparty/fix_s2_build_with_absl_and_gtest.patch' - 'thirdparty/fix_thrift_for_cpp11.patch' # TODO(yingchun): shell/* files are import from thirdparties, we can move them to thirdparty later. # Copyright (c) 2016, Adi Shavit diff --git a/cmake_modules/BaseFunctions.cmake b/cmake_modules/BaseFunctions.cmake index ad4133e0bc..1c50d630a4 100644 --- a/cmake_modules/BaseFunctions.cmake +++ b/cmake_modules/BaseFunctions.cmake @@ -353,6 +353,9 @@ function(dsn_setup_thirdparty_libs) link_libraries(${OPENSSL_CRYPTO_LIBRARY}) link_libraries(${OPENSSL_SSL_LIBRARY}) + # abseil + find_package(absl REQUIRED) + link_directories(${THIRDPARTY_INSTALL_DIR}/lib) if (NOT APPLE) link_directories(${THIRDPARTY_INSTALL_DIR}/lib64) diff --git a/src/geo/bench/CMakeLists.txt b/src/geo/bench/CMakeLists.txt index 1c4ea97a51..c55feb512c 100644 --- a/src/geo/bench/CMakeLists.txt +++ b/src/geo/bench/CMakeLists.txt @@ -28,6 +28,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + absl::flat_hash_set + absl::strings pegasus_geo_lib s2testing s2 diff --git a/src/geo/bench/bench.cpp b/src/geo/bench/bench.cpp index 18324e7673..47687b0b89 100644 --- a/src/geo/bench/bench.cpp +++ b/src/geo/bench/bench.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/lib/geo_client.h b/src/geo/lib/geo_client.h index ae3672c214..1d00dc7fe5 100644 --- a/src/geo/lib/geo_client.h +++ b/src/geo/lib/geo_client.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include #include #include diff --git a/src/geo/lib/latlng_codec.h b/src/geo/lib/latlng_codec.h index b8aa5caa33..91d986d1f3 100644 --- a/src/geo/lib/latlng_codec.h +++ b/src/geo/lib/latlng_codec.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include #include diff --git a/src/geo/test/CMakeLists.txt b/src/geo/test/CMakeLists.txt index b59934773b..69435f085d 100644 --- a/src/geo/test/CMakeLists.txt +++ b/src/geo/test/CMakeLists.txt @@ -28,6 +28,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS + absl::flat_hash_set + absl::strings pegasus_geo_lib s2testing s2 diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp index a607faa558..69074ac886 100644 --- a/src/geo/test/geo_test.cpp +++ b/src/geo/test/geo_test.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/src/geo/test/latlng_codec_test.cpp b/src/geo/test/latlng_codec_test.cpp index 174d24fe09..8ed69ebd07 100644 --- a/src/geo/test/latlng_codec_test.cpp +++ b/src/geo/test/latlng_codec_test.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include "geo/lib/latlng_codec.h" diff --git a/src/redis_protocol/proxy/CMakeLists.txt b/src/redis_protocol/proxy/CMakeLists.txt index f0584fc1a8..f93bc506aa 100644 --- a/src/redis_protocol/proxy/CMakeLists.txt +++ b/src/redis_protocol/proxy/CMakeLists.txt @@ -27,6 +27,8 @@ set(MY_PROJ_SRC "") set(MY_SRC_SEARCH_MODE "GLOB") set(MY_PROJ_LIBS pegasus.rproxylib + absl::flat_hash_set + absl::strings pegasus_geo_lib pegasus_reporter event diff --git a/src/redis_protocol/proxy/main.cpp b/src/redis_protocol/proxy/main.cpp index 7ac27e805a..21de0aa583 100644 --- a/src/redis_protocol/proxy/main.cpp +++ b/src/redis_protocol/proxy/main.cpp @@ -18,7 +18,6 @@ */ #include -#include #include #include #include diff --git a/src/redis_protocol/proxy_lib/redis_parser.h b/src/redis_protocol/proxy_lib/redis_parser.h index 92ab3af2c7..f66bce0002 100644 --- a/src/redis_protocol/proxy_lib/redis_parser.h +++ b/src/redis_protocol/proxy_lib/redis_parser.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include #include diff --git a/src/redis_protocol/proxy_ut/CMakeLists.txt b/src/redis_protocol/proxy_ut/CMakeLists.txt index 73cf0a74d0..521abddc20 100644 --- a/src/redis_protocol/proxy_ut/CMakeLists.txt +++ b/src/redis_protocol/proxy_ut/CMakeLists.txt @@ -31,6 +31,8 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex) set(MY_PROJ_LIBS pegasus.rproxylib pegasus_base + absl::flat_hash_set + absl::strings pegasus_geo_lib s2 pegasus_client_static diff --git a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp index 3436cc324e..2b76fd6e9a 100644 --- a/src/redis_protocol/proxy_ut/redis_proxy_test.cpp +++ b/src/redis_protocol/proxy_ut/redis_proxy_test.cpp @@ -32,7 +32,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/server/hotspot_partition_calculator.cpp b/src/server/hotspot_partition_calculator.cpp index 7f8920ff5d..59ffce7b63 100644 --- a/src/server/hotspot_partition_calculator.cpp +++ b/src/server/hotspot_partition_calculator.cpp @@ -84,7 +84,7 @@ void hotspot_partition_calculator::init_perf_counter(int partition_count) { for (int data_type = 0; data_type <= 1; data_type++) { for (int i = 0; i < partition_count; i++) { - string partition_desc = + std::string partition_desc = _app_name + '.' + (data_type == partition_qps_type::WRITE_HOTSPOT_DATA ? "write." : "read.") + std::to_string(i); @@ -95,7 +95,7 @@ void hotspot_partition_calculator::init_perf_counter(int partition_count) "app.pegasus", counter_name.c_str(), COUNTER_TYPE_NUMBER, counter_desc.c_str()); } - string total_desc = + std::string total_desc = _app_name + '.' + (data_type == partition_qps_type::WRITE_HOTSPOT_DATA ? "write.total" : "read.total"); std::string counter_name = fmt::format("app.stat.hotspots.{}", total_desc); diff --git a/src/server/hotspot_partition_calculator.h b/src/server/hotspot_partition_calculator.h index b497934e98..d117bdc717 100644 --- a/src/server/hotspot_partition_calculator.h +++ b/src/server/hotspot_partition_calculator.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include #include diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 258a3d6318..649d0dee09 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include // IWYU pragma: no_include @@ -233,7 +232,7 @@ class info_collector ::dsn::task_ptr _storage_size_stat_timer_task; ::dsn::utils::ex_lock_nr _capacity_unit_update_info_lock; // mapping 'node address' --> 'last updated timestamp' - std::map _capacity_unit_update_info; + std::map _capacity_unit_update_info; // _hotspot_calculator_store is to save hotspot_partition_calculator for each table, a // hotspot_partition_calculator saves historical hotspot data and alert perf_counters of // corresponding table diff --git a/src/server/info_collector_app.h b/src/server/info_collector_app.h index 3a0d552721..e21d9ca148 100644 --- a/src/server/info_collector_app.h +++ b/src/server/info_collector_app.h @@ -19,7 +19,6 @@ #pragma once -#include #include #include diff --git a/src/server/main.cpp b/src/server/main.cpp index 39ed8a7a21..7eb9253fab 100644 --- a/src/server/main.cpp +++ b/src/server/main.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 70447ed766..109df61658 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -51,6 +51,8 @@ set(MY_PROJ_LIBS PocoXML pegasus_geo_lib RocksDB::rocksdb + absl::flat_hash_set + absl::strings s2 hdfs ) diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp index 6f6f16d73b..17153dcfbe 100644 --- a/src/shell/commands/bulk_load.cpp +++ b/src/shell/commands/bulk_load.cpp @@ -23,7 +23,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/commands/cold_backup.cpp b/src/shell/commands/cold_backup.cpp index f6dbefefc7..ba159c4cea 100644 --- a/src/shell/commands/cold_backup.cpp +++ b/src/shell/commands/cold_backup.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/data_operations.cpp b/src/shell/commands/data_operations.cpp index f13b2fec99..7343e95cdb 100644 --- a/src/shell/commands/data_operations.cpp +++ b/src/shell/commands/data_operations.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/debugger.cpp b/src/shell/commands/debugger.cpp index 621fe2025a..4febb46401 100644 --- a/src/shell/commands/debugger.cpp +++ b/src/shell/commands/debugger.cpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/src/shell/commands/detect_hotkey.cpp b/src/shell/commands/detect_hotkey.cpp index b413c63b77..6d300d31d3 100644 --- a/src/shell/commands/detect_hotkey.cpp +++ b/src/shell/commands/detect_hotkey.cpp @@ -16,7 +16,6 @@ // under the License. #include -#include #include #include #include diff --git a/src/shell/commands/duplication.cpp b/src/shell/commands/duplication.cpp index caf25c9023..97bff425b0 100644 --- a/src/shell/commands/duplication.cpp +++ b/src/shell/commands/duplication.cpp @@ -18,7 +18,6 @@ */ #include -#include #include #include #include diff --git a/src/shell/commands/global_properties.cpp b/src/shell/commands/global_properties.cpp index 485570dab5..fd51bed611 100644 --- a/src/shell/commands/global_properties.cpp +++ b/src/shell/commands/global_properties.cpp @@ -17,7 +17,6 @@ * under the License. */ -#include #include #include diff --git a/src/shell/commands/node_management.cpp b/src/shell/commands/node_management.cpp index 7e51d67c79..fd9369b5c0 100644 --- a/src/shell/commands/node_management.cpp +++ b/src/shell/commands/node_management.cpp @@ -19,7 +19,6 @@ // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/shell/commands/rebalance.cpp b/src/shell/commands/rebalance.cpp index 37f0acb5f2..43a591cfc4 100644 --- a/src/shell/commands/rebalance.cpp +++ b/src/shell/commands/rebalance.cpp @@ -19,7 +19,6 @@ // IWYU pragma: no_include #include -#include #include #include #include diff --git a/src/shell/commands/recovery.cpp b/src/shell/commands/recovery.cpp index 340b1ab7f3..8df98616c6 100644 --- a/src/shell/commands/recovery.cpp +++ b/src/shell/commands/recovery.cpp @@ -20,7 +20,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/commands/table_management.cpp b/src/shell/commands/table_management.cpp index 818ca8a715..806a7a13dc 100644 --- a/src/shell/commands/table_management.cpp +++ b/src/shell/commands/table_management.cpp @@ -21,7 +21,6 @@ // IWYU pragma: no_include #include #include -#include #include #include #include diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 99ba64eb34..50c1614c10 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include diff --git a/src/test/function_test/base_api/test_copy.cpp b/src/test/function_test/base_api/test_copy.cpp index d770a4b53c..e46c8b466f 100644 --- a/src/test/function_test/base_api/test_copy.cpp +++ b/src/test/function_test/base_api/test_copy.cpp @@ -18,7 +18,6 @@ */ #include -#include #include #include #include diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 131788e5a2..b8608dc363 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -366,22 +366,29 @@ ExternalProject_Add(nlohmann_json DOWNLOAD_NO_PROGRESS true ) +ExternalProject_Add(abseil + URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip + https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip + URL_MD5 5c6193dbc82834f8e762c6a28c9cc615 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DABSL_FIND_GOOGLETEST=OFF + -DCMAKE_CXX_STANDARD=17 + DOWNLOAD_EXTRACT_TIMESTAMP true + DOWNLOAD_NO_PROGRESS true +) + ExternalProject_Add(s2geometry - URL ${OSS_URL_PREFIX}/s2geometry-e8d16637a467d9f096a92a6d81a50a9b747ca828.zip - https://github.com/neverchanje/s2geometry/archive/e8d16637a467d9f096a92a6d81a50a9b747ca828.zip - URL_MD5 75cc44c9c31382083d8a2d0e42590788 - PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_s2_build_with_cxx17.patch + URL ${OSS_URL_PREFIX}/s2geometry-0.10.0.tar.gz + https://github.com/google/s2geometry/archive/refs/tags/v0.10.0.tar.gz + URL_MD5 c68f3c5d326dde9255681b9201393a9f + PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_s2_build_with_absl_and_gtest.patch CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DGOOGLETEST_VERSION=1.14.0 - -DGTEST_ROOT=${googletest_SRC}/googletest -DBUILD_SHARED_LIBS=OFF - -DBUILD_PYTHON=OFF - -DBUILD_TESTING=OFF + -DGTEST_ROOT=${googletest_SRC}/googletest + -DWITH_PYTHON=OFF -DBUILD_EXAMPLES=OFF - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_CXX_STANDARD=17 - DEPENDS googletest + DEPENDS googletest abseil DOWNLOAD_EXTRACT_TIMESTAMP true DOWNLOAD_NO_PROGRESS true ) @@ -453,13 +460,3 @@ ExternalProject_Add(http-parser DOWNLOAD_NO_PROGRESS true ) -ExternalProject_Add(abseil - URL ${OSS_URL_PREFIX}/abseil-20230802.1.zip - https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip - URL_MD5 5c6193dbc82834f8e762c6a28c9cc615 - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${TP_OUTPUT} - -DABSL_FIND_GOOGLETEST=OFF - -DCMAKE_CXX_STANDARD=14 - DOWNLOAD_EXTRACT_TIMESTAMP true - DOWNLOAD_NO_PROGRESS true -) diff --git a/thirdparty/fix_s2_build_with_absl_and_gtest.patch b/thirdparty/fix_s2_build_with_absl_and_gtest.patch new file mode 100644 index 0000000000..43c534a137 --- /dev/null +++ b/thirdparty/fix_s2_build_with_absl_and_gtest.patch @@ -0,0 +1,42 @@ +From 3e3d469e81b5b8934abc4b15cd9e33065b5a489f Mon Sep 17 00:00:00 2001 +From: Yingchun Lai +Date: Tue, 21 Nov 2023 23:43:25 +0800 +Subject: [PATCH] Build as thirdparty of Apache Pegasus + +--- + CMakeLists.txt | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f080b4b..8e3471a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -19,7 +19,7 @@ endif() + # undefined symbol errors since ABSL_HAVE_STD_STRING_VIEW etc will + # end up defined differently. There is probably a better way to achieve + # this than assuming what absl used. +-set(CMAKE_CXX_STANDARD 11) ++set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + # No compiler-specific extensions, i.e. -std=c++11, not -std=gnu++11. + set(CMAKE_CXX_EXTENSIONS OFF) +@@ -40,6 +40,7 @@ add_feature_info(SHARED_LIBS BUILD_SHARED_LIBS + "builds shared libraries instead of static.") + + option(BUILD_EXAMPLES "Build s2 documentation examples." ON) ++option(BUILD_TESTS "Build s2 tests." OFF) + + option(WITH_PYTHON "Add python interface" OFF) + add_feature_info(PYTHON WITH_PYTHON "provides python interface to S2") +@@ -429,7 +430,7 @@ install(TARGETS ${S2_TARGETS} + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + message("GTEST_ROOT: ${GTEST_ROOT}") +-if (GTEST_ROOT) ++if (GTEST_ROOT AND BUILD_TESTS) + add_subdirectory(${GTEST_ROOT} build_gtest) + include_directories(${GTEST_ROOT}/include) + +-- +2.41.0 +