build: add rocksdb as thirdparty (#483)

apache · Jun 2, 2020 · 990ae61 · 990ae61
1 parent fb4eb4c
commit 990ae61
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 26 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,39 +1,28 @@
 sudo: required
-dist: trusty
+dist: bionic
 
 language: cpp
 
-os: linux
-
 compiler:
     - gcc
 
 cache:
     - ccache
     - apt
 
-addons:
-    apt:
-        packages:
-            - clang-format-3.9
-
 before_install:
-    - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/build-depends.tar.gz
-    - tar xfz build-depends.tar.gz
-    - rm -f build-depends.tar.gz
-    - cd packages
-    - ls | xargs sudo dpkg -i --force-depends
-    - cd ..
+    - sudo apt-get -y install libboost-all-dev libaio-dev clang-format-3.9
 
 before_script:
     - cd thirdparty
-    - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-thirdparty-prebuild.tar.gz
-    - tar xf pegasus-thirdparty-prebuild.tar.gz
-    - rm -f pegasus-thirdparty-prebuild.tar.gz
+    - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-thirdparty-prebuild.zip
+    - unzip pegasus-thirdparty-prebuild.zip
+    - rm -f pegasus-thirdparty-prebuild.zip
     - cd ..
     - ulimit -c unlimited -S
 
 script:
+    - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib/:$LD_LIBRARY_PATH
     - ./run.sh test --skip_thirdparty --check --disable_gperf
 
 after_script:

diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh
@@ -59,10 +59,17 @@ mkdir -p $ZOOKEEPER_HOME/data
 $ZOOKEEPER_HOME/bin/zkServer.sh start
 sleep 1
 
-if echo ruok | nc localhost $ZOOKEEPER_PORT | grep -q imok; then
-    echo "Zookeeper started at port $ZOOKEEPER_PORT"
-    exit 0
-else
-    echo "ERROR: start zookeeper failed"
-    exit 1
-fi
+zk_check_count=0
+while true; do
+    sleep 1 # wait until zookeeper bootstrapped
+    if echo ruok | nc localhost "$ZOOKEEPER_PORT" | grep -q imok; then
+        echo "Zookeeper started at port $ZOOKEEPER_PORT"
+        exit 0
+    fi
+    zk_check_count=$((zk_check_count+1))
+    echo "ERROR: starting zookeeper has failed ${zk_check_count} times"
+    if [ $zk_check_count -gt 30 ]; then
+        echo "ERROR: failed to start zookeeper in 30 seconds"
+        exit 1
+    fi
+done
diff --git a/src/core/tests/address.cpp b/src/core/tests/address.cpp
@@ -52,7 +52,9 @@ static inline uint32_t host_ipv4(uint8_t sec1, uint8_t sec2, uint8_t sec3, uint8
 TEST(core, rpc_address_ipv4_from_host)
 {
     // localhost --> 127.0.0.1
-    ASSERT_EQ(host_ipv4(127, 0, 0, 1), rpc_address::ipv4_from_host("localhost"));
+    // on some systems "localhost" could be "127.0.1.1" (debian)
+    ASSERT_TRUE(host_ipv4(127, 0, 0, 1) == rpc_address::ipv4_from_host("localhost") ||
+                host_ipv4(127, 0, 1, 1) == rpc_address::ipv4_from_host("localhost"));
 
     // 127.0.0.1 --> 127.0.0.1
     ASSERT_EQ(host_ipv4(127, 0, 0, 1), rpc_address::ipv4_from_host("127.0.0.1"));
@@ -115,7 +117,8 @@ TEST(core, dsn_address_build)
         ASSERT_EQ(host_ipv4(127, 0, 0, 1), addr.ip());
         ASSERT_EQ(8080, addr.port());
 
-        ASSERT_EQ(addr, dsn::rpc_address("localhost", 8080));
+        ASSERT_TRUE(dsn::rpc_address("127.0.0.1", 8080) == dsn::rpc_address("localhost", 8080) ||
+                    dsn::rpc_address("127.0.1.1", 8080) == dsn::rpc_address("localhost", 8080));
         ASSERT_EQ(addr, dsn::rpc_address("127.0.0.1", 8080));
         ASSERT_EQ(addr, dsn::rpc_address(host_ipv4(127, 0, 0, 1), 8080));
     }

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -330,3 +330,23 @@ if [ ! -d $TP_OUTPUT/include/nlohmann ]; then
 else
     echo "skip build nlohmann_json"
 fi
+
+#build rocksdb
+if [ ! -d $TP_OUTPUT/include/rocksdb ]; then
+    mkdir -p $TP_BUILD/rocksdb
+    cd $TP_BUILD/rocksdb
+    cmake $TP_SRC/pegasus-rocksdb-6.6.4-base -DCMAKE_INSTALL_PREFIX=$TP_OUTPUT \
+                                             -DWITH_LZ4=ON \
+                                             -DWITH_ZSTD=ON \
+                                             -DWITH_SNAPPY=ON \
+                                             -DWITH_BZ2=OFF \
+                                             -DWITH_TESTS=OFF \
+                                             -DWITH_GFLAGS=OFF \
+                                             -DUSE_RTTI=ON \
+                                             -DCMAKE_BUILD_TYPE=Release \
+                                             -DCMAKE_CXX_FLAGS=-g
+    #rocksdb enable jemalloc by default, but we use regular malloc.
+    make install -j4 DISABLE_JEMALLOC=1
+else
+    echo "skip build rocksdb"
+fi
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
@@ -308,6 +308,15 @@ check_and_download "${S2GEOMETRY_PKG}" \
     "bfa5f1c08f535a72fb2c92ec16332c64" \
     "${S2GEOMETRY_NAME}"
 
+# rocksdb
+# from: https://github.com/XiaoMi/pegasus-rocksdb/archive/v6.6.4-base.zip
+ROCKSDB_NAME=pegasus-rocksdb-6.6.4-base
+ROCKSDB_PKG=${ROCKSDB_NAME}.zip
+check_and_download "${ROCKSDB_PKG}" \
+    "${OSS_URL_PREFIX}/${ROCKSDB_PKG}" \
+    "454b17946ad66e1d70ab130bb1244edf" \
+    "${ROCKSDB_NAME}"
+
 ret_code=$?
 if [ $ret_code -eq 2 ]; then
     exit 2