Enable ccache to speed up build and fix dockerfile for gflags (#1510)

* Enable ccache for building and fix dockerfile for gflags Update docker hub url Use tmpfs for ccache Limit 1G tmpfs size Use container env to simplify steps environment variables Add centos6 job Fix ccache dir in centos6 Specify CC/CXX environment variables to fix node error Fix env var Try to fix cmake error Add centos cache job again Fix cache key and restore keys Fix echo string error Update ccache directory Remove write ccache conf cmd Format Use common job env Fix centos6 job Use LD_LIBRARY_PATH to fix centos 6 dep error of node 12 Setup node version Test customed actions Fix no cache for apk installation Try to fix entrypoint Fix entrypoint Use docker container to cache Use docker to run build and test Use sudo to mount tmpfs Fix tty error Fix tmpfs mount point Fix mount error Delete cache actions * Use self hosted runner * Improve cmake scripts * Add timeout seconds for ctest * Cleanup * Fix gcc warning for lower version * Execute cmake steps with condition * Limit ctest parallels Co-authored-by: dutor <[email protected]>
vesoft-inc · Jan 6, 2020 · 7b88e78 · 7b88e78
1 parent 9f8737d
commit 7b88e78
Show file tree

Hide file tree

Showing 8 changed files with 49 additions and 99 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -1,107 +1,67 @@
 name: build
 
 on:
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
   push:
     branches:
       - master
     paths-ignore:
       - 'docs/**'
+      - '**.md'
 
 jobs:
-  build-gcc:
-    name: build-gcc
-    runs-on: ubuntu-latest
+  build:
+    name: build
+    runs-on: self-hosted
     strategy:
       matrix:
         tag:
           - centos6
           - centos7
           - ubuntu1604
           - ubuntu1804
+        compiler:
+          - gcc
+          # - clang
+        exclude:
+          - tag: centos6
+            compiler: clang
+          - tag: centos7
+            compiler: clang
     container:
       image: vesoft/nebula-dev:${{ matrix.tag }}
+      env:
+        CCACHE_DIR: /tmp/ccache/${{ matrix.tag }}-${{ matrix.compiler }}
+      options: --mount type=tmpfs,destination=/tmp/ccache,tmpfs-size=1073741824 -v /tmp/ccache/${{ matrix.tag }}-${{ matrix.compiler }}:/tmp/ccache/${{ matrix.tag }}-${{ matrix.compiler }}
     steps:
       - uses: actions/checkout@v1
         with:
           fetch-depth: 1
-      - name: cmake
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          mkdir _build && cd _build
-          ${NEBULA_DEP_BIN}/cmake -DCMAKE_C_COMPILER=${NEBULA_DEP_BIN}/gcc -DCMAKE_CXX_COMPILER=${NEBULA_DEP_BIN}/g++ -DCMAKE_BUILD_TYPE=Release ..
-        shell: bash
-      - name: make
-        env:
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          cd _build && make -j $(nproc)
-        shell: bash
-      - name: test in multithreads
-        timeout-minutes: 20
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
+      - name: Prepare environment
         run: |
-          cd _build
-          ${NEBULA_DEP_BIN}/ctest -j $(nproc) --output-on-failure
+          [ -f "$CCACHE_DIR/ccache.conf" ] || cp ci/ccache.conf "$CCACHE_DIR"
+          mkdir -p build
+      - name: CMake with gcc
+        if: matrix.compiler == 'gcc'
+        run: cd build && cmake -DCMAKE_BUILD_TYPE=Release ..
+      - name: CMake with clang
+        if: matrix.compiler == 'clang'
+        run: cd build && cmake -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_C_COMPILER=clang-8 -DENABLE_ASAN=on -DENABLE_UBSAN=on ..
         shell: bash
-      - name: test in singlethread
+      - name: Make
+        run: cd build && make -j $(nproc)
+      - name: CTest with multiple threads
+        timeout-minutes: 15
+        run: cd build && ctest -j 6 --timeout 300 --output-on-failure --rerun-failed
+      - name: CTest with single thread
         if: failure()
-        timeout-minutes: 40
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          cd _build
-          ${NEBULA_DEP_BIN}/ctest --output-on-failure
-        shell: bash
-  build-clang:
-    name: build-clang
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        tag:
-          - ubuntu1604
-          - ubuntu1804
-    container:
-      image: vesoft/nebula-dev:${{ matrix.tag }}
-    steps:
-      - uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
-      - name: cmake
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          mkdir _build && cd _build
-          ${NEBULA_DEP_BIN}/cmake -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_C_COMPILER=clang-8 -DENABLE_ASAN=on -DENABLE_UBSAN=on ..
-        shell: bash
-      - name: make
-        env:
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          cd _build && make -j $(nproc)
-        shell: bash
-      - name: test in multithreads
-        timeout-minutes: 20
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          cd _build
-          ${NEBULA_DEP_BIN}/ctest -j $(nproc) --output-on-failure
-        shell: bash
-      - name: test in singlethread
-        if: failure()
-        timeout-minutes: 40
-        env:
-          NEBULA_DEP_BIN: /opt/nebula/third-party/bin
-          LIBRARY_PATH: /usr/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:${LIBRARY_PATH}
-        run: |
-          cd _build
-          ${NEBULA_DEP_BIN}/ctest --output-on-failure
-        shell: bash
-
+        timeout-minutes: 30
+        run: cd build && ctest --timeout 300 --output-on-failure --rerun-failed
+      - name: Cleanup
+        if: always()
+        run: rm -rf build
diff --git a/.gitignore b/.gitignore
@@ -9,8 +9,9 @@ _build.log
 _install
 install_manifest.txt
 
-# ccls
+# lsp
 .ccls
+.ccls-cache
 compile_commands.json
 
 src/common/base/Base.h.gch

diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
   <a href="https://github.com/vesoft-inc/nebula/actions?workflow=build">
     <img src="https://github.com/vesoft-inc/nebula/workflows/build/badge.svg" alt="build and test workflow"/>
   </a>
-  <a href="https://github.com/vesoft-inc/nebula/actions?workflow=docker">
+  <a href="https://hub.docker.com/u/vesoft">
     <img src="https://github.com/vesoft-inc/nebula/workflows/docker/badge.svg" alt="build docker image workflow"/>
   </a>
   <a href="https://github.com/vesoft-inc/nebula/actions?workflow=package">

diff --git a/ci/ccache.conf b/ci/ccache.conf
@@ -0,0 +1 @@
+max_size = 1.0G
diff --git a/docker/Dockerfile.graphd b/docker/Dockerfile.graphd
@@ -19,8 +19,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 3699 13000 13002
 
-ENTRYPOINT ["./bin/nebula-graphd", \
-  "--flagfile=./etc/nebula-graphd.conf", \
-  "--daemonize=false", \
-  "--v=0", \
-  "--minloglevel=2"]
+ENTRYPOINT ["./bin/nebula-graphd", "--flagfile=./etc/nebula-graphd.conf", "--daemonize=false"]
diff --git a/docker/Dockerfile.metad b/docker/Dockerfile.metad
@@ -19,8 +19,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 45500 45501 11000 11002
 
-ENTRYPOINT ["./bin/nebula-metad", \
-  "--flagfile=./etc/nebula-metad.conf", \
-  "--daemonize=false", \
-  "--v=0", \
-  "--minloglevel=2"]
+ENTRYPOINT ["./bin/nebula-metad", "--flagfile=./etc/nebula-metad.conf", "--daemonize=false"]
diff --git a/docker/Dockerfile.storaged b/docker/Dockerfile.storaged
@@ -19,8 +19,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 44500 44501 12000 12002
 
-ENTRYPOINT ["./bin/nebula-storaged", \
-  "--flagfile=./etc/nebula-storaged.conf", \
-  "--daemonize=false", \
-  "--v=0", \
-  "--minloglevel=2"]
+ENTRYPOINT ["./bin/nebula-storaged", "--flagfile=./etc/nebula-storaged.conf", "--daemonize=false"]
diff --git a/src/parser/CMakeLists.txt b/src/parser/CMakeLists.txt
@@ -8,7 +8,7 @@ add_custom_target(parser_target DEPENDS ${FLEX_Scanner_OUTPUTS} ${BISON_Parser_O
 
 add_flex_bison_dependency(Scanner Parser)
 
-add_compile_options(-Wno-error=sign-compare)
+add_compile_options(-Wno-sign-compare)
 
 nebula_add_library(
     parser_obj OBJECT