Use GitHub Action to build nebula toolset automatically (#3)

* Use GitHub action to build nebula toolset automatically * Cleanup ossutil URL * Refactor * login firstly * Fix dockerfile * Test toolset emulator building * Fix this dir * Remove llvm 8.0.0 * Only use centos-7 and debian-8 * Fix debian Dockerfile * replace gcc 9.3.0 with 9.2.0 * Debug * Fix data directory exists * cancel in progress * cancel in workflow group * Fix buildx Dockerfile * Restore * cancel gcc 10.0.1 * Use self-hosted runners * split different jobs * Fix concurrency group * fix concurrency
vesoft-inc · Nov 8, 2021 · b5cd9c0 · b5cd9c0
1 parent 174d1cd
commit b5cd9c0
Show file tree

Hide file tree

Showing 19 changed files with 250 additions and 41 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,88 @@
+name: build
+
+on:
+  push:
+    branches:
+      - master
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  gcc:
+    name: gcc
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos-7
+          - debian-8
+        arch:
+          - amd64
+          - arm64
+        version:
+          # - 7.5.0
+          # - 8.3.0
+          # - 9.1.0
+          - 9.2.0
+          # - 9.3.0
+          # - 10.1.0
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/images/buildx/${{ matrix.os }}.Dockerfile
+          platforms: linux/${{ matrix.arch }}
+          tags: vesoft/toolset-build:${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }}
+          push: false
+          build-args: |
+            BUILD_GCC_VERSIONS=${{ matrix.version }}
+          secrets: |
+            "ossutilconfig=${{ secrets.OSS_UTIL_CONFIG }}"
+  llvm:
+    name: llvm
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos-7
+          - debian-8
+        arch:
+          - amd64
+          - arm64
+        version:
+          # - 9.0.0
+          - 10.0.0
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/images/buildx/${{ matrix.os }}.Dockerfile
+          platforms: linux/${{ matrix.arch }}
+          tags: vesoft/toolset-build:${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }}
+          push: false
+          build-args: |
+            BUILD_LLVM_VERSIONS=${{ matrix.version }}
+          secrets: |
+            "ossutilconfig=${{ secrets.OSS_UTIL_CONFIG }}"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,10 +1,10 @@
+name: release
+
 on:
   push:
     tags:
     - 'v*'
 
-name:
-
 jobs:
   build:
     name: Release

diff --git a/docker/images/aarch64/centos-7.Dockerfile b/docker/images/aarch64/centos-7.Dockerfile
@@ -28,7 +28,7 @@ RUN yum install -y make \
 RUN yum install -y pxz || true
 
 ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
-ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64?spm=a2c63.p38356.879954.15.c0942454HuAZDI'
+ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64'
 ENV PACKAGE_DIR=/usr/src
 RUN curl -s ${NG_URL} | bash
 

diff --git a/docker/images/aarch64/debian-8.Dockerfile b/docker/images/aarch64/debian-8.Dockerfile
@@ -27,7 +27,7 @@ RUN apt-get install -y make \
 RUN apt-get install -y pxz --force-yes
 
 ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
-ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64?spm=a2c63.p38356.879954.15.c0942454HuAZDI'
+ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64'
 ENV PACKAGE_DIR=/usr/src
 RUN set -o pipefail && curl -s ${NG_URL} | bash
 

diff --git a/docker/images/build-gcc.sh b/docker/images/build-gcc.sh
@@ -6,13 +6,12 @@ nebula-gears-update
 
 release="$(lsb_release -si) $(lsb_release -sr)"
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 versions=${BUILD_GCC_VERSIONS:-all}
 
 build-gcc --version=$versions
 
+[[ -d /data ]] && cp -v toolset-build/vesoft-gcc-*.sh /data/
 
-cp -v toolset-build/vesoft-gcc-*.sh /data
-
-[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc-*.sh
+${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc-*.sh
diff --git a/docker/images/build-llvm.sh b/docker/images/build-llvm.sh
@@ -8,7 +8,7 @@ nebula-gears-update
 
 release="$(lsb_release -si) $(lsb_release -sr)"
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 versions=${BUILD_LLVM_VERSIONS:-all}
 
@@ -19,7 +19,6 @@ source /opt/vesoft/toolset/cmake/enable
 
 build-llvm --version=$versions
 
+[[ -d /data ]] && cp -v toolset-build/vesoft-llvm-*.sh /data/
 
-cp -v toolset-build/vesoft-llvm-*.sh /data
-
-[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh toolset toolset-build/vesoft-llvm-*.sh
+${this_dir}/oss-upload.sh toolset toolset-build/vesoft-llvm-*.sh
diff --git a/docker/images/buildx/centos-7.Dockerfile b/docker/images/buildx/centos-7.Dockerfile
@@ -0,0 +1,51 @@
+FROM centos:7
+
+ARG BUILD_LLVM_VERSIONS=""
+ARG BUILD_GCC_VERSIONS=""
+
+SHELL ["/bin/bash", "-c"]
+RUN yum update -y
+RUN yum install -y make \
+                   git \
+                   m4 \
+                   wget \
+                   unzip \
+                   bzip2 \
+                   xz \
+                   xz-devel \
+                   patch \
+                   python \
+                   python-devel \
+                   redhat-lsb-core \
+                   zlib-devel \
+                   gcc \
+                   gcc-c++ \
+                   libtool \
+                   autoconf \
+                   automake \
+                   bison \
+                   flex \
+                   gperf \
+                   gettext \
+                   epel-release
+
+RUN yum install -y pxz || true
+
+ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
+ENV OSS_UTIL_URL=http://gosspublic.alicdn.com/ossutil/1.7.0
+ENV PACKAGE_DIR=/usr/src
+RUN curl -s ${NG_URL} | bash
+
+RUN mkdir -p ${PACKAGE_DIR}
+WORKDIR ${PACKAGE_DIR}
+
+COPY . ${PACKAGE_DIR}
+
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-gcc.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-llvm.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/oss-upload.sh
+
+RUN [[ $(uname -m) = "aarch64" ]] && ARCH="arm"; wget -q -O /usr/bin/ossutil64 ${OSS_UTIL_URL}/ossutil${ARCH}64
+RUN chmod +x /usr/bin/ossutil64
+
+RUN --mount=type=secret,id=ossutilconfig,dst=/root/.ossutilconfig ${PACKAGE_DIR}/docker/images/run.sh
diff --git a/docker/images/buildx/debian-8.Dockerfile b/docker/images/buildx/debian-8.Dockerfile
@@ -0,0 +1,49 @@
+FROM debian:8
+
+ARG BUILD_LLVM_VERSIONS=""
+ARG BUILD_GCC_VERSIONS=""
+
+SHELL ["/bin/bash", "-c"]
+RUN apt-get update
+RUN apt-get install -y make \
+                       git \
+                       m4 \
+                       wget \
+                       unzip \
+                       curl \
+                       xz-utils \
+                       liblzma-dev \
+                       python-dev \
+                       patch \
+                       lsb-core \
+                       libz-dev \
+                       build-essential \
+                       libtool \
+                       automake \
+                       autoconf \
+                       autoconf-archive \
+                       autotools-dev \
+                       bison \
+                       flex \
+                       gperf \
+                       gettext --force-yes
+RUN apt-get install -y pxz
+
+ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
+ENV OSS_UTIL_URL=http://gosspublic.alicdn.com/ossutil/1.7.0
+ENV PACKAGE_DIR=/usr/src
+RUN set -o pipefail && curl -s ${NG_URL} | bash
+
+RUN mkdir -p ${PACKAGE_DIR}
+WORKDIR ${PACKAGE_DIR}
+
+COPY . ${PACKAGE_DIR}
+
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-gcc.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-llvm.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/oss-upload.sh
+
+RUN [[ $(uname -m) = "aarch64" ]] && ARCH="arm"; wget -q -O /usr/bin/ossutil64 ${OSS_UTIL_URL}/ossutil${ARCH}64
+RUN chmod +x /usr/bin/ossutil64
+
+RUN --mount=type=secret,id=ossutilconfig,dst=/root/.ossutilconfig ${PACKAGE_DIR}/docker/images/run.sh
diff --git a/docker/images/oss-upload.sh b/docker/images/oss-upload.sh
@@ -2,7 +2,17 @@
 
 [[ $# -lt 2 ]] && echo "$0 <subdir> <files...>" && exit 1
 
-CMD="ossutil64 -e ${OSS_ENDPOINT} -i ${OSS_ID} -k ${OSS_SECRET}"
+CMD=""
+if [[ ! -z "${OSS_ENDPOINT}" ]] && [[ ! -z "${OSS_ID}" ]] && [[ ! -z "${OSS_SECRET}" ]]; then
+    CMD="ossutil64 -e ${OSS_ENDPOINT} -i ${OSS_ID} -k ${OSS_SECRET}"
+elif [[ -f $HOME/.ossutilconfig ]]; then
+    CMD="ossutil64"
+elif [[ -f /run/secrets/ossutilconfig ]]; then
+    CMD="ossutil64 -c /run/secrets/ossutilconfig"
+else
+    echo "Could not find valid oss authentication configure file."
+    exit 1
+fi
 OSS_BASE=oss://nebula-graph
 OSS_SUBDIR=$1
 shift

diff --git a/docker/images/run.sh b/docker/images/run.sh
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+set -e
+
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+
+if [[ ! -z "${BUILD_GCC_VERSIONS}" ]]; then
+    $this_dir/build-gcc.sh
+fi
+
+if [[ ! -z "${BUILD_LLVM_VERSIONS}" ]]; then
+    $this_dir/build-llvm.sh
+fi
diff --git a/docker/images/x86_64/centos-7.Dockerfile b/docker/images/x86_64/centos-7.Dockerfile
@@ -3,26 +3,26 @@ SHELL ["/bin/bash", "-c"]
 RUN yum update -y
 RUN yum install -y make \
                    git \
-				   m4 \
-				   wget \
-				   unzip \
-				   bzip2 \
-				   xz \
-				   xz-devel \
-				   patch \
-				   python \
-				   python-devel \
-				   redhat-lsb-core \
-				   zlib-devel \
-				   gcc \
-				   gcc-c++ \
-				   libtool \
-				   autoconf \
-				   automake \
-				   bison \
-				   flex \
-				   gperf \
-				   gettext \
+                   m4 \
+                   wget \
+                   unzip \
+                   bzip2 \
+                   xz \
+                   xz-devel \
+                   patch \
+                   python \
+                   python-devel \
+                   redhat-lsb-core \
+                   zlib-devel \
+                   gcc \
+                   gcc-c++ \
+                   libtool \
+                   autoconf \
+                   automake \
+                   bison \
+                   flex \
+                   gperf \
+                   gettext \
                    epel-release
 
 RUN yum install -y pxz || true

diff --git a/scripts/build-cmake b/scripts/build-cmake
@@ -15,7 +15,7 @@ then
     exit $?
 fi
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 name=cmake
 version=3.15.7

diff --git a/scripts/build-llvm b/scripts/build-llvm
@@ -15,7 +15,7 @@ then
     exit $?
 fi
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 

diff --git a/scripts/install-cmake b/scripts/install-cmake
@@ -22,7 +22,7 @@ fi
 
 version=3.15.7
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 selected_libc=
 selected_archive=
 this_arch=$(uname -m)

diff --git a/scripts/install-gcc b/scripts/install-gcc
@@ -23,7 +23,7 @@ fi
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 version_files_dir=$this_dir/../share/nebula-gears
 [[ $# -ne 0 ]] && prefix=$(echo "$@" | sed 's;.*--prefix=(\S*).*;\1;p' -rn)

diff --git a/scripts/install-llvm b/scripts/install-llvm
@@ -23,7 +23,7 @@ fi
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 this_arch=$(uname -m)
 
 version_files_dir=$this_dir/../share/nebula-gears

diff --git a/scripts/nebula-gears-show-files b/scripts/nebula-gears-show-files
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 bin_dir_name=$(basename $this_dir)
 [[ $bin_dir_name = bin ]] || exit 1
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,7 +15,7 @@ then @@
         exit $?
     fi
-    this_dir=$(dirname $(readlink -f $0))
+    this_dir="$(cd "$(dirname "$0")" && pwd)"
     [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
@@ Expand Down @@