Improve toolset and third-party building

vesoft-inc · Mar 12, 2020 · f1befbe · f1befbe
1 parent 26aeaa1
commit f1befbe
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -195,8 +195,13 @@ $ make
 
 # Build for specific platform
 $ make centos-7
+
+# Build GCC 9.1.0 and GCC 8.3.0
+$ make BUILD_GCC_VERSIONS=8.3.0,9.1.0 centos-6
 ```
 
+After the build succeeds, all resulting packages will be located in the `$PWD/toolset-packages` subdirectory. If you want the packages uploaded to the OSS repository for public access, the `$HOME/.ossutilconfig` file must exists, which holds the necessary access tokens.
+
 # Build Nebula Third Party
 ```shell
 $ git clone https://github.com/dutor/nebula-gears.git
@@ -207,4 +212,9 @@ $ make
 
 # Build for specific platform
 $ make ubuntu-1604
+
+# Build with GCC 7.5.0 and 9.2.0
+$ make USE_GCC_VERSIONS=7.5.0,9.2.0 ubuntu-1804
 ```
+
+After the build succeeds, all resulting packages will be located in the `$PWD/third-party-packages` subdirectory. If you want the packages uploaded to the OSS repository for public access, the `$HOME/.ossutilconfig` file must exists, which holds the necessary access tokens.
diff --git a/docker/third-party/build/Makefile b/docker/third-party/build/Makefile
@@ -2,6 +2,18 @@ tags=centos-6 centos-7 ubuntu-1604 ubuntu-1804
 
 all: $(tags)
 
+oss_endpoint := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^endpoint=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+oss_id := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^accessKeyID=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+oss_secret := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^accessKeySecret=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+
+docker_cmd := docker run -v $$PWD/third-party-packages:/data
+ifneq ($(oss_endpoint),)
+	docker_cmd = docker run -e OSS_ENDPOINT=$(oss_endpoint) -e OSS_ID=$(oss_id) -e OSS_SECRET=$(oss_secret)
+endif
+ifneq ($(USE_GCC_VERSIONS),)
+	docker_cmd += -e USE_GCC_VERSIONS=$(USE_GCC_VERSIONS)
+endif
+
 %:
-	@echo '******************' Build image $* '********************'
-	@docker run --env-file=$$HOME/.oss_env --rm -it dutor/third-party-build:$* ./build-third-party.sh
+	@echo '******************' Build Nebula Third Party For $* '********************'
+	@$(docker_cmd) --rm -it dutor/third-party-build:$* ./build-third-party.sh
diff --git a/docker/third-party/images/Makefile b/docker/third-party/images/Makefile
@@ -4,7 +4,7 @@ all: $(tags)
 
 build-%: %.Dockerfile
 	@echo '******************' Build image $* '********************'
-	@docker build -t dutor/third-party-build:$* -f $*.Dockerfile .
+	@docker build --no-cache -t dutor/third-party-build:$* -f $*.Dockerfile .
 
 push-%: build-%
 	@echo '******************' Pushing image $* '********************'

diff --git a/docker/third-party/images/build-third-party.sh b/docker/third-party/images/build-third-party.sh
@@ -6,15 +6,15 @@ set -e
 
 git clone --depth=1 https://github.com/vesoft-inc/nebula.git
 
-install-gcc --version=7.1.0,7.5.0,8.3.0,9.1.0,9.2.0
+versions=${USE_GCC_VERSIONS:-7.1.0,7.5.0,8.3.0,9.1.0,9.2.0}
+install-gcc --version=$versions
 
 nebula/third-party/install-cmake.sh
 
 export PATH=$PWD/cmake-3.15.5/bin:$PATH
 
-for v in 7.1.0 7.5.0 8.3.0 9.1.0 9.2.0
+for v in $(echo $versions | tr ',' ' ')
 do
-    echo $v
     source /opt/vesoft/toolset/gcc/$v/enable;
     rm -rf /opt/vesoft/third-party
     build_package=1 disable_cxx11_abi=0 nebula/third-party/build-third-party.sh /opt/vesoft/third-party
@@ -23,4 +23,6 @@ do
     source /opt/vesoft/toolset/gcc/$v/disable;
 done
 
-${this_dir}/oss-upload.sh third-party third-party/vesoft-third-party-*.sh
+cp -v third-party/vesoft-third-party-*.sh /data
+
+[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh third-party third-party/vesoft-third-party-*.sh
diff --git a/docker/toolset/build/Makefile b/docker/toolset/build/Makefile
@@ -2,6 +2,18 @@ tags=centos-6 centos-7 debian-7 debian-8
 
 all: $(tags)
 
+oss_endpoint := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^endpoint=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+oss_id := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^accessKeyID=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+oss_secret := $(shell [[ -f $$HOME/.ossutilconfig ]] && sed 's;^accessKeySecret=(.*);\1;p' -rn $$HOME/.ossutilconfig)
+
+docker_cmd := docker run -v $$PWD/toolset-packages:/data -e LOCATION=/data
+ifneq ($(oss_endpoint),)
+	docker_cmd += docker run -e OSS_ENDPOINT=$(oss_endpoint) -e OSS_ID=$(oss_id) -e OSS_SECRET=$(oss_secret)
+endif
+ifneq ($(BUILD_GCC_VERSIONS),)
+	docker_cmd += -e BUILD_GCC_VERSIONS=$(BUILD_GCC_VERSIONS)
+endif
+
 %:
-	@echo '******************' Build image $* '********************'
-	@docker run --env-file=$$HOME/.oss_env --rm -it dutor/toolset-build:$* ./build-gcc.sh
+	@echo '******************' Build Nebula Toolset For $* '********************'
+	@$(docker_cmd) --rm -it dutor/toolset-build:$* ./build-gcc.sh
diff --git a/docker/toolset/images/Makefile b/docker/toolset/images/Makefile
@@ -3,11 +3,11 @@ tags=centos-6 centos-7 debian-7 debian-8
 all: $(tags)
 
 build-%: %.Dockerfile
-	@echo '******************' Build image $* '********************'
-	@docker build -t dutor/toolset-build:$* -f $*.Dockerfile .
+	@echo '******************' Build Image $* '********************'
+	@docker build --no-cache -t dutor/toolset-build:$* -f $*.Dockerfile .
 
 push-%: build-%
-	@echo '******************' Pushing image $* '********************'
+	@echo '******************' Pushing Image $* '********************'
 	@docker push dutor/toolset-build:$*
 
 %: push-% ;
diff --git a/docker/toolset/images/build-gcc.sh b/docker/toolset/images/build-gcc.sh
@@ -4,7 +4,11 @@ this_dir=$(dirname $(readlink -f $0))
 
 set -e
 
-build-gcc --version=all
+versions=${BUILD_GCC_VERSIONS:-all}
 
+build-gcc --version=$versions
 
-${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc*.sh
+
+cp -v toolset-build/vesoft-gcc-*.sh /data
+
+[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc-*.sh