Build scripts added

Signed-off-by: Yuvraj <[email protected]>
flyteorg · Jun 10, 2021 · a1440be · a1440be
1 parent ae37f6b
commit a1440be
Show file tree

Hide file tree

Showing 9 changed files with 213 additions and 34 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -7,11 +7,17 @@ on:
 
 jobs:
   build:
-    name: Run tests and lint
+    name: Build
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Cache CGO dependacy
+        id: env-cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/cross
+          key: ${{ hashFiles('BUILD_ENV') }}-env
       - uses: actions/setup-go@v1
         with:
           go-version: '1.16'
@@ -28,15 +34,6 @@ jobs:
           echo "::add-path::/usr/local/opt/llvm/bin"
           echo "::add-path::/usr/bin/clang"
       - name: Install system dependencies
-        env:
-          MUSL_VERSION: 1.1.24
-          MUSL_BUILD_TIME: 20210108172549
-          MUSL_CROSS_MAKE_VERSION: 0.9.9
-          GOPATH: $HOME/go
-          MUSL_CROSS_ARM64_BUILD_TIME: 20210108174735
-          OSXCROSS_VERSION: c2ad5e859d12a295c3f686a15bd7181a165bfa82
-          OSXCROSS_BUILD_TIME: 20210108174800
-          PATH: /usr/local/musl/bin:${PATH}
         run: |
           ./scripts/ci/setup.sh
       - name: Run GoReleaser dry run
@@ -46,25 +43,29 @@ jobs:
           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc
           CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: /usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang
           CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: /usr/bin/x86_64-w64-mingw32-gcc
-          PATH: /usr/local/musl/bin:${PATH}
         with:
           version: latest
-        args: --snapshot  --skip-publish --rm-dist
+          args: --snapshot  --skip-publish --rm-dist
       - name: Unit Tests
-        uses: cedrickring/[email protected]
         env:
           GO111MODULE: "on"
-        with:
-          args: make install && make test_unit_codecov
+        run: make install && make test_unit_codecov
       - name: Push CodeCov
         uses: codecov/[email protected]
         with:
           file: coverage.txt
           flags: unittests
           fail_ci_if_error: true
+  lint:
+    name: Run tests and lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
       - name: Lint
         uses: cedrickring/[email protected]
         env:
           GO111MODULE: "on"
         with:
           args: make install && make lint
+
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,10 +10,41 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+      - name: Cache CGO dependacy
+        id: env-cache
+        uses: actions/cache@v2
         with:
-          version: latest
-          args: release --rm-dist --debug
+          path: /tmp/cross
+          key: ${{ hashFiles('BUILD_ENV') }}-env
+      - uses: actions/setup-go@v1
+        with:
+          go-version: '1.16'
+      - name: setup env
+        shell: bash
+        env:
+          ACTIONS_ALLOW_UNSECURE_COMMANDS : true
+        run: |
+          echo "::add-path::${HOME}/.cargo/bin"
+          echo "::add-path::/usr/local/musl/bin:${PATH}"
+          echo "::add-path::/usr/local/musl-cross/bin"
+          echo "::add-path::/usr/local/osxcross/target/bin"
+          echo "::add-path::{GOPATH}/bin"
+          echo "::add-path::/usr/local/opt/llvm/bin"
+          echo "::add-path::/usr/bin/clang"
+      - name: Install system dependencies
+        env:
+          GOBIN: ${GOPATH}/bin
+        run: |
+          ./scripts/ci/setup.sh
+      - name: Run GoReleaser dry run
+        uses: goreleaser/goreleaser-action@v2
         env:
           GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
+          CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl/bin/musl-gcc
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc
+          CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: /usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang
+          CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: /usr/bin/x86_64-w64-mingw32-gcc
+        with:
+          version: latest
+          args: release --rm-dist
+
diff --git a/BUILD_ENV b/BUILD_ENV
@@ -0,0 +1,3 @@
+MUSL_VERSION: 1.1.24
+MUSL_CROSS_MAKE_VERSION: 0.9.9
+OSXCROSS_VERSION: c2ad5e859d12a295c3f686a15bd7181a165bfa82
diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go
@@ -3,7 +3,6 @@ package get
 import (
 	"context"
 	"fmt"
-	"io/fs"
 	"io/ioutil"
 
 	"github.com/flyteorg/flytestdlib/errors"
@@ -119,7 +118,7 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC
 			if err != nil {
 				return errors.Wrapf("VisualizationError", err, "failed to visualize workflow")
 			}
-			if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, fs.ModePerm); err != nil {
+			if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, 0600); err != nil {
 				return errors.Wrapf("FileWriteError", err, "failed to write visualization file")
 			}
 			fmt.Printf("File [%s] written", workflowconfig.DefaultConfig.OutputFile)

diff --git a/scripts/ci/setup.sh b/scripts/ci/setup.sh
@@ -9,18 +9,41 @@ sudo apt-get update
 sudo apt-get install -y --no-install-recommends bzr clang libprotobuf-dev pkg-config protobuf-compiler
 sudo apt-get install -y cmake gnupg libssl-dev libxml2-dev llvm-dev lzma-dev  mingw-w64 graphviz
 
-MUSL_ARCHIVE=musl-${MUSL_VERSION}-${MUSL_BUILD_TIME}.tar.gz
-curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/${MUSL_ARCHIVE} -O && \
-            sudo tar xzf ${MUSL_ARCHIVE} -C /usr/local && \
-            rm ${MUSL_ARCHIVE}
-MUSL_CROSS_ARM64_ARCHIVE=musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}-${MUSL_CROSS_ARM64_BUILD_TIME}.tar.gz
-curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/musl-cross/${MUSL_CROSS_MAKE_VERSION}/${MUSL_CROSS_ARM64_ARCHIVE} -O && \
-            sudo tar xzf ${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local && \
-            rm ${MUSL_CROSS_ARM64_ARCHIVE}
-OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}-${OSXCROSS_BUILD_TIME}.tar.gz
-curl https://dl.influxdata.com/influxdb-ci/osxcross/${OSXCROSS_VERSION}/${OSXCROSS_ARCHIVE} -O && \
-            sudo tar xzf ${OSXCROSS_ARCHIVE} -C /usr/local && \
-            rm ${OSXCROSS_ARCHIVE}
+declare -r BUILD_TIME=$(date -u '+%Y%m%d%H%M%S')
+
+mkdir -p /tmp/cross ${GOPATH}/bin
+
+MUSL_VERSION=1.1.24
+MUSL_CROSS_MAKE_VERSION=0.9.9
+OSXCROSS_VERSION=c2ad5e859d12a295c3f686a15bd7181a165bfa82
+
+MUSL_ARCHIVE=musl-${MUSL_VERSION}.tar.gz
+MUSL_CROSS_ARM64_ARCHIVE=musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}.tar.gz
+OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}.tar.gz
+
+if [ -f /tmp/cross/${MUSL_ARCHIVE} ]; then
+   sudo tar xzf /tmp/cross/${MUSL_ARCHIVE} -C /usr/local
+else
+   echo "Build started for musl, it will create ${MUSL_ARCHIVE}"
+   bash scripts/xcc-builds/build-amd64-musl-gcc.sh
+   sudo tar xzf /tmp/cross/${MUSL_ARCHIVE} -C /usr/local
+fi
+
+if [ -f /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} ]; then
+   sudo tar xzf /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local
+else
+   echo "Build started for musl-arrch, it will create ${MUSL_CROSS_ARM64_ARCHIVE}"
+   bash scripts/xcc-builds/build-aarch64-musl-gcc.sh
+   sudo tar xzf /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local
+fi
+
+if [ -f /tmp/cross/${OSXCROSS_ARCHIVE} ]; then
+   sudo tar xzf /tmp/cross/${OSXCROSS_ARCHIVE} -C /usr/local
+else
+   echo "Build started for musl-arrch, it will create ${MUSL_CROSS_ARM64_ARCHIVE}"
+   bash scripts/xcc-builds/build-osxcross-clang.sh
+   sudo tar xzf /tmp/cross/${OSXCROSS_ARCHIVE} -C /usr/local
+fi
 
 rustup target add \
     x86_64-unknown-linux-musl \
@@ -31,4 +54,6 @@ rustup target add \
 export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl/bin/musl-gcc
 export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc
 export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang
-export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=/usr/bin/x86_64-w64-mingw32-gcc
+export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=/usr/bin/x86_64-w64-mingw32-gcc
+
+# cp scripts/xcc.sh ${GOPATH}/bin/xcc
diff --git a/scripts/xcc-builds/build-aarch64-musl-gcc.sh b/scripts/xcc-builds/build-aarch64-musl-gcc.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r OUT_DIR=/tmp/cross
+
+declare -r BUILD_IMAGE=ubuntu:20.04
+declare -r MUSL_VERSION=1.1.24
+declare -r MUSL_CROSS_MAKE_VERSION=0.9.9
+
+docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash <<EOF
+  set -euo pipefail
+  declare -r BUILD_TIME=\$(date -u '+%Y%m%d%H%M%S')
+  export DEBIAN_FRONTEND=noninteractive
+  # Install dependencies.
+  apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    make \
+    patch
+  # Clone and build musl-cross-make's ARM64 target.
+  git clone https://github.com/richfelker/musl-cross-make.git && \
+    cd musl-cross-make && \
+    git checkout v${MUSL_CROSS_MAKE_VERSION} && \
+    make MUSL_VER=${MUSL_VERSION} TARGET=aarch64-unknown-linux-musl DL_CMD="curl -C - -L -o" install && \
+    mv output /tmp/musl-cross && \
+    cd /tmp
+  # Archive the build output.
+  tar czf /tmp/cross/musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}.tar.gz musl-cross
+EOF
diff --git a/scripts/xcc-builds/build-amd64-musl-gcc.sh b/scripts/xcc-builds/build-amd64-musl-gcc.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r OUT_DIR=/tmp/cross
+
+declare -r BUILD_IMAGE=ubuntu:20.04
+declare -r MUSL_VERSION=1.1.24
+
+docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash <<EOF
+  set -euo pipefail
+  declare -r BUILD_TIME=\$(date -u '+%Y%m%d%H%M%S')
+  # Install dependencies.
+  apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    patch
+  # Build MUSL from source.
+  curl https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz -O && \
+    tar xzf musl-${MUSL_VERSION}.tar.gz && \
+    cd musl-${MUSL_VERSION} &&
+    ./configure &&
+    make && \
+    make install && \
+    cd /tmp
+  # Archive the build output.
+  cd /usr/local && tar czf /tmp/cross/musl-${MUSL_VERSION}.tar.gz musl && cd /tmp
+EOF
diff --git a/scripts/xcc-builds/build-osxcross-clang.sh b/scripts/xcc-builds/build-osxcross-clang.sh
@@ -0,0 +1,39 @@
+
+#!/usr/bin/env bash
+set -euo pipefail
+
+declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
+declare -r OUT_DIR=/tmp/cross
+
+declare -r BUILD_IMAGE=ubuntu:20.04
+declare -r OSXCROSS_VERSION=c2ad5e859d12a295c3f686a15bd7181a165bfa82
+
+docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash <<EOF
+  set -euo pipefail
+  declare -r BUILD_TIME=\$(date -u '+%Y%m%d%H%M%S')
+  export DEBIAN_FRONTEND=noninteractive
+  # Install dependencies.
+  apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    clang \
+    cmake \
+    curl \
+    git \
+    libssl-dev \
+    libxml2-dev \
+    llvm-dev \
+    lzma-dev \
+    patch \
+    zlib1g-dev
+  # Clone and build osxcross.
+  git clone https://github.com/tpoechtrager/osxcross.git /usr/local/osxcross && \
+    cd /usr/local/osxcross && \
+    git checkout ${OSXCROSS_VERSION} && \
+    curl -L -o ./tarballs/MacOSX10.11.sdk.tar.xz https://macos-sdks.s3.amazonaws.com/MacOSX10.11.sdk.tar.xz && \
+    UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
+    rm -rf .git build tarballs && \
+    cd /tmp
+  # Archive the build output.
+  cd /usr/local && tar czf /tmp/cross/osxcross-${OSXCROSS_VERSION}.tar.gz osxcross && cd /tmp
+EOF
diff --git a/scripts/xcc.sh b/scripts/xcc.sh
@@ -0,0 +1,20 @@
+
+#!/bin/bash
+
+function die() {
+  echo "$@" 1>&2
+  exit 1
+}
+
+GOOS=${GOOS:-$(go env GOOS)}
+GOARCH=${GOARCH:-$(go env GOARCH)}
+
+case "${GOOS}_${GOARCH}" in
+  linux_amd64)   CC=musl-gcc ;;
+  linux_arm64)   CC=aarch64-unknown-linux-musl-gcc ;;
+  darwin_amd64)  CC=x86_64-apple-darwin15-clang ;;
+  windows_amd64) CC=x86_64-w64-mingw32-gcc ;;
+  *) die "No cross-compiler set for ${GOOS}_${GOARCH}" ;;
+esac
+
+exec ${CC} "$@"