diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 08a85bc6..1a08ddf0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,11 +7,17 @@ on: jobs: build: - name: Run tests and lint + name: Build runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 + - name: Cache CGO dependacy + id: env-cache + uses: actions/cache@v2 + with: + path: /tmp/cross + key: ${{ hashFiles('BUILD_ENV') }}-env - uses: actions/setup-go@v1 with: go-version: '1.16' @@ -28,15 +34,6 @@ jobs: echo "::add-path::/usr/local/opt/llvm/bin" echo "::add-path::/usr/bin/clang" - name: Install system dependencies - env: - MUSL_VERSION: 1.1.24 - MUSL_BUILD_TIME: 20210108172549 - MUSL_CROSS_MAKE_VERSION: 0.9.9 - GOPATH: $HOME/go - MUSL_CROSS_ARM64_BUILD_TIME: 20210108174735 - OSXCROSS_VERSION: c2ad5e859d12a295c3f686a15bd7181a165bfa82 - OSXCROSS_BUILD_TIME: 20210108174800 - PATH: /usr/local/musl/bin:${PATH} run: | ./scripts/ci/setup.sh - name: Run GoReleaser dry run @@ -46,25 +43,29 @@ jobs: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: /usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: /usr/bin/x86_64-w64-mingw32-gcc - PATH: /usr/local/musl/bin:${PATH} with: version: latest - args: --snapshot --skip-publish --rm-dist + args: --snapshot --skip-publish --rm-dist - name: Unit Tests - uses: cedrickring/golang-action@1.5.2 env: GO111MODULE: "on" - with: - args: make install && make test_unit_codecov + run: make install && make test_unit_codecov - name: Push CodeCov uses: codecov/codecov-action@v1.0.5 with: file: coverage.txt flags: unittests fail_ci_if_error: true + lint: + name: Run tests and lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 - name: Lint uses: cedrickring/golang-action@1.5.2 env: GO111MODULE: "on" with: args: make install && make lint + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f27d869..82145ce0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,10 +10,41 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + - name: Cache CGO dependacy + id: env-cache + uses: actions/cache@v2 with: - version: latest - args: release --rm-dist --debug + path: /tmp/cross + key: ${{ hashFiles('BUILD_ENV') }}-env + - uses: actions/setup-go@v1 + with: + go-version: '1.16' + - name: setup env + shell: bash + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS : true + run: | + echo "::add-path::${HOME}/.cargo/bin" + echo "::add-path::/usr/local/musl/bin:${PATH}" + echo "::add-path::/usr/local/musl-cross/bin" + echo "::add-path::/usr/local/osxcross/target/bin" + echo "::add-path::{GOPATH}/bin" + echo "::add-path::/usr/local/opt/llvm/bin" + echo "::add-path::/usr/bin/clang" + - name: Install system dependencies + env: + GOBIN: ${GOPATH}/bin + run: | + ./scripts/ci/setup.sh + - name: Run GoReleaser dry run + uses: goreleaser/goreleaser-action@v2 env: GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl/bin/musl-gcc + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER: /usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc + CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: /usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: /usr/bin/x86_64-w64-mingw32-gcc + with: + version: latest + args: release --rm-dist + diff --git a/BUILD_ENV b/BUILD_ENV new file mode 100644 index 00000000..84d23ca1 --- /dev/null +++ b/BUILD_ENV @@ -0,0 +1,3 @@ +MUSL_VERSION: 1.1.24 +MUSL_CROSS_MAKE_VERSION: 0.9.9 +OSXCROSS_VERSION: c2ad5e859d12a295c3f686a15bd7181a165bfa82 \ No newline at end of file diff --git a/cmd/get/workflow.go b/cmd/get/workflow.go index ab51bb44..e0f084cb 100644 --- a/cmd/get/workflow.go +++ b/cmd/get/workflow.go @@ -3,7 +3,6 @@ package get import ( "context" "fmt" - "io/fs" "io/ioutil" "github.com/flyteorg/flytestdlib/errors" @@ -119,7 +118,7 @@ func getWorkflowFunc(ctx context.Context, args []string, cmdCtx cmdCore.CommandC if err != nil { return errors.Wrapf("VisualizationError", err, "failed to visualize workflow") } - if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, fs.ModePerm); err != nil { + if err := ioutil.WriteFile(workflowconfig.DefaultConfig.OutputFile, b, 0600); err != nil { return errors.Wrapf("FileWriteError", err, "failed to write visualization file") } fmt.Printf("File [%s] written", workflowconfig.DefaultConfig.OutputFile) diff --git a/scripts/ci/setup.sh b/scripts/ci/setup.sh old mode 100644 new mode 100755 index ce28bf85..38c2ebcf --- a/scripts/ci/setup.sh +++ b/scripts/ci/setup.sh @@ -9,18 +9,41 @@ sudo apt-get update sudo apt-get install -y --no-install-recommends bzr clang libprotobuf-dev pkg-config protobuf-compiler sudo apt-get install -y cmake gnupg libssl-dev libxml2-dev llvm-dev lzma-dev mingw-w64 graphviz -MUSL_ARCHIVE=musl-${MUSL_VERSION}-${MUSL_BUILD_TIME}.tar.gz -curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/${MUSL_ARCHIVE} -O && \ - sudo tar xzf ${MUSL_ARCHIVE} -C /usr/local && \ - rm ${MUSL_ARCHIVE} -MUSL_CROSS_ARM64_ARCHIVE=musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}-${MUSL_CROSS_ARM64_BUILD_TIME}.tar.gz -curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/musl-cross/${MUSL_CROSS_MAKE_VERSION}/${MUSL_CROSS_ARM64_ARCHIVE} -O && \ - sudo tar xzf ${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local && \ - rm ${MUSL_CROSS_ARM64_ARCHIVE} -OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}-${OSXCROSS_BUILD_TIME}.tar.gz -curl https://dl.influxdata.com/influxdb-ci/osxcross/${OSXCROSS_VERSION}/${OSXCROSS_ARCHIVE} -O && \ - sudo tar xzf ${OSXCROSS_ARCHIVE} -C /usr/local && \ - rm ${OSXCROSS_ARCHIVE} +declare -r BUILD_TIME=$(date -u '+%Y%m%d%H%M%S') + +mkdir -p /tmp/cross ${GOPATH}/bin + +MUSL_VERSION=1.1.24 +MUSL_CROSS_MAKE_VERSION=0.9.9 +OSXCROSS_VERSION=c2ad5e859d12a295c3f686a15bd7181a165bfa82 + +MUSL_ARCHIVE=musl-${MUSL_VERSION}.tar.gz +MUSL_CROSS_ARM64_ARCHIVE=musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}.tar.gz +OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}.tar.gz + +if [ -f /tmp/cross/${MUSL_ARCHIVE} ]; then + sudo tar xzf /tmp/cross/${MUSL_ARCHIVE} -C /usr/local +else + echo "Build started for musl, it will create ${MUSL_ARCHIVE}" + bash scripts/xcc-builds/build-amd64-musl-gcc.sh + sudo tar xzf /tmp/cross/${MUSL_ARCHIVE} -C /usr/local +fi + +if [ -f /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} ]; then + sudo tar xzf /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local +else + echo "Build started for musl-arrch, it will create ${MUSL_CROSS_ARM64_ARCHIVE}" + bash scripts/xcc-builds/build-aarch64-musl-gcc.sh + sudo tar xzf /tmp/cross/${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local +fi + +if [ -f /tmp/cross/${OSXCROSS_ARCHIVE} ]; then + sudo tar xzf /tmp/cross/${OSXCROSS_ARCHIVE} -C /usr/local +else + echo "Build started for musl-arrch, it will create ${MUSL_CROSS_ARM64_ARCHIVE}" + bash scripts/xcc-builds/build-osxcross-clang.sh + sudo tar xzf /tmp/cross/${OSXCROSS_ARCHIVE} -C /usr/local +fi rustup target add \ x86_64-unknown-linux-musl \ @@ -31,4 +54,6 @@ rustup target add \ export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl/bin/musl-gcc export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang -export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=/usr/bin/x86_64-w64-mingw32-gcc \ No newline at end of file +export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=/usr/bin/x86_64-w64-mingw32-gcc + +# cp scripts/xcc.sh ${GOPATH}/bin/xcc diff --git a/scripts/xcc-builds/build-aarch64-musl-gcc.sh b/scripts/xcc-builds/build-aarch64-musl-gcc.sh new file mode 100644 index 00000000..392b9dec --- /dev/null +++ b/scripts/xcc-builds/build-aarch64-musl-gcc.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd) +declare -r OUT_DIR=/tmp/cross + +declare -r BUILD_IMAGE=ubuntu:20.04 +declare -r MUSL_VERSION=1.1.24 +declare -r MUSL_CROSS_MAKE_VERSION=0.9.9 + +docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash </dev/null 2>&1 && pwd) +declare -r OUT_DIR=/tmp/cross + +declare -r BUILD_IMAGE=ubuntu:20.04 +declare -r MUSL_VERSION=1.1.24 + +docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash </dev/null 2>&1 && pwd) +declare -r OUT_DIR=/tmp/cross + +declare -r BUILD_IMAGE=ubuntu:20.04 +declare -r OSXCROSS_VERSION=c2ad5e859d12a295c3f686a15bd7181a165bfa82 + +docker run --rm -i -v ${OUT_DIR}:/tmp/cross -w /tmp ${BUILD_IMAGE} bash <&2 + exit 1 +} + +GOOS=${GOOS:-$(go env GOOS)} +GOARCH=${GOARCH:-$(go env GOARCH)} + +case "${GOOS}_${GOARCH}" in + linux_amd64) CC=musl-gcc ;; + linux_arm64) CC=aarch64-unknown-linux-musl-gcc ;; + darwin_amd64) CC=x86_64-apple-darwin15-clang ;; + windows_amd64) CC=x86_64-w64-mingw32-gcc ;; + *) die "No cross-compiler set for ${GOOS}_${GOARCH}" ;; +esac + +exec ${CC} "$@" \ No newline at end of file