diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml new file mode 100644 index 0000000000000..7bb38a4e8eb9e --- /dev/null +++ b/.azure-pipelines/auto.yml @@ -0,0 +1,350 @@ +# +# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. +# + +pr: none +trigger: + - auto + +variables: +- group: prod-credentials + +jobs: +- job: Linux + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/run.yml + strategy: + matrix: + x86_64-gnu-llvm-6.0: + IMAGE: x86_64-gnu-llvm-6.0 + RUST_BACKTRACE: 1 + + dist-x86_64-linux: + IMAGE: dist-x86_64-linux + DEPLOY: 1 + + # "alternate" deployments, these are "nightlies" but have LLVM assertions + # turned on, they're deployed to a different location primarily for + # additional testing. + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 + + # Linux builders, remaining docker images + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu: + IMAGE: i686-gnu + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various + x86_64-gnu: + IMAGE: x86_64-gnu + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck + mingw-check: + IMAGE: mingw-check + +- job: macOS + pool: + vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - template: steps/run.yml + strategy: + matrix: + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. + x86_64-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-x86_64-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + + + +- job: Windows + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/run.yml + strategy: + matrix: + # 32/64 bit MSVC tests + x86_64-msvc-1: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-msvc-2: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + + # 32/64 bit MSVC and GNU deployment + dist-x86_64-msvc: + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-msvc: + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 diff --git a/.azure-pipelines/master.yml b/.azure-pipelines/master.yml new file mode 100644 index 0000000000000..3f3025ad65b66 --- /dev/null +++ b/.azure-pipelines/master.yml @@ -0,0 +1,24 @@ +# +# Azure Pipelines job to publish toolstate. Only triggers on pushes to master. +# + +pr: none +trigger: + - master + +pool: + vmImage: ubuntu-16.04 + +steps: +- checkout: self + fetchDepth: 2 + +- script: | + export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + . src/ci/docker/x86_64-gnu-tools/repo.sh + # FIXME(pietro): committing is disabled until we switch to Azure Pipelines + # as the source of truth, or until we setup a separate test repo. + #commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" + displayName: Publish toolstate + env: + TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN_SECRET) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml new file mode 100644 index 0000000000000..84c9454fee57f --- /dev/null +++ b/.azure-pipelines/pr.yml @@ -0,0 +1,22 @@ +# +# Azure Pipelines pull request build for Rust +# + +trigger: none +pr: +- master # FIXME: really just want any branch, but want an explicit "pr" property set so it's clear + +jobs: +- job: Linux + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/run.yml + strategy: + matrix: + x86_64-gnu-llvm-6.0: + RUST_BACKTRACE: 1 + +# x86_64-gnu-tools: {} +# # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) +# mingw-check: {} diff --git a/.azure-pipelines/steps/install-clang.yml b/.azure-pipelines/steps/install-clang.yml new file mode 100644 index 0000000000000..26a223282cd87 --- /dev/null +++ b/.azure-pipelines/steps/install-clang.yml @@ -0,0 +1,40 @@ +steps: + +- bash: | + set -e + curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang + echo "##vso[task.setvariable variable=CC]$CC" + + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ + echo "##vso[task.setvariable variable=CXX]$CXX" + + # Configure `AR` specifically so rustbuild doesn't try to infer it as + # `clang-ar` by accident. + echo "##vso[task.setvariable variable=AR]ar" + displayName: Install clang (OSX) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + +# If we're compiling for MSVC then we, like most other distribution builders, +# switch to clang as the compiler. This'll allow us eventually to enable LTO +# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think +# clang has an output mode compatible with MinGW that we need. If it does we +# should switch to clang for MinGW as well! +# +# Note that the LLVM installer is an NSIS installer +# +# Original downloaded here came from +# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe +- script: | + powershell -Command "iwr -outf %TEMP%\LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" + set CLANG_DIR=%CD%\citools\clang-rust + %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) + displayName: Install clang (Windows) + +# Note that we don't install clang on Linux since its compiler story is just so +# different. Each container has its own toolchain configured appropriately +# already. diff --git a/.azure-pipelines/steps/install-sccache.yml b/.azure-pipelines/steps/install-sccache.yml new file mode 100644 index 0000000000000..39f58002a7358 --- /dev/null +++ b/.azure-pipelines/steps/install-sccache.yml @@ -0,0 +1,21 @@ +steps: + +- bash: | + set -e + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin + chmod +x /usr/local/bin/sccache + displayName: Install sccache (OSX) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + +- script: | + md sccache + powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" + echo ##vso[task.prependpath]%CD%\sccache + displayName: Install sccache (Windows) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + +# Note that we don't install sccache on Linux since it's installed elsewhere +# through all the containers. +# +# FIXME: we should probably install sccache outside the containers and then +# mount it inside the containers so we can centralize all installation here. diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml new file mode 100644 index 0000000000000..037c8daa2a80a --- /dev/null +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -0,0 +1,91 @@ +steps: +# We've had issues with the default drive in use running out of space during a +# build, and it looks like the `C:` drive has more space than the default `D:` +# drive. We should probably confirm this with the azure pipelines team at some +# point, but this seems to fix our "disk space full" problems. +- script: | + mkdir c:\MORE_SPACE + mklink /J build c:\MORE_SPACE + displayName: "Ensure build happens on C:/ instead of D:/" + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + +# Download and install MSYS2, needed primarily for the test suite (run-make) but +# also used by the MinGW toolchain for assembling things. +# +# FIXME: we should probe the default azure image and see if we can use the MSYS2 +# toolchain there. (if there's even one there). For now though this gets the job +# done. +- script: | + set MSYS_PATH=%CD%\citools\msys64 + choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y + set PATH=%MSYS_PATH%\usr\bin;%PATH% + pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar + IF "%MINGW_URL%"=="" ( + IF "%MSYS_BITS%"=="32" pacman -S --noconfirm --needed mingw-w64-i686-toolchain mingw-w64-i686-cmake mingw-w64-i686-gcc mingw-w64-i686-python2 + IF "%MSYS_BITS%"=="64" pacman -S --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-python2 + ) + where rev + rev --help + where make + + echo ##vso[task.setvariable variable=MSYS_PATH]%MSYS_PATH% + echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin + displayName: Install msys2 + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + +# If we need to download a custom MinGW, do so here and set the path +# appropriately. +# +# Here we also do a pretty heinous thing which is to mangle the MinGW +# installation we just downloaded. Currently, as of this writing, we're using +# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it +# appears to be the first version which contains a fix for #40546, builds +# randomly failing during LLVM due to ar.exe/ranlib.exe failures. +# +# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds +# to contain a regression in gdb (#40184). As a result if we were to use the +# gdb provided (7.11.1) then we would fail all debuginfo tests. +# +# In order to fix spurious failures (pretty high priority) we use 6.3.0. To +# avoid disabling gdb tests we download an *old* version of gdb, specifically +# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb +# with the 6.2.0 gdb to get tests passing. +# +# Note that we don't literally overwrite the gdb.exe binary because it appears +# to just use gdborig.exe, so that's the binary we deal with instead. +- script: | + powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" + 7z x -y %MINGW_ARCHIVE% > nul + powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" + mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe + echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) + displayName: Download custom MinGW + +# Otherwise pull in the MinGW installed on appveyor +- script: | + echo ##vso[task.prependpath]%MSYS_PATH%\mingw%MSYS_BITS%\bin + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) + displayName: Add MinGW to path + +# Make sure we use the native python interpreter instead of some msys equivalent +# one way or another. The msys interpreters seem to have weird path conversions +# baked in which break LLVM's build system one way or another, so let's use the +# native version which keeps everything as native as possible. +- script: | + copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe + echo ##vso[task.prependpath]C:\Python27amd64 + displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + +# Note that this is originally from the github releases patch of Ninja +- script: | + md ninja + powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip" + 7z x -oninja 2017-03-15-ninja-win.zip + del 2017-03-15-ninja-win.zip + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + echo ##vso[task.prependpath]%CD%\ninja + displayName: Download and install ninja + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml new file mode 100644 index 0000000000000..47a4c18308512 --- /dev/null +++ b/.azure-pipelines/steps/run.yml @@ -0,0 +1,137 @@ +# FIXME(linux): need to configure core dumps, enable them, and then dump +# backtraces on failure from all core dumps: +# +# - bash: sudo apt install gdb +# - bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern' +# +# Check travis config for `gdb --batch` command to print all crash logs + +steps: + +- checkout: self + fetchDepth: 2 + +- bash: printenv | sort + displayName: Show environment variables + +- bash: | + set -e + df -h + du . | sort -nr | head -n100 + displayName: Show disk usage + # FIXME: this hasn't been tested, but maybe it works on Windows? Should test! + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + +- template: install-sccache.yml +- template: install-clang.yml + +# Install some dependencies needed to build LLDB/Clang, currently only needed +# during the `dist` target +- bash: | + set -e + brew update + brew install xz + brew install swig + displayName: Install build dependencies (OSX) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) + +# Switch to XCode 9.3 on OSX since it seems to be the last version that supports +# i686-apple-darwin. We'll eventually want to upgrade this and it will probably +# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. +- bash: | + set -e + sudo xcode-select --switch /Applications/Xcode_9.3.app + displayName: Switch to Xcode 9.3 (OSX) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) + +- template: install-windows-build-deps.yml + +# Looks like docker containers have IPv6 disabled by default, so let's turn it +# on since libstd tests require it +- bash: | + set -e + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: Enable IPv6 + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + +# Check out all our submodules, but more quickly than using git by using one of +# our custom scripts +- bash: | + set -e + mkdir -p $HOME/rustsrc + $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) + displayName: Check out submodules (Unix) +- script: | + if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc + sh src/ci/init_repo.sh . /d/cache/rustsrc + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + displayName: Check out submodules (Windows) + +# Ensure the `aws` CLI is installed so we can deploy later on, cache docker +# images, etc. +- bash: | + set -e + sudo apt-get install -y python3-setuptools + pip3 install awscli --upgrade --user + echo "##vso[task.prependpath]$HOME/.local/bin" + displayName: Install awscli (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) +- script: pip install awscli + displayName: Install awscli (non-Linux) + condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) + +# Configure our CI_JOB_NAME variable which log analyzers can use for the main +# step to see what's going on. +- bash: echo "##vso[task.setvariable variable=CI_JOB_NAME]$SYSTEM_JOBNAME" + displayName: Configure Job Name + +# As a quick smoke check on the otherwise very fast mingw-check linux builder +# check our own internal scripts. +- bash: | + set -e + git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git + cd rust-toolstate + python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" + cd .. + rm -rf rust-toolstate + condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check')) + displayName: Verify the publish_toolstate script works + +- bash: | + set -e + if [ "$IMAGE" = "" ]; then + src/ci/run.sh + else + src/ci/docker/run.sh $IMAGE + fi + #timeoutInMinutes: 180 + timeoutInMinutes: 600 + env: + CI: true + SRC: . + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + displayName: Run build + +# If we're a deploy builder, use the `aws` command to publish everything to our +# bucket. +- bash: | + set -e + if [ "$AGENT_OS" = "Linux" ]; then + rm -rf obj/build/dist/doc + upload_dir=obj/build/dist + else + rm -rf build/dist/doc + upload_dir=build/dist + fi + ls -la $upload_dir + deploy_dir=rustc-builds + if [ "$DEPLOY_ALT" == "1" ]; then + deploy_dir=rustc-builds-alt + fi + aws s3 cp --no-progress --recursive --acl public-read ./$upload_dir s3://$DEPLOY_BUCKET/$deploy_dir/$BUILD_SOURCEVERSION + env: + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) + displayName: Upload artifacts diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml new file mode 100644 index 0000000000000..f37bb7d26ed6d --- /dev/null +++ b/.azure-pipelines/try.yml @@ -0,0 +1,78 @@ +# +# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. +# + +pr: none +trigger: +- try + +variables: +- group: prod-credentials + +jobs: +- job: Linux + timeoutInMinutes: 600 + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/run.yml + strategy: + matrix: + dist-x86_64-linux: + IMAGE: dist-x86_64-linux + DEPLOY: 1 + + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 + +- job: macOS + timeoutInMinutes: 600 + pool: + vmImage: macos-10.13 + steps: + - template: steps/run.yml + strategy: + matrix: + dist-x86_64-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + +- job: Windows + timeoutInMinutes: 600 + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/run.yml + strategy: + matrix: + dist-x86_64-msvc: + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 diff --git a/appveyor.yml b/appveyor.yml index dffd79c56e48a..c35da8927ce93 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,11 +20,11 @@ environment: - CI_JOB_NAME: i686-msvc-1 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-1 + SCRIPT: make ci-subset-1 - CI_JOB_NAME: i686-msvc-2 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-2 + SCRIPT: make ci-subset-2 # MSVC aux tests - CI_JOB_NAME: x86_64-msvc-aux @@ -58,7 +58,7 @@ environment: - CI_JOB_NAME: i686-mingw-1 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-1 + SCRIPT: make ci-subset-1 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 @@ -68,7 +68,7 @@ environment: - CI_JOB_NAME: i686-mingw-2 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-2 + SCRIPT: make ci-subset-2 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 07be27c2f5a02..ea05b30eceff9 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -80,9 +80,9 @@ TESTS_IN_2 := \ src/test/run-pass-fulldeps \ src/tools/linkchecker -appveyor-subset-1: +ci-subset-1: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) -appveyor-subset-2: +ci-subset-2: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index a162c65672f82..f22f0559265b1 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -326,6 +326,8 @@ pub enum CiEnv { Travis, /// The AppVeyor environment, for Windows builds. AppVeyor, + /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds. + AzurePipelines, } impl CiEnv { @@ -335,6 +337,8 @@ impl CiEnv { CiEnv::Travis } else if env::var("APPVEYOR").ok().map_or(false, |e| &*e == "True") { CiEnv::AppVeyor + } else if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") { + CiEnv::AzurePipelines } else { CiEnv::None } diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile index 869ee9883c100..d19862d9c583b 100644 --- a/src/ci/docker/dist-various-2/Dockerfile +++ b/src/ci/docker/dist-various-2/Dockerfile @@ -32,9 +32,10 @@ COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/ RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh -COPY dist-various-2/build-solaris-toolchain.sh /tmp/ -RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 -RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc +# FIXME(#61022) - reenable solaris +# COPY dist-various-2/build-solaris-toolchain.sh /tmp/ +# RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 +# RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/ # We pass the commit id of the port of LLVM's libunwind to the build script. # Any update to the commit id here, should cause the container image to be re-built from this point on. @@ -73,10 +74,11 @@ ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \ ENV TARGETS=x86_64-fuchsia ENV TARGETS=$TARGETS,aarch64-fuchsia -ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,wasm32-wasi -ENV TARGETS=$TARGETS,x86_64-sun-solaris +# FIXME(#61022) - reenable solaris +# ENV TARGETS=$TARGETS,sparcv9-sun-solaris +# ENV TARGETS=$TARGETS,x86_64-sun-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi ENV TARGETS=$TARGETS,x86_64-fortanix-unknown-sgx diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 98a765e1cec95..c9642dbf60c66 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -40,9 +40,12 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then docker --version >> $hash_key cksum=$(sha512sum $hash_key | \ awk '{print $1}') + s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" - echo "Attempting to download $s3url" + url="https://$SCCACHE_BUCKET.s3.amazonaws.com/docker/$cksum" + upload="aws s3 cp - $s3url" + + echo "Attempting to download $url" rm -f /tmp/rustci_docker_cache set +e retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url" @@ -65,17 +68,17 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then -f "$dockerfile" \ "$context" - if [ "$s3url" != "" ]; then + if [ "$upload" != "" ]; then digest=$(docker inspect rust-ci --format '{{.Id}}') echo "Built container $digest" if ! grep -q "$digest" <(echo "$loaded_images"); then - echo "Uploading finished image to $s3url" + echo "Uploading finished image to $url" set +e docker history -q rust-ci | \ grep -v missing | \ xargs docker save | \ gzip | \ - aws s3 cp - $s3url + $upload set -e else echo "Looks like docker image is the same as before, not uploading" @@ -87,8 +90,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then echo "$digest" >>"$info" fi elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then - if [ -n "$TRAVIS_OS_NAME" ]; then - echo Cannot run disabled images on travis! + if isCI; then + echo Cannot run disabled images on CI! exit 1 fi # retry messes with the pipe from tar to docker. Not needed on non-travis @@ -140,8 +143,11 @@ exec docker \ --env DEPLOY \ --env DEPLOY_ALT \ --env LOCAL_USER_ID=`id -u` \ + --env CI \ --env TRAVIS \ --env TRAVIS_BRANCH \ + --env TF_BUILD \ + --env BUILD_SOURCEBRANCHNAME \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ --volume "$HOME/.cargo:/cargo" \ diff --git a/src/ci/run.sh b/src/ci/run.sh index 42d0d7db5964c..c996dcb14af38 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -23,7 +23,9 @@ fi ci_dir=`cd $(dirname $0) && pwd` source "$ci_dir/shared.sh" -if [ "$TRAVIS" != "true" ] || [ "$TRAVIS_BRANCH" == "auto" ]; then +branch_name=$(getCIBranch) + +if [ ! isCI ] || [ "$branch_name" = "auto" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" fi @@ -44,7 +46,7 @@ fi # FIXME: need a scheme for changing this `nightly` value to `beta` and `stable` # either automatically or manually. export RUST_RELEASE_CHANNEL=nightly -if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then +if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo" @@ -109,7 +111,7 @@ travis_time_finish # Display the CPU and memory information. This helps us know why the CI timing # is fluctuating. travis_fold start log-system-info -if [ "$TRAVIS_OS_NAME" = "osx" ]; then +if isOSX; then system_profiler SPHardwareDataType || true sysctl hw || true ncpus=$(sysctl -n hw.ncpu) @@ -138,3 +140,5 @@ else do_make all do_make "$RUST_CHECK_TARGET" fi + +sccache --show-stats || true diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 3ba64ad412064..1e667fca5875a 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -24,6 +24,22 @@ function retry { done } +function isCI { + [ "$CI" = "true" ] || [ "$TRAVIS" = "true" ] || [ "$TF_BUILD" = "True" ] +} + +function isOSX { + [ "$TRAVIS_OS_NAME" = "osx" ] || [ "$AGENT_OS" = "Darwin" ] +} + +function getCIBranch { + if [ "$TRAVIS" = "true" ]; then + echo "$TRAVIS_BRANCH" + else + echo "$BUILD_SOURCEBRANCHNAME" + fi; +} + if ! declare -F travis_fold; then if [ "${TRAVIS-false}" = 'true' ]; then # This is a trimmed down copy of diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs index 3126b44f0d618..14035eedbb44b 100644 --- a/src/tools/cargotest/main.rs +++ b/src/tools/cargotest/main.rs @@ -61,7 +61,7 @@ const TEST_REPOS: &'static [Test] = &[ Test { name: "webrender", repo: "https://github.com/servo/webrender", - sha: "57250b2b8fa63934f80e5376a29f7dcb3f759ad6", + sha: "cdadd068f4c7218bd983d856981d561e605270ab", lock: None, packages: &[], },