Merge pull request #230 from microsoft/master

[vcpkg] Fix OSX CI by ensuring the downloads directory exists (microsoft#11839)

ports/arrow/CONTROL

@@ -1,5 +1,5 @@
 Source: arrow
-Version: 0.17.0-2
+Version: 0.17.1
 Build-Depends: boost-system, boost-filesystem, boost-multiprecision, boost-algorithm, flatbuffers, rapidjson, zlib, lz4, brotli, bzip2, zstd, snappy, gflags, thrift, double-conversion, glog, uriparser, openssl
 Homepage: https://github.com/apache/arrow
 Description: Apache Arrow is a columnar in-memory analytics layer designed to accelerate big data. It houses a set of canonical in-memory representations of flat and hierarchical data along with multiple language-bindings for structure manipulation. It also provides IPC and common algorithm implementations.

ports/arrow/all.patch

@@ -1,8 +1,8 @@
 diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
-index f5f0ad7..3dca82e 100644
+index e4e13cb70..58ca626da 100644
 --- a/cpp/cmake_modules/BuildUtils.cmake
 +++ b/cpp/cmake_modules/BuildUtils.cmake
-@@ -305,7 +305,7 @@ function(ADD_ARROW_LIB LIB_NAME)
+@@ -335,7 +335,7 @@ function(ADD_ARROW_LIB LIB_NAME)
        target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES})
      endif()
 
@@ -11,8 +11,22 @@ index f5f0ad7..3dca82e 100644
        set(LIB_NAME_STATIC ${LIB_NAME}_static)
      else()
        set(LIB_NAME_STATIC ${LIB_NAME})
+diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake
+index bf47915c4..053e605a0 100644
+--- a/cpp/cmake_modules/FindBrotli.cmake
++++ b/cpp/cmake_modules/FindBrotli.cmake
+@@ -64,8 +64,7 @@ if(BROTLI_ROOT)
+             PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES}
+             NO_DEFAULT_PATH)
+ else()
+-  pkg_check_modules(BROTLI_PC libbrotlicommon libbrotlienc libbrotlidec)
+-  if(BROTLI_PC_FOUND)
++  if(0) # Find via pkg_check_modules disabled as incompatible with vcpkg
+     set(BROTLI_INCLUDE_DIR "${BROTLI_PC_libbrotlicommon_INCLUDEDIR}")
+
+     # Some systems (e.g. Fedora) don't fill Brotli_LIBRARY_DIRS, so add the other dirs here.
 diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake
-index 8410916..a196b25 100644
+index 841091643..a196b251d 100644
 --- a/cpp/cmake_modules/FindLz4.cmake
 +++ b/cpp/cmake_modules/FindLz4.cmake
 @@ -19,14 +19,16 @@ if(MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX)
@@ -53,10 +67,10 @@ index 8410916..a196b25 100644
        PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
      find_path(LZ4_INCLUDE_DIR NAMES lz4.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES})
 diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake
-index f9d6296..82b8d22 100644
+index bb3eb5608..0b03d37d3 100644
 --- a/cpp/cmake_modules/FindThrift.cmake
 +++ b/cpp/cmake_modules/FindThrift.cmake
-@@ -54,6 +54,10 @@ if(MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX)
+@@ -43,6 +43,10 @@ if(MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX)
    set(THRIFT_MSVC_STATIC_LIB_SUFFIX md)
  endif()
 
@@ -67,7 +81,7 @@ index f9d6296..82b8d22 100644
  if(Thrift_ROOT)
    find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX}
                 PATHS ${Thrift_ROOT}
-@@ -74,16 +78,14 @@ else()
+@@ -61,16 +65,14 @@ else()
 
      list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}")
 
@@ -88,7 +102,7 @@ index f9d6296..82b8d22 100644
      find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include")
      find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin")
 diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/FindZSTD.cmake
-index 8e47086..d7ce559 100644
+index 8e47086e8..d87906a25 100644
 --- a/cpp/cmake_modules/FindZSTD.cmake
 +++ b/cpp/cmake_modules/FindZSTD.cmake
 @@ -19,14 +19,18 @@ if(MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX)
@@ -112,7 +126,13 @@ index 8e47086..d7ce559 100644
                       "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}"
                 PATHS ${ZSTD_ROOT}
                 PATH_SUFFIXES ${LIB_PATH_SUFFIXES}
-@@ -44,14 +48,14 @@ else()
+@@ -39,19 +43,18 @@ if(ZSTD_ROOT)
+
+ else()
+   # Second, find via pkg_check_modules
+-  pkg_check_modules(ZSTD_PC libzstd)
+-  if(ZSTD_PC_FOUND)
++  if(0) # Disabled as incompatible with vcpkg
      set(ZSTD_INCLUDE_DIR "${ZSTD_PC_INCLUDEDIR}")
 
      list(APPEND ZSTD_PC_LIBRARY_DIRS "${ZSTD_PC_LIBDIR}")
@@ -130,10 +150,10 @@ index 8e47086..d7ce559 100644
                   PATH_SUFFIXES ${LIB_PATH_SUFFIXES})
      find_path(ZSTD_INCLUDE_DIR NAMES zstd.h PATH_SUFFIXES ${INCLUDE_PATH_SUFFIXES})
 diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake
-index 75b33c2..80cac9a 100644
+index 6110a5aa5..3270d74a9 100644
 --- a/cpp/cmake_modules/SetupCxxFlags.cmake
 +++ b/cpp/cmake_modules/SetupCxxFlags.cmake
-@@ -128,7 +128,9 @@ macro(arrow_add_werror_if_debug)
+@@ -163,7 +163,9 @@ macro(arrow_add_werror_if_debug)
    if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
      # Treat all compiler warnings as errors
      if(MSVC)

ports/arrow/portfile.cmake

@@ -1,12 +1,10 @@
-if(NOT VCPKG_TARGET_ARCHITECTURE STREQUAL "x64")
-  message(FATAL_ERROR "Apache Arrow only supports x64")
-endif()
+vcpkg_fail_port_install(ON_ARCH "x86" "arm" "arm64")
 
 vcpkg_from_github(
     OUT_SOURCE_PATH SOURCE_PATH
     REPO apache/arrow
-    REF apache-arrow-0.17.0
-    SHA512 293737db80defa0f8766f726dc228ace50936f7124647de15c2e024c2901ded1cda893d771d38aa4e9ab19ff7eb06b11dfc230587ca5b17cdd87e681fc3009ca
+    REF apache-arrow-0.17.1
+    SHA512 2a1a637d6df08e19d0c8313c51e1baf8902db677b072f8787c4f9faf8bdec94357ac8af839718d449377b508fe4f6e31b011cbdc6ccf029b6a66f567172569aa
     HEAD_REF master
     PATCHES
         all.patch

ports/mlpack/CONTROL

@@ -1,8 +1,8 @@
 Source: mlpack
-Version: 3.2.2-2
+Version: 3.2.2-3
 Homepage: https://github.com/mlpack/mlpack
 Description: mlpack is a fast, flexible machine learning library, written in C++, that aims to provide fast, extensible implementations of cutting-edge machine learning algorithms.
-Build-Depends: openblas (!osx), clapack (!osx), boost, armadillo, ensmallen
+Build-Depends: openblas (!osx), clapack (!osx), boost, armadillo, ensmallen, stb
 
 Feature: tools
 Description: Build command-line executables.

ports/mlpack/portfile.cmake

@@ -28,6 +28,10 @@ vcpkg_configure_cmake(
     PREFER_NINJA
     OPTIONS
         -DBUILD_TESTS=OFF
+        -DDOWNLOAD_STB_IMAGE=OFF
+        -DDOWNLOAD_ENSMALLEN=OFF
+        -DBUILD_PYTHON_BINDINGS=OFF
+        -DCMAKE_DISABLE_FIND_PACKAGE_Git=ON
         ${FEATURE_OPTIONS}
 )
 vcpkg_install_cmake()

ports/tensorflow-cc/CONTROL

@@ -1,5 +1,5 @@
 Source: tensorflow-cc
-Version: 1.14-1
+Version: 1.14-2
 Description: Library for computation using data flow graphs for scalable machine learning
 Build-Depends: c-ares
-Supports: !x86
+Supports: !x86

ports/tensorflow-cc/portfile.cmake

@@ -1,9 +1,7 @@
-include(vcpkg_common_functions)
-
 message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.")
 
 if (VCPKG_TARGET_ARCHITECTURE STREQUAL x86)
-    message(FATAL_ERROR "TensorFlow does not support 32bit system.")
+    message(FATAL_ERROR "TensorFlow does not support 32bit systems.")
 endif()
 
 vcpkg_from_github(
@@ -76,6 +74,7 @@ set(ENV{TF_NCCL_VERSION} 2.3)
 set(ENV{NCCL_INSTALL_PATH} "")
 set(ENV{CC_OPT_FLAGS} "/arch:AVX")
 set(ENV{TF_NEED_CUDA} 0)
+set(ENV{TF_CONFIGURE_IOS} 0)
 
 message(STATUS "Configuring TensorFlow")
 
@@ -118,4 +117,4 @@ file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tenso
 file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright)
 
 file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc)
-file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)
+file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)

scripts/azure-pipelines/clean-tombstones.yml

@@ -14,27 +14,19 @@ jobs:
       displayName: 'Initialize Environment'
       inputs:
         filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
-    - script: rmdir /s /q archives\fail
-      displayName: 'Delete archives\fail'
+    - script: rmdir /s /q W:\fail
+      displayName: 'Delete W:\fail'
   - job: linux
     displayName: 'Clean Linux Tombstones'
     pool: $(linux-pool)
     steps:
-    - task: Bash@3
-      displayName: 'Initialize Environment'
-      inputs:
-        filePath: scripts/azure-pipelines/linux/initialize-environment.sh
-    - bash: rm -rf archives/fail
-      displayName: 'Delete archives/fail'
+    - bash: rm -rf /archives/fail
+      displayName: 'Delete /archives/fail'
   - job: osx
     displayName: 'Clean MacOS Tombstones'
     pool:
       name: vcpkgAgentPool
       demands: Agent.OS -equals Darwin
     steps:
-    - task: Bash@3
-      displayName: 'Initialize Environment'
-      inputs:
-        filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'
-    - bash: rm -rf archives/fail
-      displayName: 'Delete archives/fail'
+    - bash: rm -rf /Users/vagrant/Data/archives/fail
+      displayName: 'Delete /Users/vagrant/Data/archives/fail'

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -33,11 +33,20 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - powershell: |
-      ./vcpkg x-ci-clean --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages
+      $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,/archives,upload',
+        '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees',
+        '--x-install-root=/mnt/vcpkg-ci/install',
+        '--x-packages-root=/mnt/vcpkg-ci/packages'
+      )
+
+      ./vcpkg x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
       mkdir $(System.ArtifactsDirectory)/xml-results
       $env:HOME = '/home/agent'
-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads
+      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -13,22 +13,19 @@ jobs:
   steps:
   - bash: |
       df -h
-    displayName: 'Report on Disk Space Before Environment'
+    displayName: 'Report on Disk Space'
     condition: always()
-  - task: Bash@3
-    displayName: 'Initialize Environment'
-    inputs:
-      filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'
   - bash: |
       brew list autoconf || brew install autoconf
       brew list automake || brew install automake
       brew list libtool || brew install libtool
       brew list gfortran || brew cask install gfortran
     displayName: 'Install brew dependencies'
   - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Environment'
-    condition: always()
+      sudo mkdir /Users/vagrant/Data/downloads || 0
+      sudo chmod 777 /Users/vagrant/Data/downloads || 0
+      exit 0
+    displayName: 'Create /Users/vagrant/Data/downloads'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
@@ -39,10 +36,19 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - powershell: |
-      ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads
+      $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload',
+        '--x-buildtrees-root=/Users/vagrant/Data/buildtrees',
+        '--x-install-root=/Users/vagrant/Data/installed',
+        '--x-packages-root=/Users/vagrant/Data/packages'
+      )
+
+      ./vcpkg x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
       mkdir $(System.ArtifactsDirectory)/xml-results
-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads
+      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -23,17 +23,25 @@ jobs:
         set VCPKG_DOWNLOADS=D:\downloads
         .\bootstrap-vcpkg.bat
   - powershell: |
-      .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
+      $env:VCPKG_DOWNLOADS='D:\downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,W:\,upload',
+        '--x-buildtrees-root=E:\buildtrees',
+        '--x-install-root=E:\install',
+        '--x-packages-root=E:\packages'
+      )
+
+      .\vcpkg.exe x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt
       mkdir $(System.ArtifactsDirectory)\xml-results
       # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.
       # Install them so the CI succeeds:
       if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {
-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
+          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs
       }
 
-      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
-      Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII
+      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

scripts/ci.baseline.txt

@@ -75,7 +75,6 @@ apr:arm64-windows=fail
 argtable2:arm-uwp=fail
 argtable2:x64-uwp=fail
 arrow:arm64-windows=fail
-arrow:x64-linux=fail
 arrow:x86-windows=fail
 asiosdk:x64-linux=fail
 asiosdk:x64-osx=fail
@@ -621,7 +620,7 @@ ignition-msgs5:x64-linux=fail
 ignition-msgs5:arm64-windows=fail
 ignition-msgs5:arm-uwp=fail
 ignition-msgs5:x64-uwp=fail
-ignition-msgs5:x64-osx=fail
+ignition-msgs5:x64-osx=skip
 imgui-sfml:x64-linux=ignore
 intel-ipsec:arm64-windows=fail
 intel-ipsec:arm-uwp=fail
@@ -1092,10 +1091,6 @@ microsoft-signalr:x64-uwp=skip
 microsoft-signalr:x64-windows=skip
 microsoft-signalr:x64-windows-static=skip
 microsoft-signalr:x86-windows=skip
-# conflicts with stb
-mlpack:x86-windows=skip
-mlpack:x64-windows=skip
-mlpack:x64-windows-static=skip
 mman:x64-linux=fail
 mman:x64-osx=fail
 mmloader:arm64-windows=fail