From 2b71a8ed2f872537151b92e8527358e9e14bb1a9 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Thu, 29 Jul 2021 15:46:03 -0700 Subject: [PATCH 1/2] [scripts-audit] vcpkg_download_distfile depends on PR microsoft/vcpkg-tool#142 --- scripts/cmake/vcpkg_download_distfile.cmake | 381 ++++++++++---------- 1 file changed, 191 insertions(+), 190 deletions(-) diff --git a/scripts/cmake/vcpkg_download_distfile.cmake b/scripts/cmake/vcpkg_download_distfile.cmake index 557024351ee204..dfc986170758b6 100644 --- a/scripts/cmake/vcpkg_download_distfile.cmake +++ b/scripts/cmake/vcpkg_download_distfile.cmake @@ -58,221 +58,222 @@ The helper [`vcpkg_from_github`](vcpkg_from_github.md) should be used for downlo * [freetype](https://github.com/Microsoft/vcpkg/blob/master/ports/freetype/portfile.cmake) #]===] -include(vcpkg_execute_in_download_mode) +function(z_vcpkg_download_distfile_test_hash path kind error_advice sha512 skip_sha512) + if(_VCPKG_INTERNAL_NO_HASH_CHECK) + # When using the internal hash skip, do not output an explicit message. + return() + endif() + if(skip_sha512) + message(STATUS "Skipping hash check for ${file_path}.") + return() + endif() + + file(SHA512 "${path}" file_hash) + if(NOT "${file_hash}" STREQUAL "${sha512}") + message(FATAL_ERROR + "\nFile does not have expected hash:\n" + " File path: [ ${file_path} ]\n" + " Expected hash: [ ${sha512} ]\n" + " Actual hash: [ ${file_hash} ]\n" + "${CUSTOM_ERROR_ADVICE}\n") + endif() +endfunction() + +function(z_vcpkg_download_distfile_show_proxy_and_fail) + message(FATAL_ERROR + " \n" + " Failed to download file.\n" + " If you use a proxy, please check your proxy setting. Possible causes are:\n" + " \n" + " 1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable\n" + " to `https://address:port`. This is not correct, because `https://` prefix\n" + " claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr\n" + " , etc..) is an HTTP proxy. Try setting `http://address:port` to both\n" + " HTTP_PROXY and HTTPS_PROXY instead.\n" + " \n" + " 2. You are using Fiddler. Currently a bug (https://github.com/microsoft/vcpkg/issues/17752)\n" + " will set HTTPS_PROXY to `https://fiddler_address:port` which lead to problem 1 above.\n" + " Workaround is open Windows 10 Settings App, and search for Proxy Configuration page,\n" + " Change `http=address:port;https=address:port` to `address`, and fill the port number.\n" + " \n" + " 3. You proxy's remote server is out of service.\n" + " \n" + " In future vcpkg releases, if you are using Windows, you no longer need to set\n" + " HTTP(S)_PROXY environment variables. Vcpkg will simply apply Windows IE Proxy\n" + " Settings set by your proxy software. See (https://github.com/microsoft/vcpkg-tool/pull/49)\n" + " and (https://github.com/microsoft/vcpkg-tool/pull/77)\n" + " \n" + " Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n") +endfunction() + +function(z_vcpkg_download_distfile_via_aria filename urls headers sha512 skip_sha512) + vcpkg_find_acquire_program(ARIA2) + message(STATUS "Downloading ${filename}...") + + vcpkg_list(SET headers_param) + foreach(header IN LISTS headers) + vcpkg_list(APPEND headers_param "--header=${header}") + endforeach() + + vcpkg_execute_in_download_mode( + COMMAND ${ARIA2} ${urls} + -o temp/${filename} + -l download-${filename}-detailed.log + ${headers_param} + OUTPUT_FILE download-${filename}-out.log + ERROR_FILE download-${filename}-err.log + RESULT_VARIABLE error_code + WORKING_DIRECTORY "${DOWNLOADS}" + ) + if (NOT "${error_code}" STREQUAL "0") + message(STATUS + "Downloading ${filename}... Failed.\n" + " Exit Code: ${error_code}\n" + " See logs for more information:\n" + " ${DOWNLOADS}/download-${filename}-out.log\n" + " ${DOWNLOADS}/download-${filename}-err.log\n" + " ${DOWNLOADS}/download-${filename}-detailed.log\n" + ) + z_vcpkg_download_distfile_show_proxy_and_fail() + else() + z_vcpkg_download_distfile_test_hash( + "${DOWNLOADS}/temp/${filename}" + "downloaded file" + "The file may have been corrupted in transit." + ) + file(REMOVE + ${DOWNLOADS}/download-${filename}-out.log + ${DOWNLOADS}/download-${filename}-err.log + ${DOWNLOADS}/download-${filename}-detailed.log + ) + get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) + file(MAKE_DIRECTORY "${downloaded_file_dir}") + file(RENAME "${DOWNLOADS}/temp/${filename}" "${downloaded_file_path}") + endif() +endfunction() -function(vcpkg_download_distfile VAR) - set(options SKIP_SHA512 SILENT_EXIT QUIET ALWAYS_REDOWNLOAD) - set(oneValueArgs FILENAME SHA512) - set(multipleValuesArgs URLS HEADERS) - # parse parameters such that semicolons in options arguments to COMMAND don't get erased - cmake_parse_arguments(PARSE_ARGV 1 vcpkg_download_distfile "${options}" "${oneValueArgs}" "${multipleValuesArgs}") +function(vcpkg_download_distfile out_var) + cmake_parse_arguments(PARSE_ARGV 1 arg + "SKIP_SHA512;SILENT_EXIT;QUIET;ALWAYS_REDOWNLOAD" + "FILENAME;SHA512" + "URLS;HEADERS" + ) - if(NOT DEFINED vcpkg_download_distfile_URLS) + if(NOT DEFINED arg_URLS) message(FATAL_ERROR "vcpkg_download_distfile requires a URLS argument.") endif() - if(NOT DEFINED vcpkg_download_distfile_FILENAME) + if(NOT DEFINED arg_FILENAME) message(FATAL_ERROR "vcpkg_download_distfile requires a FILENAME argument.") endif() - if(vcpkg_download_distfile_SILENT_EXIT) + if(arg_SILENT_EXIT) message(WARNING "SILENT_EXIT has been deprecated as an argument to vcpkg_download_distfile -- remove the argument to resolve this warning") endif() - if(vcpkg_download_distfile_ALWAYS_REDOWNLOAD AND NOT vcpkg_download_distfile_SKIP_SHA512) + if(arg_ALWAYS_REDOWNLOAD AND NOT arg_SKIP_SHA512) message(FATAL_ERROR "ALWAYS_REDOWNLOAD option requires SKIP_SHA512 as well") endif() + + if(NOT arg_SKIP_SHA512 AND NOT DEFINED arg_SHA512) + message(FATAL_ERROR "vcpkg_download_distfile requires a SHA512 argument. +If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.") + elseif(arg_SKIP_SHA512 AND DEFINED arg_SHA512) + message(FATAL_ERROR "vcpkg_download_distfile must not be passed both SHA512 and SKIP_SHA512.") + endif() + if(_VCPKG_INTERNAL_NO_HASH_CHECK) - set(vcpkg_download_distfile_SKIP_SHA512 1) - else() - if(NOT vcpkg_download_distfile_SKIP_SHA512 AND NOT DEFINED vcpkg_download_distfile_SHA512) - message(FATAL_ERROR "vcpkg_download_distfile requires a SHA512 argument. If you do not know the SHA512, add it as 'SHA512 0' and re-run this command.") - endif() - if(vcpkg_download_distfile_SKIP_SHA512 AND DEFINED vcpkg_download_distfile_SHA512) - message(FATAL_ERROR "vcpkg_download_distfile must not be passed both SHA512 and SKIP_SHA512.") - endif() + set(arg_SKIP_SHA512 1) endif() - if(NOT vcpkg_download_distfile_SKIP_SHA512) - if(vcpkg_download_distfile_SHA512 STREQUAL "0") - string(REPEAT "0" 128 vcpkg_download_distfile_SHA512) - endif() - string(LENGTH "${vcpkg_download_distfile_SHA512}" vcpkg_download_distfile_SHA512_length) - if(NOT vcpkg_download_distfile_SHA512_length EQUAL "128") - message(FATAL_ERROR "Invalid SHA512: ${vcpkg_download_distfile_SHA512}. If you do not know the file's SHA512, set this to \"0\".") + + if(NOT arg_SKIP_SHA512) + if("${arg_SHA512}" STREQUAL "0") + string(REPEAT 0 128 arg_SHA512) + else() + string(LENGTH "${arg_SHA512}" arg_SHA512_length) + if(NOT "${arg_SHA512_length}" EQUAL "128" OR NOT "${arg_SHA512}" MATCHES "^[a-zA-Z0-9]*$") + message(FATAL_ERROR "Invalid SHA512: ${arg_SHA512}. + If you do not know the file's SHA512, set this to \"0\".") + endif() endif() endif() - set(downloaded_file_path ${DOWNLOADS}/${vcpkg_download_distfile_FILENAME}) - set(download_file_path_part "${DOWNLOADS}/temp/${vcpkg_download_distfile_FILENAME}") + set(downloaded_file_path "${DOWNLOADS}/${arg_FILENAME}") + set(download_file_path_part "${DOWNLOADS}/temp/${arg_FILENAME}") # Works around issue #3399 - if(IS_DIRECTORY "${DOWNLOADS}/temp") - # Delete "temp0" directory created by the old version of vcpkg - file(REMOVE_RECURSE "${DOWNLOADS}/temp0") - - file(GLOB temp_files "${DOWNLOADS}/temp") - file(REMOVE_RECURSE ${temp_files}) - else() - file(MAKE_DIRECTORY "${DOWNLOADS}/temp") - endif() - - function(test_hash FILE_PATH FILE_KIND CUSTOM_ERROR_ADVICE) - if(_VCPKG_INTERNAL_NO_HASH_CHECK) - # When using the internal hash skip, do not output an explicit message. - return() - endif() - if(vcpkg_download_distfile_SKIP_SHA512) - message(STATUS "Skipping hash check for ${FILE_PATH}.") - return() - endif() - - file(SHA512 ${FILE_PATH} FILE_HASH) - if(NOT FILE_HASH STREQUAL vcpkg_download_distfile_SHA512) - message(FATAL_ERROR - "\nFile does not have expected hash:\n" - " File path: [ ${FILE_PATH} ]\n" - " Expected hash: [ ${vcpkg_download_distfile_SHA512} ]\n" - " Actual hash: [ ${FILE_HASH} ]\n" - "${CUSTOM_ERROR_ADVICE}\n") - endif() - endfunction() + # Delete "temp0" directory created by the old version of vcpkg + file(REMOVE_RECURSE "${DOWNLOADS}/temp0") + file(REMOVE_RECURSE "${DOWNLOADS}/temp") + file(MAKE_DIRECTORY "${DOWNLOADS}/temp") # vcpkg_download_distfile_ALWAYS_REDOWNLOAD only triggers when NOT _VCPKG_NO_DOWNLOADS # this could be de-morgan'd out but it's more clear this way - if(EXISTS "${downloaded_file_path}" AND NOT (vcpkg_download_distfile_ALWAYS_REDOWNLOAD AND NOT _VCPKG_NO_DOWNLOADS)) - if(NOT vcpkg_download_distfile_QUIET) + if(_VCPKG_NO_DOWNLOADS) + if(NOT EXISTS "${downloaded_file_path}") + message(FATAL_ERROR "Downloads are disabled, but '${downloaded_file_path}' does not exist.") + endif() + if(NOT arg_QUIET) message(STATUS "Using ${downloaded_file_path}") endif() - test_hash("${downloaded_file_path}" "cached file" "Please delete the file and retry if this file should be downloaded again.") + + z_vcpkg_download_distfile_test_hash( + "${downloaded_file_path}" + "cached file" + "Please delete the file and retry if this file should be downloaded again." + ) + set("${out_var}" "${downloaded_file_path}" PARENT_SCOPE) + return() + endif() + + if(_VCPKG_DOWNLOAD_TOOL STREQUAL "ARIA2" AND NOT EXISTS "${downloaded_file_path}") + z_vcpkg_download_distfile_via_aria( + "${arg_FILENAME}" + "${arg_URLS}" + "${arg_HEADERS}" + "${arg_SHA512}" + "${arg_skip_sha512}" + ) + set("${out_var}" "${downloaded_file_path}" PARENT_SCOPE) + return() + endif() + + vcpkg_list(SET urls_param) + foreach(url IN LISTS arg_URLS) + vcpkg_list(APPEND urls_param "--url=${url}") + endforeach() + if(NOT vcpkg_download_distfile_QUIET) + message(STATUS "Downloading ${arg_URLS} -> ${arg_FILENAME}...") + endif() + + vcpkg_list(SET headers_param) + foreach(header IN LISTS arg_HEADERS) + list(APPEND headers_param "--header=${header}") + endforeach() + + if(arg_SKIP_SHA512) + vcpkg_list(SET sha512_param "--skip-sha512") else() - if(_VCPKG_NO_DOWNLOADS) - message(FATAL_ERROR "Downloads are disabled, but '${downloaded_file_path}' does not exist.") - endif() + vcpkg_list(SET sha512_param "--sha512=${arg_SHA512}") + endif() - # Tries to download the file. - list(GET vcpkg_download_distfile_URLS 0 SAMPLE_URL) - if(_VCPKG_DOWNLOAD_TOOL STREQUAL "ARIA2" AND NOT SAMPLE_URL MATCHES "aria2") - vcpkg_find_acquire_program("ARIA2") - message(STATUS "Downloading ${vcpkg_download_distfile_FILENAME}...") - if(vcpkg_download_distfile_HEADERS) - foreach(header IN LISTS vcpkg_download_distfile_HEADERS) - list(APPEND request_headers "--header=${header}") - endforeach() - endif() - vcpkg_execute_in_download_mode( - COMMAND ${ARIA2} ${vcpkg_download_distfile_URLS} - -o temp/${vcpkg_download_distfile_FILENAME} - -l download-${vcpkg_download_distfile_FILENAME}-detailed.log - ${request_headers} - OUTPUT_FILE download-${vcpkg_download_distfile_FILENAME}-out.log - ERROR_FILE download-${vcpkg_download_distfile_FILENAME}-err.log - RESULT_VARIABLE error_code - WORKING_DIRECTORY "${DOWNLOADS}" - ) - if (NOT "${error_code}" STREQUAL "0") - message(STATUS - "Downloading ${vcpkg_download_distfile_FILENAME}... Failed.\n" - " Exit Code: ${error_code}\n" - " See logs for more information:\n" - " ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-out.log\n" - " ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-err.log\n" - " ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-detailed.log\n" - ) - set(download_success 0) - else() - test_hash("${DOWNLOADS}/temp/${vcpkg_download_distfile_FILENAME}" "downloaded file" "The file may have been corrupted in transit.") - file(REMOVE - ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-out.log - ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-err.log - ${DOWNLOADS}/download-${vcpkg_download_distfile_FILENAME}-detailed.log - ) - get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) - file(MAKE_DIRECTORY "${downloaded_file_dir}") - file(RENAME "${DOWNLOADS}/temp/${vcpkg_download_distfile_FILENAME}" "${downloaded_file_path}") - set(download_success 1) - endif() - elseif(vcpkg_download_distfile_SKIP_SHA512 OR vcpkg_download_distfile_HEADERS) - # This is a workaround until the vcpkg tool supports downloading files without SHA512 and with headers - set(download_success 0) - set(request_headers) - if(vcpkg_download_distfile_HEADERS) - foreach(header IN LISTS vcpkg_download_distfile_HEADERS) - list(APPEND request_headers HTTPHEADER ${header}) - endforeach() - endif() - foreach(url IN LISTS vcpkg_download_distfile_URLS) - message(STATUS "Downloading ${url} -> ${vcpkg_download_distfile_FILENAME}...") - file(DOWNLOAD "${url}" "${download_file_path_part}" STATUS download_status ${request_headers}) - list(GET download_status 0 status_code) - if (NOT "${status_code}" STREQUAL "0") - message(STATUS "Downloading ${url}... Failed. Status: ${download_status}") - else() - test_hash("${download_file_path_part}" "downloaded file" "The file may have been corrupted in transit. This can be caused by proxies. If you use a proxy, please set the HTTPS_PROXY and HTTP_PROXY environment variables to \"https://user:password@your-proxy-ip-address:port/\".\n") - get_filename_component(downloaded_file_dir "${downloaded_file_path}" DIRECTORY) - file(MAKE_DIRECTORY "${downloaded_file_dir}") - file(RENAME ${download_file_path_part} ${downloaded_file_path}) - set(download_success 1) - break() - endif() - endforeach(url) - else() - set(urls) - foreach(url IN LISTS vcpkg_download_distfile_URLS) - list(APPEND urls "--url=${url}") - endforeach() - if(NOT vcpkg_download_distfile_QUIET) - message(STATUS "Downloading ${vcpkg_download_distfile_URLS} -> ${vcpkg_download_distfile_FILENAME}...") - endif() - set(request_headers) - if(vcpkg_download_distfile_HEADERS) - foreach(header IN LISTS vcpkg_download_distfile_HEADERS) - list(APPEND request_headers "--header=${header}") - endforeach() - endif() - vcpkg_execute_in_download_mode( - COMMAND "$ENV{VCPKG_COMMAND}" x-download - "${downloaded_file_path}" - "${vcpkg_download_distfile_SHA512}" - ${urls} - ${request_headers} - --debug - --feature-flags=-manifests # there's a bug in vcpkg x-download when it finds a manifest-root - OUTPUT_VARIABLE output - ERROR_VARIABLE output - RESULT_VARIABLE failure - WORKING_DIRECTORY "${DOWNLOADS}" - ) - if(failure) - message("${output}") - set(download_success 0) - else() - set(download_success 1) - endif() - endif() + if(NOT EXISTS "${downloaded_file_path}" OR arg_ALWAYS_REDOWNLOAD) + vcpkg_execute_in_download_mode( + COMMAND "$ENV{VCPKG_COMMAND}" x-download + "${downloaded_file_path}" + ${sha512_param} + ${urls_param} + ${headers_param} + --debug + --feature-flags=-manifests # there's a bug in vcpkg x-download when it finds a manifest-root + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE failure + WORKING_DIRECTORY "${DOWNLOADS}" + ) - if(NOT download_success) - message(FATAL_ERROR - " \n" - " Failed to download file.\n" - " If you use a proxy, please check your proxy setting. Possible causes are:\n" - " \n" - " 1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable\n" - " to `https://address:port`. This is not correct, because `https://` prefix\n" - " claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr\n" - " , etc..) is an HTTP proxy. Try setting `http://address:port` to both\n" - " HTTP_PROXY and HTTPS_PROXY instead.\n" - " \n" - " 2. You are using Fiddler. Currently a bug (https://github.com/microsoft/vcpkg/issues/17752)\n" - " will set HTTPS_PROXY to `https://fiddler_address:port` which lead to problem 1 above.\n" - " Workaround is open Windows 10 Settings App, and search for Proxy Configuration page,\n" - " Change `http=address:port;https=address:port` to `address`, and fill the port number.\n" - " \n" - " 3. You proxy's remote server is out of service.\n" - " \n" - " In future vcpkg releases, if you are using Windows, you no longer need to set\n" - " HTTP(S)_PROXY environment variables. Vcpkg will simply apply Windows IE Proxy\n" - " Settings set by your proxy software. See (https://github.com/microsoft/vcpkg-tool/pull/49)\n" - " and (https://github.com/microsoft/vcpkg-tool/pull/77)\n" - " \n" - " Otherwise, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n") + if(NOT "${failure}" EQUAL 0) + message("${output}") + z_vcpkg_download_distfile_show_proxy_and_fail() endif() endif() - set(${VAR} ${downloaded_file_path} PARENT_SCOPE) + + set("${out_var}" "${downloaded_file_path}" PARENT_SCOPE) endfunction() From 1a9237a28858be3d62f6b23a4a51b15c2473e3b8 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Fri, 6 Aug 2021 09:48:05 -0700 Subject: [PATCH 2/2] update vcpkg_minimum_required --- scripts/ports.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ports.cmake b/scripts/ports.cmake index c444634110235d..76eac6de8d837d 100644 --- a/scripts/ports.cmake +++ b/scripts/ports.cmake @@ -26,7 +26,7 @@ endif() list(APPEND CMAKE_MODULE_PATH "${SCRIPTS}/cmake") include("${SCRIPTS}/cmake/vcpkg_minimum_required.cmake") -vcpkg_minimum_required(VERSION 2021-05-05) +vcpkg_minimum_required(VERSION 2021-08-03) file(TO_CMAKE_PATH "${BUILDTREES_DIR}" BUILDTREES_DIR) file(TO_CMAKE_PATH "${PACKAGES_DIR}" PACKAGES_DIR)