From ec2158f4198c93cd3135364e5902164da7c0fbde Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Mon, 20 May 2024 10:28:37 -0400 Subject: [PATCH 01/25] clarify asset caching output --- .vscode/settings.json | 19 ++++++++++++++++++- include/vcpkg/base/message-data.inc.h | 8 ++++++++ src/vcpkg/base/downloads.cpp | 11 +++++++++++ src/vcpkg/tools.cpp | 6 ------ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f1da23c28e..24918c3d2e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -33,6 +33,7 @@ "eslint.format.enable": true, "files.eol": "\n", "files.associations": { + "error_message_analysis": "plaintext", "any": "cpp", "array": "cpp", "atomic": "cpp", @@ -98,6 +99,22 @@ "cinttypes": "cpp", "typeindex": "cpp", "typeinfo": "cpp", - "variant": "cpp" + "variant": "cpp", + "__bit_reference": "cpp", + "__config": "cpp", + "__hash_table": "cpp", + "__locale": "cpp", + "__node_handle": "cpp", + "__split_buffer": "cpp", + "__threading_support": "cpp", + "__tree": "cpp", + "__verbose_abort": "cpp", + "charconv": "cpp", + "execution": "cpp", + "ios": "cpp", + "locale": "cpp", + "queue": "cpp", + "source_location": "cpp", + "stack": "cpp" } } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 8c276c2b09..f95c0d7675 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,10 +236,18 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") +DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url), "", "Attempting to download from: \"{url}\".") +DECLARE_MESSAGE(AssetCacheHit, (), "", "Asset cache hit!") +DECLARE_MESSAGE(AssetCacheMiss, (), "", "Asset cache miss!") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (), "", "Successfully stored back to mirror.") +DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") +DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") +DECLARE_MESSAGE(AssetDownloadsBlocked, (), "`x-block-origin` is a vcpkg option. Do not translate. ", "External asset downloads are blocked (x-block-origin is enabled)..") +DECLARE_MESSAGE(AssetDownloadsEnabled, (), "`x-block-origin` is a vcpkg option. Do not translate. ", "External asset downloads are allowed (x-block-origin is disabled).") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 572318a832..15a0d8d0a5 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -878,6 +878,9 @@ namespace vcpkg const Optional& sha512, MessageSink& progress_sink) const { + m_config.m_block_origin ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCachingEnabled) : msg::println(msgAssetCachingNotConfigured); + this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); } @@ -915,8 +918,12 @@ namespace vcpkg errors, progress_sink)) { + msg::println(msgAssetCacheHit); return read_url; } + else{ + msg::println(msgAssetCacheMiss); + } } else if (auto script = m_config.m_script.get()) { @@ -970,6 +977,7 @@ namespace vcpkg { if (urls.size() != 0) { + msg::println(msgAssetCacheAttemptingDownload, msg::url = urls[0]); auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) @@ -982,6 +990,9 @@ namespace vcpkg msg::println_warning(msgFailedToStoreBackToMirror); msg::println(maybe_push.error()); } + else{ + msg::println(msgAssetCacheSuccesfullyStored); + } } return *url; diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 9181632fda..72e980c844 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -677,12 +677,6 @@ namespace vcpkg const auto download_path = downloads / tool_data.download_subpath; if (!fs.exists(download_path, IgnoreErrors{})) { - status_sink.println(Color::none, - msgDownloadingTool, - msg::tool_name = tool_data.name, - msg::url = tool_data.url, - msg::path = download_path); - downloader->download_file(fs, tool_data.url, {}, download_path, tool_data.sha512, null_sink); } else From 019f3be55a2775337857a7177cc9fbb7f1ca8289 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Mon, 20 May 2024 10:35:42 -0400 Subject: [PATCH 02/25] Ignore .vscode/settings.json --- .gitignore | 1 + .vscode/settings.json | 120 ------------------------------------------ 2 files changed, 1 insertion(+), 120 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 3b7ff43bcc..016dbdb38f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.idea /cmake-build-* /.vs +.vscode/settings.json /CMakeSettings.json /out /OneLocBuild diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 24918c3d2e..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "typescript.tsdk": "ce/ce/common/temp/node_modules/typescript/lib", - "mochaExplorer.files": "./ce/test/**/*.js", - "mochaExplorer.ignore": [ - "./ce/test/**/*.d.ts", - "**/node_modules/**", - "./ce/ce/**", - "./ce/common/**/*.ts", - "./ce/common/**/*.js" - ], - "mochaExplorer.cwd": "./ce/test", - "mochaExplorer.mochaPath": "./ce/test/node_modules/mocha", - "mochaExplorer.timeout": 500000, - "mochaExplorer.require": [ - "source-map-support/register", - ], - "mochaExplorer.monkeyPatch": true, - "mochaExplorer.debuggerConfig": "MochaTest", - "eslint.workingDirectories": [ - { - "changeProcessCWD": true, - "directory": "./ce/ce", - }, - { - "changeProcessCWD": true, - "directory": "./ce/test", - } - ], - "editor.codeActionsOnSave": { - "source.fixAll.eslint": "explicit", - "source.organizeImports": "explicit" - }, - "eslint.format.enable": true, - "files.eol": "\n", - "files.associations": { - "error_message_analysis": "plaintext", - "any": "cpp", - "array": "cpp", - "atomic": "cpp", - "bit": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "cctype": "cpp", - "chrono": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "codecvt": "cpp", - "complex": "cpp", - "condition_variable": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdint": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "deque": "cpp", - "list": "cpp", - "map": "cpp", - "set": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "vector": "cpp", - "exception": "cpp", - "algorithm": "cpp", - "functional": "cpp", - "iterator": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "regex": "cpp", - "string": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "utility": "cpp", - "fstream": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "ostream": "cpp", - "shared_mutex": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "cinttypes": "cpp", - "typeindex": "cpp", - "typeinfo": "cpp", - "variant": "cpp", - "__bit_reference": "cpp", - "__config": "cpp", - "__hash_table": "cpp", - "__locale": "cpp", - "__node_handle": "cpp", - "__split_buffer": "cpp", - "__threading_support": "cpp", - "__tree": "cpp", - "__verbose_abort": "cpp", - "charconv": "cpp", - "execution": "cpp", - "ios": "cpp", - "locale": "cpp", - "queue": "cpp", - "source_location": "cpp", - "stack": "cpp" - } -} From 0e70a8a7d64236a54ce73eeead17f5bbc6a1b8e6 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Mon, 20 May 2024 10:36:55 -0400 Subject: [PATCH 03/25] undo --- .gitignore | 1 - .vscode/settings.json | 103 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 016dbdb38f..3b7ff43bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ /.idea /cmake-build-* /.vs -.vscode/settings.json /CMakeSettings.json /out /OneLocBuild diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..f1da23c28e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,103 @@ +{ + "typescript.tsdk": "ce/ce/common/temp/node_modules/typescript/lib", + "mochaExplorer.files": "./ce/test/**/*.js", + "mochaExplorer.ignore": [ + "./ce/test/**/*.d.ts", + "**/node_modules/**", + "./ce/ce/**", + "./ce/common/**/*.ts", + "./ce/common/**/*.js" + ], + "mochaExplorer.cwd": "./ce/test", + "mochaExplorer.mochaPath": "./ce/test/node_modules/mocha", + "mochaExplorer.timeout": 500000, + "mochaExplorer.require": [ + "source-map-support/register", + ], + "mochaExplorer.monkeyPatch": true, + "mochaExplorer.debuggerConfig": "MochaTest", + "eslint.workingDirectories": [ + { + "changeProcessCWD": true, + "directory": "./ce/ce", + }, + { + "changeProcessCWD": true, + "directory": "./ce/test", + } + ], + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit", + "source.organizeImports": "explicit" + }, + "eslint.format.enable": true, + "files.eol": "\n", + "files.associations": { + "any": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "codecvt": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "regex": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "shared_mutex": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "variant": "cpp" + } +} From 1f5a61f7530f2d4072081990dd6c848ce915154f Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Mon, 20 May 2024 10:43:11 -0400 Subject: [PATCH 04/25] remove unused message --- include/vcpkg/base/message-data.inc.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index f95c0d7675..f2baefe908 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1037,7 +1037,6 @@ DECLARE_MESSAGE(DownloadingPortableToolVersionX, "", "A suitable version of {tool_name} was not found (required v{version}) Downloading " "portable {tool_name} {version}...") -DECLARE_MESSAGE(DownloadingTool, (msg::tool_name, msg::url, msg::path), "", "Downloading {tool_name}...\n{url}->{path}") DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(DownloadWinHttpError, (msg::system_api, msg::exit_code, msg::url), From 44287ef6b689ea564348ca5b7f047012fa27c483 Mon Sep 17 00:00:00 2001 From: Javier Matos Denizac Date: Mon, 20 May 2024 14:47:26 +0000 Subject: [PATCH 05/25] Format and regenerate messages --- include/vcpkg/base/message-data.inc.h | 10 ++++++++-- locales/messages.json | 13 +++++++++++-- src/vcpkg/base/downloads.cpp | 13 ++++++++----- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index f2baefe908..2586eef5a2 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -246,8 +246,14 @@ DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (), "", "Successfully stored back to mirror.") DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") -DECLARE_MESSAGE(AssetDownloadsBlocked, (), "`x-block-origin` is a vcpkg option. Do not translate. ", "External asset downloads are blocked (x-block-origin is enabled)..") -DECLARE_MESSAGE(AssetDownloadsEnabled, (), "`x-block-origin` is a vcpkg option. Do not translate. ", "External asset downloads are allowed (x-block-origin is disabled).") +DECLARE_MESSAGE(AssetDownloadsBlocked, + (), + "`x-block-origin` is a vcpkg option. Do not translate. ", + "External asset downloads are blocked (x-block-origin is enabled)..") +DECLARE_MESSAGE(AssetDownloadsEnabled, + (), + "`x-block-origin` is a vcpkg option. Do not translate. ", + "External asset downloads are allowed (x-block-origin is disabled).") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") diff --git a/locales/messages.json b/locales/messages.json index 0d82c56f8a..ed8b7dbd8b 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -172,8 +172,19 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", + "AssetCacheAttemptingDownload": "Attempting to download from: \"{url}\".", + "_AssetCacheAttemptingDownload.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheHit": "Asset cache hit!", + "AssetCacheMiss": "Asset cache miss!", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", + "AssetCacheSuccesfullyStored": "Successfully stored back to mirror.", + "AssetCachingEnabled": "Asset caching is enabled.", + "AssetCachingNotConfigured": "Asset caching is not configured.", + "AssetDownloadsBlocked": "External asset downloads are blocked (x-block-origin is enabled)..", + "_AssetDownloadsBlocked.comment": "`x-block-origin` is a vcpkg option. Do not translate. ", + "AssetDownloadsEnabled": "External asset downloads are allowed (x-block-origin is disabled).", + "_AssetDownloadsEnabled.comment": "`x-block-origin` is a vcpkg option. Do not translate. ", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -599,8 +610,6 @@ "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}) Downloading portable {tool_name} {version}...", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", - "DownloadingTool": "Downloading {tool_name}...\n{url}->{path}", - "_DownloadingTool.comment": "An example of {tool_name} is aria2. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", "DownloadingUrl": "Downloading {url}", "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 15a0d8d0a5..60030f766a 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -878,9 +878,10 @@ namespace vcpkg const Optional& sha512, MessageSink& progress_sink) const { - m_config.m_block_origin ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); - m_config.m_read_url_template.has_value() ? msg::println(msgAssetCachingEnabled) : msg::println(msgAssetCachingNotConfigured); - + m_config.m_block_origin ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCachingEnabled) + : msg::println(msgAssetCachingNotConfigured); + this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); } @@ -921,7 +922,8 @@ namespace vcpkg msg::println(msgAssetCacheHit); return read_url; } - else{ + else + { msg::println(msgAssetCacheMiss); } } @@ -990,7 +992,8 @@ namespace vcpkg msg::println_warning(msgFailedToStoreBackToMirror); msg::println(maybe_push.error()); } - else{ + else + { msg::println(msgAssetCacheSuccesfullyStored); } } From 70313b6f8c8adb781d4db5db13c3f03035a4ad16 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Fri, 24 May 2024 12:39:10 -0400 Subject: [PATCH 06/25] propogate filename --- include/vcpkg/base/message-data.inc.h | 4 ++-- src/vcpkg/base/downloads.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 2586eef5a2..a5bb537984 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,14 +236,14 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url), "", "Attempting to download from: \"{url}\".") +DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url, msg::path), "", "Attempting to download from: {url} -> {path}...") DECLARE_MESSAGE(AssetCacheHit, (), "", "Asset cache hit!") DECLARE_MESSAGE(AssetCacheMiss, (), "", "Asset cache miss!") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (), "", "Successfully stored back to mirror.") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} back to mirror.") DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") DECLARE_MESSAGE(AssetDownloadsBlocked, diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 60030f766a..40bbcf8372 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -979,7 +979,7 @@ namespace vcpkg { if (urls.size() != 0) { - msg::println(msgAssetCacheAttemptingDownload, msg::url = urls[0]); + msg::println(msgAssetCacheAttemptingDownload, msg::url = urls[0], msg::path = download_path.filename()); auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) @@ -994,7 +994,7 @@ namespace vcpkg } else { - msg::println(msgAssetCacheSuccesfullyStored); + msg::println(msgAssetCacheSuccesfullyStored, msg::path = download_path.filename()); } } From 9a1be75ced5c85153ac985ca5c11e578a975b082 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Fri, 24 May 2024 14:21:50 -0400 Subject: [PATCH 07/25] print asset cache and x-block-origin info when --debug --- include/vcpkg/base/downloads.h | 3 +++ include/vcpkg/base/message-data.inc.h | 7 ++++--- src/vcpkg/base/downloads.cpp | 13 +++++++++++-- src/vcpkg/tools.cpp | 1 + src/vcpkg/vcpkgpaths.cpp | 5 +++-- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index a361e24f76..93b4163ee5 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -92,6 +92,9 @@ namespace vcpkg const Path& file_to_put, StringView sha512) const; + bool get_block_origin() const; + bool asset_cache_configured() const; + private: DownloadManagerConfig m_config; }; diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index a5bb537984..257d10019a 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,7 +236,7 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url, msg::path), "", "Attempting to download from: {url} -> {path}...") +DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url, msg::path), "", "Attempting to download: {url} -> {path}...") DECLARE_MESSAGE(AssetCacheHit, (), "", "Asset cache hit!") DECLARE_MESSAGE(AssetCacheMiss, (), "", "Asset cache miss!") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, @@ -244,6 +244,7 @@ DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} back to mirror.") +DECLARE_MESSAGE(AssetCachingDisabled, (), "", "Asset caching is not configured.") DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") DECLARE_MESSAGE(AssetDownloadsBlocked, @@ -252,8 +253,8 @@ DECLARE_MESSAGE(AssetDownloadsBlocked, "External asset downloads are blocked (x-block-origin is enabled)..") DECLARE_MESSAGE(AssetDownloadsEnabled, (), - "`x-block-origin` is a vcpkg option. Do not translate. ", - "External asset downloads are allowed (x-block-origin is disabled).") + "`x-block-origin` is a vcpkg option. Do not translate.", + "External asset downloads are allowed (x-block-origin is disabled)...") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 40bbcf8372..6f37818057 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -871,6 +871,15 @@ namespace vcpkg return s_headers; } + bool DownloadManager::get_block_origin() const { + return m_config.m_block_origin; + } + + bool DownloadManager::asset_cache_configured() const + { + return m_config.m_read_url_template.has_value(); + } + void DownloadManager::download_file(const Filesystem& fs, const std::string& url, View headers, @@ -878,8 +887,8 @@ namespace vcpkg const Optional& sha512, MessageSink& progress_sink) const { - m_config.m_block_origin ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); - m_config.m_read_url_template.has_value() ? msg::println(msgAssetCachingEnabled) + get_block_origin() ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); + asset_cache_configured() ? msg::println(msgAssetCachingEnabled) : msg::println(msgAssetCachingNotConfigured); this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 72e980c844..300b80fdf9 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 7d48e87932..ac56bec5c3 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -316,7 +316,7 @@ namespace , scripts(process_input_directory(fs, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO)) , m_registries_cache(compute_registries_cache_root(fs, args)) { - Debug::println("Using builtin-ports: ", m_builtin_ports); + Debug::println("Using builtin-ports: ", m_builtin_ports); } const Filesystem& m_fs; @@ -655,9 +655,10 @@ namespace vcpkg , community_triplets(filesystem.almost_canonical(triplets / "community", VCPKG_LINE_INFO)) { Debug::print("Using vcpkg-root: ", root, '\n'); - Debug::print("Using scripts-root: ", scripts, '\n'); Debug::print("Using builtin-registry: ", builtin_registry_versions, '\n'); Debug::print("Using downloads-root: ", downloads, '\n'); + m_pimpl->m_download_manager->get_block_origin() ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); + m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") : Debug::println("Asset cache is not configured."); { const auto config_path = m_pimpl->m_config_dir / "vcpkg-configuration.json"; From 94eae199b97d2dc99b37a8e760e06bd7294f345a Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Fri, 24 May 2024 17:48:29 -0400 Subject: [PATCH 08/25] consolidate messaging --- include/vcpkg/base/message-data.inc.h | 14 ++++++-------- src/vcpkg/base/downloads.cpp | 27 +++++++++++---------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 257d10019a..982f1b11c4 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,14 +236,13 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheAttemptingDownload, (msg::url, msg::path), "", "Attempting to download: {url} -> {path}...") -DECLARE_MESSAGE(AssetCacheHit, (), "", "Asset cache hit!") -DECLARE_MESSAGE(AssetCacheMiss, (), "", "Asset cache miss!") +DECLARE_MESSAGE(AssetCacheHit, (msg::path), "", "Asset cache hit for {path}.") +DECLARE_MESSAGE(AssetCacheMiss, (msg::path), "", "Asset cache miss for {path}.") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} back to mirror.") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} to mirror.") DECLARE_MESSAGE(AssetCachingDisabled, (), "", "Asset caching is not configured.") DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") @@ -1042,8 +1041,7 @@ DECLARE_MESSAGE(DownloadFailedStatusCode, DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", - "A suitable version of {tool_name} was not found (required v{version}) Downloading " - "portable {tool_name} {version}...") + "A suitable version of {tool_name} was not found (required v{version}).") DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(DownloadWinHttpError, (msg::system_api, msg::exit_code, msg::url), @@ -1212,7 +1210,7 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (), "", "Failed to download from mirror set") +DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "", "Failed to download {path}.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, @@ -1266,7 +1264,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, "Additional information, such as the command line output, if any, will be appended on " "the line after this message", "Failed to run \"{path}\" to determine the {tool_name} version.") -DECLARE_MESSAGE(FailedToStoreBackToMirror, (), "", "failed to store back to mirror:") +DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path), "", "Asset Caching is not configured. Failed to store {path} to mirror.") DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}") DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock") DECLARE_MESSAGE(FailedToWriteManifest, (msg::path), "", "Failed to write manifest file {path}") diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 6f37818057..8aeb823c51 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -887,10 +887,6 @@ namespace vcpkg const Optional& sha512, MessageSink& progress_sink) const { - get_block_origin() ? msg::println(msgAssetDownloadsBlocked) : msg::println(msgAssetDownloadsEnabled); - asset_cache_configured() ? msg::println(msgAssetCachingEnabled) - : msg::println(msgAssetCachingNotConfigured); - this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); } @@ -928,12 +924,12 @@ namespace vcpkg errors, progress_sink)) { - msg::println(msgAssetCacheHit); + msg::println(msgAssetCacheHit, msg::path = download_path.filename()); return read_url; } else { - msg::println(msgAssetCacheMiss); + msg::println(msgAssetCacheMiss, msg::path = download_path.filename()); } } else if (auto script = m_config.m_script.get()) @@ -988,30 +984,29 @@ namespace vcpkg { if (urls.size() != 0) { - msg::println(msgAssetCacheAttemptingDownload, msg::url = urls[0], msg::path = download_path.filename()); + msg::println(LocalizedString::from_raw(fmt::format("Downloading {}", urls[0]))); + auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { if (auto hash = sha512.get()) { - auto maybe_push = put_file_to_mirror(fs, download_path, *hash); - if (!maybe_push) - { - msg::println_warning(msgFailedToStoreBackToMirror); - msg::println(maybe_push.error()); - } - else + if(m_config.m_write_url_template.has_value()) { + put_file_to_mirror(fs, download_path, *hash); msg::println(msgAssetCacheSuccesfullyStored, msg::path = download_path.filename()); - } + + }else{ + msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); + } } return *url; } } } - msg::println_error(msgFailedToDownloadFromMirrorSet); + msg::println_error(msgFailedToDownloadFromMirrorSet, msg::path = download_path.filename()); for (LocalizedString& error : errors) { msg::println(error); From 9676b5345c134b415bfcb9f6d0f158bd92ca0fc0 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 15:14:18 -0400 Subject: [PATCH 09/25] add e2e tests --- .../end-to-end-tests-dir/asset-caching.ps1 | 99 +++++++++++++++++++ azure-pipelines/end-to-end-tests-prelude.ps1 | 10 ++ 2 files changed, 109 insertions(+) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index fcf1792bbe..d8f5d5155b 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -1,6 +1,105 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 +# Testing x-script Refresh-TestRoot Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed + +# Test Asset Cache not configured + x-block-origin enabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) + +$expected = @( +"Computing installation plan..." +"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"error: Failed to download cmake-[0-9\.]+-.*\.tar\.gz\." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Asset cache not configured + x-block-origin enabled failed" +} + +# Test Asset Cache not configured + x-block-origin disabled +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) + +$expected = @( +"Computing installation plan..." +"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"Downloading .*" +"warning: Asset Caching is not configured. Failed to store cmake-[0-9\.]+-.*\.tar\.gz to mirror." +"Extracting cmake..." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Asset cache not configured + x-block-origin disabled failed" +} + +# Test Asset Cache Miss + x-block-origin enabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) + +$expected = @( +"Computing installation plan..." +"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"Asset cache miss for cmake-[0-9\.]+-.*\.tar\.gz." +"error: Failed to download cmake-[0-9\.]+-.*\.tar\.gz." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Asset cache miss + x-block-origin enabled failed" +} + +# Test Asset Cache Miss + x-block-origin disabled +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) + +$expected = @( +"Computing installation plan..." +"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"Asset cache miss for cmake-[0-9\.]+-.*\.tar\.gz." +"Downloading .*" +"Successfully stored cmake-[0-9\.]+-.*\.tar\.gz to mirror." +"Extracting cmake..." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Asset cache miss + x-block-origin disabled failed" +} + +# Test Asset Cache Hit +Refresh-Downloads +Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +$expected = @( +"Computing installation plan..." +"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"Asset cache hit for cmake-[0-9\.]+-.*\.tar\.gz." +"Extracting cmake..." +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Asset cache hit failed" +} + +# Test asset caching && x-block-orgin promises when --debug is passed (enabled) +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) +# Define the regex pattern that accounts for multiline input +$expectedPattern = "(?s)" + + ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.?" + + ".*\[DEBUG\] Asset caching is enabled\..*" + +if (-not ($actual -match $expectedPattern)) { + throw "Test failed: Debug messages mismatch" +} + +# Test asset caching && x-block-orgin promises when --debug is passed (disabled) +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) +$expectedPattern = "(?s)" + + ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" + + ".*\[DEBUG\] Asset cache is not configured.*" + +if (-not ($actual -match $expectedPattern)) { + throw "Test failed: Debug messages mismatch" +} \ No newline at end of file diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index fe7e6095f2..e38c1e7065 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -6,6 +6,9 @@ $NuGetRoot = Join-Path $TestingRoot 'nuget' $NuGetRoot2 = Join-Path $TestingRoot 'nuget2' $ArchiveRoot = Join-Path $TestingRoot 'archives' $VersionFilesRoot = Join-Path $TestingRoot 'version-test' +$DownloadsRoot = Join-Path $TestingRoot 'downloads' +$AssetCache = Join-Path $TestingRoot 'asset-cache' + $directoryArgs = @( "--x-buildtrees-root=$buildtreesRoot", "--x-install-root=$installRoot", @@ -34,6 +37,13 @@ function Refresh-TestRoot { Remove-Item -Recurse -Force $TestingRoot -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force $TestingRoot | Out-Null New-Item -ItemType Directory -Force $NuGetRoot | Out-Null + New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null + New-Item -ItemType Directory -Force $AssetCache | Out-Null +} + +function Refresh-Downloads{ + Remove-Item -Recurse -Force $DownloadsRoot -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force $DownloadsRoot | Out-Null } function Write-Stack { From 4441c2bb9d7d5d8ebfec0cdd868b6d3fe518d0b3 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 16:31:51 -0400 Subject: [PATCH 10/25] generalize expected output --- .../end-to-end-tests-dir/asset-caching.ps1 | 29 +++++++++---------- include/vcpkg/base/message-data.inc.h | 2 +- src/vcpkg/base/downloads.cpp | 11 +++---- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index d8f5d5155b..8aa0a59594 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -12,8 +12,8 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." -"A suitable version of cmake was not found \(required v[0-9\.]+\)." -"error: Failed to download cmake-[0-9\.]+-.*\.tar\.gz\." +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"error: Failed to download .*-[0-9\.]+-.*\.tar\.gz\." ) -join "`n" if (-not ($actual -match $expected)) { @@ -25,10 +25,9 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." -"A suitable version of cmake was not found \(required v[0-9\.]+\)." +"A suitable version of .* was not found \(required v[0-9\.]+\)." "Downloading .*" -"warning: Asset Caching is not configured. Failed to store cmake-[0-9\.]+-.*\.tar\.gz to mirror." -"Extracting cmake..." +"Extracting .*..." ) -join "`n" if (-not ($actual -match $expected)) { @@ -41,9 +40,9 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." -"A suitable version of cmake was not found \(required v[0-9\.]+\)." -"Asset cache miss for cmake-[0-9\.]+-.*\.tar\.gz." -"error: Failed to download cmake-[0-9\.]+-.*\.tar\.gz." +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache miss for .*-[0-9\.]+-.*\.tar\.gz." +"error: Failed to download .*[0-9\.]+-.*\.tar\.gz." ) -join "`n" if (-not ($actual -match $expected)) { @@ -55,11 +54,11 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." -"A suitable version of cmake was not found \(required v[0-9\.]+\)." -"Asset cache miss for cmake-[0-9\.]+-.*\.tar\.gz." +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache miss for .*-[0-9\.]+-.*\.tar\.gz." "Downloading .*" -"Successfully stored cmake-[0-9\.]+-.*\.tar\.gz to mirror." -"Extracting cmake..." +"Successfully stored .*-[0-9\.]+-.*\.tar\.gz to mirror." +"Extracting .*..." ) -join "`n" if (-not ($actual -match $expected)) { @@ -72,9 +71,9 @@ Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( "Computing installation plan..." -"A suitable version of cmake was not found \(required v[0-9\.]+\)." -"Asset cache hit for cmake-[0-9\.]+-.*\.tar\.gz." -"Extracting cmake..." +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache hit for .*-[0-9\.]+-.*\.tar\.gz." +"Extracting .*..." ) -join "`n" if (-not ($actual -match $expected)) { diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 982f1b11c4..0a24c6de6a 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1264,7 +1264,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, "Additional information, such as the command line output, if any, will be appended on " "the line after this message", "Failed to run \"{path}\" to determine the {tool_name} version.") -DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path), "", "Asset Caching is not configured. Failed to store {path} to mirror.") +DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path), "", "Failed to store {path} to mirror.") DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}") DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock") DECLARE_MESSAGE(FailedToWriteManifest, (msg::path), "", "Failed to write manifest file {path}") diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 8aeb823c51..6897d8f821 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -984,7 +984,7 @@ namespace vcpkg { if (urls.size() != 0) { - msg::println(LocalizedString::from_raw(fmt::format("Downloading {}", urls[0]))); + msg::println(msgDownloadingUrl, msg::url = urls[0]); auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); @@ -994,12 +994,9 @@ namespace vcpkg { if(m_config.m_write_url_template.has_value()) { - put_file_to_mirror(fs, download_path, *hash); - msg::println(msgAssetCacheSuccesfullyStored, msg::path = download_path.filename()); - - }else{ - msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); - } + + put_file_to_mirror(fs, download_path, *hash) ? msg::println(msgAssetCacheSuccesfullyStored, msg::path = download_path.filename()) : msg::println(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); + } } return *url; From 8d149a838422c9264e73be514860d1a563b0f6f0 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 16:45:24 -0400 Subject: [PATCH 11/25] generalize even more --- .../end-to-end-tests-dir/asset-caching.ps1 | 13 +++++++------ include/vcpkg/base/message-data.inc.h | 2 +- src/vcpkg/base/downloads.cpp | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 8aa0a59594..06eec986ed 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -13,7 +13,7 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Failed to download .*-[0-9\.]+-.*\.tar\.gz\." +"error: Failed to download .*." ) -join "`n" if (-not ($actual -match $expected)) { @@ -41,8 +41,8 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .*-[0-9\.]+-.*\.tar\.gz." -"error: Failed to download .*[0-9\.]+-.*\.tar\.gz." +"Asset cache miss for .*." +"error: Failed to download .*" ) -join "`n" if (-not ($actual -match $expected)) { @@ -55,9 +55,9 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .*-[0-9\.]+-.*\.tar\.gz." +"Asset cache miss for .*." "Downloading .*" -"Successfully stored .*-[0-9\.]+-.*\.tar\.gz to mirror." +"Successfully stored .* to mirror." "Extracting .*..." ) -join "`n" @@ -72,7 +72,8 @@ $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg $expected = @( "Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache hit for .*-[0-9\.]+-.*\.tar\.gz." +"Asset cache hit for .*." +"Downloading: .*" "Extracting .*..." ) -join "`n" diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 0a24c6de6a..c4df89f25e 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,7 +236,7 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheHit, (msg::path), "", "Asset cache hit for {path}.") +DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}.\nDownloading: {url}") DECLARE_MESSAGE(AssetCacheMiss, (msg::path), "", "Asset cache miss for {path}.") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 6897d8f821..a67a6ccf56 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -924,7 +924,7 @@ namespace vcpkg errors, progress_sink)) { - msg::println(msgAssetCacheHit, msg::path = download_path.filename()); + msg::println(msgAssetCacheHit, msg::path = download_path.filename(), msg::url = read_url); return read_url; } else From 35795b250b0ab26544b42c1f8acb9e62ed589609 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 16:53:19 -0400 Subject: [PATCH 12/25] try again --- .../end-to-end-tests-dir/asset-caching.ps1 | 19 +++++++------------ include/vcpkg/base/message-data.inc.h | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 06eec986ed..eb6b88d2e5 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -6,12 +6,11 @@ Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed -# Test Asset Cache not configured + x-block-origin enabled +# Testing asset cache not configured + x-block-origin enabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) $expected = @( -"Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." "error: Failed to download .*." ) -join "`n" @@ -20,11 +19,10 @@ if (-not ($actual -match $expected)) { throw "Asset cache not configured + x-block-origin enabled failed" } -# Test Asset Cache not configured + x-block-origin disabled +# Testing asset cache not configured + x-block-origin disabled $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) $expected = @( -"Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." "Downloading .*" "Extracting .*..." @@ -34,12 +32,11 @@ if (-not ($actual -match $expected)) { throw "Asset cache not configured + x-block-origin disabled failed" } -# Test Asset Cache Miss + x-block-origin enabled +# Testing asset cache miss + x-block-origin enabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) $expected = @( -"Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache miss for .*." "error: Failed to download .*" @@ -49,11 +46,10 @@ if (-not ($actual -match $expected)) { throw "Asset cache miss + x-block-origin enabled failed" } -# Test Asset Cache Miss + x-block-origin disabled +# Testing asset cache miss + x-block-origin disabled $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( -"Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache miss for .*." "Downloading .*" @@ -65,12 +61,11 @@ if (-not ($actual -match $expected)) { throw "Asset cache miss + x-block-origin disabled failed" } -# Test Asset Cache Hit +# Testing asset cache hit Refresh-Downloads Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( -"Computing installation plan..." "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache hit for .*." "Downloading: .*" @@ -81,7 +76,7 @@ if (-not ($actual -match $expected)) { throw "Asset cache hit failed" } -# Test asset caching && x-block-orgin promises when --debug is passed (enabled) +# Testing asset caching && x-block-orgin promises when --debug is passed (enabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) # Define the regex pattern that accounts for multiline input @@ -93,7 +88,7 @@ if (-not ($actual -match $expectedPattern)) { throw "Test failed: Debug messages mismatch" } -# Test asset caching && x-block-orgin promises when --debug is passed (disabled) +# Testing asset caching && x-block-orgin promises when --debug is passed (disabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) $expectedPattern = "(?s)" + diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index c4df89f25e..83a73a68ae 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -237,7 +237,7 @@ DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when a DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}.\nDownloading: {url}") -DECLARE_MESSAGE(AssetCacheMiss, (msg::path), "", "Asset cache miss for {path}.") +DECLARE_MESSAGE(AssetCacheMiss, (msg::path), "", "Asset cache miss for {path}.\nDownloading: {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", From 6b20afa37cc6d9e876fc116fc43fb53a21083dee Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 17:34:53 -0400 Subject: [PATCH 13/25] maybe --- .../end-to-end-tests-dir/asset-caching.ps1 | 31 +++++-------------- include/vcpkg/base/message-data.inc.h | 6 ++-- src/vcpkg/base/downloads.cpp | 17 +++++----- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index eb6b88d2e5..5b4aa79772 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -6,55 +6,41 @@ Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed -# Testing asset cache not configured + x-block-origin enabled +# Testing asset cache miss + x-block-origin enabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Failed to download .*." +"error: Asset cache miss for .* and external downloads are blocked by x-block-origin." ) -join "`n" if (-not ($actual -match $expected)) { throw "Asset cache not configured + x-block-origin enabled failed" } -# Testing asset cache not configured + x-block-origin disabled +# Testing asset cache miss (not configured) + x-block-origin disabled $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Downloading .*" -"Extracting .*..." +"Asset cache miss for .*." +"Downloading: .*" ) -join "`n" if (-not ($actual -match $expected)) { throw "Asset cache not configured + x-block-origin disabled failed" } -# Testing asset cache miss + x-block-origin enabled +# Testing asset cache miss (configured) + x-block-origin disabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) - -$expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .*." -"error: Failed to download .*" -) -join "`n" - -if (-not ($actual -match $expected)) { - throw "Asset cache miss + x-block-origin enabled failed" -} - -# Testing asset cache miss + x-block-origin disabled $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache miss for .*." -"Downloading .*" -"Successfully stored .* to mirror." -"Extracting .*..." +"Downloading: .*" +"Successfully stored .* to .*." ) -join "`n" if (-not ($actual -match $expected)) { @@ -69,7 +55,6 @@ $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache hit for .*." "Downloading: .*" -"Extracting .*..." ) -join "`n" if (-not ($actual -match $expected)) { diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 83a73a68ae..81d6db17d4 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -237,12 +237,12 @@ DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when a DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}.\nDownloading: {url}") -DECLARE_MESSAGE(AssetCacheMiss, (msg::path), "", "Asset cache miss for {path}.\nDownloading: {url}") +DECLARE_MESSAGE(AssetCacheMiss, (msg::path, msg::url), "", "Asset cache miss for {path}.\nDownloading: {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} to mirror.") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") DECLARE_MESSAGE(AssetCachingDisabled, (), "", "Asset caching is not configured.") DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") @@ -1210,7 +1210,7 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "", "Failed to download {path}.") +DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index a67a6ccf56..0002b031cd 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -927,10 +927,6 @@ namespace vcpkg msg::println(msgAssetCacheHit, msg::path = download_path.filename(), msg::url = read_url); return read_url; } - else - { - msg::println(msgAssetCacheMiss, msg::path = download_path.filename()); - } } else if (auto script = m_config.m_script.get()) { @@ -984,7 +980,7 @@ namespace vcpkg { if (urls.size() != 0) { - msg::println(msgDownloadingUrl, msg::url = urls[0]); + msg::println(msgAssetCacheMiss, msg::path = download_path.filename(), msg::url = urls[0]); auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); @@ -992,16 +988,18 @@ namespace vcpkg { if (auto hash = sha512.get()) { - if(m_config.m_write_url_template.has_value()) + auto maybe_push = put_file_to_mirror(fs, download_path, *hash); + if (!maybe_push) { - - put_file_to_mirror(fs, download_path, *hash) ? msg::println(msgAssetCacheSuccesfullyStored, msg::path = download_path.filename()) : msg::println(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); - } + msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); + msg::println(maybe_push.error()); + } } return *url; } } + } msg::println_error(msgFailedToDownloadFromMirrorSet, msg::path = download_path.filename()); for (LocalizedString& error : errors) @@ -1019,6 +1017,7 @@ namespace vcpkg auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "", sha512); if (!maybe_mirror_url.empty()) { + msg::println(msgAssetCacheSuccesfullyStored, msg::path = file_to_put.filename(), msg::url = maybe_mirror_url); return put_file(fs, maybe_mirror_url, m_config.m_secrets, m_config.m_write_headers, file_to_put); } return 0; From fde427155a345517aa2efa027789781a2f73260e Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 18:28:45 -0400 Subject: [PATCH 14/25] try setting VCPKG_FORCE_DOWNLOADED_BINARIES in script --- azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 | 4 ++-- include/vcpkg/base/message-data.inc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 5b4aa79772..7d2216a333 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -5,8 +5,8 @@ Refresh-TestRoot Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed +$env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON" -# Testing asset cache miss + x-block-origin enabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) @@ -66,7 +66,7 @@ Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) # Define the regex pattern that accounts for multiline input $expectedPattern = "(?s)" + - ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.?" + + ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" + ".*\[DEBUG\] Asset caching is enabled\..*" if (-not ($actual -match $expectedPattern)) { diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 81d6db17d4..1906aa1808 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -249,7 +249,7 @@ DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configu DECLARE_MESSAGE(AssetDownloadsBlocked, (), "`x-block-origin` is a vcpkg option. Do not translate. ", - "External asset downloads are blocked (x-block-origin is enabled)..") + "External asset downloads are blocked (x-block-origin is enabled)...") DECLARE_MESSAGE(AssetDownloadsEnabled, (), "`x-block-origin` is a vcpkg option. Do not translate.", From 36411d1a47a76d2084e713aaac7dc1e64a037eef Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 18:51:17 -0400 Subject: [PATCH 15/25] normalize line endings --- .../end-to-end-tests-dir/asset-caching.ps1 | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 7d2216a333..82c180b7b0 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -5,10 +5,13 @@ Refresh-TestRoot Run-Vcpkg -TestArgs ($commonArgs + @('fetch', 'cmake')) Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-binarysource=clear", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin")) Throw-IfFailed + $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON" +# Testing asset cache miss (not configured) + x-block-origin enabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." @@ -16,11 +19,12 @@ $expected = @( ) -join "`n" if (-not ($actual -match $expected)) { - throw "Asset cache not configured + x-block-origin enabled failed" + throw "Failure: asset cache miss (not configured) + x-block-origin enabled" } # Testing asset cache miss (not configured) + x-block-origin disabled $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." @@ -29,12 +33,13 @@ $expected = @( ) -join "`n" if (-not ($actual -match $expected)) { - throw "Asset cache not configured + x-block-origin disabled failed" + throw "Failure: asset cache miss (not configured) + x-block-origin disabled" } # Testing asset cache miss (configured) + x-block-origin disabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." @@ -44,13 +49,15 @@ $expected = @( ) -join "`n" if (-not ($actual -match $expected)) { - throw "Asset cache miss + x-block-origin disabled failed" + throw "Failure: asset cache miss (configured) + x-block-origin disabled" } # Testing asset cache hit Refresh-Downloads Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache hit for .*." @@ -58,28 +65,32 @@ $expected = @( ) -join "`n" if (-not ($actual -match $expected)) { - throw "Asset cache hit failed" + throw "Failure: asset cache hit" } # Testing asset caching && x-block-orgin promises when --debug is passed (enabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) +$actual = $actual -replace "`r`n", "`n" + # Define the regex pattern that accounts for multiline input $expectedPattern = "(?s)" + ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" + ".*\[DEBUG\] Asset caching is enabled\..*" if (-not ($actual -match $expectedPattern)) { - throw "Test failed: Debug messages mismatch" + throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)" } # Testing asset caching && x-block-orgin promises when --debug is passed (disabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) +$actual = $actual -replace "`r`n", "`n" + $expectedPattern = "(?s)" + ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" + ".*\[DEBUG\] Asset cache is not configured.*" if (-not ($actual -match $expectedPattern)) { - throw "Test failed: Debug messages mismatch" + throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" } \ No newline at end of file From ab0f7264c2b69d76c43dd011b0eda78605aad3f8 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 19:03:03 -0400 Subject: [PATCH 16/25] remove extra downloading message on windows --- src/vcpkg/base/downloads.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 0002b031cd..c02bce01e0 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -717,7 +717,6 @@ namespace vcpkg fs.create_directories(dir, VCPKG_LINE_INFO); const auto sanitized_url = replace_secrets(url, secrets); - msg::println(msgDownloadingUrl, msg::url = sanitized_url); static auto s = WinHttpSession::make(sanitized_url).value_or_exit(VCPKG_LINE_INFO); for (size_t trials = 0; trials < 4; ++trials) { From ebc59fd9a6f4a34a2ac926b123b7d27e9c727f57 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Tue, 28 May 2024 19:28:27 -0400 Subject: [PATCH 17/25] remove unused messages --- include/vcpkg/base/message-data.inc.h | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 1906aa1808..044f6c62fe 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -243,17 +243,6 @@ DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") -DECLARE_MESSAGE(AssetCachingDisabled, (), "", "Asset caching is not configured.") -DECLARE_MESSAGE(AssetCachingEnabled, (), "", "Asset caching is enabled.") -DECLARE_MESSAGE(AssetCachingNotConfigured, (), "", "Asset caching is not configured.") -DECLARE_MESSAGE(AssetDownloadsBlocked, - (), - "`x-block-origin` is a vcpkg option. Do not translate. ", - "External asset downloads are blocked (x-block-origin is enabled)...") -DECLARE_MESSAGE(AssetDownloadsEnabled, - (), - "`x-block-origin` is a vcpkg option. Do not translate.", - "External asset downloads are allowed (x-block-origin is disabled)...") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1042,7 +1031,6 @@ DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", "A suitable version of {tool_name} was not found (required v{version}).") -DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(DownloadWinHttpError, (msg::system_api, msg::exit_code, msg::url), "", @@ -1210,7 +1198,7 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, @@ -2607,7 +2595,7 @@ DECLARE_MESSAGE(UnexpectedArgument, DECLARE_MESSAGE( UnexpectedAssetCacheProvider, (), - "", + "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate", "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'") DECLARE_MESSAGE(UnexpectedByteSize, (msg::expected, msg::actual), From beed0fe1f1b16511832157661a9bd14e8b24c53e Mon Sep 17 00:00:00 2001 From: Javier Matos Denizac Date: Tue, 28 May 2024 23:33:46 +0000 Subject: [PATCH 18/25] Format and regenerate messages --- include/vcpkg/base/message-data.inc.h | 5 ++++- locales/messages.json | 28 ++++++++++++--------------- src/vcpkg/base/downloads.cpp | 15 +++++--------- src/vcpkg/tools.cpp | 2 +- src/vcpkg/vcpkgpaths.cpp | 9 ++++++--- 5 files changed, 28 insertions(+), 31 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 044f6c62fe..db38b56305 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1198,7 +1198,10 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, + (msg::path), + "x-block-origin is a vcpkg term. Do not translate", + "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/locales/messages.json b/locales/messages.json index ed8b7dbd8b..a26eed7f2e 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -172,19 +172,14 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", - "AssetCacheAttemptingDownload": "Attempting to download from: \"{url}\".", - "_AssetCacheAttemptingDownload.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheHit": "Asset cache hit!", - "AssetCacheMiss": "Asset cache miss!", + "AssetCacheHit": "Asset cache hit for {path}.\nDownloading: {url}", + "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMiss": "Asset cache miss for {path}.\nDownloading: {url}", + "_AssetCacheMiss.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", - "AssetCacheSuccesfullyStored": "Successfully stored back to mirror.", - "AssetCachingEnabled": "Asset caching is enabled.", - "AssetCachingNotConfigured": "Asset caching is not configured.", - "AssetDownloadsBlocked": "External asset downloads are blocked (x-block-origin is enabled)..", - "_AssetDownloadsBlocked.comment": "`x-block-origin` is a vcpkg option. Do not translate. ", - "AssetDownloadsEnabled": "External asset downloads are allowed (x-block-origin is disabled).", - "_AssetDownloadsEnabled.comment": "`x-block-origin` is a vcpkg option. Do not translate. ", + "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", + "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -608,10 +603,8 @@ "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", - "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}) Downloading portable {tool_name} {version}...", + "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", - "DownloadingUrl": "Downloading {url}", - "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", "_DownloadingVcpkgStandaloneBundle.comment": "An example of {version} is 1.3.8.", "DownloadingVcpkgStandaloneBundleLatest": "Downloading latest standalone bundle.", @@ -729,7 +722,8 @@ "FailedToDeleteInsideDueToFile": "failed to remove_all_inside({value}) due to {path}: ", "_FailedToDeleteInsideDueToFile.comment": "{value} is the parent path of {path} we tried to delete; the underlying Windows error message is printed after this An example of {path} is /foo/bar.", "FailedToDetermineCurrentCommit": "Failed to determine the current commit:", - "FailedToDownloadFromMirrorSet": "Failed to download from mirror set", + "FailedToDownloadFromMirrorSet": "Asset cache miss for {path} and external downloads are blocked by x-block-origin.", + "_FailedToDownloadFromMirrorSet.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "FailedToExtract": "Failed to extract \"{path}\":", "_FailedToExtract.comment": "An example of {path} is /foo/bar.", "FailedToFetchRepo": "Failed to fetch {url}.", @@ -773,7 +767,8 @@ "_FailedToRemoveControl.comment": "An example of {path} is /foo/bar.", "FailedToRunToolToDetermineVersion": "Failed to run \"{path}\" to determine the {tool_name} version.", "_FailedToRunToolToDetermineVersion.comment": "Additional information, such as the command line output, if any, will be appended on the line after this message An example of {tool_name} is aria2. An example of {path} is /foo/bar.", - "FailedToStoreBackToMirror": "failed to store back to mirror:", + "FailedToStoreBackToMirror": "Failed to store {path} to mirror.", + "_FailedToStoreBackToMirror.comment": "An example of {path} is /foo/bar.", "FailedToStoreBinaryCache": "Failed to store binary cache {path}", "_FailedToStoreBinaryCache.comment": "An example of {path} is /foo/bar.", "FailedToTakeFileSystemLock": "Failed to take the filesystem lock", @@ -1455,6 +1450,7 @@ "UnexpectedArgument": "unexpected argument: {option}", "_UnexpectedArgument.comment": "Argument is literally what the user passed on the command line. An example of {option} is editable.", "UnexpectedAssetCacheProvider": "unknown asset provider type: valid source types are 'x-azurl', 'x-script', 'x-block-origin', and 'clear'", + "_UnexpectedAssetCacheProvider.comment": "'x-azurl', 'x-script', 'clear' are valid source types. Do not translate", "UnexpectedByteSize": "Expected {expected} bytes to be written, but {actual} were written.", "_UnexpectedByteSize.comment": "{expected} is the expected byte size and {actual} is the actual byte size.", "UnexpectedCharExpectedCloseBrace": "Unexpected character; expected property or close brace", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index c02bce01e0..68473fd3a3 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -870,14 +870,9 @@ namespace vcpkg return s_headers; } - bool DownloadManager::get_block_origin() const { - return m_config.m_block_origin; - } + bool DownloadManager::get_block_origin() const { return m_config.m_block_origin; } - bool DownloadManager::asset_cache_configured() const - { - return m_config.m_read_url_template.has_value(); - } + bool DownloadManager::asset_cache_configured() const { return m_config.m_read_url_template.has_value(); } void DownloadManager::download_file(const Filesystem& fs, const std::string& url, @@ -980,7 +975,7 @@ namespace vcpkg if (urls.size() != 0) { msg::println(msgAssetCacheMiss, msg::path = download_path.filename(), msg::url = urls[0]); - + auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) @@ -998,7 +993,6 @@ namespace vcpkg return *url; } } - } msg::println_error(msgFailedToDownloadFromMirrorSet, msg::path = download_path.filename()); for (LocalizedString& error : errors) @@ -1016,7 +1010,8 @@ namespace vcpkg auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "", sha512); if (!maybe_mirror_url.empty()) { - msg::println(msgAssetCacheSuccesfullyStored, msg::path = file_to_put.filename(), msg::url = maybe_mirror_url); + msg::println( + msgAssetCacheSuccesfullyStored, msg::path = file_to_put.filename(), msg::url = maybe_mirror_url); return put_file(fs, maybe_mirror_url, m_config.m_secrets, m_config.m_write_headers, file_to_put); } return 0; diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 300b80fdf9..0fb43745d7 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -9,9 +9,9 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index ac56bec5c3..5a70351708 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -316,7 +316,7 @@ namespace , scripts(process_input_directory(fs, root, args.scripts_root_dir.get(), "scripts", VCPKG_LINE_INFO)) , m_registries_cache(compute_registries_cache_root(fs, args)) { - Debug::println("Using builtin-ports: ", m_builtin_ports); + Debug::println("Using builtin-ports: ", m_builtin_ports); } const Filesystem& m_fs; @@ -657,8 +657,11 @@ namespace vcpkg Debug::print("Using vcpkg-root: ", root, '\n'); Debug::print("Using builtin-registry: ", builtin_registry_versions, '\n'); Debug::print("Using downloads-root: ", downloads, '\n'); - m_pimpl->m_download_manager->get_block_origin() ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); - m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") : Debug::println("Asset cache is not configured."); + m_pimpl->m_download_manager->get_block_origin() + ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") + : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); + m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") + : Debug::println("Asset cache is not configured."); { const auto config_path = m_pimpl->m_config_dir / "vcpkg-configuration.json"; From 8e3bd454c4106a23d8c3c355a1625c75b3317858 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Fri, 31 May 2024 15:39:42 -0400 Subject: [PATCH 19/25] solidify asset caching output;add more tests; mirror url hidden --- .../end-to-end-tests-dir/asset-caching.ps1 | 37 +++++++++++++++---- include/vcpkg/base/message-data.inc.h | 12 +++--- src/vcpkg/base/downloads.cpp | 29 +++++++++++---- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 82c180b7b0..91ea2ccce8 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -15,7 +15,7 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Asset cache miss for .* and external downloads are blocked by x-block-origin." +"error: Missing .* and external downloads are blocked by x-block-origin." ) -join "`n" if (-not ($actual -match $expected)) { @@ -28,14 +28,27 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .*." -"Downloading: .*" +"Downloading .*." ) -join "`n" if (-not ($actual -match $expected)) { throw "Failure: asset cache miss (not configured) + x-block-origin disabled" } +# Testing asset cache miss (configured) + x-block-origin enabled +Refresh-TestRoot +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) +$actual = $actual -replace "`r`n", "`n" + +$expected = @( +"A suitable version of .* was not found \(required v[0-9\.]+\)." +"Asset cache miss for .* and external downloads are blocked by x-block-origin" +) -join "`n" + +if (-not ($actual -match $expected)) { + throw "Failure: asset cache miss (configured) + x-block-origin disabled" +} + # Testing asset cache miss (configured) + x-block-origin disabled Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) @@ -43,9 +56,8 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .*." -"Downloading: .*" -"Successfully stored .* to .*." +"Asset cache miss; downloading from .*" +"Successfully stored .* to mirror." ) -join "`n" if (-not ($actual -match $expected)) { @@ -61,13 +73,24 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache hit for .*." -"Downloading: .*" ) -join "`n" if (-not ($actual -match $expected)) { throw "Failure: asset cache hit" } +# Testing asset cache hit with --debug +Refresh-Downloads +Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot", "--debug")) +$actual = $actual -replace "`r`n", "`n" + +$expectedPattern = "(?s)" + ".*\[DEBUG\] Downloaded from: .*" + +if (-not ($actual -match $expectedPattern)) { + throw "Failure: asset cache hit (--debug)" +} + # Testing asset caching && x-block-orgin promises when --debug is passed (enabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index db38b56305..d0951c8cd0 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,13 +236,14 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}.\nDownloading: {url}") -DECLARE_MESSAGE(AssetCacheMiss, (msg::path, msg::url), "", "Asset cache miss for {path}.\nDownloading: {url}") +DECLARE_MESSAGE(AssetCacheHit, (msg::path), "", "Asset cache hit for {path}.") +DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") +DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} to mirror.") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1198,10 +1199,11 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(FailedToDownloadFromMirrorSet, +DECLARE_MESSAGE(MissingAssetBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", - "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") + "Missing {path} and external downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(AssetCacheMissBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 68473fd3a3..9bba8c4a81 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -580,7 +580,9 @@ namespace vcpkg msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); } } - + msg::println( + msgAssetCacheSuccesfullyStored, msg::path = file.filename()); + Debug::println(fmt::format("Uploaded {} to {} ", file.filename(), url)); return res; } @@ -918,7 +920,8 @@ namespace vcpkg errors, progress_sink)) { - msg::println(msgAssetCacheHit, msg::path = download_path.filename(), msg::url = read_url); + msg::println(msgAssetCacheHit, msg::path = download_path.filename()); + Debug::println("Downloaded from: ", read_url); return read_url; } } @@ -974,12 +977,12 @@ namespace vcpkg { if (urls.size() != 0) { - msg::println(msgAssetCacheMiss, msg::path = download_path.filename(), msg::url = urls[0]); - auto maybe_url = try_download_file( fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) : msg::println(msgDownloadingUrl, msg::url = urls[0]); + if (auto hash = sha512.get()) { auto maybe_push = put_file_to_mirror(fs, download_path, *hash); @@ -994,10 +997,22 @@ namespace vcpkg } } } - msg::println_error(msgFailedToDownloadFromMirrorSet, msg::path = download_path.filename()); + // Asset cache is not configured and x-block-origin enabled + if (m_config.m_read_url_template.has_value()) + { + msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); + } + else + { + msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); + } + for (LocalizedString& error : errors) { - msg::println(error); + // Debug only since: + // 1. curl already prints the error + // 2. the printed url can contain sensitive information like username/password + Debug::println(error); } Checks::exit_fail(VCPKG_LINE_INFO); @@ -1010,8 +1025,6 @@ namespace vcpkg auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "", sha512); if (!maybe_mirror_url.empty()) { - msg::println( - msgAssetCacheSuccesfullyStored, msg::path = file_to_put.filename(), msg::url = maybe_mirror_url); return put_file(fs, maybe_mirror_url, m_config.m_secrets, m_config.m_write_headers, file_to_put); } return 0; From 1932451002e3e9a0eb104f193aa7d1860a477e54 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 5 Jun 2024 12:04:44 -0700 Subject: [PATCH 20/25] clang-format --- include/vcpkg/base/message-data.inc.h | 5 ++++- locales/messages.json | 20 ++++++++++++-------- src/vcpkg/base/downloads.cpp | 10 +++++----- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index d0951c8cd0..7932d1ed81 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1203,7 +1203,10 @@ DECLARE_MESSAGE(MissingAssetBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Missing {path} and external downloads are blocked by x-block-origin.") -DECLARE_MESSAGE(AssetCacheMissBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(AssetCacheMissBlockOrigin, + (msg::path), + "x-block-origin is a vcpkg term. Do not translate", + "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/locales/messages.json b/locales/messages.json index a26eed7f2e..6ec1d7878a 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -172,14 +172,16 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", - "AssetCacheHit": "Asset cache hit for {path}.\nDownloading: {url}", - "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMiss": "Asset cache miss for {path}.\nDownloading: {url}", - "_AssetCacheMiss.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheHit": "Asset cache hit for {path}.", + "_AssetCacheHit.comment": "An example of {path} is /foo/bar.", + "AssetCacheMiss": "Asset cache miss; downloading from {url}", + "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and external downloads are blocked by x-block-origin.", + "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", - "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", - "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheSuccesfullyStored": "Successfully stored {path} to mirror.", + "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar.", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -605,6 +607,8 @@ "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", + "DownloadingUrl": "Downloading {url}", + "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", "_DownloadingVcpkgStandaloneBundle.comment": "An example of {version} is 1.3.8.", "DownloadingVcpkgStandaloneBundleLatest": "Downloading latest standalone bundle.", @@ -722,8 +726,6 @@ "FailedToDeleteInsideDueToFile": "failed to remove_all_inside({value}) due to {path}: ", "_FailedToDeleteInsideDueToFile.comment": "{value} is the parent path of {path} we tried to delete; the underlying Windows error message is printed after this An example of {path} is /foo/bar.", "FailedToDetermineCurrentCommit": "Failed to determine the current commit:", - "FailedToDownloadFromMirrorSet": "Asset cache miss for {path} and external downloads are blocked by x-block-origin.", - "_FailedToDownloadFromMirrorSet.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "FailedToExtract": "Failed to extract \"{path}\":", "_FailedToExtract.comment": "An example of {path} is /foo/bar.", "FailedToFetchRepo": "Failed to fetch {url}.", @@ -1141,6 +1143,8 @@ "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}", "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.", "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.", + "MissingAssetBlockOrigin": "Missing {path} and external downloads are blocked by x-block-origin.", + "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "MissingClosingParen": "missing closing )", "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.", "_MissingDependency.comment": "An example of {spec} is zlib:x64-windows. An example of {package_name} is zlib.", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 9bba8c4a81..1b7a317a66 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -580,8 +580,7 @@ namespace vcpkg msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); } } - msg::println( - msgAssetCacheSuccesfullyStored, msg::path = file.filename()); + msg::println(msgAssetCacheSuccesfullyStored, msg::path = file.filename()); Debug::println(fmt::format("Uploaded {} to {} ", file.filename(), url)); return res; } @@ -981,8 +980,9 @@ namespace vcpkg fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { - m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) : msg::println(msgDownloadingUrl, msg::url = urls[0]); - + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) + : msg::println(msgDownloadingUrl, msg::url = urls[0]); + if (auto hash = sha512.get()) { auto maybe_push = put_file_to_mirror(fs, download_path, *hash); @@ -1006,7 +1006,7 @@ namespace vcpkg { msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); } - + for (LocalizedString& error : errors) { // Debug only since: From 4f553d79b0930e1e192db6a1d7bc4ddc8a068c50 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Wed, 12 Jun 2024 11:09:56 -0400 Subject: [PATCH 21/25] sanitize mirror url, print mirror whenever applicable, restore curl output --- .../end-to-end-tests-dir/asset-caching.ps1 | 16 ++-------------- include/vcpkg/base/message-data.inc.h | 6 +++--- src/vcpkg/base/downloads.cpp | 13 ++++--------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 91ea2ccce8..9b805b8614 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -57,7 +57,7 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." "Asset cache miss; downloading from .*" -"Successfully stored .* to mirror." +"Successfully stored .* to .*." ) -join "`n" if (-not ($actual -match $expected)) { @@ -72,25 +72,13 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache hit for .*." +"Asset cache hit for .*; downloaded from: .*" ) -join "`n" if (-not ($actual -match $expected)) { throw "Failure: asset cache hit" } -# Testing asset cache hit with --debug -Refresh-Downloads -Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot", "--debug")) -$actual = $actual -replace "`r`n", "`n" - -$expectedPattern = "(?s)" + ".*\[DEBUG\] Downloaded from: .*" - -if (-not ($actual -match $expectedPattern)) { - throw "Failure: asset cache hit (--debug)" -} - # Testing asset caching && x-block-orgin promises when --debug is passed (enabled) Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index d0951c8cd0..b4bea46a16 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -236,14 +236,14 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheHit, (msg::path), "", "Asset cache hit for {path}.") +DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}") DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path), "", "Successfully stored {path} to mirror.") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1257,7 +1257,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, "Additional information, such as the command line output, if any, will be appended on " "the line after this message", "Failed to run \"{path}\" to determine the {tool_name} version.") -DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path), "", "Failed to store {path} to mirror.") +DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path, msg::url), "", "Failed to store {path} to {path}.") DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}") DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock") DECLARE_MESSAGE(FailedToWriteManifest, (msg::path), "", "Failed to write manifest file {path}") diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 9bba8c4a81..4ed8a66b43 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -581,8 +581,7 @@ namespace vcpkg } } msg::println( - msgAssetCacheSuccesfullyStored, msg::path = file.filename()); - Debug::println(fmt::format("Uploaded {} to {} ", file.filename(), url)); + msgAssetCacheSuccesfullyStored, msg::path = file.filename(), msg::url = replace_secrets(url.to_string(), secrets)); return res; } @@ -920,8 +919,7 @@ namespace vcpkg errors, progress_sink)) { - msg::println(msgAssetCacheHit, msg::path = download_path.filename()); - Debug::println("Downloaded from: ", read_url); + msg::println(msgAssetCacheHit, msg::path = download_path.filename(), msg::url = replace_secrets(read_url, m_config.m_secrets)); return read_url; } } @@ -988,7 +986,7 @@ namespace vcpkg auto maybe_push = put_file_to_mirror(fs, download_path, *hash); if (!maybe_push) { - msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename()); + msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename(), msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); msg::println(maybe_push.error()); } } @@ -1009,10 +1007,7 @@ namespace vcpkg for (LocalizedString& error : errors) { - // Debug only since: - // 1. curl already prints the error - // 2. the printed url can contain sensitive information like username/password - Debug::println(error); + msg::println(error); } Checks::exit_fail(VCPKG_LINE_INFO); From ff453cd86cc2afe8884bb7ff3ea0bc7026d876b5 Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Wed, 12 Jun 2024 11:37:57 -0400 Subject: [PATCH 22/25] update messaging --- azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 | 4 ++-- include/vcpkg/base/message-data.inc.h | 4 ++-- src/vcpkg/base/downloads.cpp | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 9b805b8614..2a767d28c3 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -15,7 +15,7 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Missing .* and external downloads are blocked by x-block-origin." +"error: Missing .* and downloads are blocked by x-block-origin." ) -join "`n" if (-not ($actual -match $expected)) { @@ -42,7 +42,7 @@ $actual = $actual -replace "`r`n", "`n" $expected = @( "A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .* and external downloads are blocked by x-block-origin" +"Asset cache miss for .* and downloads are blocked by x-block-origin" ) -join "`n" if (-not ($actual -match $expected)) { diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index b4bea46a16..ae3d338959 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1202,8 +1202,8 @@ DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the DECLARE_MESSAGE(MissingAssetBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", - "Missing {path} and external downloads are blocked by x-block-origin.") -DECLARE_MESSAGE(AssetCacheMissBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and external downloads are blocked by x-block-origin.") + "Missing {path} and downloads are blocked by x-block-origin.") +DECLARE_MESSAGE(AssetCacheMissBlockOrigin, (msg::path), "x-block-origin is a vcpkg term. Do not translate", "Asset cache miss for {path} and downloads are blocked by x-block-origin.") DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":") DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.") DECLARE_MESSAGE(FailedToFindPortFeature, diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 4ed8a66b43..f8b1321b01 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -979,6 +979,7 @@ namespace vcpkg fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); if (auto url = maybe_url.get()) { + m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0]) : msg::println(msgDownloadingUrl, msg::url = urls[0]); if (auto hash = sha512.get()) From 86d622d3a7838086da685b4936154d2b5badc06c Mon Sep 17 00:00:00 2001 From: Javier Matos Date: Wed, 12 Jun 2024 12:56:36 -0400 Subject: [PATCH 23/25] fix message signature --- include/vcpkg/base/message-data.inc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index de38b34af7..aacb0b1b97 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1260,7 +1260,7 @@ DECLARE_MESSAGE(FailedToRunToolToDetermineVersion, "Additional information, such as the command line output, if any, will be appended on " "the line after this message", "Failed to run \"{path}\" to determine the {tool_name} version.") -DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path, msg::url), "", "Failed to store {path} to {path}.") +DECLARE_MESSAGE(FailedToStoreBackToMirror, (msg::path, msg::url), "", "Failed to store {path} to {url}.") DECLARE_MESSAGE(FailedToStoreBinaryCache, (msg::path), "", "Failed to store binary cache {path}") DECLARE_MESSAGE(FailedToTakeFileSystemLock, (), "", "Failed to take the filesystem lock") DECLARE_MESSAGE(FailedToWriteManifest, (msg::path), "", "Failed to write manifest file {path}") From 1de2f3fe8924f88bbca51444392e1df9a39356bf Mon Sep 17 00:00:00 2001 From: Javier Matos Denizac Date: Wed, 12 Jun 2024 17:00:49 +0000 Subject: [PATCH 24/25] Format and regenerate messages --- include/vcpkg/base/expected.h | 8 ++++---- include/vcpkg/base/optional.h | 4 ++-- locales/messages.json | 16 ++++++++-------- src/vcpkg/base/downloads.cpp | 13 +++++++++---- src/vcpkg/base/system.process.cpp | 4 +--- src/vcpkg/commands.integrate.cpp | 13 ++++++++----- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/include/vcpkg/base/expected.h b/include/vcpkg/base/expected.h index 49ced29d1c..d58c7dfaa9 100644 --- a/include/vcpkg/base/expected.h +++ b/include/vcpkg/base/expected.h @@ -102,8 +102,8 @@ namespace vcpkg { } - ExpectedT(const ExpectedT& other) noexcept( - std::is_nothrow_copy_constructible_v&& std::is_nothrow_copy_constructible_v>) + ExpectedT(const ExpectedT& other) noexcept(std::is_nothrow_copy_constructible_v && + std::is_nothrow_copy_constructible_v>) : value_is_error(other.value_is_error) { if (value_is_error) @@ -116,8 +116,8 @@ namespace vcpkg } } - ExpectedT(ExpectedT&& other) noexcept( - std::is_nothrow_move_constructible_v&& std::is_nothrow_move_constructible_v>) + ExpectedT(ExpectedT&& other) noexcept(std::is_nothrow_move_constructible_v && + std::is_nothrow_move_constructible_v>) : value_is_error(other.value_is_error) { if (value_is_error) diff --git a/include/vcpkg/base/optional.h b/include/vcpkg/base/optional.h index 50a186866d..12c192957c 100644 --- a/include/vcpkg/base/optional.h +++ b/include/vcpkg/base/optional.h @@ -76,8 +76,8 @@ namespace vcpkg } } - OptionalStorage& operator=(const OptionalStorage& o) noexcept( - std::is_nothrow_copy_constructible_v&& std::is_nothrow_copy_assignable_v) + OptionalStorage& operator=(const OptionalStorage& o) noexcept(std::is_nothrow_copy_constructible_v && + std::is_nothrow_copy_assignable_v) { if (m_is_present && o.m_is_present) { diff --git a/locales/messages.json b/locales/messages.json index 6ec1d7878a..e9ac23467a 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -172,16 +172,16 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", - "AssetCacheHit": "Asset cache hit for {path}.", - "_AssetCacheHit.comment": "An example of {path} is /foo/bar.", + "AssetCacheHit": "Asset cache hit for {path}; downloaded from: {url}", + "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "AssetCacheMiss": "Asset cache miss; downloading from {url}", "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and external downloads are blocked by x-block-origin.", + "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and downloads are blocked by x-block-origin.", "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", - "AssetCacheSuccesfullyStored": "Successfully stored {path} to mirror.", - "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar.", + "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", + "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -769,8 +769,8 @@ "_FailedToRemoveControl.comment": "An example of {path} is /foo/bar.", "FailedToRunToolToDetermineVersion": "Failed to run \"{path}\" to determine the {tool_name} version.", "_FailedToRunToolToDetermineVersion.comment": "Additional information, such as the command line output, if any, will be appended on the line after this message An example of {tool_name} is aria2. An example of {path} is /foo/bar.", - "FailedToStoreBackToMirror": "Failed to store {path} to mirror.", - "_FailedToStoreBackToMirror.comment": "An example of {path} is /foo/bar.", + "FailedToStoreBackToMirror": "Failed to store {path} to {url}.", + "_FailedToStoreBackToMirror.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "FailedToStoreBinaryCache": "Failed to store binary cache {path}", "_FailedToStoreBinaryCache.comment": "An example of {path} is /foo/bar.", "FailedToTakeFileSystemLock": "Failed to take the filesystem lock", @@ -1143,7 +1143,7 @@ "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}", "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.", "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.", - "MissingAssetBlockOrigin": "Missing {path} and external downloads are blocked by x-block-origin.", + "MissingAssetBlockOrigin": "Missing {path} and downloads are blocked by x-block-origin.", "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "MissingClosingParen": "missing closing )", "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.", diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 4e45fa2e2c..95e3838b31 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -580,8 +580,9 @@ namespace vcpkg msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); } } - msg::println( - msgAssetCacheSuccesfullyStored, msg::path = file.filename(), msg::url = replace_secrets(url.to_string(), secrets)); + msg::println(msgAssetCacheSuccesfullyStored, + msg::path = file.filename(), + msg::url = replace_secrets(url.to_string(), secrets)); return res; } @@ -919,7 +920,9 @@ namespace vcpkg errors, progress_sink)) { - msg::println(msgAssetCacheHit, msg::path = download_path.filename(), msg::url = replace_secrets(read_url, m_config.m_secrets)); + msg::println(msgAssetCacheHit, + msg::path = download_path.filename(), + msg::url = replace_secrets(read_url, m_config.m_secrets)); return read_url; } } @@ -987,7 +990,9 @@ namespace vcpkg auto maybe_push = put_file_to_mirror(fs, download_path, *hash); if (!maybe_push) { - msg::println_warning(msgFailedToStoreBackToMirror, msg::path = download_path.filename(), msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); + msg::println_warning(msgFailedToStoreBackToMirror, + msg::path = download_path.filename(), + msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); msg::println(maybe_push.error()); } } diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index 38f003fefc..ac4215365c 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -1885,9 +1885,7 @@ namespace vcpkg { std::string output; return cmd_execute_and_stream_data(cmd, settings, [&](StringView sv) { Strings::append(output, sv); }) - .map([&](int exit_code) { - return ExitCodeAndOutput{exit_code, std::move(output)}; - }); + .map([&](int exit_code) { return ExitCodeAndOutput{exit_code, std::move(output)}; }); } uint64_t get_subproccess_stats() { return g_subprocess_stats.load(); } diff --git a/src/vcpkg/commands.integrate.cpp b/src/vcpkg/commands.integrate.cpp index a6dcc419f2..83495fdf90 100644 --- a/src/vcpkg/commands.integrate.cpp +++ b/src/vcpkg/commands.integrate.cpp @@ -574,13 +574,16 @@ namespace vcpkg static std::vector valid_arguments(const VcpkgPaths&) { // Note that help lists all supported args, but we only want to autocomplete the ones valid on this platform - return - { - INSTALL.to_string(), REMOVE.to_string(), + return { + INSTALL.to_string(), + REMOVE.to_string(), #if defined(_WIN32) - PROJECT.to_string(), POWERSHELL.to_string(), + PROJECT.to_string(), + POWERSHELL.to_string(), #else - BASH.to_string(), FISH.to_string(), ZSH.to_string() + BASH.to_string(), + FISH.to_string(), + ZSH.to_string() #endif }; } From 1c7397e38cf0d115a8992fe25368bc92747746d3 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 5 Jul 2024 21:58:42 -0700 Subject: [PATCH 25/25] Fix missing trailing newline in test. --- azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index 2a767d28c3..dfdd86e127 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -104,4 +104,4 @@ $expectedPattern = "(?s)" + if (-not ($actual -match $expectedPattern)) { throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" -} \ No newline at end of file +}