Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[x-download] teach to not need a sha512 #142

Merged
merged 5 commits into from
Jul 30, 2021
Merged

[x-download] teach to not need a sha512 #142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3ab96bf
[x-download] teach to not need a sha512
strega-nil Jul 29, 2021
0f753a0
Billy CRs
strega-nil Jul 29, 2021
a980c11
Sha512Check -> Optional<std::string>
strega-nil Jul 29, 2021
9195a3a
format + fix build error
strega-nil Jul 29, 2021
8cee9c2
x-download always redownloads anyways
strega-nil Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions include/vcpkg/base/downloads.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ namespace vcpkg::Downloads
void download_file(Filesystem& fs,
const std::string& url,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
this->download_file(fs, url, {}, download_path, sha512);
}
Expand All @@ -64,18 +64,16 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const;
const Optional<std::string>& sha512) const;

// Returns url that was successfully downloaded from
std::string download_file(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const;
const Optional<std::string>& sha512) const;

ExpectedS<int> put_file_to_mirror(const Filesystem& fs,
const path& file_to_put,
const std::string& sha512) const;
ExpectedS<int> put_file_to_mirror(const Filesystem& fs, const path& file_to_put, StringView sha512) const;

private:
DownloadManagerConfig m_config;
Expand Down
85 changes: 51 additions & 34 deletions src/vcpkg/base/downloads.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,9 @@ namespace vcpkg::Downloads
}

static Optional<std::string> try_verify_downloaded_file_hash(const Filesystem& fs,
const std::string& sanitized_url,
StringView sanitized_url,
const path& downloaded_path,
const std::string& sha512)
StringView sha512)
{
std::string actual_hash =
vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, downloaded_path, Hash::Algorithm::Sha512);
Expand Down Expand Up @@ -232,6 +232,24 @@ namespace vcpkg::Downloads
}
}

static bool check_downloaded_file_hash(Filesystem& fs,
const Optional<std::string>& hash,
StringView sanitized_url,
const path& download_part_path,
std::string& errors)
{
if (auto p = hash.get())
{
auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err, '\n');
return false;
}
}
return true;
}

static void url_heads_inner(View<std::string> urls, View<std::string> headers, std::vector<int>* out)
{
static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e";
Expand Down Expand Up @@ -481,7 +499,7 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
const Optional<std::string>& sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
Expand All @@ -504,22 +522,13 @@ namespace vcpkg::Downloads
{
if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors))
{
auto maybe_error = try_verify_downloaded_file_hash(fs, url, download_path_part_path, sha512);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
else
if (check_downloaded_file_hash(fs, sha512, url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
}
}
else
{
return false;
}
return false;
}
}
}
Expand All @@ -544,24 +553,19 @@ namespace vcpkg::Downloads
return false;
}

auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_path_part_path, sha512);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
else
if (check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
}
return false;
}

static Optional<const std::string&> try_download_files(vcpkg::Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
const Optional<std::string>& sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
Expand All @@ -582,7 +586,7 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
this->download_file(fs, View<std::string>(&url, 1), headers, download_path, sha512);
}
Expand All @@ -591,33 +595,46 @@ namespace vcpkg::Downloads
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
std::string errors;
if (auto read_template = m_config.m_read_url_template.get())
if (auto hash = sha512.get())
{
auto read_url = Strings::replace_all(std::string(*read_template), "<SHA>", sha512);
if (Downloads::try_download_file(
fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
return read_url;
if (auto read_template = m_config.m_read_url_template.get())
{
auto read_url = Strings::replace_all(std::string(*read_template), "<SHA>", *hash);
if (Downloads::try_download_file(
fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
return read_url;
}
}

if (!m_config.m_block_origin)
{
if (urls.size() == 0)
{
Strings::append(errors, "Error: No urls specified to download SHA: ", sha512);
if (auto hash = sha512.get())
{
Strings::append(errors, "Error: No urls specified to download SHA: ", *hash, '\n');
}
else
{
Strings::append(errors, "Error: No urls specified\n");
}
}
else
{
auto maybe_url =
try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors);
if (auto url = maybe_url.get())
{
auto maybe_push = put_file_to_mirror(fs, download_path, sha512);
if (!maybe_push.has_value())
if (auto hash = sha512.get())
{
print2(Color::warning, "Warning: failed to store back to mirror:\n", maybe_push.error());
auto maybe_push = put_file_to_mirror(fs, download_path, *hash);
if (!maybe_push.has_value())
{
print2(Color::warning, "Warning: failed to store back to mirror:\n", maybe_push.error());
}
}
return *url;
}
Expand All @@ -628,7 +645,7 @@ namespace vcpkg::Downloads

ExpectedS<int> DownloadManager::put_file_to_mirror(const Filesystem& fs,
const path& file_to_put,
const std::string& sha512) const
StringView sha512) const
{
auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "<SHA>", sha512);
if (!maybe_mirror_url.empty())
Expand Down
102 changes: 78 additions & 24 deletions src/vcpkg/commands.xdownload.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,32 +12,84 @@
namespace vcpkg::Commands::X_Download
{
static constexpr StringLiteral OPTION_STORE = "store";
static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512";
static constexpr StringLiteral OPTION_SHA512 = "sha512";
static constexpr StringLiteral OPTION_URL = "url";
static constexpr StringLiteral OPTION_HEADER = "header";

static constexpr CommandSwitch FETCH_SWITCHES[] = {
{OPTION_STORE, "Indicates the file should be stored instead of fetched"},
{OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"},
};
static constexpr CommandSetting FETCH_SETTINGS[] = {
{OPTION_SHA512, "The hash of the file to be downloaded"},
};
static constexpr CommandMultiSetting FETCH_MULTISETTINGS[] = {
{OPTION_URL, "URL to download and store if missing from cache"},
{OPTION_HEADER, "Additional header to use when fetching from URLs"},
};

const CommandStructure COMMAND_STRUCTURE = {
Strings::format("The argument must be at least a file path and a SHA512\n%s",
create_example_string("x-download <filepath> <sha512> [--url=https://...]...")),
2,
Strings::format("%s\n%s",
create_example_string("x-download <filepath> [--sha512=]<sha512> [--url=https://...]..."),
create_example_string("x-download <filepath> --skip-sha512 [--url=https://...]...")),
1,
2,
{{FETCH_SWITCHES}, {}, FETCH_MULTISETTINGS},
{FETCH_SWITCHES, FETCH_SETTINGS, FETCH_MULTISETTINGS},
nullptr,
};

static bool is_lower_hex(StringView sha)
static bool is_hex(StringView sha)
{
return std::all_of(sha.begin(), sha.end(), [](char ch) {
return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
});
}
static bool is_sha512(StringView sha) { return sha.size() == 128 && is_hex(sha); }

static Optional<std::string> get_sha512_check(const VcpkgCmdArguments& args, const ParsedArguments& parsed)
{
return std::all_of(
sha.begin(), sha.end(), [](char ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'); });
Optional<std::string> sha = nullopt;
auto sha_it = parsed.settings.find(OPTION_SHA512);
if (args.command_arguments.size() > 1)
{
if (sha_it != parsed.settings.end())
{
Checks::exit_with_message(
VCPKG_LINE_INFO,
"Error: SHA512 passed as both an argument and as an option. Only pass one of these.");
}
sha = args.command_arguments[1];
}
else if (sha_it != parsed.settings.end())
{
sha = sha_it->second;
}

if (Util::Sets::contains(parsed.switches, OPTION_SKIP_SHA512))
{
if (sha.has_value())
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "SHA512 passed, but --skip-sha512 was also passed; only do one or the other.");
}
}
else if (!sha.has_value())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Required argument --sha512 was not passed.");
}

if (auto p = sha.get())
{
if (!is_sha512(*p))
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", *p);
}
Strings::ascii_to_lowercase(p->begin(), p->end());
}

return sha;
}
static bool is_lower_sha512(StringView sha) { return sha.size() == 128 && is_lower_hex(sha); }

void perform_and_exit(const VcpkgCmdArguments& args, Filesystem& fs)
{
Expand All @@ -46,26 +98,29 @@ namespace vcpkg::Commands::X_Download
parse_download_configuration(args.asset_sources_template).value_or_exit(VCPKG_LINE_INFO)};
path file = fs.absolute(vcpkg::u8path(args.command_arguments[0]), VCPKG_LINE_INFO);

std::string sha = Strings::ascii_to_lowercase(std::string(args.command_arguments[1]));
if (!is_lower_sha512(sha))
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", args.command_arguments[1]);
}
auto sha = get_sha512_check(args, parsed);

// Is this a store command?
if (Util::Sets::contains(parsed.switches, OPTION_STORE))
{
auto hash = sha.get();
if (!hash)
{
Checks::exit_with_message(VCPKG_LINE_INFO, "--store option is invalid without a sha512.");
}

auto s = fs.status(file, VCPKG_LINE_INFO);
if (s.type() != vcpkg::file_type::regular)
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: path was not a regular file: %s", vcpkg::u8string(file));
}
auto hash =
Strings::ascii_to_lowercase(Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512));
if (hash != sha) Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
download_manager.put_file_to_mirror(fs, file, sha).value_or_exit(VCPKG_LINE_INFO);
auto actual_hash = Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512);
if (*hash != actual_hash)
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
}
download_manager.put_file_to_mirror(fs, file, actual_hash).value_or_exit(VCPKG_LINE_INFO);
Checks::exit_success(VCPKG_LINE_INFO);
}
else
Expand All @@ -79,14 +134,13 @@ namespace vcpkg::Commands::X_Download
}

auto it_urls = parsed.multisettings.find(OPTION_URL);
if (it_urls == parsed.multisettings.end())
{
download_manager.download_file(fs, View<std::string>{}, headers, file, sha);
}
else
View<std::string> urls{};
if (it_urls != parsed.multisettings.end())
{
download_manager.download_file(fs, it_urls->second, headers, file, sha);
urls = it_urls->second;
}

download_manager.download_file(fs, urls, headers, file, sha);
Checks::exit_success(VCPKG_LINE_INFO);
}
}
Expand Down