Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[x-download] teach to not need a sha512 #142

Merged
merged 5 commits into from
Jul 30, 2021
Merged

[x-download] teach to not need a sha512 #142

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3ab96bf
[x-download] teach to not need a sha512
strega-nil Jul 29, 2021
0f753a0
Billy CRs
strega-nil Jul 29, 2021
a980c11
Sha512Check -> Optional<std::string>
strega-nil Jul 29, 2021
9195a3a
format + fix build error
strega-nil Jul 29, 2021
8cee9c2
x-download always redownloads anyways
strega-nil Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions include/vcpkg/base/downloads.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,22 @@ namespace vcpkg::Downloads
explicit DownloadManager(const DownloadManagerConfig& config) : m_config(config) { }
explicit DownloadManager(DownloadManagerConfig&& config) : m_config(std::move(config)) { }

void download_file_without_hash_check(Filesystem& fs,
strega-nil-ms marked this conversation as resolved.
Show resolved Hide resolved
const std::string& url,
const path& download_path) const
{
this->download_file_without_hash_check(fs, url, {}, download_path);
}
void download_file_without_hash_check(Filesystem& fs,
const std::string& url,
View<std::string> headers,
const path& download_path) const;
// Returns url that was successfully downloaded from
std::string download_file_without_hash_check(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path) const;

void download_file(Filesystem& fs,
const std::string& url,
const path& download_path,
Expand Down
38 changes: 27 additions & 11 deletions src/vcpkg/base/downloads.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
Optional<const std::string&> sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
Expand Down Expand Up @@ -544,24 +544,25 @@ namespace vcpkg::Downloads
return false;
}

auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_path_part_path, sha512);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
else
if (auto p = sha512.get())
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_path_part_path, *p);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
}

fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
}

static Optional<const std::string&> try_download_files(vcpkg::Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
Optional<const std::string&> sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
Expand All @@ -587,6 +588,21 @@ namespace vcpkg::Downloads
this->download_file(fs, View<std::string>(&url, 1), headers, download_path, sha512);
}

std::string DownloadManager::download_file_without_hash_check(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path) const
{
std::string errors;
auto maybe_url =
try_download_files(fs, urls, headers, download_path, nullopt, m_config.m_secrets, errors);
if (auto url = maybe_url.get())
{
return *url;
}
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: Failed to download from mirror set:\n%s", errors);
}

std::string DownloadManager::download_file(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
Expand Down
85 changes: 67 additions & 18 deletions src/vcpkg/commands.xdownload.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,32 +12,41 @@
namespace vcpkg::Commands::X_Download
{
static constexpr StringLiteral OPTION_STORE = "store";
static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512";
static constexpr StringLiteral OPTION_ALWAYS_REDOWNLOAD = "always-redownload";
static constexpr StringLiteral OPTION_SHA512 = "sha512";
static constexpr StringLiteral OPTION_URL = "url";
static constexpr StringLiteral OPTION_HEADER = "header";

static constexpr CommandSwitch FETCH_SWITCHES[] = {
{OPTION_STORE, "Indicates the file should be stored instead of fetched"},
{OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"},
{OPTION_ALWAYS_REDOWNLOAD, "Always download, and overwrite any file that already exists"}
};
static constexpr CommandSetting FETCH_SETTINGS[] = {
{OPTION_SHA512, "The hash of the file to be downloaded"},
};
static constexpr CommandMultiSetting FETCH_MULTISETTINGS[] = {
{OPTION_URL, "URL to download and store if missing from cache"},
{OPTION_HEADER, "Additional header to use when fetching from URLs"},
};

const CommandStructure COMMAND_STRUCTURE = {
Strings::format("The argument must be at least a file path and a SHA512\n%s",
create_example_string("x-download <filepath> <sha512> [--url=https://...]...")),
2,
Strings::format("%s\n%s",
create_example_string("x-download <filepath> [--sha512=]<sha512> [--url=https://...]..."),
create_example_string("x-download <filepath> --skip-sha512 [--url=https://...]...")),
1,
2,
{{FETCH_SWITCHES}, {}, FETCH_MULTISETTINGS},
{FETCH_SWITCHES, FETCH_SETTINGS, FETCH_MULTISETTINGS},
nullptr,
};

static bool is_lower_hex(StringView sha)
static bool is_hex(StringView sha)
{
return std::all_of(
sha.begin(), sha.end(), [](char ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'); });
sha.begin(), sha.end(), [](char ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); });
}
static bool is_lower_sha512(StringView sha) { return sha.size() == 128 && is_lower_hex(sha); }
static bool is_sha512(StringView sha) { return sha.size() == 128 && is_hex(sha); }

void perform_and_exit(const VcpkgCmdArguments& args, Filesystem& fs)
{
Expand All @@ -46,26 +55,60 @@ namespace vcpkg::Commands::X_Download
parse_download_configuration(args.asset_sources_template).value_or_exit(VCPKG_LINE_INFO)};
path file = fs.absolute(vcpkg::u8path(args.command_arguments[0]), VCPKG_LINE_INFO);

std::string sha = Strings::ascii_to_lowercase(std::string(args.command_arguments[1]));
if (!is_lower_sha512(sha))
Optional<std::string> sha;
auto sha_it = parsed.settings.find(OPTION_SHA512);
if (args.command_arguments.size() > 1)
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", args.command_arguments[1]);
if (sha_it != parsed.settings.end())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: SHA512 passed as both an argument and as an option. Only pass one of these.");
}
sha = args.command_arguments[1];
}
else if (sha_it != parsed.settings.end())
{
sha = sha_it->second;
}

if (Util::Sets::contains(parsed.switches, OPTION_SKIP_SHA512))
{
if (sha.has_value())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "SHA512 passed, but --skip-sha512 was also passed; only do one or the other.");
}
}
else if (!sha.has_value())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Required argument --sha512 was not passed.");
}
else
{
auto p = sha.get();
if (!is_sha512(*p))
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", *p);
}
Strings::ascii_to_lowercase(p->begin(), p->end());
strega-nil-ms marked this conversation as resolved.
Show resolved Hide resolved
}

// Is this a store command?
if (Util::Sets::contains(parsed.switches, OPTION_STORE))
{
if (!sha.has_value())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "--store option is invalid without a sha512.");
}

auto s = fs.status(file, VCPKG_LINE_INFO);
if (s.type() != vcpkg::file_type::regular)
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: path was not a regular file: %s", vcpkg::u8string(file));
}
auto hash =
Strings::ascii_to_lowercase(Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512));
if (hash != sha) Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
download_manager.put_file_to_mirror(fs, file, sha).value_or_exit(VCPKG_LINE_INFO);
auto hash = Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512);
strega-nil-ms marked this conversation as resolved.
Show resolved Hide resolved
if (hash != *sha.get()) Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
download_manager.put_file_to_mirror(fs, file, hash).value_or_exit(VCPKG_LINE_INFO);
Checks::exit_success(VCPKG_LINE_INFO);
}
else
Expand All @@ -79,13 +122,19 @@ namespace vcpkg::Commands::X_Download
}

auto it_urls = parsed.multisettings.find(OPTION_URL);
if (it_urls == parsed.multisettings.end())
View<std::string> urls{};
if (it_urls != parsed.multisettings.end())
{
urls = it_urls->second;
}

if (auto p = sha.get())
{
download_manager.download_file(fs, View<std::string>{}, headers, file, sha);
download_manager.download_file(fs, urls, headers, file, *p);
}
else
{
download_manager.download_file(fs, it_urls->second, headers, file, sha);
download_manager.download_file_without_hash_check(fs, urls, headers, file);
}
Checks::exit_success(VCPKG_LINE_INFO);
}
Expand Down