microsoft
/
vcpkg-tool
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[x-download] teach to not need a sha512 (#142)

This commit is contained in:
nicole mazzuca 2021-07-30 13:42:23 -07:00 коммит произвёл GitHub
Родитель 816d21f303
Коммит 1e4448e073
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
3 изменённых файлов: 133 добавлений и 64 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
include/vcpkg/base/downloads.h
Просмотреть файл

@ -55,7 +55,7 @@ namespace vcpkg::Downloads
void download_file(Filesystem& fs,
const std::string& url,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
this->download_file(fs, url, {}, download_path, sha512);
}
@ -64,18 +64,16 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const;
const Optional<std::string>& sha512) const;
// Returns url that was successfully downloaded from
std::string download_file(Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const;
const Optional<std::string>& sha512) const;
ExpectedS<int> put_file_to_mirror(const Filesystem& fs,
const path& file_to_put,
const std::string& sha512) const;
ExpectedS<int> put_file_to_mirror(const Filesystem& fs, const path& file_to_put, StringView sha512) const;
private:
DownloadManagerConfig m_config;

85
src/vcpkg/base/downloads.cpp
Просмотреть файл

@ -195,9 +195,9 @@ namespace vcpkg::Downloads
}
static Optional<std::string> try_verify_downloaded_file_hash(const Filesystem& fs,
const std::string& sanitized_url,
StringView sanitized_url,
const path& downloaded_path,
const std::string& sha512)
StringView sha512)
{
std::string actual_hash =
vcpkg::Hash::get_file_hash(VCPKG_LINE_INFO, fs, downloaded_path, Hash::Algorithm::Sha512);
@ -232,6 +232,24 @@ namespace vcpkg::Downloads
}
}
static bool check_downloaded_file_hash(Filesystem& fs,
const Optional<std::string>& hash,
StringView sanitized_url,
const path& download_part_path,
std::string& errors)
{
if (auto p = hash.get())
{
auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err, '\n');
return false;
}
}
return true;
}
static void url_heads_inner(View<std::string> urls, View<std::string> headers, std::vector<int>* out)
{
static constexpr StringLiteral guid_marker = "8a1db05f-a65d-419b-aa72-037fb4d0672e";
@ -481,7 +499,7 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
const Optional<std::string>& sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
@ -504,22 +522,13 @@ namespace vcpkg::Downloads
{
if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors))
{
auto maybe_error = try_verify_downloaded_file_hash(fs, url, download_path_part_path, sha512);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
else
if (check_downloaded_file_hash(fs, sha512, url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
}
}
else
{
return false;
}
return false;
}
}
}
@ -544,24 +553,19 @@ namespace vcpkg::Downloads
return false;
}
auto maybe_error = try_verify_downloaded_file_hash(fs, sanitized_url, download_path_part_path, sha512);
if (auto err = maybe_error.get())
{
Strings::append(errors, *err);
return false;
}
else
if (check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path, errors))
{
fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO);
return true;
}
return false;
}
static Optional<const std::string&> try_download_files(vcpkg::Filesystem& fs,
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512,
const Optional<std::string>& sha512,
const std::vector<std::string>& secrets,
std::string& errors)
{
@ -582,7 +586,7 @@ namespace vcpkg::Downloads
const std::string& url,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
this->download_file(fs, View<std::string>(&url, 1), headers, download_path, sha512);
}
@ -591,22 +595,32 @@ namespace vcpkg::Downloads
View<std::string> urls,
View<std::string> headers,
const path& download_path,
const std::string& sha512) const
const Optional<std::string>& sha512) const
{
std::string errors;
if (auto read_template = m_config.m_read_url_template.get())
if (auto hash = sha512.get())
{
auto read_url = Strings::replace_all(std::string(*read_template), "<SHA>", sha512);
if (Downloads::try_download_file(
fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
return read_url;
if (auto read_template = m_config.m_read_url_template.get())
{
auto read_url = Strings::replace_all(std::string(*read_template), "<SHA>", *hash);
if (Downloads::try_download_file(
fs, read_url, m_config.m_read_headers, download_path, sha512, m_config.m_secrets, errors))
return read_url;
}
}
if (!m_config.m_block_origin)
{
if (urls.size() == 0)
{
Strings::append(errors, "Error: No urls specified to download SHA: ", sha512);
if (auto hash = sha512.get())
{
Strings::append(errors, "Error: No urls specified to download SHA: ", *hash, '\n');
}
else
{
Strings::append(errors, "Error: No urls specified\n");
}
}
else
{
@ -614,10 +628,13 @@ namespace vcpkg::Downloads
try_download_files(fs, urls, headers, download_path, sha512, m_config.m_secrets, errors);
if (auto url = maybe_url.get())
{
auto maybe_push = put_file_to_mirror(fs, download_path, sha512);
if (!maybe_push.has_value())
if (auto hash = sha512.get())
{
print2(Color::warning, "Warning: failed to store back to mirror:\n", maybe_push.error());
auto maybe_push = put_file_to_mirror(fs, download_path, *hash);
if (!maybe_push.has_value())
{
print2(Color::warning, "Warning: failed to store back to mirror:\n", maybe_push.error());
}
}
return *url;
}
@ -628,7 +645,7 @@ namespace vcpkg::Downloads
ExpectedS<int> DownloadManager::put_file_to_mirror(const Filesystem& fs,
const path& file_to_put,
const std::string& sha512) const
StringView sha512) const
{
auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "<SHA>", sha512);
if (!maybe_mirror_url.empty())

102
src/vcpkg/commands.xdownload.cpp
Просмотреть файл

@ -12,11 +12,17 @@
namespace vcpkg::Commands::X_Download
{
static constexpr StringLiteral OPTION_STORE = "store";
static constexpr StringLiteral OPTION_SKIP_SHA512 = "skip-sha512";
static constexpr StringLiteral OPTION_SHA512 = "sha512";
static constexpr StringLiteral OPTION_URL = "url";
static constexpr StringLiteral OPTION_HEADER = "header";
static constexpr CommandSwitch FETCH_SWITCHES[] = {
{OPTION_STORE, "Indicates the file should be stored instead of fetched"},
{OPTION_SKIP_SHA512, "Do not check the SHA512 of the downloaded file"},
};
static constexpr CommandSetting FETCH_SETTINGS[] = {
{OPTION_SHA512, "The hash of the file to be downloaded"},
};
static constexpr CommandMultiSetting FETCH_MULTISETTINGS[] = {
{OPTION_URL, "URL to download and store if missing from cache"},
@ -24,20 +30,66 @@ namespace vcpkg::Commands::X_Download
};
const CommandStructure COMMAND_STRUCTURE = {
Strings::format("The argument must be at least a file path and a SHA512\n%s",
create_example_string("x-download <filepath> <sha512> [--url=https://...]...")),
Strings::format("%s\n%s",
create_example_string("x-download <filepath> [--sha512=]<sha512> [--url=https://...]..."),
create_example_string("x-download <filepath> --skip-sha512 [--url=https://...]...")),
1,
2,
2,
{{FETCH_SWITCHES}, {}, FETCH_MULTISETTINGS},
{FETCH_SWITCHES, FETCH_SETTINGS, FETCH_MULTISETTINGS},
nullptr,
};
static bool is_lower_hex(StringView sha)
static bool is_hex(StringView sha)
{
return std::all_of(
sha.begin(), sha.end(), [](char ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'); });
return std::all_of(sha.begin(), sha.end(), [](char ch) {
return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
});
}
static bool is_sha512(StringView sha) { return sha.size() == 128 && is_hex(sha); }
static Optional<std::string> get_sha512_check(const VcpkgCmdArguments& args, const ParsedArguments& parsed)
{
Optional<std::string> sha = nullopt;
auto sha_it = parsed.settings.find(OPTION_SHA512);
if (args.command_arguments.size() > 1)
{
if (sha_it != parsed.settings.end())
{
Checks::exit_with_message(
VCPKG_LINE_INFO,
"Error: SHA512 passed as both an argument and as an option. Only pass one of these.");
}
sha = args.command_arguments[1];
}
else if (sha_it != parsed.settings.end())
{
sha = sha_it->second;
}
if (Util::Sets::contains(parsed.switches, OPTION_SKIP_SHA512))
{
if (sha.has_value())
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "SHA512 passed, but --skip-sha512 was also passed; only do one or the other.");
}
}
else if (!sha.has_value())
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Required argument --sha512 was not passed.");
}
if (auto p = sha.get())
{
if (!is_sha512(*p))
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", *p);
}
Strings::ascii_to_lowercase(p->begin(), p->end());
}
return sha;
}
static bool is_lower_sha512(StringView sha) { return sha.size() == 128 && is_lower_hex(sha); }
void perform_and_exit(const VcpkgCmdArguments& args, Filesystem& fs)
{
@ -46,26 +98,29 @@ namespace vcpkg::Commands::X_Download
parse_download_configuration(args.asset_sources_template).value_or_exit(VCPKG_LINE_INFO)};
path file = fs.absolute(vcpkg::u8path(args.command_arguments[0]), VCPKG_LINE_INFO);
std::string sha = Strings::ascii_to_lowercase(std::string(args.command_arguments[1]));
if (!is_lower_sha512(sha))
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: SHA512's must be 128 hex characters: '%s'", args.command_arguments[1]);
}
auto sha = get_sha512_check(args, parsed);
// Is this a store command?
if (Util::Sets::contains(parsed.switches, OPTION_STORE))
{
auto hash = sha.get();
if (!hash)
{
Checks::exit_with_message(VCPKG_LINE_INFO, "--store option is invalid without a sha512.");
}
auto s = fs.status(file, VCPKG_LINE_INFO);
if (s.type() != vcpkg::file_type::regular)
{
Checks::exit_with_message(
VCPKG_LINE_INFO, "Error: path was not a regular file: %s", vcpkg::u8string(file));
}
auto hash =
Strings::ascii_to_lowercase(Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512));
if (hash != sha) Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
download_manager.put_file_to_mirror(fs, file, sha).value_or_exit(VCPKG_LINE_INFO);
auto actual_hash = Hash::get_file_hash(VCPKG_LINE_INFO, fs, file, Hash::Algorithm::Sha512);
if (*hash != actual_hash)
{
Checks::exit_with_message(VCPKG_LINE_INFO, "Error: file to store does not match hash");
}
download_manager.put_file_to_mirror(fs, file, actual_hash).value_or_exit(VCPKG_LINE_INFO);
Checks::exit_success(VCPKG_LINE_INFO);
}
else
@ -79,14 +134,13 @@ namespace vcpkg::Commands::X_Download
}
auto it_urls = parsed.multisettings.find(OPTION_URL);
if (it_urls == parsed.multisettings.end())
View<std::string> urls{};
if (it_urls != parsed.multisettings.end())
{
download_manager.download_file(fs, View<std::string>{}, headers, file, sha);
}
else
{
download_manager.download_file(fs, it_urls->second, headers, file, sha);
urls = it_urls->second;
}
download_manager.download_file(fs, urls, headers, file, sha);
Checks::exit_success(VCPKG_LINE_INFO);
}
}