Fix all download failures claiming that the download was disabled by x-block-origin. (#1513)

This commit is contained in:
Javier Matos Denizac 2024-11-11 16:36:51 -05:00 коммит произвёл GitHub
Родитель af3caca31b
Коммит 1393ad8082
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
4 изменённых файлов: 155 добавлений и 31 удалений

Просмотреть файл

@ -57,6 +57,8 @@ $actual = $actual -replace "`r`n", "`n"
$expected = @(
"A suitable version of .* was not found \(required v[0-9\.]+\)."
"Asset cache miss; downloading from .*"
"Downloading .*"
"Successfully downloaded .*."
"Successfully stored .* to .*."
) -join "`n"
@ -82,26 +84,116 @@ if (-not ($actual -match $expected)) {
# Testing asset caching && x-block-orgin promises when --debug is passed (enabled)
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug"))
$actual = $actual -replace "`r`n", "`n"
# Define the regex pattern that accounts for multiline input
$expectedPattern = "(?s)" +
".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" +
".*\[DEBUG\] Asset caching is enabled\..*"
if (-not ($actual -match $expectedPattern)) {
if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) {
throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)"
}
# Testing asset caching && x-block-orgin promises when --debug is passed (disabled)
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug"))
$actual = $actual -replace "`r`n", "`n"
$expectedPattern = "(?s)" +
".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" +
".*\[DEBUG\] Asset cache is not configured.*"
if (-not ($actual -match $expectedPattern)) {
if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) {
throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
}
# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)
# Expected: Download failure message, nothing about asset caching
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html"))
if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) {
throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)"
}
#azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)
#Expected: Download message with the "you might need to configure a proxy" message and with expected/actual sha
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com"))
if (-not ($actual.Contains("Failed to download example3.html.") -and
$actual.Contains("If you are using a proxy, please ensure your proxy settings are correct.") -and
$actual.Contains("error: File does not have the expected hash:") -and
$actual.Contains("url: https://example.com") -and
$actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
$actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)"
}
# azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)
# Expected: Download success message, nothing about asset caching
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com"))
if (-not ($actual.Contains("Downloading example3.html") -and
$actual.Contains("Successfully downloaded example3.html."))) {
throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)"
}
# azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)
# Expected: Download failure message, nothing about asset caching, x-block-origin complaint
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin"))
if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)"
}
# azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)
# Expected: Download failure message, asset cache named, nothing about x-block-origin
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and
$actual.Contains("Downloading example3.html") -and
$actual.Contains("error: file://$AssetCache") -and
$actual.Contains("curl failed to download with exit code 37"))) {
throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)"
}
# azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)
# Expected: Download success message, asset cache named, nothing about x-block-origin
Refresh-TestRoot
$actual = $actual -replace "`r`n", "`n"
Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)"
}
# azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)
# Expected: Download message with "you might need to configure a proxy" and expected/actual sha
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
$actual.Contains("Downloading example3.html") -and
$actual.Contains("error: file://$AssetCache") -and
$actual.Contains("curl failed to download with exit code 37") -and
$actual.Contains("error: File does not have the expected hash:") -and
$actual.Contains("url: https://example.com") -and
$actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
$actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)"
}
# azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)
# Expected: Download success message, asset cache upload, nothing about x-block-origin
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
$actual.Contains("Downloading example3.html") -and
$actual.Contains("Successfully downloaded example3.html.") -and
$actual.Contains("Successfully stored example3.html to file://$AssetCache"))) {
throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)"
}
# azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)
# Expected: Download failure message, which asset cache was tried, x-block-origin complaint
Refresh-TestRoot
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and
$actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)"
}
# azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)
# Expected: Download success message, asset cache named, nothing about x-block-origin
Refresh-TestRoot
Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)"
}

Просмотреть файл

@ -248,6 +248,11 @@ DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when a
DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts")
DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}")
DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}")
DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
(msg::path),
"x-block-origin is a vcpkg term. Do not translate",
"Asset cache miss for {path} and downloads are blocked by x-block-origin.")
DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.")
DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}")
DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments,
(msg::value),
@ -1046,6 +1051,22 @@ DECLARE_MESSAGE(DownloadFailedStatusCode,
(msg::url, msg::value),
"{value} is an HTTP status code",
"{url}: failed: status code {value}")
DECLARE_MESSAGE(DownloadFailedProxySettings,
(msg::path, msg::url),
"",
"Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are "
"correct.\nPossible causes are:\n"
"1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable "
"to `https//address:port`.\nThis is not correct, because `https://` prefix "
"claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n"
"Try setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n"
"2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings "
"set by your proxy software. See, {url}\n"
"The value set by your proxy might be wrong, or have same `https://` prefix issue.\n"
"3. Your proxy's remote server is our of service.\n"
"If you've tried directly download the link, and believe this is not a temporay download server "
"failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n"
"to report this upstream download server failure.")
DECLARE_MESSAGE(DownloadingPortableToolVersionX,
(msg::tool_name, msg::version),
"",
@ -1240,10 +1261,6 @@ DECLARE_MESSAGE(MissingShaVariable,
(),
"{{sha}} should not be translated",
"The {{sha}} variable must be used in the template if other variables are used.")
DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
(msg::path),
"x-block-origin is a vcpkg term. Do not translate",
"Asset cache miss for {path} and downloads are blocked by x-block-origin.")
DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":")
DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.")
DECLARE_MESSAGE(FailedToFindPortFeature,

Просмотреть файл

@ -595,12 +595,16 @@
"_DownloadFailedCurl.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {exit_code} is 127.",
"DownloadFailedHashMismatch": "File does not have the expected hash:\nurl: {url}\nFile: {path}\nExpected hash: {expected}\nActual hash: {actual}",
"_DownloadFailedHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.",
"DownloadFailedProxySettings": "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See, {url}\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporay download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.",
"_DownloadFailedProxySettings.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
"DownloadFailedRetrying": "Download failed -- retrying after {value}ms",
"_DownloadFailedRetrying.comment": "{value} is a number of milliseconds",
"DownloadFailedStatusCode": "{url}: failed: status code {value}",
"_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.",
"DownloadRootsDir": "Downloads directory (default: {env_var})",
"_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.",
"DownloadSuccesful": "Successfully downloaded {path}.",
"_DownloadSuccesful.comment": "An example of {path} is /foo/bar.",
"DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}",
"_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
"DownloadedSources": "Downloaded sources for {spec}",

Просмотреть файл

@ -905,6 +905,8 @@ namespace vcpkg
MessageSink& progress_sink) const
{
std::vector<LocalizedString> errors;
bool block_origin_enabled = m_config.m_block_origin;
if (urls.size() == 0)
{
if (auto hash = sha512.get())
@ -936,6 +938,14 @@ namespace vcpkg
msg::url = replace_secrets(read_url, m_config.m_secrets));
return read_url;
}
else if (block_origin_enabled)
{
msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename());
}
else
{
msg::println(msgAssetCacheMiss, msg::url = urls[0]);
}
}
else if (auto script = m_config.m_script.get())
{
@ -985,16 +995,20 @@ namespace vcpkg
}
}
if (!m_config.m_block_origin)
if (block_origin_enabled)
{
msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename());
}
else
{
if (urls.size() != 0)
{
msg::println(msgDownloadingUrl, msg::url = download_path.filename());
auto maybe_url = try_download_file(
fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink);
if (auto url = maybe_url.get())
{
m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0])
: msg::println(msgDownloadingUrl, msg::url = urls[0]);
msg::println(msgDownloadSuccesful, msg::path = download_path.filename());
if (auto hash = sha512.get())
{
@ -1010,17 +1024,14 @@ namespace vcpkg
return *url;
}
else
{
msg::println(msgDownloadFailedProxySettings,
msg::path = download_path.filename(),
msg::url = "https://github.com/microsoft/vcpkg-tool/pull/77");
}
}
}
// Asset cache is not configured and x-block-origin enabled
if (m_config.m_read_url_template.has_value())
{
msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename());
}
else
{
msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename());
}
for (LocalizedString& error : errors)
{