diff --git a/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 new file mode 100644 index 0000000000..0f30f60271 --- /dev/null +++ b/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 @@ -0,0 +1,3 @@ +Param([string]$File) +Write-Host "Creating file with the wrong hash" +Set-Content -Path $File -Value "This is a file with the wrong hash" -Encoding Ascii -NoNewline diff --git a/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 index c232cc056d..fafc73a496 100644 --- a/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 +++ b/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 @@ -1 +1,2 @@ -throw "Script download error" +Write-Host "Script download error" +exit 1 diff --git a/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 new file mode 100644 index 0000000000..16bcc8c21d --- /dev/null +++ b/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 @@ -0,0 +1 @@ +Write-Host "Not creating a file" diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index e00df0f361..098373e28e 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -7,15 +7,16 @@ Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-bina Throw-IfFailed $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON" +$assetCacheRegex = [regex]::Escape($AssetCache) # Testing asset cache miss (not configured) + x-block-origin enabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) +Throw-IfNotFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Missing .* and downloads are blocked by x-block-origin." +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Downloading cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+" ) -join "`n" if (-not ($actual -match $expected)) { @@ -23,12 +24,12 @@ if (-not ($actual -match $expected)) { } # Testing asset cache miss (not configured) + x-block-origin disabled -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) +Throw-IfFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Downloading .*." +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Downloading https://github\.com/Kitware/CMake/releases/download/[^ ]+ -> cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)", +"Successfully downloaded cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)" ) -join "`n" if (-not ($actual -match $expected)) { @@ -37,12 +38,23 @@ if (-not ($actual -match $expected)) { # Testing asset cache miss (configured) + x-block-origin enabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) +Throw-IfNotFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .* and downloads are blocked by x-block-origin" +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." ) -join "`n" if (-not ($actual -match $expected)) { @@ -51,15 +63,14 @@ if (-not ($actual -match $expected)) { # Testing asset cache miss (configured) + x-block-origin disabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +Throw-IfFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss; downloading from .*" -"Downloading .*" -"Successfully downloaded .*." -"Successfully stored .* to .*." +"A suitable version of cmake was not found \(required v[0-9\.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-f]+" +"Asset cache miss; trying authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", +"Successfully downloaded cmake-[0-9.]+-[^.]+\.(zip|tar\.gz), storing to file://$assetCacheRegex/[0-9a-f]+", +"Store success" ) -join "`n" if (-not ($actual -match $expected)) { @@ -69,159 +80,252 @@ if (-not ($actual -match $expected)) { # Testing asset cache hit Refresh-Downloads Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache hit for .*; downloaded from: .*" +"A suitable version of cmake was not found \(required v[0-9\.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+" ) -join "`n" if (-not ($actual -match $expected)) { throw "Failure: asset cache hit" } -# Testing asset caching && x-block-orgin promises when --debug is passed (enabled) -Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) -if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) { - throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)" -} - -# Testing asset caching && x-block-orgin promises when --debug is passed (disabled) -Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) -if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) { - throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" -} - # azurl (no), x-block-origin (no), asset-cache (n/a), download (fail) # Expected: Download failure message, nothing about asset caching Refresh-TestRoot $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html")) -if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) { +Throw-IfNotFailed +if (-not ($actual -match "curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)")) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)" } - #azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch) #Expected: Download message with the "you might need to configure a proxy" message and with expected/actual sha Refresh-TestRoot +$expected = @( +"Downloading https://example\.com -> example3\.html", +"[^\n]+example3\.html\.\d+\.part: error: download from https://example\.com had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com")) -if (-not ($actual.Contains("Failed to download example3.html.") -and - $actual.Contains("If you are using a proxy, please ensure your proxy settings are correct.") -and - $actual.Contains("error: File does not have the expected hash:") -and - $actual.Contains("url: https://example.com") -and - $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and - $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)" } # azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed) # Expected: Download success message, nothing about asset caching Refresh-TestRoot +$expected = @( +"^Downloading https://example\.com -> example3\.html", +"Successfully downloaded example3\.html", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com")) -if (-not ($actual.Contains("Downloading example3.html") -and - $actual.Contains("Successfully downloaded example3.html."))) { +Throw-IfFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" } # azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a) # Expected: Download failure message, nothing about asset caching, x-block-origin complaint Refresh-TestRoot +$expected = @( +"^Downloading example3\.html", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin")) -if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { +Throw-IfNotFailed + if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)" } + # azurl (yes), x-block-origin (no), asset-cache (miss), download (fail) # Expected: Download failure message, asset cache named, nothing about x-block-origin Refresh-TestRoot +$expected = @( +"Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://localhost:1234/foobar\.html", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("error: file://$AssetCache") -and - $actual.Contains("curl failed to download with exit code 37"))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)" } # azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a) # Expected: Download success message, asset cache named, nothing about x-block-origin Refresh-TestRoot -$actual = $actual -replace "`r`n", "`n" -Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Run-Vcpkg -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfFailed +Remove-Item "$downloadsRoot/example3.html" +$expected = @( +"Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" } # azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch) # Expected: Download message with "you might need to configure a proxy" and expected/actual sha Refresh-TestRoot +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"[^\n]+example3\.html\.\d+\.part: error: download from https://example\.com had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("error: file://$AssetCache") -and - $actual.Contains("curl failed to download with exit code 37") -and - $actual.Contains("error: File does not have the expected hash:") -and - $actual.Contains("url: https://example.com") -and - $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and - $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)" } # azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed) # Expected: Download success message, asset cache upload, nothing about x-block-origin Refresh-TestRoot +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"Successfully downloaded example3\.html, storing to file://$assetCacheRegex/[0-9a-z]+", +"Store success", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("Successfully downloaded example3.html.") -and - $actual.Contains("Successfully stored example3.html to file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)" } # azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a) # Expected: Download failure message, which asset cache was tried, x-block-origin complaint Refresh-TestRoot +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) -if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and - $actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)" } # azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a) # Expected: Download success message, asset cache named, nothing about x-block-origin Refresh-TestRoot -Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com", +"$" +) -join "`n" +Run-Vcpkg -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfFailed +Remove-Item "$downloadsRoot/example3.html" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) -if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" } # Testing x-download failure with asset cache (x-script) and x-block-origin settings $env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/failing-script.ps1 {url} {sha512} {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Script download error", +"error: the asset cache script returned nonzero exit code 1", +"note: the full script command line was: pwsh .+/failing-script\.ps1 https://example\.com d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a `"?[^`"]+example3\.html\.\d+\.part`"?", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" +} + +Refresh-TestRoot +$env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/no-file-script.ps1 {url} {sha512} {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Not creating a file", +"[^\n]+example3\.html\.\d+\.part: error: the asset cache script returned success but did not create expected result file", +"note: the full script command line was: pwsh .+/no-file-script\.ps1 https://example\.com d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a `"?[^`"]+example3\.html\.\d+\.part`"?", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) -# Check for the expected messages in order -$expectedOrder = @( - "error: failed with exit code: (1).", - "error: Missing example3.html and downloads are blocked by x-block-origin." -) - -# Verify order -$index = 0 -foreach ($message in $expectedOrder) { - $index = $actual.IndexOf($message, $index) - if ($index -lt 0) { - throw "Failure: Expected message '$message' not found in the correct order." - } - $index += $message.Length +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" +} + +Refresh-TestRoot +$env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/bad-hash-script.ps1 -File {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Creating file with the wrong hash", +"[^\n]+example3\.html\.\d+\.part: error: the asset cache script returned success but the resulting file has an unexpected hash", +"note: the full script command line was: pwsh .+/bad-hash-script\.ps1 -File `"?[^`"]+example3\.html\.\d+\.part`"?", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"note: Actual : cc9c9070d8a54bfc32d6be2eb01b531f22f657d868200fbcdc7c4cc5f31e92909bd7c83971bebefa918c2c34e53d859ed49a79f4a943f36ec521fc0544b30d9e", +"$" +) -join "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" } # Testing x-download success with asset cache (x-script) and x-block-origin settings Refresh-TestRoot +$expected = @( +"^Trying to download example3.html using asset cache script", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com/hello-world.txt", +"$" +) -join "`n" $env:X_VCPKG_ASSET_SOURCES = "clear;x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com/hello-world.txt", "--sha512", "09e1e2a84c92b56c8280f4a1203c7cffd61b162cfe987278d4d6be9afbf38c0e8934cdadf83751f4e99d111352bffefc958e5a4852c8a7a29c95742ce59288a8")) -if (-not ($actual.Contains("Successfully downloaded example3.html."))) { - throw "Failure: x-script download success message" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: x-script download success message" } diff --git a/include/vcpkg/base/api-stable-format.h b/include/vcpkg/base/api-stable-format.h index 17094f5058..ae61b87ac9 100644 --- a/include/vcpkg/base/api-stable-format.h +++ b/include/vcpkg/base/api-stable-format.h @@ -1,6 +1,7 @@ #pragma once -#include +#include +#include #include #include @@ -10,21 +11,22 @@ namespace vcpkg namespace details { template - void api_stable_format_cb(void* f, std::string& s, StringView sv) + bool api_stable_format_cb(void* f, std::string& s, StringView sv) { - (*(F*)(f))(s, sv); + return (*(F*)(f))(s, sv); } - ExpectedL api_stable_format_impl(StringView fmtstr, - void (*cb)(void*, std::string&, StringView), - void* data); + Optional api_stable_format_impl(DiagnosticContext& context, + StringView fmtstr, + bool (*cb)(void*, std::string&, StringView), + void* data); } // This function exists in order to provide an API-stable formatting function similar to `std::format()` that does // not depend on the feature set of fmt or the C++ standard library and thus can be contractual for user interfaces. template - ExpectedL api_stable_format(StringView fmtstr, F&& handler) + Optional api_stable_format(DiagnosticContext& context, StringView fmtstr, F&& handler) { - return details::api_stable_format_impl(fmtstr, &details::api_stable_format_cb, &handler); + return details::api_stable_format_impl(context, fmtstr, &details::api_stable_format_cb, &handler); } } diff --git a/include/vcpkg/base/diagnostics.h b/include/vcpkg/base/diagnostics.h index d7d676f5fe..f5f5415109 100644 --- a/include/vcpkg/base/diagnostics.h +++ b/include/vcpkg/base/diagnostics.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -66,11 +67,22 @@ namespace vcpkg std::string to_string() const; void to_string(std::string& target) const; + MessageLine to_message_line() const; + LocalizedString to_json_reader_string(const std::string& path, const LocalizedString& type) const; DiagKind kind() const noexcept { return m_kind; } + // Returns this DiagnosticLine with kind == Error reduced to Warning. + DiagnosticLine reduce_to_warning() const&; + DiagnosticLine reduce_to_warning() &&; private: + DiagnosticLine(DiagKind kind, + const Optional& origin, + TextRowCol position, + const LocalizedString& message); + DiagnosticLine(DiagKind kind, Optional&& origin, TextRowCol position, LocalizedString&& message); + DiagKind m_kind; Optional m_origin; TextRowCol m_position; @@ -79,8 +91,11 @@ namespace vcpkg struct DiagnosticContext { + // The `report` family are used to report errors or warnings that may result in a function failing + // to do what it is intended to do. Data sent to the `report` family is expected to not be printed + // to the console if a caller decides to handle an error. virtual void report(const DiagnosticLine& line) = 0; - virtual void report(DiagnosticLine&& line) { report(line); } + virtual void report(DiagnosticLine&& line); void report_error(const LocalizedString& message) { report(DiagnosticLine{DiagKind::Error, message}); } void report_error(LocalizedString&& message) { report(DiagnosticLine{DiagKind::Error, std::move(message)}); } @@ -92,15 +107,64 @@ namespace vcpkg this->report_error(std::move(message)); } + template + void report_error_with_log(StringView log_content, VCPKG_DECL_MSG_ARGS) + { + LocalizedString message; + msg::format_to(message, VCPKG_EXPAND_MSG_ARGS); + message.append_raw('\n'); + message.append_raw(log_content); + this->report_error(std::move(message)); + } + + void report_system_error(StringLiteral system_api_name, int error_value); + + // The `status` family are used to report status or progress information that callers are expected + // to show on the console, even if it would decide to handle errors or warnings itself. + // Examples: + // * "Downloading file..." + // * "Building package 1 of 47..." + // + // Some implementations of DiagnosticContext may buffer these messages *anyway* if that makes sense, + // for example, if the work is happening on a background thread. + virtual void statusln(const LocalizedString& message) = 0; + virtual void statusln(LocalizedString&& message) = 0; + virtual void statusln(const MessageLine& message) = 0; + virtual void statusln(MessageLine&& message) = 0; + protected: ~DiagnosticContext() = default; }; + struct PrintingDiagnosticContext final : DiagnosticContext + { + PrintingDiagnosticContext(MessageSink& sink) : sink(sink) { } + + virtual void report(const DiagnosticLine& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + private: + MessageSink& sink; + }; + + // Stores all diagnostics into a vector, while passing through status lines to an underlying MessageSink. struct BufferedDiagnosticContext final : DiagnosticContext { + BufferedDiagnosticContext(MessageSink& status_sink) : status_sink(status_sink) { } + virtual void report(const DiagnosticLine& line) override; virtual void report(DiagnosticLine&& line) override; + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + MessageSink& status_sink; std::vector lines; // Prints all diagnostics to the supplied sink. @@ -111,9 +175,75 @@ namespace vcpkg void to_string(std::string& target) const; bool any_errors() const noexcept; + bool empty() const noexcept; + }; + + // Stores all diagnostics and status messages into a vector. This is generally used for background thread or similar + // scenarios where even status messages can't be immediately printed. + struct FullyBufferedDiagnosticContext final : DiagnosticContext + { + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + std::vector lines; + + // Prints all diagnostics to the supplied sink. + void print_to(MessageSink& sink) const; + // Converts this message into a string + // Prefer print() if possible because it applies color + std::string to_string() const; + void to_string(std::string& target) const; + + bool empty() const noexcept; + }; + + // DiagnosticContext for attempted operations that may be recovered. + // Stores all diagnostics and passes through all status messages. Afterwards, call commit() to report all + // diagnostics to the outer DiagnosticContext, or handle() to forget them. + struct AttemptDiagnosticContext final : DiagnosticContext + { + AttemptDiagnosticContext(DiagnosticContext& inner_context) : inner_context(inner_context) { } + + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + void commit(); + void handle(); + + ~AttemptDiagnosticContext(); + + DiagnosticContext& inner_context; + std::vector lines; + }; + + // Wraps another DiagnosticContext and reduces the severity of any reported diagnostics to warning from error. + struct WarningDiagnosticContext final : DiagnosticContext + { + WarningDiagnosticContext(DiagnosticContext& inner_context) : inner_context(inner_context) { } + + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + DiagnosticContext& inner_context; }; extern DiagnosticContext& console_diagnostic_context; + extern DiagnosticContext& status_only_diagnostic_context; extern DiagnosticContext& null_diagnostic_context; // The following overloads are implementing @@ -191,7 +321,7 @@ namespace vcpkg { using Unwrapper = AdaptContextUnwrapOptional>; using ReturnType = ExpectedL; - BufferedDiagnosticContext bdc; + BufferedDiagnosticContext bdc{out_sink}; decltype(auto) maybe_result = functor(bdc, std::forward(args)...); if (auto result = maybe_result.get()) { @@ -261,7 +391,7 @@ namespace vcpkg { using ReturnType = ExpectedL< typename AdaptContextDetectUniquePtr>::type>; - BufferedDiagnosticContext bdc; + BufferedDiagnosticContext bdc{out_sink}; decltype(auto) maybe_result = functor(bdc, std::forward(args)...); if (maybe_result) { diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index ad7fc72124..6349177a90 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -15,7 +15,17 @@ namespace vcpkg { - struct SplitURIView + struct SanitizedUrl + { + SanitizedUrl() = default; + SanitizedUrl(StringView raw_url, View secrets); + const std::string& to_string() const noexcept { return m_sanitized_url; } + + private: + std::string m_sanitized_url; + }; + + struct SplitUrlView { StringView scheme; Optional authority; @@ -23,12 +33,7 @@ namespace vcpkg }; // e.g. {"https","//example.org", "/index.html"} - ExpectedL split_uri_view(StringView uri); - - void verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView sanitized_url, - const Path& downloaded_path, - StringView sha512); + Optional parse_split_url_view(StringView raw_url); View azure_blob_headers(); @@ -36,33 +41,37 @@ namespace vcpkg // -w "PREFIX%{http_code} %{exitcode} %{errormsg}" // with specific handling for curl version < 7.75.0 which does not understand %{exitcode} %{errormsg} // If the line is malformed for any reason, no entry to http_codes is added. - void parse_curl_status_line(std::vector>& http_codes, StringLiteral prefix, StringView this_line); - - std::vector> download_files(View> url_pairs, - View headers, - View secrets); - - bool submit_github_dependency_graph_snapshot(const Optional& maybe_github_server_url, + // Returns: true if the new version of curl's output with exitcode and errormsg was parsed; otherwise, false. + bool parse_curl_status_line(DiagnosticContext& context, + std::vector& http_codes, + StringLiteral prefix, + StringView this_line); + + std::vector download_files_no_cache(DiagnosticContext& context, + View> url_pairs, + View headers, + View secrets); + + bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, + const Optional& maybe_github_server_url, const std::string& github_token, const std::string& github_repository, const Json::Object& snapshot); - ExpectedL put_file(const ReadOnlyFilesystem&, - StringView url, - const std::vector& secrets, - View headers, - const Path& file, - StringView method = "PUT"); - - ExpectedL invoke_http_request(StringView method, - View headers, - StringView url, - StringView data = {}); + + Optional invoke_http_request(DiagnosticContext& context, + StringLiteral method, + View headers, + StringView url, + StringView data = {}); std::string format_url_query(StringView base_url, View query_params); - std::vector> url_heads(View urls, View headers, View secrets); + std::vector url_heads(DiagnosticContext& context, + View urls, + View headers, + View secrets); - struct DownloadManagerConfig + struct AssetCachingSettings { Optional m_read_url_template; std::vector m_read_headers; @@ -74,37 +83,35 @@ namespace vcpkg }; // Handles downloading and uploading to a content addressable mirror - struct DownloadManager - { - DownloadManager() = default; - explicit DownloadManager(const DownloadManagerConfig& config) : m_config(config) { } - explicit DownloadManager(DownloadManagerConfig&& config) : m_config(std::move(config)) { } - - void download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const; - - // Returns url that was successfully downloaded from - std::string download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const; - - ExpectedL put_file_to_mirror(const ReadOnlyFilesystem& fs, - const Path& file_to_put, - StringView sha512) const; - - bool get_block_origin() const; - bool asset_cache_configured() const; - - private: - DownloadManagerConfig m_config; - }; + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const std::string& url, + View headers, + const Path& download_path, + const Optional& maybe_sha512); + + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View urls, + View headers, + const Path& download_path, + const Optional& maybe_sha512); + + bool store_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file); + + bool store_to_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Path& file_to_put, + StringView sha512); Optional try_parse_curl_max5_size(StringView sv); @@ -112,8 +119,8 @@ namespace vcpkg { unsigned int total_percent; unsigned long long total_size; - unsigned int recieved_percent; - unsigned long long recieved_size; + unsigned int received_percent; + unsigned long long received_size; unsigned int transfer_percent; unsigned long long transfer_size; unsigned long long average_download_speed; // bytes per second @@ -134,3 +141,5 @@ namespace vcpkg // is likely to contain query parameters or similar. std::string url_encode_spaces(StringView url); } + +VCPKG_FORMAT_WITH_TO_STRING(vcpkg::SanitizedUrl); diff --git a/include/vcpkg/base/file_sink.h b/include/vcpkg/base/file_sink.h index 7f2c20c9ed..ef1730104b 100644 --- a/include/vcpkg/base/file_sink.h +++ b/include/vcpkg/base/file_sink.h @@ -14,6 +14,7 @@ namespace vcpkg : m_log_file(log_file), m_out_file(fs.open_for_write(m_log_file, append_to_file, VCPKG_LINE_INFO)) { } - void print(Color c, StringView sv) override; + void println(const MessageLine& line) override; + void println(MessageLine&& line) override; }; } diff --git a/include/vcpkg/base/fwd/downloads.h b/include/vcpkg/base/fwd/downloads.h index 1f7a59ca3a..c51accb232 100644 --- a/include/vcpkg/base/fwd/downloads.h +++ b/include/vcpkg/base/fwd/downloads.h @@ -2,8 +2,8 @@ namespace vcpkg { - struct SplitURIView; - struct DownloadManager; - struct DownloadManagerConfig; + struct SanitizedUrl; + struct SplitUrlView; + struct AssetCachingSettings; struct CurlProgressData; } diff --git a/include/vcpkg/base/fwd/message_sinks.h b/include/vcpkg/base/fwd/message_sinks.h index 541fa35e6a..6def368015 100644 --- a/include/vcpkg/base/fwd/message_sinks.h +++ b/include/vcpkg/base/fwd/message_sinks.h @@ -2,6 +2,8 @@ namespace vcpkg { + struct MessageLineSegment; + struct MessageLine; struct MessageSink; extern MessageSink& null_sink; @@ -10,5 +12,5 @@ namespace vcpkg extern MessageSink& stderr_sink; struct FileSink; - struct CombiningSink; + struct TeeSink; } diff --git a/include/vcpkg/base/hash.h b/include/vcpkg/base/hash.h index f8a5f71e69..baedbd2b97 100644 --- a/include/vcpkg/base/hash.h +++ b/include/vcpkg/base/hash.h @@ -4,12 +4,26 @@ #include #include +#include #include #include namespace vcpkg::Hash { + enum class HashPrognosis + { + Success, + FileNotFound, + OtherError, + }; + + struct HashResult + { + HashPrognosis prognosis = HashPrognosis::Success; + std::string hash; + }; + enum class Algorithm { Sha256, @@ -33,5 +47,28 @@ namespace vcpkg::Hash std::string get_bytes_hash(const void* first, const void* last, Algorithm algo); std::string get_string_hash(StringView s, Algorithm algo); std::string get_string_sha256(StringView s); - ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& target, Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // Returns a HashResult with the following outcomes: + // HashPrognosis::Success: The entire file was read and hashed. The result hash is stored in `hash`. + // HashPrognosis::FileNotFound: The file does not exist. `hash` is empty string. + // HashPrognosis::OtherError: An error occurred while reading the file. `hash` is empty string. + HashResult get_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // If the file exists and could be completely read, returns an engaged optional with the stringized hash. + // Otherwise, returns an disengaged optional. + // Note that the file not existing is interpreted as an error that will be reported to `context`. + Optional get_file_hash_required(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // If the file exists and could be completely read, returns an engaged optional with the stringized hash. + // Otherwise, returns the read operation error. + ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo); } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 7357d2a412..696c11dfcf 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -248,19 +248,56 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}") -DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") +DECLARE_MESSAGE(AssetCacheConsult, (msg::path, msg::url), "", "Trying to download {path} using asset cache {url}") +DECLARE_MESSAGE(AssetCacheConsultScript, (msg::path), "", "Trying to download {path} using asset cache script") +DECLARE_MESSAGE(AssetCacheHit, (), "", "Download successful! Asset cache hit.") +DECLARE_MESSAGE(AssetCacheHitUrl, + (msg::url), + "", + "Download successful! Asset cache hit, did not try authoritative source {url}") +DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; trying authoritative source {url}") DECLARE_MESSAGE(AssetCacheMissBlockOrigin, - (msg::path), + (msg::url), "x-block-origin is a vcpkg term. Do not translate", - "Asset cache miss for {path} and downloads are blocked by x-block-origin.") -DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.") -DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") + "there were no asset cache hits, and x-block-origin blocks trying the authoritative source {url}") +DECLARE_MESSAGE(AssetCacheMissNoUrls, + (msg::sha), + "", + "Asset cache missed looking for {sha} and no authoritative URL is known") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") +DECLARE_MESSAGE(AssetCacheScriptBadVariable, + (msg::value, msg::list), + "{value} is the script template passed to x-script, {list} is the name of the unknown replacement", + "the script template {value} contains unknown replacement {list}") +DECLARE_MESSAGE(AssetCacheScriptBadVariableHint, + (msg::list), + "{list} is the name of the unknown replacement", + "if you want this on the literal command line, use {{{list}}}") +DECLARE_MESSAGE(AssetCacheScriptCommandLine, (), "", "the full script command line was") +DECLARE_MESSAGE(AssetCacheScriptNeedsSha, + (msg::value, msg::url), + "{value} is the script template the user supplied to x-script", + "the script template {value} requires a SHA, but no SHA is known for attempted download of {url}") +DECLARE_MESSAGE(AssetCacheScriptNeedsUrl, + (msg::value, msg::sha), + "{value} is the script template the user supplied to x-script", + "the script template {value} requires a URL, but no URL is known for attempted download of {sha}") +DECLARE_MESSAGE(AssetCacheScriptFailed, + (msg::exit_code), + "", + "the asset cache script returned nonzero exit code {exit_code}") +DECLARE_MESSAGE(AssetCacheScriptFailedToWriteFile, + (), + "", + "the asset cache script returned success but did not create expected result file") +DECLARE_MESSAGE(AssetCacheScriptFailedToWriteCorrectHash, + (), + "", + "the asset cache script returned success but the resulting file has an unexpected hash") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (), "", "Store success") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1034,18 +1071,9 @@ DECLARE_MESSAGE(DownloadAvailable, "", "A downloadable copy of this tool is available and can be used by unsetting {env_var}.") DECLARE_MESSAGE(DownloadedSources, (msg::spec), "", "Downloaded sources for {spec}") -DECLARE_MESSAGE(DownloadFailedCurl, - (msg::url, msg::exit_code), - "", - "{url}: curl failed to download with exit code {exit_code}") -DECLARE_MESSAGE(DownloadFailedHashMismatch, - (msg::url, msg::path, msg::expected, msg::actual), - "{expected} and {actual} are SHA512 hashes in hex format.", - "File does not have the expected hash:\n" - "url: {url}\n" - "File: {path}\n" - "Expected hash: {expected}\n" - "Actual hash: {actual}") +DECLARE_MESSAGE(DownloadFailedHashMismatch, (msg::url), "", "download from {url} had an unexpected hash") +DECLARE_MESSAGE(DownloadFailedHashMismatchExpectedHash, (msg::sha), "", "Expected: {sha}") +DECLARE_MESSAGE(DownloadFailedHashMismatchActualHash, (msg::sha), "", "Actual : {sha}") DECLARE_MESSAGE(DownloadFailedRetrying, (msg::value), "{value} is a number of milliseconds", @@ -1055,32 +1083,47 @@ DECLARE_MESSAGE(DownloadFailedStatusCode, "{value} is an HTTP status code", "{url}: failed: status code {value}") DECLARE_MESSAGE(DownloadFailedProxySettings, - (msg::path, msg::url), + (), "", - "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are " - "correct.\nPossible causes are:\n" - "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable " - "to `https//address:port`.\nThis is not correct, because `https://` prefix " - "claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n" + "If you are using a proxy, please ensure your proxy settings are correct.\n" + "Possible causes are:\n" + "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to " + "`https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS " + "proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n" "Try setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n" - "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings " - "set by your proxy software. See, {url}\n" + "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your " + "proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\n" "The value set by your proxy might be wrong, or have same `https://` prefix issue.\n" "3. Your proxy's remote server is our of service.\n" - "If you've tried directly download the link, and believe this is not a temporay download server " + "If you've tried directly download the link, and believe this is not a temporary download server " "failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n" "to report this upstream download server failure.") DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", "A suitable version of {tool_name} was not found (required v{version}).") +DECLARE_MESSAGE(DownloadingAssetShaToFile, (msg::sha, msg::path), "", "Downloading asset cache entry {sha} -> {path}") +DECLARE_MESSAGE(DownloadingAssetShaWithoutAssetCache, + (msg::sha, msg::path), + "", + "requested download of asset cache entry {sha} -> {path}, but no asset caches are configured") +DECLARE_MESSAGE(DownloadingFile, (msg::path), "", "Downloading {path}") +DECLARE_MESSAGE(DownloadingFileFirstAuthoritativeSource, (msg::path, msg::url), "", "Downloading {path}, trying {url}") +DECLARE_MESSAGE(DownloadingUrlToFile, (msg::url, msg::path), "", "Downloading {url} -> {path}") +DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloading standalone bundle {version}.") +DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.") +DECLARE_MESSAGE(DownloadOrUrl, (), "", "or") +DECLARE_MESSAGE(DownloadTryingAuthoritativeSource, (msg::url), "", "Trying authoritative source {url}") +DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})") +DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}") +DECLARE_MESSAGE(DownloadSuccesfulUploading, + (msg::path, msg::url), + "", + "Successfully downloaded {path}, storing to {url}") DECLARE_MESSAGE(DownloadWinHttpError, (msg::system_api, msg::exit_code, msg::url), "", "{url}: {system_api} failed with exit code {exit_code}") -DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloading standalone bundle {version}.") -DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.") -DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})") DECLARE_MESSAGE(DuplicatedKeyInObj, (msg::value), "{value} is a json property/object", @@ -1256,10 +1299,6 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(MissingAssetBlockOrigin, - (msg::path), - "x-block-origin is a vcpkg term. Do not translate", - "Missing {path} and downloads are blocked by x-block-origin.") DECLARE_MESSAGE(MissingShaVariable, (), "{{sha}} should not be translated", @@ -1347,7 +1386,7 @@ DECLARE_MESSAGE(FetchingRegistryInfo, (msg::url, msg::value), "{value} is a reference", "Fetching registry information from {url} ({value})...") -DECLARE_MESSAGE(FileNotFound, (msg::path), "", "{path}: file not found") +DECLARE_MESSAGE(FileNotFound, (), "", "file not found") DECLARE_MESSAGE(FileReadFailed, (msg::path, msg::byte_offset, msg::count), "", @@ -1422,10 +1461,6 @@ DECLARE_MESSAGE(GraphCycleDetected, (msg::package_name), "A list of package names comprising the cycle will be printed after this message.", "Cycle detected within graph at {package_name}:") -DECLARE_MESSAGE(HashFileFailureToRead, - (msg::path), - "Printed after ErrorMessage and before the specific failing filesystem operation (like file not found)", - "failed to read file \"{path}\" for hashing: ") DECLARE_MESSAGE(HashPortManyFiles, (msg::package_name, msg::count), "", @@ -2206,7 +2241,6 @@ DECLARE_MESSAGE(NonZeroRemainingArgs, "the command '{command_name}' does not accept any additional arguments") DECLARE_MESSAGE(NoOutdatedPackages, (), "", "There are no outdated packages.") DECLARE_MESSAGE(NoRegistryForPort, (msg::package_name), "", "no registry configured for port {package_name}") -DECLARE_MESSAGE(NoUrlsAndHashSpecified, (msg::sha), "", "No urls specified to download SHA: {sha}") DECLARE_MESSAGE(NoUrlsAndNoHashSpecified, (), "", "No urls specified and no hash specified.") DECLARE_MESSAGE(NugetOutputNotCapturedBecauseInteractiveSpecified, (), @@ -2600,6 +2634,7 @@ DECLARE_MESSAGE(ProgramReturnedNonzeroExitCode, (msg::tool_name, msg::exit_code), "The program's console output is appended after this.", "{tool_name} failed with exit code: ({exit_code}).") +DECLARE_MESSAGE(ProgramPathReturnedNonzeroExitCode, (msg::exit_code), "", "failed with exit code {exit_code}") DECLARE_MESSAGE( ProvideExportType, (), @@ -2739,6 +2774,11 @@ DECLARE_MESSAGE(SystemRootMustAlwaysBePresent, "", "Expected the SystemRoot environment variable to be always set on Windows.") DECLARE_MESSAGE(SystemTargetsInstallFailed, (msg::path), "", "failed to install system targets file to {path}") +DECLARE_MESSAGE( + ToolHashMismatch, + (msg::tool_name, msg::expected, msg::actual), + "{expected} and {actual} are SHA512 hashes in hex format.", + "{tool_name} appears to be already downloaded, but has an incorrect hash. Expected {expected} but was {actual}") DECLARE_MESSAGE(ToolFetchFailed, (msg::tool_name), "", "Could not fetch {tool_name}.") DECLARE_MESSAGE(ToolInWin10, (), "", "This utility is bundled with Windows 10 or later.") DECLARE_MESSAGE(ToolOfVersionXNotFound, @@ -3227,6 +3267,7 @@ DECLARE_MESSAGE(WhileParsingVersionsForPort, (msg::package_name, msg::path), "", "while parsing versions for {package_name} from {path}") +DECLARE_MESSAGE(WhileRunningAssetCacheScriptCommandLine, (), "", "while running asset cache script command line") DECLARE_MESSAGE(WhileValidatingVersion, (msg::version), "", "while validating version: {version}") DECLARE_MESSAGE(WindowsOnlyCommand, (), "", "This command only supports Windows.") DECLARE_MESSAGE(WroteNuGetPkgConfInfo, (msg::path), "", "Wrote NuGet package config information to {path}") diff --git a/include/vcpkg/base/message_sinks.h b/include/vcpkg/base/message_sinks.h index 4d854364a0..85cd345840 100644 --- a/include/vcpkg/base/message_sinks.h +++ b/include/vcpkg/base/message_sinks.h @@ -4,56 +4,54 @@ #include +#include +#include + namespace vcpkg { + struct MessageLineSegment + { + Color color; + std::string text; + }; + + struct MessageLine + { + MessageLine() = default; + MessageLine(const MessageLine&) = default; + MessageLine(MessageLine&&) = default; + + explicit MessageLine(const LocalizedString& ls); + explicit MessageLine(LocalizedString&& ls); + + void print(Color color, StringView text); + void print(StringView text); + const std::vector& get_segments() const noexcept; + + std::string to_string() const; + void to_string(std::string& target) const; + + private: + std::vector segments; + }; struct MessageSink { - virtual void print(Color c, StringView sv) = 0; + virtual void println(const MessageLine& line) = 0; + virtual void println(MessageLine&& line) = 0; - void println() { this->print(Color::none, "\n"); } - void print(const LocalizedString& s) { this->print(Color::none, s); } - void println(Color c, const LocalizedString& s) - { - this->print(c, s); - this->print(Color::none, "\n"); - } - inline void println(const LocalizedString& s) - { - this->print(Color::none, s); - this->print(Color::none, "\n"); - } - inline void println(Color c, LocalizedString&& s) { this->print(c, s.append_raw('\n')); } - inline void println(LocalizedString&& s) { this->print(Color::none, s.append_raw('\n')); } - void println_warning(const LocalizedString& s); - void println_error(const LocalizedString& s); + virtual void println(const LocalizedString& s); + virtual void println(LocalizedString&& s); + + virtual void println(Color c, const LocalizedString& s); + virtual void println(Color c, LocalizedString&& s); - template - void print(VCPKG_DECL_MSG_ARGS) - { - this->print(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } template void println(VCPKG_DECL_MSG_ARGS) { this->println(msg::format(VCPKG_EXPAND_MSG_ARGS)); } - template - void println_warning(VCPKG_DECL_MSG_ARGS) - { - this->println_warning(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } - template - void println_error(VCPKG_DECL_MSG_ARGS) - { - this->println_error(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } - template - void print(Color c, VCPKG_DECL_MSG_ARGS) - { - this->print(c, msg::format(VCPKG_EXPAND_MSG_ARGS)); - } template void println(Color c, VCPKG_DECL_MSG_ARGS) { @@ -68,11 +66,17 @@ namespace vcpkg ~MessageSink() = default; }; - struct CombiningSink : MessageSink + struct TeeSink final : MessageSink { MessageSink& m_first; MessageSink& m_second; - CombiningSink(MessageSink& first, MessageSink& second) : m_first(first), m_second(second) { } - void print(Color c, StringView sv) override; + TeeSink(MessageSink& first, MessageSink& second) : m_first(first), m_second(second) { } + + virtual void println(const MessageLine& line) override; + virtual void println(MessageLine&& line) override; + virtual void println(const LocalizedString& line) override; + virtual void println(LocalizedString&& line) override; + virtual void println(Color color, const LocalizedString& line) override; + virtual void println(Color color, LocalizedString&& line) override; }; } diff --git a/include/vcpkg/base/strings.h b/include/vcpkg/base/strings.h index af7ca77acf..236d5c8b56 100644 --- a/include/vcpkg/base/strings.h +++ b/include/vcpkg/base/strings.h @@ -129,6 +129,7 @@ namespace vcpkg::Strings void inplace_ascii_to_lowercase(std::string& s); [[nodiscard]] std::string ascii_to_lowercase(StringView s); [[nodiscard]] std::string ascii_to_uppercase(StringView s); + void append_ascii_lowercase(std::string& target, StringView s); bool case_insensitive_ascii_starts_with(StringView s, StringView pattern); bool case_insensitive_ascii_ends_with(StringView s, StringView pattern); diff --git a/include/vcpkg/base/system.process.h b/include/vcpkg/base/system.process.h index e362dd997c..198d2e1f3f 100644 --- a/include/vcpkg/base/system.process.h +++ b/include/vcpkg/base/system.process.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -119,8 +120,23 @@ namespace vcpkg std::string stdin_content; }; - ExpectedL cmd_execute(const Command& cmd); - ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings); + Optional cmd_execute(DiagnosticContext& context, const Command& cmd); + inline ExpectedL cmd_execute(const Command& cmd) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, const Command&)>(cmd_execute), cmd); + } + Optional cmd_execute(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings); + inline ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const ProcessLaunchSettings&)>(cmd_execute), + cmd, + settings); + } #if defined(_WIN32) Environment cmd_execute_and_capture_environment(const Command& cmd, const Environment& env); @@ -128,25 +144,97 @@ namespace vcpkg void cmd_execute_background(const Command& cmd_line); - ExpectedL cmd_execute_and_capture_output(const Command& cmd); - ExpectedL cmd_execute_and_capture_output(const Command& cmd, - const RedirectedProcessLaunchSettings& settings); + Optional cmd_execute_and_capture_output(DiagnosticContext& context, const Command& cmd); + inline ExpectedL cmd_execute_and_capture_output(const Command& cmd) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, const Command&)>( + cmd_execute_and_capture_output), + cmd); + } + Optional cmd_execute_and_capture_output(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings); + inline ExpectedL cmd_execute_and_capture_output(const Command& cmd, + const RedirectedProcessLaunchSettings& settings) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const RedirectedProcessLaunchSettings&)>( + cmd_execute_and_capture_output), + cmd, + settings); + } std::vector> cmd_execute_and_capture_output_parallel(View commands); std::vector> cmd_execute_and_capture_output_parallel( View commands, const RedirectedProcessLaunchSettings& settings); - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const std::function& per_line_cb); - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& per_line_cb); - - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const std::function& data_cb); - ExpectedL cmd_execute_and_stream_data(const Command& cmd, + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const std::function& per_line_cb); + inline ExpectedL cmd_execute_and_stream_lines(const Command& cmd, + const std::function& per_line_cb) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const std::function&)>( + cmd_execute_and_stream_lines), + cmd, + per_line_cb); + } + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb); + const std::function& per_line_cb); + inline ExpectedL cmd_execute_and_stream_lines(const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& per_line_cb) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, + const Command&, + const RedirectedProcessLaunchSettings&, + const std::function&)>( + cmd_execute_and_stream_lines), + cmd, + settings, + per_line_cb); + } + + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const std::function& data_cb); + inline ExpectedL cmd_execute_and_stream_data(const Command& cmd, + const std::function& data_cb) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const std::function&)>( + cmd_execute_and_stream_data), + cmd, + data_cb); + } + + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb); + + inline ExpectedL cmd_execute_and_stream_data(const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, + const Command&, + const RedirectedProcessLaunchSettings&, + const std::function&)>( + cmd_execute_and_stream_data), + cmd, + settings, + data_cb); + } uint64_t get_subproccess_stats(); @@ -170,9 +258,15 @@ namespace vcpkg // If exit code is 0, returns a 'success' ExpectedL. // Otherwise, returns an ExpectedL containing error text - ExpectedL flatten(const ExpectedL&, StringView tool_name); + ExpectedL flatten(const ExpectedL& maybe_exit, StringView tool_name); // If exit code is 0, returns a 'success' ExpectedL containing the output // Otherwise, returns an ExpectedL containing error text - ExpectedL flatten_out(ExpectedL&&, StringView tool_name); + ExpectedL flatten_out(ExpectedL&& maybe_exit, StringView tool_name); + + // Checks that `maybe_exit` implies a process that returned 0. If so, returns a pointer to the process' output. + // Otherwise, records an error in `context` and returns nullptr. + std::string* check_zero_exit_code(DiagnosticContext& context, + Optional& maybe_exit, + StringView exe_path); } diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index f9057847df..dbd0e82a6c 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -225,7 +225,7 @@ namespace vcpkg bool m_needs_zip_file = false; }; - ExpectedL parse_download_configuration(const Optional& arg); + ExpectedL parse_download_configuration(const Optional& arg); std::string generate_nuget_packages_config(const ActionPlan& action, StringView prefix); diff --git a/include/vcpkg/configure-environment.h b/include/vcpkg/configure-environment.h index 74807999f6..3f53416a30 100644 --- a/include/vcpkg/configure-environment.h +++ b/include/vcpkg/configure-environment.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include @@ -16,9 +18,10 @@ namespace vcpkg { - ExpectedL download_vcpkg_standalone_bundle(const DownloadManager& download_manager, - const Filesystem& fs, - const Path& download_root); + Optional download_vcpkg_standalone_bundle(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_root); int run_configure_environment_command(const VcpkgPaths& paths, View args); diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h index 6542f324e3..32b5860a02 100644 --- a/include/vcpkg/tools.h +++ b/include/vcpkg/tools.h @@ -60,7 +60,7 @@ namespace vcpkg ExpectedL find_system_cmake(const ReadOnlyFilesystem& fs); std::unique_ptr get_tool_cache(const Filesystem& fs, - std::shared_ptr downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index f0fb8b95f6..bea70d50c2 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -106,7 +106,7 @@ namespace vcpkg std::string get_toolver_diagnostics() const; const Filesystem& get_filesystem() const; - const DownloadManager& get_download_manager() const; + const AssetCachingSettings& get_asset_cache_settings() const; const ToolCache& get_tool_cache() const; const Path& get_tool_exe(StringView tool, MessageSink& status_messages) const; const std::string& get_tool_version(StringView tool, MessageSink& status_messages) const; diff --git a/locales/messages.json b/locales/messages.json index 779bc18bd9..d5b518921b 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -178,16 +178,35 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", - "AssetCacheHit": "Asset cache hit for {path}; downloaded from: {url}", - "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMiss": "Asset cache miss; downloading from {url}", + "AssetCacheConsult": "Trying to download {path} using asset cache {url}", + "_AssetCacheConsult.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheConsultScript": "Trying to download {path} using asset cache script", + "_AssetCacheConsultScript.comment": "An example of {path} is /foo/bar.", + "AssetCacheHit": "Download successful! Asset cache hit.", + "AssetCacheHitUrl": "Download successful! Asset cache hit, did not try authoritative source {url}", + "_AssetCacheHitUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMiss": "Asset cache miss; trying authoritative source {url}", "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and downloads are blocked by x-block-origin.", - "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", + "AssetCacheMissBlockOrigin": "there were no asset cache hits, and x-block-origin blocks trying the authoritative source {url}", + "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMissNoUrls": "Asset cache missed looking for {sha} and no authoritative URL is known", + "_AssetCacheMissNoUrls.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", - "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", - "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheScriptBadVariable": "the script template {value} contains unknown replacement {list}", + "_AssetCacheScriptBadVariable.comment": "{value} is the script template passed to x-script, {list} is the name of the unknown replacement", + "AssetCacheScriptBadVariableHint": "if you want this on the literal command line, use {{{list}}}", + "_AssetCacheScriptBadVariableHint.comment": "{list} is the name of the unknown replacement", + "AssetCacheScriptCommandLine": "the full script command line was", + "AssetCacheScriptFailed": "the asset cache script returned nonzero exit code {exit_code}", + "_AssetCacheScriptFailed.comment": "An example of {exit_code} is 127.", + "AssetCacheScriptFailedToWriteCorrectHash": "the asset cache script returned success but the resulting file has an unexpected hash", + "AssetCacheScriptFailedToWriteFile": "the asset cache script returned success but did not create expected result file", + "AssetCacheScriptNeedsSha": "the script template {value} requires a SHA, but no SHA is known for attempted download of {url}", + "_AssetCacheScriptNeedsSha.comment": "{value} is the script template the user supplied to x-script An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheScriptNeedsUrl": "the script template {value} requires a URL, but no URL is known for attempted download of {sha}", + "_AssetCacheScriptNeedsUrl.comment": "{value} is the script template the user supplied to x-script An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "AssetCacheSuccesfullyStored": "Store success", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -594,28 +613,42 @@ "DocumentedFieldsSuggestUpdate": "If these are documented fields that should be recognized try updating the vcpkg tool.", "DownloadAvailable": "A downloadable copy of this tool is available and can be used by unsetting {env_var}.", "_DownloadAvailable.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", - "DownloadFailedCurl": "{url}: curl failed to download with exit code {exit_code}", - "_DownloadFailedCurl.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {exit_code} is 127.", - "DownloadFailedHashMismatch": "File does not have the expected hash:\nurl: {url}\nFile: {path}\nExpected hash: {expected}\nActual hash: {actual}", - "_DownloadFailedHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", - "DownloadFailedProxySettings": "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See, {url}\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporay download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.", - "_DownloadFailedProxySettings.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadFailedHashMismatch": "download from {url} had an unexpected hash", + "_DownloadFailedHashMismatch.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadFailedHashMismatchActualHash": "Actual : {sha}", + "_DownloadFailedHashMismatchActualHash.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "DownloadFailedHashMismatchExpectedHash": "Expected: {sha}", + "_DownloadFailedHashMismatchExpectedHash.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "DownloadFailedProxySettings": "If you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.", "DownloadFailedRetrying": "Download failed -- retrying after {value}ms", "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds", "DownloadFailedStatusCode": "{url}: failed: status code {value}", "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadOrUrl": "or", "DownloadRootsDir": "Downloads directory (default: {env_var})", "_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", - "DownloadSuccesful": "Successfully downloaded {path}.", + "DownloadSuccesful": "Successfully downloaded {path}", "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.", + "DownloadSuccesfulUploading": "Successfully downloaded {path}, storing to {url}", + "_DownloadSuccesfulUploading.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTryingAuthoritativeSource": "Trying authoritative source {url}", + "_DownloadTryingAuthoritativeSource.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}", "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", + "DownloadingAssetShaToFile": "Downloading asset cache entry {sha} -> {path}", + "_DownloadingAssetShaToFile.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e. An example of {path} is /foo/bar.", + "DownloadingAssetShaWithoutAssetCache": "requested download of asset cache entry {sha} -> {path}, but no asset caches are configured", + "_DownloadingAssetShaWithoutAssetCache.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e. An example of {path} is /foo/bar.", + "DownloadingFile": "Downloading {path}", + "_DownloadingFile.comment": "An example of {path} is /foo/bar.", + "DownloadingFileFirstAuthoritativeSource": "Downloading {path}, trying {url}", + "_DownloadingFileFirstAuthoritativeSource.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", - "DownloadingUrl": "Downloading {url}", - "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadingUrlToFile": "Downloading {url} -> {path}", + "_DownloadingUrlToFile.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", "_DownloadingVcpkgStandaloneBundle.comment": "An example of {version} is 1.3.8.", "DownloadingVcpkgStandaloneBundleLatest": "Downloading latest standalone bundle.", @@ -785,8 +818,7 @@ "FieldKindDidNotHaveExpectedValue": "\"kind\" did not have an expected value: (expected one of: {expected}; found {actual})", "_FieldKindDidNotHaveExpectedValue.comment": "{expected} is a list of literal kinds the user must type, separated by commas, {actual} is what the user supplied", "FileIsNotExecutable": "this file does not appear to be executable", - "FileNotFound": "{path}: file not found", - "_FileNotFound.comment": "An example of {path} is /foo/bar.", + "FileNotFound": "file not found", "FileReadFailed": "Failed to read {count} bytes from {path} at offset {byte_offset}.", "_FileReadFailed.comment": "An example of {path} is /foo/bar. An example of {byte_offset} is 42. An example of {count} is 42.", "FileSeekFailed": "Failed to seek to position {byte_offset} in {path}.", @@ -847,8 +879,6 @@ "_GitUnexpectedCommandOutputCmd.comment": "An example of {command_line} is vcpkg install zlib.", "GraphCycleDetected": "Cycle detected within graph at {package_name}:", "_GraphCycleDetected.comment": "A list of package names comprising the cycle will be printed after this message. An example of {package_name} is zlib.", - "HashFileFailureToRead": "failed to read file \"{path}\" for hashing: ", - "_HashFileFailureToRead.comment": "Printed after ErrorMessage and before the specific failing filesystem operation (like file not found) An example of {path} is /foo/bar.", "HashPortManyFiles": "{package_name} contains {count} files. Hashing these contents may take a long time when determining the ABI hash for binary caching. Consider reducing the number of files. Common causes of this are accidentally checking out source or build files into a port's directory.", "_HashPortManyFiles.comment": "An example of {package_name} is zlib. An example of {count} is 42.", "HeaderOnlyUsage": "{package_name} is header-only and can be used from CMake via:", @@ -1160,8 +1190,6 @@ "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}", "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.", "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.", - "MissingAssetBlockOrigin": "Missing {path} and downloads are blocked by x-block-origin.", - "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "MissingClosingParen": "missing closing )", "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.", "_MissingDependency.comment": "An example of {spec} is zlib:x64-windows. An example of {package_name} is zlib.", @@ -1199,8 +1227,6 @@ "NoOutdatedPackages": "There are no outdated packages.", "NoRegistryForPort": "no registry configured for port {package_name}", "_NoRegistryForPort.comment": "An example of {package_name} is zlib.", - "NoUrlsAndHashSpecified": "No urls specified to download SHA: {sha}", - "_NoUrlsAndHashSpecified.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "NoUrlsAndNoHashSpecified": "No urls specified and no hash specified.", "NonExactlyArgs": "the command '{command_name}' requires exactly {expected} arguments, but {actual} were provided", "_NonExactlyArgs.comment": "{expected} and {actual} are integers An example of {command_name} is install.", @@ -1363,6 +1389,8 @@ "_PortsUpdated.comment": "An example of {count} is 42.", "PrebuiltPackages": "There are packages that have not been built. To build them run:", "PreviousIntegrationFileRemains": "Previous integration file was not removed.", + "ProgramPathReturnedNonzeroExitCode": "failed with exit code {exit_code}", + "_ProgramPathReturnedNonzeroExitCode.comment": "An example of {exit_code} is 127.", "ProgramReturnedNonzeroExitCode": "{tool_name} failed with exit code: ({exit_code}).", "_ProgramReturnedNonzeroExitCode.comment": "The program's console output is appended after this. An example of {tool_name} is aria2. An example of {exit_code} is 127.", "ProvideExportType": "At least one of the following options are required: --raw --nuget --ifw --zip --7zip --chocolatey --prefab.", @@ -1448,6 +1476,8 @@ "_ToolDataFileSchemaVersionNotSupported.comment": "An example of {version} is 1.3.8.", "ToolFetchFailed": "Could not fetch {tool_name}.", "_ToolFetchFailed.comment": "An example of {tool_name} is aria2.", + "ToolHashMismatch": "{tool_name} appears to be already downloaded, but has an incorrect hash. Expected {expected} but was {actual}", + "_ToolHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {tool_name} is aria2.", "ToolInWin10": "This utility is bundled with Windows 10 or later.", "ToolOfVersionXNotFound": "A suitable version of {tool_name} was not found (required v{version}) and unable to automatically download a portable one. Please install a newer version of {tool_name}", "_ToolOfVersionXNotFound.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", @@ -1703,6 +1733,7 @@ "_WhileLookingForSpec.comment": "An example of {spec} is zlib:x64-windows.", "WhileParsingVersionsForPort": "while parsing versions for {package_name} from {path}", "_WhileParsingVersionsForPort.comment": "An example of {package_name} is zlib. An example of {path} is /foo/bar.", + "WhileRunningAssetCacheScriptCommandLine": "while running asset cache script command line", "WhileValidatingVersion": "while validating version: {version}", "_WhileValidatingVersion.comment": "An example of {version} is 1.3.8.", "WindowsOnlyCommand": "This command only supports Windows.", diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp index cc8832b344..8e1a60d367 100644 --- a/src/vcpkg-test/configparser.cpp +++ b/src/vcpkg-test/configparser.cpp @@ -614,19 +614,19 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(parse_download_configuration("x-azurl,ftp://magic,none")); { - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(empty.m_write_headers.empty()); CHECK(empty.m_read_headers.empty()); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); CHECK(dm.m_write_url_template == nullopt); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123/,foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -634,7 +634,7 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(dm.m_secrets == std::vector{"foo"}); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,?foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -642,14 +642,14 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(dm.m_secrets == std::vector{"?foo"}); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/"); CHECK(dm.m_read_headers.empty()); CHECK(dm.m_write_url_template == nullopt); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,,readwrite").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/"); CHECK(dm.m_read_headers.empty()); @@ -657,7 +657,7 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") Test::check_ranges(dm.m_write_headers, azure_blob_headers()); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,foo,readwrite").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -665,6 +665,16 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") Test::check_ranges(dm.m_write_headers, azure_blob_headers()); CHECK(dm.m_secrets == std::vector{"foo"}); } + { + AssetCachingSettings dm = + parse_download_configuration("x-script,powershell {SHA} {URL}").value_or_exit(VCPKG_LINE_INFO); + CHECK(!dm.m_read_url_template.has_value()); + CHECK(dm.m_read_headers.empty()); + CHECK(!dm.m_write_url_template.has_value()); + CHECK(dm.m_write_headers.empty()); + CHECK(dm.m_secrets.empty()); + CHECK(dm.m_script.value_or_exit(VCPKG_LINE_INFO) == "powershell {SHA} {URL}"); + } } TEST_CASE ("AssetConfigParser clear provider", "[assetconfigparser]") @@ -679,7 +689,7 @@ TEST_CASE ("AssetConfigParser clear provider", "[assetconfigparser]") return std::move(v); }; - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(value_or(parse_download_configuration("x-azurl,https://abc/123,foo;clear"), empty).m_read_url_template == nullopt); @@ -698,7 +708,7 @@ TEST_CASE ("AssetConfigParser x-block-origin provider", "[assetconfigparser]") return std::move(v); }; - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(!value_or(parse_download_configuration({}), empty).m_block_origin); CHECK(value_or(parse_download_configuration("x-block-origin"), empty).m_block_origin); diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 2dcb46abb5..31eef37073 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -5,63 +5,111 @@ using namespace vcpkg; -TEST_CASE ("split_uri_view", "[downloads]") +TEST_CASE ("parse_split_url_view", "[downloads]") { { - auto x = split_uri_view("https://github.com/Microsoft/vcpkg"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "https"); - REQUIRE(x.get()->authority.value_or("") == "//github.com"); - REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg"); + auto x = parse_split_url_view("https://github.com/Microsoft/vcpkg"); + if (auto v = x.get()) + { + REQUIRE(v->scheme == "https"); + REQUIRE(v->authority.value_or("") == "//github.com"); + REQUIRE(v->path_query_fragment == "/Microsoft/vcpkg"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view(""); - REQUIRE(!x.has_value()); + REQUIRE(!parse_split_url_view("").has_value()); + REQUIRE(!parse_split_url_view("hello").has_value()); } { - auto x = split_uri_view("hello"); - REQUIRE(!x.has_value()); + auto x = parse_split_url_view("file:"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == ""); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == ""); + auto x = parse_split_url_view("file:path"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == "path"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:path"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == "path"); + auto x = parse_split_url_view("file:/path"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == "/path"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:/path"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == "/path"); + auto x = parse_split_url_view("file://user:pw@host"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(y->authority.value_or("") == "//user:pw@host"); + REQUIRE(y->path_query_fragment == ""); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file://user:pw@host"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host"); - REQUIRE(x.get()->path_query_fragment == ""); + auto x = parse_split_url_view("ftp://host:port/"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "ftp"); + REQUIRE(y->authority.value_or("") == "//host:port"); + REQUIRE(y->path_query_fragment == "/"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("ftp://host:port/"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "ftp"); - REQUIRE(x.get()->authority.value_or({}) == "//host:port"); - REQUIRE(x.get()->path_query_fragment == "/"); + auto x = parse_split_url_view("file://D:\\work\\testing\\asset-cache/" + "562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42b69e4aeb3" + "1a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(y->authority.value_or("") == "//D:\\work\\testing\\asset-cache"); + REQUIRE(y->path_query_fragment == "/562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42" + "b69e4aeb31a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); + } + else + { + FAIL(); + } } } TEST_CASE ("parse_curl_status_line", "[downloads]") { - std::vector> http_codes; + std::vector http_codes; StringLiteral malformed_examples[] = { "asdfasdf", // wrong prefix "curl: unknown --write-out variable: 'exitcode'", // wrong prefixes, and also what old curl does @@ -72,43 +120,46 @@ TEST_CASE ("parse_curl_status_line", "[downloads]") "prefix42 2a", // non numeric exitcode }; + FullyBufferedDiagnosticContext bdc; for (auto&& malformed : malformed_examples) { - parse_curl_status_line(http_codes, "prefix", malformed); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", malformed)); REQUIRE(http_codes.empty()); + REQUIRE(bdc.empty()); } // old curl output - parse_curl_status_line(http_codes, "prefix", "prefix200 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 200); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 ")); + REQUIRE(http_codes == std::vector{200}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, "prefix", "prefix404 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 404); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix404 ")); + REQUIRE(http_codes == std::vector{404}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, "prefix", "prefix0 "); // a failure, but we don't know that yet - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 0); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix0 ")); // a failure, but we don't know that yet + REQUIRE(http_codes == std::vector{0}); + REQUIRE(bdc.empty()); http_codes.clear(); // current curl output - parse_curl_status_line(http_codes, "prefix", "prefix200 0 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 200); + REQUIRE(parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 0 ")); + REQUIRE(http_codes == std::vector{200}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, - "prefix", - "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " - "- The target principal name is incorrect."); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].error().data() == - "curl operation failed with error code 60. schannel: SNI or certificate check failed: " + REQUIRE(parse_curl_status_line( + bdc, + http_codes, + "prefix", + "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " + "- The target principal name is incorrect.")); + REQUIRE(http_codes == std::vector{0}); + REQUIRE(bdc.to_string() == + "error: curl operation failed with error code 60. schannel: SNI or certificate check failed: " "SEC_E_WRONG_PRINCIPAL (0x80090322) - The target principal name is incorrect."); - http_codes.clear(); } TEST_CASE ("download_files", "[downloads]") @@ -116,40 +167,29 @@ TEST_CASE ("download_files", "[downloads]") auto const dst = Test::base_temporary_directory() / "download_files"; auto const url = [&](std::string l) -> auto { return std::pair(l, dst); }; + FullyBufferedDiagnosticContext bdc; std::vector headers; std::vector secrets; - auto results = - download_files(std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, - headers, - secrets); - REQUIRE(results.size() == 2); - if (auto first_result = results[0].get()) - { - // old curl - REQUIRE(*first_result == 0); - } - else - { - // current curl - REQUIRE(results[0].error().data() == - "curl operation failed with error code 1. Protocol \"unknown\" not supported"); - } - - auto&& second_error = results[1].error().data(); - std::puts(second_error.c_str()); - // curl operation failed with error code 7. Failed to connect to localhost port 9 after 2241 ms: Could not connect - // to server - if (second_error == "curl operation failed with error code 7.") + auto results = download_files_no_cache( + bdc, + std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, + headers, + secrets); + REQUIRE(results == std::vector{0, 0}); + auto all_errors = bdc.to_string(); + if (all_errors == "error: curl operation failed with error code 7.") { - // old curl + // old curl, this is OK! } else { // new curl REQUIRE_THAT( - second_error, - Catch::Matches("curl operation failed with error code 7. Failed to connect to localhost port 9 after " - "[0-9]+ ms: Could not connect to server", + all_errors, + Catch::Matches("error: curl operation failed with error code 1\\. Protocol \"unknown\" not supported( or " + "disabled in libcurl)?\n" + "error: curl operation failed with error code 7\\. Failed to connect to localhost port 9 " + "after [0-9]+ ms: (Could not|Couldn't) connect to server", Catch::CaseSensitive::Yes)); } } @@ -235,8 +275,8 @@ TEST_CASE ("try_parse_curl_progress_data", "[downloads]") .value_or_exit(VCPKG_LINE_INFO); REQUIRE(out.total_percent == 0); REQUIRE(out.total_size == 0); - REQUIRE(out.recieved_percent == 0); - REQUIRE(out.recieved_size == 0); + REQUIRE(out.received_percent == 0); + REQUIRE(out.received_size == 0); REQUIRE(out.transfer_percent == 0); REQUIRE(out.transfer_size == 0); REQUIRE(out.average_upload_speed == 0); @@ -250,8 +290,8 @@ TEST_CASE ("try_parse_curl_progress_data", "[downloads]") .value_or_exit(VCPKG_LINE_INFO); REQUIRE(out.total_percent == 2); REQUIRE(out.total_size == 190 * 1024 * 1024); - REQUIRE(out.recieved_percent == 2); - REQUIRE(out.recieved_size == 3935 * 1024); + REQUIRE(out.received_percent == 2); + REQUIRE(out.received_size == 3935 * 1024); REQUIRE(out.transfer_percent == 0); REQUIRE(out.transfer_size == 0); REQUIRE(out.average_upload_speed == 0); diff --git a/src/vcpkg-test/strings.cpp b/src/vcpkg-test/strings.cpp index de6a284db9..d31a7d956f 100644 --- a/src/vcpkg-test/strings.cpp +++ b/src/vcpkg-test/strings.cpp @@ -204,36 +204,62 @@ TEST_CASE ("inplace_replace_all(char)", "[strings]") TEST_CASE ("api_stable_format(sv,append_f)", "[strings]") { - std::string target; - auto res = api_stable_format("{", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("{ {", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("{ {}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - - res = api_stable_format("}}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(*res.get() == "}"); - res = api_stable_format("{{", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(*res.get() == "{"); - - res = api_stable_format("{x}{y}{z}", [](std::string& out, StringView t) { - CHECK((t == "x" || t == "y" || t == "z")); - Strings::append(out, t, t); - }); - REQUIRE(*res.get() == "xxyyzz"); - res = api_stable_format("{x}}}", [](std::string& out, StringView t) { - CHECK(t == "x"); - Strings::append(out, "hello"); - }); - REQUIRE(*res.get() == "hello}"); - res = api_stable_format("123{x}456", [](std::string& out, StringView t) { - CHECK(t == "x"); - Strings::append(out, "hello"); - }); - REQUIRE(*res.get() == "123hello456"); + for (auto&& invalid_format_string : {"{", "}", "{ {", "{ {}"}) + { + FullyBufferedDiagnosticContext bdc_invalid{}; + auto res = api_stable_format(bdc_invalid, invalid_format_string, [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc_invalid.to_string() == fmt::format("error: invalid format string: {}", invalid_format_string)); + } + + FullyBufferedDiagnosticContext bdc{}; + { + auto res = api_stable_format(bdc, "}}", [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "}"); + } + { + auto res = api_stable_format(bdc, "{{", [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "{"); + } + { + auto res = api_stable_format(bdc, "{x}{y}{z}", [](std::string& out, StringView t) { + CHECK((t == "x" || t == "y" || t == "z")); + Strings::append(out, t, t); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "xxyyzz"); + } + { + auto res = api_stable_format(bdc, "{x}}}", [](std::string& out, StringView t) { + CHECK(t == "x"); + Strings::append(out, "hello"); + return true; + }); + + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "hello}"); + } + { + auto res = api_stable_format(bdc, "123{x}456", [](std::string& out, StringView t) { + CHECK(t == "x"); + Strings::append(out, "hello"); + return true; + }); + + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "123hello456"); + } } TEST_CASE ("lex compare less", "[strings]") diff --git a/src/vcpkg/base/diagnostics.cpp b/src/vcpkg/base/diagnostics.cpp index 2ecb4e8052..1cc08058a1 100644 --- a/src/vcpkg/base/diagnostics.cpp +++ b/src/vcpkg/base/diagnostics.cpp @@ -1,6 +1,9 @@ #include #include #include +#include + +#include #include @@ -48,42 +51,56 @@ namespace const auto prefix = prefixes[diag_index]; target.append(prefix->data(), prefix->size()); } -} -namespace vcpkg -{ - void DiagnosticLine::print_to(MessageSink& sink) const + template + void joined_line_to_string(const std::vector& lines, std::string& target) { - std::string buf; - append_file_prefix(buf, m_origin, m_position); - switch (m_kind) + auto first = lines.begin(); + const auto last = lines.end(); + if (first == last) { - case DiagKind::None: - // intentionally blank - break; - case DiagKind::Message: buf.append(MessagePrefix.data(), MessagePrefix.size()); break; - case DiagKind::Error: + return; + } + + for (;;) + { + first->to_string(target); + if (++first == last) { - sink.print(Color::none, buf); - sink.print(Color::error, "error"); - buf.assign(ColonSpace.data(), ColonSpace.size()); + return; } - break; - case DiagKind::Warning: + + target.push_back('\n'); + } + } + + bool diagnostic_lines_any_errors(const std::vector& lines) + { + for (auto&& line : lines) + { + if (line.kind() == DiagKind::Error) { - sink.print(Color::none, buf); - sink.print(Color::warning, "warning"); - buf.assign(ColonSpace.data(), ColonSpace.size()); + return true; } - break; - case DiagKind::Note: buf.append(NotePrefix.data(), NotePrefix.size()); break; - default: Checks::unreachable(VCPKG_LINE_INFO); } - buf.append(m_message.data()); - buf.push_back('\n'); - sink.print(Color::none, buf); + return false; } +} + +namespace vcpkg +{ + void DiagnosticContext::report(DiagnosticLine&& line) { report(line); } + + void DiagnosticContext::report_system_error(StringLiteral system_api_name, int error_value) + { + report_error(msgSystemApiErrorMessage, + msg::system_api = system_api_name, + msg::exit_code = error_value, + msg::error_msg = std::system_category().message(error_value)); + } + + void DiagnosticLine::print_to(MessageSink& sink) const { sink.println(to_message_line()); } std::string DiagnosticLine::to_string() const { std::string result; @@ -97,6 +114,40 @@ namespace vcpkg target.append(m_message.data()); } + MessageLine DiagnosticLine::to_message_line() const + { + MessageLine ret; + { + std::string file_prefix; + append_file_prefix(file_prefix, m_origin, m_position); + ret.print(file_prefix); + } + switch (m_kind) + { + case DiagKind::None: + // intentionally blank + break; + case DiagKind::Message: ret.print(MessagePrefix); break; + case DiagKind::Error: + { + ret.print(Color::error, "error"); + ret.print(ColonSpace); + } + break; + case DiagKind::Warning: + { + ret.print(Color::warning, "warning"); + ret.print(ColonSpace); + } + break; + case DiagKind::Note: ret.print(NotePrefix); break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + + ret.print(m_message); + return ret; + } + LocalizedString DiagnosticLine::to_json_reader_string(const std::string& path, const LocalizedString& type) const { std::string result; @@ -110,8 +161,41 @@ namespace vcpkg return LocalizedString::from_raw(result); } - void BufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } + DiagnosticLine DiagnosticLine::reduce_to_warning() const& + { + return DiagnosticLine{m_kind == DiagKind::Error ? DiagKind::Warning : m_kind, m_origin, m_position, m_message}; + } + DiagnosticLine DiagnosticLine::reduce_to_warning() && + { + return DiagnosticLine{m_kind == DiagKind::Error ? DiagKind::Warning : m_kind, + std::move(m_origin), + m_position, + std::move(m_message)}; + } + DiagnosticLine::DiagnosticLine(DiagKind kind, + const Optional& origin, + TextRowCol position, + const LocalizedString& message) + : m_kind(kind), m_origin(origin), m_position(position), m_message(message) + { + } + DiagnosticLine::DiagnosticLine(DiagKind kind, + Optional&& origin, + TextRowCol position, + LocalizedString&& message) + : m_kind(kind), m_origin(std::move(origin)), m_position(position), m_message(std::move(message)) + { + } + + void PrintingDiagnosticContext::report(const DiagnosticLine& line) { line.print_to(sink); } + + void PrintingDiagnosticContext::statusln(const LocalizedString& message) { sink.println(message); } + void PrintingDiagnosticContext::statusln(LocalizedString&& message) { sink.println(std::move(message)); } + void PrintingDiagnosticContext::statusln(const MessageLine& message) { sink.println(message); } + void PrintingDiagnosticContext::statusln(MessageLine&& message) { sink.println(std::move(message)); } + + void BufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } void BufferedDiagnosticContext::report(DiagnosticLine&& line) { lines.push_back(std::move(line)); } void BufferedDiagnosticContext::print_to(MessageSink& sink) const { @@ -120,79 +204,132 @@ namespace vcpkg line.print_to(sink); } } + // Converts this message into a string // Prefer print() if possible because it applies color // Not safe to use in the face of concurrent calls to report() - std::string BufferedDiagnosticContext::to_string() const + std::string BufferedDiagnosticContext::to_string() const { return adapt_to_string(*this); } + void BufferedDiagnosticContext::to_string(std::string& target) const { joined_line_to_string(lines, target); } + + bool BufferedDiagnosticContext::any_errors() const noexcept { return diagnostic_lines_any_errors(lines); } + bool BufferedDiagnosticContext::empty() const noexcept { return lines.empty(); } + + void BufferedDiagnosticContext::statusln(const LocalizedString& message) { status_sink.println(message); } + void BufferedDiagnosticContext::statusln(LocalizedString&& message) { status_sink.println(std::move(message)); } + void BufferedDiagnosticContext::statusln(const MessageLine& message) { status_sink.println(message); } + void BufferedDiagnosticContext::statusln(MessageLine&& message) { status_sink.println(std::move(message)); } + + void FullyBufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line.to_message_line()); } + void FullyBufferedDiagnosticContext::report(DiagnosticLine&& line) { - std::string result; - this->to_string(result); - return result; + lines.push_back(std::move(line).to_message_line()); } - void BufferedDiagnosticContext::to_string(std::string& target) const + + void FullyBufferedDiagnosticContext::statusln(const LocalizedString& message) { lines.emplace_back(message); } + void FullyBufferedDiagnosticContext::statusln(LocalizedString&& message) { lines.emplace_back(std::move(message)); } + void FullyBufferedDiagnosticContext::statusln(const MessageLine& message) { lines.emplace_back(message); } + void FullyBufferedDiagnosticContext::statusln(MessageLine&& message) { lines.emplace_back(std::move(message)); } + + void FullyBufferedDiagnosticContext::print_to(MessageSink& sink) const { - auto first = lines.begin(); - const auto last = lines.end(); - if (first == last) + for (auto&& line : lines) { - return; + sink.println(line); } + } - for (;;) - { - first->to_string(target); - if (++first == last) - { - return; - } + std::string FullyBufferedDiagnosticContext::to_string() const { return adapt_to_string(*this); } + void FullyBufferedDiagnosticContext::to_string(std::string& target) const { joined_line_to_string(lines, target); } - target.push_back('\n'); + bool FullyBufferedDiagnosticContext::empty() const noexcept { return lines.empty(); } + + void AttemptDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } + void AttemptDiagnosticContext::report(DiagnosticLine&& line) { lines.push_back(std::move(line)); } + + void AttemptDiagnosticContext::statusln(const LocalizedString& message) { inner_context.statusln(message); } + void AttemptDiagnosticContext::statusln(LocalizedString&& message) { inner_context.statusln(std::move(message)); } + void AttemptDiagnosticContext::statusln(const MessageLine& message) { inner_context.statusln(message); } + void AttemptDiagnosticContext::statusln(MessageLine&& message) { inner_context.statusln(std::move(message)); } + + void AttemptDiagnosticContext::commit() + { + for (auto& line : lines) + { + inner_context.report(std::move(line)); } + + lines.clear(); } - bool BufferedDiagnosticContext::any_errors() const noexcept + void AttemptDiagnosticContext::handle() { lines.clear(); } + + AttemptDiagnosticContext::~AttemptDiagnosticContext() { - for (auto&& line : lines) + if (!lines.empty()) { - if (line.kind() == DiagKind::Error) + Debug::println("Uncommitted diagnostics in ~AttemptDiagnosticContext"); + for (auto& line : lines) { - return true; + inner_context.report(std::move(line)); } } + } - return false; + void WarningDiagnosticContext::report(const DiagnosticLine& line) + { + inner_context.report(line.reduce_to_warning()); + } + void WarningDiagnosticContext::report(DiagnosticLine&& line) + { + inner_context.report(std::move(line).reduce_to_warning()); } + + void WarningDiagnosticContext::statusln(const LocalizedString& message) { inner_context.statusln(message); } + void WarningDiagnosticContext::statusln(LocalizedString&& message) { inner_context.statusln(std::move(message)); } + void WarningDiagnosticContext::statusln(const MessageLine& message) { inner_context.statusln(message); } + void WarningDiagnosticContext::statusln(MessageLine&& message) { inner_context.statusln(std::move(message)); } } // namespace vcpkg namespace { - struct ConsoleDiagnosticContext : DiagnosticContext + struct NullDiagnosticContext final : DiagnosticContext + { + // these are all intentionally empty + virtual void report(const DiagnosticLine&) override { } + virtual void statusln(const LocalizedString&) override { } + virtual void statusln(LocalizedString&&) override { } + virtual void statusln(const MessageLine&) override { } + virtual void statusln(MessageLine&&) override { } + }; + + NullDiagnosticContext null_diagnostic_context_instance; + + struct ConsoleDiagnosticContext final : DiagnosticContext { virtual void report(const DiagnosticLine& line) override { line.print_to(out_sink); } + virtual void statusln(const LocalizedString& message) override { out_sink.println(message); } + virtual void statusln(LocalizedString&& message) override { out_sink.println(std::move(message)); } + virtual void statusln(const MessageLine& message) override { out_sink.println(message); } + virtual void statusln(MessageLine&& message) override { out_sink.println(std::move(message)); } }; ConsoleDiagnosticContext console_diagnostic_context_instance; -} // unnamed namespace - -namespace vcpkg -{ - DiagnosticContext& console_diagnostic_context = console_diagnostic_context_instance; -} -namespace -{ - struct NullDiagnosticContext : DiagnosticContext + struct StatusOnlyDiagnosticContext final : DiagnosticContext { - virtual void report(const DiagnosticLine&) override - { - // intentionally empty - } + virtual void report(const DiagnosticLine&) override { } + virtual void statusln(const LocalizedString& message) override { out_sink.println(message); } + virtual void statusln(LocalizedString&& message) override { out_sink.println(std::move(message)); } + virtual void statusln(const MessageLine& message) override { out_sink.println(message); } + virtual void statusln(MessageLine&& message) override { out_sink.println(std::move(message)); } }; - NullDiagnosticContext null_diagnostic_context_instance; -} + StatusOnlyDiagnosticContext status_only_diagnostic_context_instance; +} // unnamed namespace namespace vcpkg { DiagnosticContext& null_diagnostic_context = null_diagnostic_context_instance; + DiagnosticContext& console_diagnostic_context = console_diagnostic_context_instance; + DiagnosticContext& status_only_diagnostic_context = status_only_diagnostic_context_instance; } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 8f573a3a36..23039ea3b3 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -15,60 +17,58 @@ #include -namespace vcpkg +using namespace vcpkg; + +namespace { - static std::string replace_secrets(std::string input, View secrets) + constexpr StringLiteral vcpkg_curl_user_agent_header = + "User-Agent: vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; + + void add_curl_headers(Command& cmd, View headers) { - const auto replacement = msg::format(msgSecretBanner); - for (const auto& secret : secrets) + cmd.string_arg("-H").string_arg(vcpkg_curl_user_agent_header); + for (auto&& header : headers) { - Strings::inplace_replace_all(input, secret, replacement); + cmd.string_arg("-H").string_arg(header); } - - return input; } -#if defined(_WIN32) - struct WinHttpHandle + void replace_secrets(std::string& target, View secrets) { - HINTERNET h; - - WinHttpHandle() : h(0) { } - explicit WinHttpHandle(HINTERNET h_) : h(h_) { } - WinHttpHandle(const WinHttpHandle&) = delete; - WinHttpHandle(WinHttpHandle&& other) : h(other.h) { other.h = 0; } - WinHttpHandle& operator=(const WinHttpHandle&) = delete; - WinHttpHandle& operator=(WinHttpHandle&& other) + const auto replacement = msg::format(msgSecretBanner); + for (const auto& secret : secrets) { - auto cpy = std::move(other); - std::swap(h, cpy.h); - return *this; + Strings::inplace_replace_all(target, secret, replacement); } + } +} - ~WinHttpHandle() - { - if (h) - { - WinHttpCloseHandle(h); - } - } - }; +namespace vcpkg +{ + SanitizedUrl::SanitizedUrl(StringView raw_url, View secrets) + : m_sanitized_url(raw_url.data(), raw_url.size()) + { + replace_secrets(m_sanitized_url, secrets); + } - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, StringView url, DWORD last_error) +#if defined(_WIN32) + static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, + const SanitizedUrl& sanitized_url, + DWORD last_error) { - return msg::format_error( - msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = url); + return msg::format( + msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = sanitized_url); } - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, StringView url) + static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, const SanitizedUrl& sanitized_url) { - return format_winhttp_last_error_message(api_name, url, GetLastError()); + return format_winhttp_last_error_message(api_name, sanitized_url, GetLastError()); } - static void maybe_emit_winhttp_progress(const Optional& maybe_content_length, + static void maybe_emit_winhttp_progress(MessageSink& machine_readable_progress, + const Optional& maybe_content_length, std::chrono::steady_clock::time_point& last_write, - unsigned long long total_downloaded_size, - MessageSink& progress_sink) + unsigned long long total_downloaded_size) { if (const auto content_length = maybe_content_length.get()) { @@ -77,174 +77,240 @@ namespace vcpkg { const double percent = (static_cast(total_downloaded_size) / static_cast(*content_length)) * 100; - progress_sink.print(Color::none, fmt::format("{:.2f}%\n", percent)); + machine_readable_progress.println(LocalizedString::from_raw(fmt::format("{:.2f}%", percent))); last_write = now; } } } - struct WinHttpRequest + struct WinHttpHandle { - static ExpectedL make(HINTERNET hConnect, - StringView url_path, - StringView sanitized_url, - bool https, - const wchar_t* method = L"GET") - { - WinHttpRequest ret; - ret.m_sanitized_url.assign(sanitized_url.data(), sanitized_url.size()); - // Create an HTTP request handle. - { - auto h = WinHttpOpenRequest(hConnect, - method, - Strings::to_utf16(url_path).c_str(), - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - https ? WINHTTP_FLAG_SECURE : 0); - if (!h) - { - return format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url); - } + WinHttpHandle() = default; + WinHttpHandle(const WinHttpHandle&) = delete; + WinHttpHandle& operator=(const WinHttpHandle&) = delete; - ret.m_hRequest = WinHttpHandle{h}; + void require_null_handle() const + { + if (h) + { + Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle type confusion"); } + } - // Send a request. - auto bResults = WinHttpSendRequest( - ret.m_hRequest.h, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); - - if (!bResults) + void require_created_handle() const + { + if (!h) { - return format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url); + Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle not created"); } + } - // End the request. - bResults = WinHttpReceiveResponse(ret.m_hRequest.h, NULL); - if (!bResults) + bool Connect(DiagnosticContext& context, + const WinHttpHandle& session, + StringView hostname, + INTERNET_PORT port, + const SanitizedUrl& sanitized_url) + { + require_null_handle(); + session.require_created_handle(); + h = WinHttpConnect(session.h, Strings::to_utf16(hostname).c_str(), port, 0); + if (h) { - return format_winhttp_last_error_message("WinHttpReceiveResponse", sanitized_url); + return true; } - return ret; + context.report_error(format_winhttp_last_error_message("WinHttpConnect", sanitized_url)); + return false; } - ExpectedL query_status() const + bool Open(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + _In_opt_z_ LPCWSTR pszAgentW, + _In_ DWORD dwAccessType, + _In_opt_z_ LPCWSTR pszProxyW, + _In_opt_z_ LPCWSTR pszProxyBypassW, + _In_ DWORD dwFlags) { - DWORD status_code; - DWORD size = sizeof(status_code); + require_null_handle(); + h = WinHttpOpen(pszAgentW, dwAccessType, pszProxyW, pszProxyBypassW, dwFlags); + if (h) + { + return true; + } - auto succeeded = WinHttpQueryHeaders(m_hRequest.h, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - WINHTTP_HEADER_NAME_BY_INDEX, - &status_code, - &size, - WINHTTP_NO_HEADER_INDEX); - if (succeeded) + context.report_error(format_winhttp_last_error_message("WinHttpOpen", sanitized_url)); + return false; + } + + bool OpenRequest(DiagnosticContext& context, + const WinHttpHandle& hConnect, + const SanitizedUrl& sanitized_url, + IN LPCWSTR pwszVerb, + StringView path_query_fragment, + IN LPCWSTR pwszVersion, + IN LPCWSTR pwszReferrer OPTIONAL, + IN LPCWSTR FAR* ppwszAcceptTypes OPTIONAL, + IN DWORD dwFlags) + { + require_null_handle(); + h = WinHttpOpenRequest(hConnect.h, + pwszVerb, + Strings::to_utf16(path_query_fragment).c_str(), + pwszVersion, + pwszReferrer, + ppwszAcceptTypes, + dwFlags); + if (h) { - return status_code; + return true; } - return format_winhttp_last_error_message("WinHttpQueryHeaders", m_sanitized_url); + context.report_error(format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url)); + return false; } - ExpectedL> query_content_length() const + bool SendRequest(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + _In_reads_opt_(dwHeadersLength) LPCWSTR lpszHeaders, + IN DWORD dwHeadersLength, + _In_reads_bytes_opt_(dwOptionalLength) LPVOID lpOptional, + IN DWORD dwOptionalLength, + IN DWORD dwTotalLength, + IN DWORD_PTR dwContext) const { - static constexpr DWORD buff_characters = 21; // 18446744073709551615 - wchar_t buff[buff_characters]; - DWORD size = sizeof(buff); - auto succeeded = WinHttpQueryHeaders(m_hRequest.h, - WINHTTP_QUERY_CONTENT_LENGTH, - WINHTTP_HEADER_NAME_BY_INDEX, - buff, - &size, - WINHTTP_NO_HEADER_INDEX); - if (succeeded) + require_created_handle(); + if (WinHttpSendRequest( + h, lpszHeaders, dwHeadersLength, lpOptional, dwOptionalLength, dwTotalLength, dwContext)) { - return Strings::strto(Strings::to_utf8(buff, size >> 1)); + return true; } - const DWORD last_error = GetLastError(); - if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) + context.report_error(format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url)); + return false; + } + + bool ReceiveResponse(DiagnosticContext& context, const SanitizedUrl& url) + { + require_created_handle(); + if (WinHttpReceiveResponse(h, NULL)) { - return Optional{nullopt}; + return true; } - return format_winhttp_last_error_message("WinHttpQueryHeaders", m_sanitized_url, last_error); + context.report_error(format_winhttp_last_error_message("WinHttpReceiveResponse", url)); + return false; } - ExpectedL write_response_body(WriteFilePointer& file, MessageSink& progress_sink) + bool SetTimeouts(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + int nResolveTimeout, + int nConnectTimeout, + int nSendTimeout, + int nReceiveTimeout) const { - static constexpr DWORD buff_size = 65535; - std::unique_ptr buff{new char[buff_size]}; - Optional maybe_content_length; - auto last_write = std::chrono::steady_clock::now(); + require_created_handle(); + if (WinHttpSetTimeouts(h, nResolveTimeout, nConnectTimeout, nSendTimeout, nReceiveTimeout)) + { + return true; + } + context.report_error(format_winhttp_last_error_message("WinHttpSetTimeouts", sanitized_url)); + return false; + } + + bool SetOption(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + DWORD dwOption, + LPVOID lpBuffer, + DWORD dwBufferLength) const + { + require_created_handle(); + if (WinHttpSetOption(h, dwOption, lpBuffer, dwBufferLength)) { - auto maybe_maybe_content_length = query_content_length(); - if (const auto p = maybe_maybe_content_length.get()) - { - maybe_content_length = *p; - } - else - { - return std::move(maybe_maybe_content_length).error(); - } + return true; } - unsigned long long total_downloaded_size = 0; - for (;;) + context.report_error(format_winhttp_last_error_message("WinHttpSetOption", sanitized_url)); + return false; + } + + DWORD QueryHeaders(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + DWORD dwInfoLevel, + LPWSTR pwszName, + LPVOID lpBuffer, + LPDWORD lpdwBufferLength, + LPDWORD lpdwIndex) const + { + require_created_handle(); + if (WinHttpQueryHeaders(h, dwInfoLevel, pwszName, lpBuffer, lpdwBufferLength, lpdwIndex)) { - DWORD this_read; - if (!WinHttpReadData(m_hRequest.h, buff.get(), buff_size, &this_read)) - { - return format_winhttp_last_error_message("WinHttpReadData", m_sanitized_url); - } + return 0; + } - if (this_read == 0) - { - return Unit{}; - } + DWORD last_error = GetLastError(); + context.report_error(format_winhttp_last_error_message("WinHttpQueryHeaders", sanitized_url, last_error)); + return last_error; + } - do - { - const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); - if (this_write == 0) - { - return format_winhttp_last_error_message("fwrite", m_sanitized_url); - } + bool ReadData(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + LPVOID buffer, + DWORD dwNumberOfBytesToRead, + DWORD* numberOfBytesRead) + { + require_created_handle(); + if (WinHttpReadData(h, buffer, dwNumberOfBytesToRead, numberOfBytesRead)) + { + return true; + } - maybe_emit_winhttp_progress(maybe_content_length, last_write, total_downloaded_size, progress_sink); - this_read -= this_write; - total_downloaded_size += this_write; - } while (this_read > 0); + context.report_error(format_winhttp_last_error_message("WinHttpReadData", sanitized_url)); + return false; + } + + ~WinHttpHandle() + { + if (h) + { + // intentionally ignore failures + (void)WinHttpCloseHandle(h); } } - WinHttpHandle m_hRequest; - std::string m_sanitized_url; + private: + HINTERNET h{}; + }; + + enum class WinHttpTrialResult + { + failed, + succeeded, + retry }; struct WinHttpSession { - static ExpectedL make(StringView sanitized_url) + bool open(DiagnosticContext& context, const SanitizedUrl& sanitized_url) { - WinHttpSession ret; + if (!m_hSession.Open(context, + sanitized_url, + L"vcpkg/1.0", + WINHTTP_ACCESS_TYPE_NO_PROXY, + WINHTTP_NO_PROXY_NAME, + WINHTTP_NO_PROXY_BYPASS, + 0)) { - auto h = WinHttpOpen( - L"vcpkg/1.0", WINHTTP_ACCESS_TYPE_NO_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); - if (!h) - { - return format_winhttp_last_error_message("WinHttpOpen", sanitized_url); - } - - ret.m_hSession = WinHttpHandle{h}; + return false; } // Increase default timeouts to help connections behind proxies // WinHttpSetTimeouts(HINTERNET hInternet, int nResolveTimeout, int nConnectTimeout, int nSendTimeout, int // nReceiveTimeout); - WinHttpSetTimeouts(ret.m_hSession.h, 0, 120000, 120000, 120000); + if (!m_hSession.SetTimeouts(context, sanitized_url, 0, 120000, 120000, 120000)) + { + return false; + } // If the environment variable HTTPS_PROXY is set // use that variable as proxy. This situation might exist when user is in a company network @@ -257,8 +323,10 @@ namespace vcpkg proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; proxy.lpszProxy = env_proxy_settings.data(); proxy.lpszProxyBypass = nullptr; - - WinHttpSetOption(ret.m_hSession.h, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) + { + return false; + } } // IE Proxy fallback, this works on Windows 10 else @@ -272,24 +340,36 @@ namespace vcpkg proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; proxy.lpszProxy = ieProxy.get()->server.data(); proxy.lpszProxyBypass = ieProxy.get()->bypass.data(); - WinHttpSetOption(ret.m_hSession.h, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) + { + return false; + } } } // Use Windows 10 defaults on Windows 7 DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - WinHttpSetOption( - ret.m_hSession.h, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); + if (!m_hSession.SetOption(context, + sanitized_url, + WINHTTP_OPTION_SECURE_PROTOCOLS, + &secure_protocols, + sizeof(secure_protocols))) + { + return false; + } // Many open source mirrors such as https://download.gnome.org/ will redirect to http mirrors. // `curl.exe -L` does follow https -> http redirection. // Additionally, vcpkg hash checks the resulting archive. DWORD redirect_policy(WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS); - WinHttpSetOption( - ret.m_hSession.h, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy)); + if (!m_hSession.SetOption( + context, sanitized_url, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy))) + { + return false; + } - return ret; + return true; } WinHttpHandle m_hSession; @@ -297,85 +377,223 @@ namespace vcpkg struct WinHttpConnection { - static ExpectedL make(HINTERNET hSession, - StringView hostname, - INTERNET_PORT port, - StringView sanitized_url) + bool connect(DiagnosticContext& context, + const WinHttpSession& hSession, + StringView hostname, + INTERNET_PORT port, + const SanitizedUrl& sanitized_url) { // Specify an HTTP server. - auto h = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), port, 0); - if (!h) + return m_hConnect.Connect(context, hSession.m_hSession, hostname, port, sanitized_url); + } + + WinHttpHandle m_hConnect; + }; + + struct WinHttpRequest + { + bool open(DiagnosticContext& context, + const WinHttpConnection& hConnect, + StringView path_query_fragment, + const SanitizedUrl& sanitized_url, + bool https, + const wchar_t* method = L"GET") + { + if (!m_hRequest.OpenRequest(context, + hConnect.m_hConnect, + sanitized_url, + method, + path_query_fragment, + nullptr, + WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, + https ? WINHTTP_FLAG_SECURE : 0)) { - return format_winhttp_last_error_message("WinHttpConnect", sanitized_url); + return false; + } + + // Send a request. + if (!m_hRequest.SendRequest( + context, sanitized_url, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0)) + { + return false; + } + + // End the request. + if (!m_hRequest.ReceiveResponse(context, sanitized_url)) + { + return false; } - return WinHttpConnection{WinHttpHandle{h}}; + return true; } - WinHttpHandle m_hConnect; + Optional query_status(DiagnosticContext& context, const SanitizedUrl& sanitized_url) const + { + DWORD status_code; + DWORD size = sizeof(status_code); + DWORD last_error = m_hRequest.QueryHeaders(context, + sanitized_url, + WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + WINHTTP_HEADER_NAME_BY_INDEX, + &status_code, + &size, + WINHTTP_NO_HEADER_INDEX); + if (last_error) + { + return nullopt; + } + + return status_code; + } + + bool query_content_length(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + Optional& result) const + { + static constexpr DWORD buff_characters = 21; // 18446744073709551615 + wchar_t buff[buff_characters]; + DWORD size = sizeof(buff); + AttemptDiagnosticContext adc{context}; + DWORD last_error = m_hRequest.QueryHeaders(adc, + sanitized_url, + WINHTTP_QUERY_CONTENT_LENGTH, + WINHTTP_HEADER_NAME_BY_INDEX, + buff, + &size, + WINHTTP_NO_HEADER_INDEX); + if (!last_error) + { + adc.commit(); + result = Strings::strto(Strings::to_utf8(buff, size >> 1)); + return true; + } + + if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) + { + adc.handle(); + return true; + } + + adc.commit(); + return false; + } + + WinHttpTrialResult write_response_body(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const SanitizedUrl& sanitized_url, + const WriteFilePointer& file) + { + static constexpr DWORD buff_size = 65535; + std::unique_ptr buff{new char[buff_size]}; + Optional maybe_content_length; + auto last_write = std::chrono::steady_clock::now(); + if (!query_content_length(context, sanitized_url, maybe_content_length)) + { + return WinHttpTrialResult::retry; + } + + unsigned long long total_downloaded_size = 0; + for (;;) + { + DWORD this_read; + if (!m_hRequest.ReadData(context, sanitized_url, buff.get(), buff_size, &this_read)) + { + return WinHttpTrialResult::retry; + } + + if (this_read == 0) + { + return WinHttpTrialResult::succeeded; + } + + do + { + const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); + if (this_write == 0) + { + context.report_error(format_filesystem_call_error( + std::error_code{errno, std::generic_category()}, "fwrite", {file.path()})); + return WinHttpTrialResult::failed; + } + + maybe_emit_winhttp_progress( + machine_readable_progress, maybe_content_length, last_write, total_downloaded_size); + this_read -= this_write; + total_downloaded_size += this_write; + } while (this_read > 0); + } + } + + WinHttpHandle m_hRequest; }; #endif - ExpectedL split_uri_view(StringView uri) + Optional parse_split_url_view(StringView raw_url) { - auto sep = std::find(uri.begin(), uri.end(), ':'); - if (sep == uri.end()) return msg::format_error(msgInvalidUri, msg::value = uri); + auto sep = std::find(raw_url.begin(), raw_url.end(), ':'); + if (sep == raw_url.end()) + { + return nullopt; + } - StringView scheme(uri.begin(), sep); - if (Strings::starts_with({sep + 1, uri.end()}, "//")) + StringView scheme(raw_url.begin(), sep); + if (Strings::starts_with({sep + 1, raw_url.end()}, "//")) { - auto path_start = std::find(sep + 3, uri.end(), '/'); - return SplitURIView{scheme, StringView{sep + 1, path_start}, {path_start, uri.end()}}; + auto path_start = std::find(sep + 3, raw_url.end(), '/'); + return SplitUrlView{scheme, StringView{sep + 1, path_start}, StringView{path_start, raw_url.end()}}; } + // no authority - return SplitURIView{scheme, {}, {sep + 1, uri.end()}}; + return SplitUrlView{scheme, {}, StringView{sep + 1, raw_url.end()}}; } - static ExpectedL try_verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView sanitized_url, - const Path& downloaded_path, - StringView sha512) + static bool check_downloaded_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const SanitizedUrl& sanitized_url, + const Path& downloaded_path, + StringView sha512) { - std::string actual_hash = - vcpkg::Hash::get_file_hash(fs, downloaded_path, Hash::Algorithm::Sha512).value_or_exit(VCPKG_LINE_INFO); - if (!Strings::case_insensitive_ascii_equals(sha512, actual_hash)) + auto maybe_actual_hash = + vcpkg::Hash::get_file_hash_required(context, fs, downloaded_path, Hash::Algorithm::Sha512); + if (auto actual_hash = maybe_actual_hash.get()) { - return msg::format_error(msgDownloadFailedHashMismatch, - msg::url = sanitized_url, - msg::path = downloaded_path, - msg::expected = sha512, - msg::actual = actual_hash); - } + if (Strings::case_insensitive_ascii_equals(sha512, *actual_hash)) + { + return true; + } - return Unit{}; - } + context.report(DiagnosticLine{DiagKind::Error, + downloaded_path, + msg::format(msgDownloadFailedHashMismatch, msg::url = sanitized_url)}); + context.report(DiagnosticLine{DiagKind::Note, + msg::format(msgDownloadFailedHashMismatchExpectedHash, msg::sha = sha512)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgDownloadFailedHashMismatchActualHash, msg::sha = *actual_hash)}); + } - void verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView url, - const Path& downloaded_path, - StringView sha512) - { - try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512).value_or_exit(VCPKG_LINE_INFO); + return false; } - static ExpectedL check_downloaded_file_hash(const ReadOnlyFilesystem& fs, - const Optional& hash, - StringView sanitized_url, - const Path& download_part_path) + static bool check_downloaded_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const SanitizedUrl& sanitized_url, + const Path& downloaded_path, + Optional maybe_sha512) { - if (auto p = hash.get()) + if (auto sha512 = maybe_sha512.get()) { - return try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p); + return check_downloaded_file_hash(context, fs, sanitized_url, downloaded_path, *sha512); } - Debug::println("Skipping hash check because none was specified."); - return Unit{}; + return true; } - static std::vector> curl_bulk_operation(View operation_args, - StringLiteral prefixArgs, - View headers, - View secrets) + static std::vector curl_bulk_operation(DiagnosticContext& context, + View operation_args, + StringLiteral prefixArgs, + View headers, + View secrets) { #define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c" static constexpr StringLiteral guid_marker = GUID_MARKER; @@ -389,14 +607,9 @@ namespace vcpkg GUID_MARKER "%{http_code} %{exitcode} %{errormsg}\\n"); #undef GUID_MARKER - std::vector> ret; + std::vector ret; ret.reserve(operation_args.size()); - - for (auto&& header : headers) - { - prefix_cmd.string_arg("-H").string_arg(header); - } - + add_curl_headers(prefix_cmd, headers); while (ret.size() != operation_args.size()) { // there's an edge case that we aren't handling here where not even one operation fits with the configured @@ -411,47 +624,36 @@ namespace vcpkg } // actually run curl + bool new_curl_seen = false; std::vector debug_lines; - auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(batch_cmd, [&](StringView line) { + auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(context, batch_cmd, [&](StringView line) { debug_lines.emplace_back(line.data(), line.size()); - parse_curl_status_line(ret, guid_marker, line); + new_curl_seen |= parse_curl_status_line(context, ret, guid_marker, line); }); if (auto this_batch_exit_code = maybe_this_batch_exit_code.get()) { - if (!ret.empty()) + if (!new_curl_seen) { - if (auto last_http_code = ret.back().get()) - { - if (*last_http_code == 0 && *this_batch_exit_code) - { - // old version of curl, we only have the result code for the last operation - ret.back() = msg::format(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); - } - } + // old version of curl, we only have the result code for the last operation + context.report_error(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); } if (ret.size() != last_try_op) { // curl didn't process everything we asked of it; this usually means curl crashed - auto full_failure = - msg::format_error(msgCurlFailedToReturnExpectedNumberOfExitCodes, - msg::exit_code = *this_batch_exit_code, - msg::command_line = replace_secrets(std::move(batch_cmd).extract(), secrets)); - for (const auto& debug_line : debug_lines) - { - full_failure.append_raw('\n'); - full_failure.append_raw(debug_line); - } - - ret.emplace_back(std::move(full_failure)); + auto command_line = std::move(batch_cmd).extract(); + replace_secrets(command_line, secrets); + context.report_error_with_log(Strings::join("\n", debug_lines), + msgCurlFailedToReturnExpectedNumberOfExitCodes, + msg::exit_code = *this_batch_exit_code, + msg::command_line = command_line); return ret; } } else { // couldn't even launch curl, record this as the last fatal error and give up - ret.emplace_back(std::move(maybe_this_batch_exit_code).error()); return ret; } } @@ -459,20 +661,26 @@ namespace vcpkg return ret; } - std::vector> url_heads(View urls, View headers, View secrets) + std::vector url_heads(DiagnosticContext& context, + View urls, + View headers, + View secrets) { return curl_bulk_operation( + context, Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }), "--head", headers, secrets); } - std::vector> download_files(View> url_pairs, - View headers, - View secrets) + std::vector download_files_no_cache(DiagnosticContext& context, + View> url_pairs, + View headers, + View secrets) { - return curl_bulk_operation(Util::fmap(url_pairs, + return curl_bulk_operation(context, + Util::fmap(url_pairs, [](const std::pair& url_pair) { return Command{} .string_arg(url_encode_spaces(url_pair.first)) @@ -484,7 +692,8 @@ namespace vcpkg secrets); } - bool submit_github_dependency_graph_snapshot(const Optional& maybe_github_server_url, + bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, + const Optional& maybe_github_server_url, const std::string& github_token, const std::string& github_repository, const Json::Object& snapshot) @@ -508,26 +717,26 @@ namespace vcpkg auto cmd = Command{"curl"}; cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); cmd.string_arg("-X").string_arg("POST"); - cmd.string_arg("-H").string_arg("Accept: application/vnd.github+json"); + { + std::string headers[] = { + "Accept: application/vnd.github+json", + "Authorization: Bearer " + github_token, + "X-GitHub-Api-Version: 2022-11-28", + }; + add_curl_headers(cmd, headers); + } - std::string res = "Authorization: Bearer " + github_token; - cmd.string_arg("-H").string_arg(res); - cmd.string_arg("-H").string_arg("X-GitHub-Api-Version: 2022-11-28"); cmd.string_arg(uri); cmd.string_arg("-d").string_arg("@-"); RedirectedProcessLaunchSettings settings; settings.stdin_content = Json::stringify(snapshot); int code = 0; - auto result = cmd_execute_and_stream_lines(cmd, settings, [&code](StringView line) { + auto result = cmd_execute_and_stream_lines(context, cmd, settings, [&code](StringView line) { if (Strings::starts_with(line, guid_marker)) { code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); } - else - { - Debug::println(line); - } }); auto r = result.get(); @@ -538,93 +747,100 @@ namespace vcpkg return false; } - ExpectedL put_file(const ReadOnlyFilesystem&, - StringView url, - const std::vector& secrets, - View headers, - const Path& file, - StringView method) + static bool store_to_asset_cache_impl(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file) { static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - if (Strings::starts_with(url, "ftp://")) + if (Strings::starts_with(raw_url, "ftp://")) { // HTTP headers are ignored for FTP clients auto ftp_cmd = Command{"curl"}; - ftp_cmd.string_arg(url_encode_spaces(url)); + ftp_cmd.string_arg(url_encode_spaces(raw_url)); ftp_cmd.string_arg("-T").string_arg(file); - auto maybe_res = cmd_execute_and_capture_output(ftp_cmd); + auto maybe_res = cmd_execute_and_capture_output(context, ftp_cmd); if (auto res = maybe_res.get()) { if (res->exit_code == 0) { - return 0; + return true; } - Debug::print(res->output, '\n'); - return msg::format_error(msgCurlFailedToPut, - msg::exit_code = res->exit_code, - msg::url = replace_secrets(url.to_string(), secrets)); + context.report_error_with_log( + res->output, msgCurlFailedToPut, msg::exit_code = res->exit_code, msg::url = sanitized_url); + return false; } - return std::move(maybe_res).error(); + return false; } auto http_cmd = Command{"curl"}.string_arg("-X").string_arg(method); - for (auto&& header : headers) - { - http_cmd.string_arg("-H").string_arg(header); - } - + add_curl_headers(http_cmd, headers); http_cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - http_cmd.string_arg(url); + http_cmd.string_arg(raw_url); http_cmd.string_arg("-T").string_arg(file); int code = 0; - auto res = cmd_execute_and_stream_lines(http_cmd, [&code](StringView line) { + auto res = cmd_execute_and_stream_lines(context, http_cmd, [&code](StringView line) { if (Strings::starts_with(line, guid_marker)) { code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); } }); - if (auto pres = res.get()) + auto pres = res.get(); + if (!pres) + { + return false; + } + + if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) + { + context.report_error(msg::format( + msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); + return false; + } + + return true; + } + + bool store_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file) + { + if (store_to_asset_cache_impl(context, raw_url, sanitized_url, method, headers, file)) { - if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) - { - return msg::format_error( - msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); - } + context.statusln(msg::format(msgAssetCacheSuccesfullyStored)); + return true; } - msg::println(msgAssetCacheSuccesfullyStored, - msg::path = file.filename(), - msg::url = replace_secrets(url.to_string(), secrets)); - return 0; + + return false; } std::string format_url_query(StringView base_url, View query_params) { - auto url = base_url.to_string(); if (query_params.empty()) { - return url; + return base_url.to_string(); } - return url + "?" + Strings::join("&", query_params); + return fmt::format(FMT_COMPILE("{}?{}"), base_url, fmt::join(query_params, "&")); } - ExpectedL invoke_http_request(StringView method, - View headers, - StringView url, - StringView data) + Optional invoke_http_request(DiagnosticContext& context, + StringLiteral method, + View headers, + StringView raw_url, + StringView data) { auto cmd = Command{"curl"}.string_arg("-s").string_arg("-L"); - cmd.string_arg("-H").string_arg( - fmt::format("User-Agent: vcpkg/{}-{} (curl)", VCPKG_BASE_VERSION_AS_STRING, VCPKG_VERSION_AS_STRING)); - - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } + add_curl_headers(cmd, headers); cmd.string_arg("-X").string_arg(method); @@ -633,92 +849,78 @@ namespace vcpkg cmd.string_arg("--data-raw").string_arg(data); } - cmd.string_arg(url_encode_spaces(url)); + cmd.string_arg(url_encode_spaces(raw_url)); + + auto maybe_output = cmd_execute_and_capture_output(context, cmd); + if (auto output = check_zero_exit_code(context, maybe_output, "curl")) + { + return *output; + } - return flatten_out(cmd_execute_and_capture_output(cmd), "curl"); + return nullopt; } #if defined(_WIN32) - enum class WinHttpTrialResult - { - failed, - succeeded, - retry - }; - - static WinHttpTrialResult download_winhttp_trial(const Filesystem& fs, - WinHttpSession& s, + static WinHttpTrialResult download_winhttp_trial(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, + const WinHttpSession& s, const Path& download_path_part_path, - SplitURIView split_uri, + SplitUrlView split_uri_view, StringView hostname, INTERNET_PORT port, - StringView sanitized_url, - std::vector& errors, - MessageSink& progress_sink) + const SanitizedUrl& sanitized_url) { - auto maybe_conn = WinHttpConnection::make(s.m_hSession.h, hostname, port, sanitized_url); - const auto conn = maybe_conn.get(); - if (!conn) + WinHttpConnection conn; + if (!conn.connect(context, s, hostname, port, sanitized_url)) { - errors.push_back(std::move(maybe_conn).error()); return WinHttpTrialResult::retry; } - auto maybe_req = WinHttpRequest::make( - conn->m_hConnect.h, split_uri.path_query_fragment, sanitized_url, split_uri.scheme == "https"); - const auto req = maybe_req.get(); - if (!req) + WinHttpRequest req; + if (!req.open( + context, conn, split_uri_view.path_query_fragment, sanitized_url, split_uri_view.scheme == "https")) { - errors.push_back(std::move(maybe_req).error()); return WinHttpTrialResult::retry; } - auto maybe_status = req->query_status(); + auto maybe_status = req.query_status(context, sanitized_url); const auto status = maybe_status.get(); if (!status) { - errors.push_back(std::move(maybe_status).error()); return WinHttpTrialResult::retry; } if (*status < 200 || *status >= 300) { - errors.push_back( - msg::format_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status)); + context.report_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status); return WinHttpTrialResult::failed; } - auto f = fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO); - auto maybe_write = req->write_response_body(f, progress_sink); - const auto write = maybe_write.get(); - if (!write) - { - errors.push_back(std::move(maybe_write).error()); - return WinHttpTrialResult::retry; - } - - return WinHttpTrialResult::succeeded; + return req.write_response_body(context, + machine_readable_progress, + sanitized_url, + fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO)); } /// /// Download a file using WinHTTP -- only supports HTTP and HTTPS /// - static bool download_winhttp(const Filesystem& fs, + static bool download_winhttp(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, const Path& download_path_part_path, - SplitURIView split_uri, - const std::string& url, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) + SplitUrlView split_url_view, + const SanitizedUrl& sanitized_url) { // `download_winhttp` does not support user or port syntax in authorities - auto hostname = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); + auto hostname = split_url_view.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); INTERNET_PORT port; - if (split_uri.scheme == "https") + if (split_url_view.scheme == "https") { port = INTERNET_DEFAULT_HTTPS_PORT; } - else if (split_uri.scheme == "http") + else if (split_url_view.scheme == "http") { port = INTERNET_DEFAULT_HTTP_PORT; } @@ -731,39 +933,111 @@ namespace vcpkg const auto dir = download_path_part_path.parent_path(); fs.create_directories(dir, VCPKG_LINE_INFO); - const auto sanitized_url = replace_secrets(url, secrets); - static auto s = WinHttpSession::make(sanitized_url).value_or_exit(VCPKG_LINE_INFO); - for (size_t trials = 0; trials < 4; ++trials) + WinHttpSession s; + if (!s.open(context, sanitized_url)) { - if (trials > 0) - { - // 1s, 2s, 4s - const auto trialMs = 500 << trials; - msg::println_warning(msgDownloadFailedRetrying, msg::value = trialMs); - std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); - } + return false; + } + + AttemptDiagnosticContext adc{context}; + switch (download_winhttp_trial(context, + machine_readable_progress, + fs, + s, + download_path_part_path, + split_url_view, + hostname, + port, + sanitized_url)) + { + case WinHttpTrialResult::succeeded: adc.commit(); return true; + case WinHttpTrialResult::failed: adc.commit(); return false; + case WinHttpTrialResult::retry: break; + } - switch (download_winhttp_trial( - fs, s, download_path_part_path, split_uri, hostname, port, sanitized_url, errors, progress_sink)) + for (size_t trials = 1; trials < 4; ++trials) + { + // 1s, 2s, 4s + const auto trialMs = 500 << trials; + adc.handle(); + adc.statusln(msg::format_warning(msgDownloadFailedRetrying, msg::value = trialMs)); + std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); + switch (download_winhttp_trial(context, + machine_readable_progress, + fs, + s, + download_path_part_path, + split_url_view, + hostname, + port, + sanitized_url)) { - case WinHttpTrialResult::failed: return false; - case WinHttpTrialResult::succeeded: return true; + case WinHttpTrialResult::succeeded: adc.commit(); return true; + case WinHttpTrialResult::failed: adc.commit(); return false; case WinHttpTrialResult::retry: break; } } + adc.commit(); return false; } #endif - static bool try_download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) + enum class DownloadPrognosis + { + Success, + OtherError, + NetworkErrorProxyMightHelp, + HashMismatch + }; + + static bool check_combine_download_prognosis(DownloadPrognosis& target, DownloadPrognosis individual_call) + { + switch (individual_call) + { + case DownloadPrognosis::Success: return true; + case DownloadPrognosis::OtherError: + if (target == DownloadPrognosis::Success) + { + target = DownloadPrognosis::OtherError; + } + + return false; + case DownloadPrognosis::NetworkErrorProxyMightHelp: + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::OtherError) + { + target = DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + return false; + case DownloadPrognosis::HashMismatch: + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::OtherError || + target == DownloadPrognosis::NetworkErrorProxyMightHelp) + { + target = DownloadPrognosis::HashMismatch; + } + + return false; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + static void maybe_report_proxy_might_help(DiagnosticContext& context, DownloadPrognosis prognosis) + { + if (prognosis == DownloadPrognosis::NetworkErrorProxyMightHelp) + { + context.report(DiagnosticLine{DiagKind::Note, msg::format(msgDownloadFailedProxySettings)}); + } + } + + static DownloadPrognosis try_download_file(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, + StringView raw_url, + const SanitizedUrl& sanitized_url, + View headers, + const Path& download_path, + const Optional& maybe_sha512) { auto download_path_part_path = download_path; download_path_part_path += "."; @@ -777,40 +1051,51 @@ namespace vcpkg #if defined(_WIN32) auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); bool needs_proxy_auth = false; - if (maybe_https_proxy_env) + if (auto proxy_url = maybe_https_proxy_env.get()) { - const auto& proxy_url = maybe_https_proxy_env.value_or_exit(VCPKG_LINE_INFO); - needs_proxy_auth = proxy_url.find('@') != std::string::npos; + needs_proxy_auth = proxy_url->find('@') != std::string::npos; } if (headers.size() == 0 && !needs_proxy_auth) { - auto split_uri = split_uri_view(url).value_or_exit(VCPKG_LINE_INFO); - if (split_uri.scheme == "https" || split_uri.scheme == "http") + auto maybe_split_uri_view = parse_split_url_view(raw_url); + auto split_uri_view = maybe_split_uri_view.get(); + if (!split_uri_view) + { + context.report_error(msgInvalidUri, msg::value = sanitized_url); + return DownloadPrognosis::OtherError; + } + + if (split_uri_view->scheme == "https" || split_uri_view->scheme == "http") { - auto maybe_authority = split_uri.authority.get(); + auto maybe_authority = split_uri_view->authority.get(); if (!maybe_authority) { - Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgInvalidUri, msg::value = url); + context.report_error(msg::format(msgInvalidUri, msg::value = sanitized_url)); + return DownloadPrognosis::OtherError; } - auto authority = maybe_authority->substr(2); - // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to curl.exe + auto authority = StringView{*maybe_authority}.substr(2); + // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to + // curl.exe if (Strings::find_first_of(authority, ":@") == authority.end()) { - if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors, progress_sink)) + if (!download_winhttp(context, + machine_readable_progress, + fs, + download_path_part_path, + *split_uri_view, + sanitized_url)) { - auto maybe_hash_check = check_downloaded_file_hash(fs, sha512, url, download_path_part_path); - if (maybe_hash_check.has_value()) - { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return true; - } - else - { - errors.push_back(std::move(maybe_hash_check).error()); - } + return DownloadPrognosis::NetworkErrorProxyMightHelp; } - return false; + + if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512)) + { + return DownloadPrognosis::HashMismatch; + } + + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } } } @@ -818,78 +1103,49 @@ namespace vcpkg auto cmd = Command{"curl"} .string_arg("--fail") .string_arg("-L") - .string_arg(url_encode_spaces(url)) + .string_arg(url_encode_spaces(raw_url)) .string_arg("--create-dirs") .string_arg("--output") .string_arg(download_path_part_path); - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } - - std::string non_progress_content; - auto maybe_exit_code = cmd_execute_and_stream_lines(cmd, [&](StringView line) { + add_curl_headers(cmd, headers); + std::string likely_curl_errors; + auto maybe_exit_code = cmd_execute_and_stream_lines(context, cmd, [&](StringView line) { const auto maybe_parsed = try_parse_curl_progress_data(line); if (const auto parsed = maybe_parsed.get()) { - progress_sink.print(Color::none, fmt::format("{}%\n", parsed->total_percent)); + machine_readable_progress.println(Color::none, + LocalizedString::from_raw(fmt::format("{}%", parsed->total_percent))); } - else + else if (Strings::starts_with(line, "curl: ")) { - non_progress_content.append(line.data(), line.size()); - non_progress_content.push_back('\n'); + if (!likely_curl_errors.empty()) + { + likely_curl_errors.push_back('\n'); + } + + likely_curl_errors.append(line.data(), line.size()); } }); - const auto sanitized_url = replace_secrets(url, secrets); - if (const auto exit_code = maybe_exit_code.get()) + const auto exit_code = maybe_exit_code.get(); + if (!exit_code) { - if (*exit_code != 0) - { - errors.push_back( - msg::format_error(msgDownloadFailedCurl, msg::url = sanitized_url, msg::exit_code = *exit_code) - .append_raw('\n') - .append_raw(non_progress_content)); - return false; - } - - auto maybe_hash_check = check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path); - if (maybe_hash_check.has_value()) - { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return true; - } - else - { - errors.push_back(std::move(maybe_hash_check).error()); - } + return DownloadPrognosis::OtherError; } - else + + if (*exit_code != 0) { - errors.push_back(std::move(maybe_exit_code).error()); + context.report(DiagnosticLine{DiagKind::Error, LocalizedString::from_raw(std::move(likely_curl_errors))}); + return DownloadPrognosis::NetworkErrorProxyMightHelp; } - return false; - } - - static Optional try_download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) - { - for (auto&& url : urls) + if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512)) { - if (try_download_file(fs, url, headers, download_path, sha512, secrets, errors, progress_sink)) - { - return url; - } + return DownloadPrognosis::HashMismatch; } - return nullopt; + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } View azure_blob_headers() @@ -898,11 +1154,14 @@ namespace vcpkg return s_headers; } - void parse_curl_status_line(std::vector>& http_codes, StringLiteral prefix, StringView this_line) + bool parse_curl_status_line(DiagnosticContext& context, + std::vector& http_codes, + StringLiteral prefix, + StringView this_line) { if (!Strings::starts_with(this_line, prefix)) { - return; + return false; } auto first = this_line.begin(); @@ -917,7 +1176,7 @@ namespace vcpkg { // this output is broken, even if we don't know %{exit_code} or ${errormsg}, the spaces in front // of them should still be printed. - return; + return false; } if (!ParserBase::is_ascii_digit(*first)) @@ -930,20 +1189,20 @@ namespace vcpkg if (*first != ' ' || ++first == last) { // didn't see the space after the http_code - return; + return false; } if (*first == ' ') { // old curl that doesn't understand %{exit_code}, this is the space after it http_codes.emplace_back(http_code); - return; + return false; } if (!ParserBase::is_ascii_digit(*first)) { // not exit_code - return; + return false; } const auto first_exit_code = first; @@ -952,204 +1211,521 @@ namespace vcpkg if (++first == last) { // didn't see the space after %{exit_code} - return; + return false; } if (*first == ' ') { // the space after exit_code, everything after this space is the error message if any + http_codes.emplace_back(http_code); auto exit_code = Strings::strto(StringView{first_exit_code, first}).value_or_exit(VCPKG_LINE_INFO); - if (exit_code == 0) + // note that this gets the space out of the output :) + if (exit_code != 0) { - // success! - http_codes.emplace_back(http_code); - return; + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code) + .append_raw(StringView{first, last})); } - // note that this gets the space out of the output :) - http_codes.emplace_back( - msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code).append_raw(StringView{first, last})); - return; + return true; } if (!ParserBase::is_ascii_digit(*first)) { // non numeric exit_code? - return; + return false; } } } - bool DownloadManager::get_block_origin() const { return m_config.m_block_origin; } + static DownloadPrognosis download_file_azurl_asset_cache(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + auto read_template = asset_cache_settings.m_read_url_template.get(); + auto sha512 = maybe_sha512.get(); + if (!read_template || !sha512) + { + // can't use http asset caches when none are configured or we don't have a SHA + return DownloadPrognosis::OtherError; + } - bool DownloadManager::asset_cache_configured() const { return m_config.m_read_url_template.has_value(); } + auto raw_read_url = Strings::replace_all(*read_template, "", *sha512); + SanitizedUrl sanitized_read_url{raw_read_url, asset_cache_settings.m_secrets}; + context.statusln(msg::format(msgAssetCacheConsult, msg::path = target_filename, msg::url = sanitized_read_url)); + return try_download_file(context, + machine_readable_progress, + fs, + raw_read_url, + sanitized_read_url, + asset_cache_settings.m_read_headers, + download_path, + maybe_sha512); + } - void DownloadManager::download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const + static void report_script_while_command_line(DiagnosticContext& context, const std::string& raw_command) { - this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgWhileRunningAssetCacheScriptCommandLine).append_raw(": ").append_raw(raw_command)}); } - std::string DownloadManager::download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const + static void report_script_failed_to_make_file(DiagnosticContext& context, + const std::string& raw_command, + const Path& download_path_part_path) { - std::vector errors; - bool block_origin_enabled = m_config.m_block_origin; + context.report(DiagnosticLine{ + DiagKind::Error, download_path_part_path, msg::format(msgAssetCacheScriptFailedToWriteFile)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(raw_command)}); + } - if (urls.size() == 0) + static DownloadPrognosis download_file_script_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + const std::vector& sanitized_urls, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + using Hash::HashPrognosis; + auto script = asset_cache_settings.m_script.get(); + if (!script) { - if (auto hash = sha512.get()) - { - errors.push_back(msg::format_error(msgNoUrlsAndHashSpecified, msg::sha = *hash)); - } - else - { - errors.push_back(msg::format_error(msgNoUrlsAndNoHashSpecified)); - } + return DownloadPrognosis::OtherError; } - if (auto hash = sha512.get()) + if (raw_urls.empty() && !maybe_sha512.has_value()) { - if (auto read_template = m_config.m_read_url_template.get()) + Checks::unreachable(VCPKG_LINE_INFO); + } + + context.statusln(msg::format(msgAssetCacheConsultScript, msg::path = target_filename)); + const auto download_path_part_path = fmt::format("{}.{}.part", download_path, get_process_id()); + Lazy escaped_url; + const auto escaped_dpath = Command(download_path_part_path).extract(); + auto maybe_raw_command = api_stable_format(context, *script, [&](std::string& out, StringView key) { + if (key == "url") { - auto read_url = Strings::replace_all(*read_template, "", *hash); - if (try_download_file(fs, - read_url, - m_config.m_read_headers, - download_path, - sha512, - m_config.m_secrets, - errors, - progress_sink)) - { - msg::println(msgAssetCacheHit, - msg::path = download_path.filename(), - msg::url = replace_secrets(read_url, m_config.m_secrets)); - return read_url; - } - else if (block_origin_enabled) + if (raw_urls.empty()) { - msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); + context.report_error(msg::format(msgAssetCacheScriptNeedsUrl, + msg::value = *script, + msg::sha = maybe_sha512.value_or_exit(VCPKG_LINE_INFO))); + return false; } - else + + Strings::append(out, escaped_url.get_lazy([&] { return Command(raw_urls[0]).extract(); })); + return true; + } + + if (key == "sha512") + { + if (auto sha512 = maybe_sha512.get()) { - msg::println(msgAssetCacheMiss, msg::url = urls[0]); + Strings::append_ascii_lowercase(out, *sha512); + return true; } + + context.report_error( + msg::format(msgAssetCacheScriptNeedsSha, msg::value = *script, msg::url = sanitized_urls[0])); + return false; } - else if (auto script = m_config.m_script.get()) + + if (key == "dst") { - if (urls.size() != 0) - { - const auto download_path_part_path = download_path + fmt::format(".{}.part", get_process_id()); - const auto escaped_url = Command(urls[0]).extract(); - auto escaped_sha512 = Command(*hash).extract(); - Strings::inplace_ascii_to_lowercase(escaped_sha512); - const auto escaped_dpath = Command(download_path_part_path).extract(); - Command cmd; - cmd.raw_arg(api_stable_format(*script, [&](std::string& out, StringView key) { - if (key == "url") - { - Strings::append(out, escaped_url); - } - else if (key == "sha512") - { - Strings::append(out, escaped_sha512); - } - else if (key == "dst") - { - Strings::append(out, escaped_dpath); - } - }).value_or_exit(VCPKG_LINE_INFO)); - - RedirectedProcessLaunchSettings settings; - settings.environment = get_clean_environment(); - settings.echo_in_debug = EchoInDebug::Show; - - auto maybe_res = flatten(cmd_execute_and_capture_output(cmd, settings), ""); - if (maybe_res) - { - auto maybe_success = - try_verify_downloaded_file_hash(fs, "", download_path_part_path, *hash); - if (maybe_success) - { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - msg::println(msgDownloadSuccesful, msg::path = download_path.filename()); - return urls[0]; - } - msg::println_error(maybe_success.error()); - } - else + Strings::append(out, escaped_dpath); + return true; + } + + context.report_error(msg::format(msgAssetCacheScriptBadVariable, msg::value = *script, msg::list = key)); + context.report( + DiagnosticLine{DiagKind::Note, msg::format(msgAssetCacheScriptBadVariableHint, msg::list = key)}); + return false; + }); + + auto raw_command = maybe_raw_command.get(); + if (!raw_command) + { + return DownloadPrognosis::OtherError; + } + + Command cmd; + cmd.raw_arg(*raw_command); + RedirectedProcessLaunchSettings settings; + settings.environment = get_clean_environment(); + auto maybe_res = cmd_execute_and_stream_lines( + context, cmd, settings, [&](StringView line) { context.statusln(LocalizedString::from_raw(line)); }); + auto res = maybe_res.get(); + if (!res) + { + report_script_while_command_line(context, *raw_command); + return DownloadPrognosis::OtherError; + } + + if (*res != 0) + { + context.report_error(msg::format(msgAssetCacheScriptFailed, msg::exit_code = *res)); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(*raw_command)}); + return DownloadPrognosis::OtherError; + } + + auto sha512 = maybe_sha512.get(); + if (sha512) + { + auto hash_result = Hash::get_file_hash(context, fs, download_path_part_path, Hash::Algorithm::Sha512); + switch (hash_result.prognosis) + { + case HashPrognosis::Success: + if (Strings::case_insensitive_ascii_equals(*sha512, hash_result.hash)) { - msg::println_error(maybe_res.error()); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } - } + + context.report(DiagnosticLine{DiagKind::Error, + download_path_part_path, + msg::format(msgAssetCacheScriptFailedToWriteCorrectHash)}); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(*raw_command)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgDownloadFailedHashMismatchExpectedHash, msg::sha = *sha512)}); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgDownloadFailedHashMismatchActualHash, msg::sha = hash_result.hash)}); + return DownloadPrognosis::HashMismatch; + case HashPrognosis::FileNotFound: + report_script_failed_to_make_file(context, *raw_command, download_path_part_path); + return DownloadPrognosis::OtherError; + case HashPrognosis::OtherError: + report_script_while_command_line(context, *raw_command); + return DownloadPrognosis::OtherError; + default: Checks::unreachable(VCPKG_LINE_INFO); } } - if (block_origin_enabled) + if (fs.exists(download_path_part_path, VCPKG_LINE_INFO)) { - msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } - else + + report_script_failed_to_make_file(context, *raw_command, download_path_part_path); + return DownloadPrognosis::OtherError; + } + + static DownloadPrognosis download_file_asset_cache(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + const std::vector& sanitized_urls, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + switch (download_file_azurl_asset_cache( + context, machine_readable_progress, asset_cache_settings, fs, download_path, target_filename, maybe_sha512)) + { + case DownloadPrognosis::Success: return DownloadPrognosis::Success; + case DownloadPrognosis::OtherError: + return download_file_script_asset_cache(context, + asset_cache_settings, + fs, + raw_urls, + sanitized_urls, + download_path, + target_filename, + maybe_sha512); + case DownloadPrognosis::NetworkErrorProxyMightHelp: return DownloadPrognosis::NetworkErrorProxyMightHelp; + case DownloadPrognosis::HashMismatch: return DownloadPrognosis::HashMismatch; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + static void report_download_success_and_maybe_upload(DiagnosticContext& context, + const Path& download_path, + StringView target_filename, + const AssetCachingSettings& asset_cache_settings, + const Optional& maybe_sha512) + { + auto hash = maybe_sha512.get(); + auto url_template = asset_cache_settings.m_write_url_template.get(); + if (hash && url_template && !url_template->empty()) { - if (urls.size() != 0) + auto raw_upload_url = Strings::replace_all(*url_template, "", *hash); + SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; + context.statusln(msg::format( + msgDownloadSuccesfulUploading, msg::path = target_filename, msg::url = sanitized_upload_url)); + WarningDiagnosticContext wdc{context}; + if (!store_to_asset_cache(wdc, + raw_upload_url, + sanitized_upload_url, + "PUT", + asset_cache_settings.m_write_headers, + download_path)) { - msg::println(msgDownloadingUrl, msg::url = download_path.filename()); - auto maybe_url = try_download_file( - fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); - if (auto url = maybe_url.get()) - { - msg::println(msgDownloadSuccesful, msg::path = download_path.filename()); + context.report(DiagnosticLine{DiagKind::Warning, + msg::format(msgFailedToStoreBackToMirror, + msg::path = target_filename, + msg::url = sanitized_upload_url)}); + } + } + else + { + context.statusln(msg::format(msgDownloadSuccesful, msg::path = target_filename)); + } + } - if (auto hash = sha512.get()) - { - auto maybe_push = put_file_to_mirror(fs, download_path, *hash); - if (!maybe_push) - { - msg::println_warning(msgFailedToStoreBackToMirror, - msg::path = download_path.filename(), - msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); - msg::println(maybe_push.error()); - } - } + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const std::string& url, + View headers, + const Path& download_path, + const Optional& maybe_sha512) + { + return download_file_asset_cached(context, + machine_readable_progress, + asset_cache_settings, + fs, + View(&url, 1), + headers, + download_path, + maybe_sha512); + } + + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + View headers, + const Path& download_path, + const Optional& maybe_sha512) + { + // Note: no secrets for the input URLs + std::vector sanitized_urls = + Util::fmap(raw_urls, [&](const std::string& url) { return SanitizedUrl{url, {}}; }); + const auto last_sanitized_url = sanitized_urls.end(); + const auto target_filename = download_path.filename(); + bool can_read_asset_cache = false; + if (asset_cache_settings.m_read_url_template.has_value() && maybe_sha512.has_value()) + { + // url asset cache reads need a hash + can_read_asset_cache = true; + } + + if (asset_cache_settings.m_script.has_value() && (maybe_sha512.has_value() || !raw_urls.empty())) + { + // script asset cache reads need either a hash or a URL + can_read_asset_cache = true; + } - return *url; + if (raw_urls.empty()) + { + // try to fetch from asset cache only without a known URL + if (auto sha512 = maybe_sha512.get()) + { + if (can_read_asset_cache) + { + context.statusln( + msg::format(msgDownloadingAssetShaToFile, msg::sha = *sha512, msg::path = download_path)); } else { - msg::println(msgDownloadFailedProxySettings, - msg::path = download_path.filename(), - msg::url = "https://github.com/microsoft/vcpkg-tool/pull/77"); + context.report_error(msg::format( + msgDownloadingAssetShaWithoutAssetCache, msg::sha = *sha512, msg::path = download_path)); + return false; } } + else + { + context.report_error(msgNoUrlsAndNoHashSpecified); + return false; + } + } + + if (asset_cache_settings.m_block_origin && !can_read_asset_cache) + { + // this will emit msgAssetCacheMissBlockOrigin below, this message just ensures the filename is mentioned in + // the output at all + context.statusln(msg::format(msgDownloadingFile, msg::path = target_filename)); + } + + DownloadPrognosis prognosis = DownloadPrognosis::Success; + // the asset cache downloads might fail, but that's OK if we can download the file from an authoritative source + AttemptDiagnosticContext asset_cache_attempt_context{context}; + if (check_combine_download_prognosis(prognosis, + download_file_asset_cache(asset_cache_attempt_context, + machine_readable_progress, + asset_cache_settings, + fs, + raw_urls, + sanitized_urls, + download_path, + target_filename, + maybe_sha512))) + { + asset_cache_attempt_context.commit(); + if (raw_urls.empty()) + { + context.statusln(msg::format(msgAssetCacheHit)); + return true; + } + + auto first_sanitized_url = sanitized_urls.begin(); + LocalizedString overall_url; + overall_url.append_raw(first_sanitized_url->to_string()); + while (++first_sanitized_url != last_sanitized_url) + { + overall_url.append_raw(' ') + .append(msgDownloadOrUrl) + .append_raw(' ') + .append_raw(first_sanitized_url->to_string()); + } + + context.statusln(msg::format(msgAssetCacheHitUrl, msg::url = overall_url)); + return true; + } + + if (prognosis == DownloadPrognosis::HashMismatch) + { + return false; + } + + if (raw_urls.empty()) + { + asset_cache_attempt_context.commit(); + context.report_error( + msg::format(msgAssetCacheMissNoUrls, msg::sha = maybe_sha512.value_or_exit(VCPKG_LINE_INFO))); + maybe_report_proxy_might_help(context, prognosis); + return false; + } + + if (asset_cache_settings.m_block_origin) + { + asset_cache_attempt_context.commit(); + auto first_sanitized_url = sanitized_urls.begin(); + context.report_error(msg::format(msgAssetCacheMissBlockOrigin, msg::url = *first_sanitized_url)); + while (++first_sanitized_url != last_sanitized_url) + { + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgDownloadOrUrl).append_raw(' ').append_raw(first_sanitized_url->to_string())}); + } + + maybe_report_proxy_might_help(context, prognosis); + return false; + } + + auto first_raw_url = raw_urls.begin(); + const auto last_raw_url = raw_urls.end(); + auto first_sanitized_url = sanitized_urls.begin(); + AttemptDiagnosticContext authoritative_attempt_context{context}; + if (can_read_asset_cache) + { + context.statusln(msg::format(msgAssetCacheMiss, msg::url = *first_sanitized_url)); + } + else if (raw_urls.size() == 1) + { + context.statusln( + msg::format(msgDownloadingUrlToFile, msg::url = *first_sanitized_url, msg::path = target_filename)); + } + else + { + context.statusln(msg::format(msgDownloadingFileFirstAuthoritativeSource, + msg::path = target_filename, + msg::url = *first_sanitized_url)); + } + + if (check_combine_download_prognosis(prognosis, + try_download_file(authoritative_attempt_context, + machine_readable_progress, + fs, + *first_raw_url, + *first_sanitized_url, + headers, + download_path, + maybe_sha512))) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.handle(); + report_download_success_and_maybe_upload( + context, download_path, target_filename, asset_cache_settings, maybe_sha512); + return true; + } + + if (prognosis == DownloadPrognosis::HashMismatch) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.commit(); + return false; } - for (LocalizedString& error : errors) + while (++first_sanitized_url, ++first_raw_url != last_raw_url) { - msg::println(error); + context.statusln(msg::format(msgDownloadTryingAuthoritativeSource, msg::url = *first_sanitized_url)); + if (check_combine_download_prognosis(prognosis, + try_download_file(authoritative_attempt_context, + machine_readable_progress, + fs, + *first_raw_url, + *first_sanitized_url, + headers, + download_path, + maybe_sha512))) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.handle(); + report_download_success_and_maybe_upload( + context, download_path, target_filename, asset_cache_settings, maybe_sha512); + return true; + } + + if (prognosis == DownloadPrognosis::HashMismatch) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.commit(); + return false; + } } - Checks::exit_fail(VCPKG_LINE_INFO); + asset_cache_attempt_context.commit(); + authoritative_attempt_context.commit(); + maybe_report_proxy_might_help(context, prognosis); + return false; } - ExpectedL DownloadManager::put_file_to_mirror(const ReadOnlyFilesystem& fs, - const Path& file_to_put, - StringView sha512) const + bool store_to_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Path& file_to_put, + StringView sha512) { - auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "", sha512); - if (!maybe_mirror_url.empty()) + if (auto url_template = asset_cache_settings.m_write_url_template.get()) { - return put_file(fs, maybe_mirror_url, m_config.m_secrets, m_config.m_write_headers, file_to_put); + if (url_template->empty()) + { + return true; + } + + auto raw_upload_url = Strings::replace_all(*url_template, "", sha512); + SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; + return store_to_asset_cache(context, + raw_upload_url, + sanitized_upload_url, + "PUT", + asset_cache_settings.m_write_headers, + file_to_put); } - return 0; + + return true; } Optional try_parse_curl_max5_size(StringView sv) @@ -1267,8 +1843,8 @@ namespace vcpkg const auto last = curl_progress_line.end(); if (parse_curl_uint_impl(result.total_percent, first, last) || parse_curl_max5_impl(result.total_size, first, last) || - parse_curl_uint_impl(result.recieved_percent, first, last) || - parse_curl_max5_impl(result.recieved_size, first, last) || + parse_curl_uint_impl(result.received_percent, first, last) || + parse_curl_max5_impl(result.received_size, first, last) || parse_curl_uint_impl(result.transfer_percent, first, last) || parse_curl_max5_impl(result.transfer_size, first, last) || parse_curl_max5_impl(result.average_download_speed, first, last) || diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index e466e97163..73139ad88c 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -1374,7 +1374,7 @@ namespace vcpkg FilePointer::FilePointer(const Path& path) : m_fs(nullptr), m_path(path) { } FilePointer::FilePointer() noexcept : m_fs(nullptr), m_path{} { } - FilePointer::FilePointer(FilePointer&& other) noexcept : m_fs(other.m_fs), m_path(std ::move(other.m_path)) + FilePointer::FilePointer(FilePointer&& other) noexcept : m_fs(other.m_fs), m_path(std::move(other.m_path)) { other.m_fs = nullptr; other.m_path = {}; @@ -3912,8 +3912,7 @@ namespace vcpkg ls.append_indent().append_raw(as_preferred).append_raw('\n'); } - ls.append_raw('\n'); - msg_sink.print(ls); + msg_sink.println(ls); } uint64_t get_filesystem_stats() { return g_us_filesystem_stats.load(); } diff --git a/src/vcpkg/base/hash.cpp b/src/vcpkg/base/hash.cpp index c87bc6a1ec..c895108541 100644 --- a/src/vcpkg/base/hash.cpp +++ b/src/vcpkg/base/hash.cpp @@ -142,10 +142,14 @@ namespace vcpkg::Hash virtual std::string get_hash() override { + static constexpr unsigned long static_max_hash_size = 64; // SHA512, 512/8 = 64 const auto hash_size = get_hash_buffer_size(); - const auto buffer = std::make_unique(hash_size); - const auto hash = buffer.get(); + if (hash_size > static_max_hash_size) + { + Checks::unreachable(VCPKG_LINE_INFO, "hash buffer too small"); + } + uchar hash[static_max_hash_size]; const NTSTATUS error_code = BCryptFinishHash(hash_handle, hash, hash_size, 0); Checks::check_exit(VCPKG_LINE_INFO, NT_SUCCESS(error_code), "Failed to finalize the hash"); return to_hex(hash, hash + hash_size); @@ -518,20 +522,17 @@ namespace vcpkg::Hash static ReturnType do_hash(Algorithm algo, const F& f) { #if defined(_WIN32) - auto hasher = BCryptHasher(algo); - return f(hasher); + return f(BCryptHasher(algo)); #else switch (algo) { case Algorithm::Sha256: { - auto hasher = ShaHasher(); - return f(hasher); + return f(ShaHasher()); } case Algorithm::Sha512: { - auto hasher = ShaHasher(); - return f(hasher); + return f(ShaHasher()); } default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -540,7 +541,7 @@ namespace vcpkg::Hash std::string get_bytes_hash(const void* first, const void* last, Algorithm algo) { - return do_hash(algo, [first, last](Hasher& hasher) { + return do_hash(algo, [first, last](Hasher&& hasher) { hasher.add_bytes(first, last); return hasher.get_hash(); }); @@ -553,17 +554,25 @@ namespace vcpkg::Hash std::string get_string_sha256(StringView s) { return get_string_hash(s, Hash::Algorithm::Sha256); } - ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) + HashResult get_file_hash(DiagnosticContext& context, const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) { - Debug::println("Trying to hash ", path); + HashResult result; std::error_code ec; auto file = fs.open_for_read(path, ec); if (ec) { - return error_prefix().append(msgHashFileFailureToRead, msg::path = path).append_raw(ec.message()); + if (ec == std::errc::no_such_file_or_directory || ec == std::errc::not_a_directory) + { + result.prognosis = HashPrognosis::FileNotFound; + return result; + } + + context.report_error(format_filesystem_call_error(ec, "open_for_read", {path})); + result.prognosis = HashPrognosis::OtherError; + return result; } - return do_hash>(algo, [&](Hasher& hasher) -> ExpectedL { + result.hash = do_hash(algo, [&](Hasher&& hasher) { constexpr std::size_t buffer_size = 1024 * 32; char buffer[buffer_size]; do @@ -575,13 +584,37 @@ namespace vcpkg::Hash } else if ((ec = file.error())) { - return error_prefix().append(msgHashFileFailureToRead, msg::path = path).append_raw(ec.message()); + result.prognosis = HashPrognosis::OtherError; + context.report_error(format_filesystem_call_error(ec, "read", {path})); + return std::string(); } } while (!file.eof()); - auto result_hash = hasher.get_hash(); - Debug::print(fmt::format("{} has hash {}\n", path, result_hash)); - return result_hash; + return std::move(hasher).get_hash(); }); + + return result; + } + + Optional get_file_hash_required(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo) + { + auto result = get_file_hash(context, fs, path, algo); + switch (result.prognosis) + { + case HashPrognosis::Success: return result.hash; + case HashPrognosis::FileNotFound: + context.report(DiagnosticLine{DiagKind::Error, path, msg::format(msgFileNotFound)}); + return nullopt; + case HashPrognosis::OtherError: return nullopt; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) + { + return adapt_context_to_expected(get_file_hash_required, fs, path, algo); } } diff --git a/src/vcpkg/base/message_sinks.cpp b/src/vcpkg/base/message_sinks.cpp index f28f109550..c73c0c851c 100644 --- a/src/vcpkg/base/message_sinks.cpp +++ b/src/vcpkg/base/message_sinks.cpp @@ -6,28 +6,165 @@ namespace using namespace vcpkg; struct NullMessageSink : MessageSink { - virtual void print(Color, StringView) override { } + virtual void println(const MessageLine&) override { } + virtual void println(MessageLine&&) override { } + virtual void println(const LocalizedString&) override { } + virtual void println(LocalizedString&&) override { } + virtual void println(Color, const LocalizedString&) override { } + virtual void println(Color, LocalizedString&&) override { } }; NullMessageSink null_sink_instance; - struct OutMessageSink : MessageSink + struct OutMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text(segment.color, segment.text); + } + + msg::write_unlocalized_text(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text(Color::none, text); + msg::write_unlocalized_text(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text(color, text); + msg::write_unlocalized_text(Color::none, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text(color, text); + } }; OutMessageSink out_sink_instance; - struct StdOutMessageSink : MessageSink + struct StdOutMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text_to_stdout(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stdout(segment.color, segment.text); + } + + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stdout(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stdout(Color::none, text); + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stdout(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stdout(color, text); + msg::write_unlocalized_text_to_stdout(color, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stdout(color, text); + } }; StdOutMessageSink stdout_sink_instance; - struct StdErrMessageSink : MessageSink + struct StdErrMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text_to_stderr(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stderr(segment.color, segment.text); + } + + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stderr(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stderr(Color::none, text); + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stderr(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stderr(color, text); + msg::write_unlocalized_text_to_stderr(color, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stderr(color, text); + } }; StdErrMessageSink stderr_sink_instance; @@ -36,26 +173,110 @@ namespace namespace vcpkg { - void MessageSink::println_warning(const LocalizedString& s) { println(Color::warning, warning_prefix().append(s)); } - void MessageSink::println_error(const LocalizedString& s) { println(Color::error, error_prefix().append(s)); } + MessageLine::MessageLine(const LocalizedString& ls) { segments.push_back({Color::none, ls.data()}); } + MessageLine::MessageLine(LocalizedString&& ls) { segments.push_back({Color::none, std::move(ls).extract_data()}); } + void MessageLine::print(Color color, StringView text) + { + if (!segments.empty() && segments.back().color == color) + { + segments.back().text.append(text.data(), text.size()); + } + else + { + segments.push_back({color, std::string(text)}); + } + } + void MessageLine::print(StringView text) { print(Color::none, text); } + const std::vector& MessageLine::get_segments() const noexcept { return segments; } + + std::string MessageLine::to_string() const { return adapt_to_string(*this); } + void MessageLine::to_string(std::string& target) const + { + for (const auto& segment : segments) + { + target.append(segment.text); + } + } + + void MessageSink::println(const LocalizedString& s) + { + MessageLine line; + line.print(Color::none, s); + this->println(line); + } + + void MessageSink::println(LocalizedString&& s) { this->println(s); } + + void MessageSink::println(Color c, const LocalizedString& s) + { + MessageLine line; + line.print(c, s); + this->println(line); + } + + void MessageSink::println(Color c, LocalizedString&& s) { this->println(c, s); } MessageSink& null_sink = null_sink_instance; MessageSink& out_sink = out_sink_instance; MessageSink& stderr_sink = stderr_sink_instance; MessageSink& stdout_sink = stdout_sink_instance; - void FileSink::print(Color, StringView sv) + void FileSink::println(const MessageLine& line) { + std::string whole_line; + line.to_string(whole_line); + whole_line.push_back('\n'); Checks::msg_check_exit(VCPKG_LINE_INFO, - m_out_file.write(sv.data(), 1, sv.size()) == sv.size(), + m_out_file.write(whole_line.data(), 1, whole_line.size()) == whole_line.size(), msgErrorWhileWriting, msg::path = m_log_file); } - void CombiningSink::print(Color c, StringView sv) + void FileSink::println(MessageLine&& line) + { + line.print("\n"); + for (auto&& segment : line.get_segments()) + { + Checks::msg_check_exit(VCPKG_LINE_INFO, + m_out_file.write(segment.text.data(), 1, segment.text.size()) == segment.text.size(), + msgErrorWhileWriting, + msg::path = m_log_file); + } + } + + void TeeSink::println(const MessageLine& line) { - m_first.print(c, sv); - m_second.print(c, sv); + m_first.println(line); + m_second.println(line); } + void TeeSink::println(MessageLine&& line) + { + m_first.println(line); + m_second.println(std::move(line)); + } + + void TeeSink::println(const LocalizedString& line) + { + m_first.println(line); + m_second.println(line); + } + + void TeeSink::println(LocalizedString&& line) + { + m_first.println(line); + m_second.println(std::move(line)); + } + + void TeeSink::println(Color color, const LocalizedString& line) + { + m_first.println(color, line); + m_second.println(color, line); + } + + void TeeSink::println(Color color, LocalizedString&& line) + { + m_first.println(color, line); + m_second.println(color, std::move(line)); + } } diff --git a/src/vcpkg/base/strings.cpp b/src/vcpkg/base/strings.cpp index 16d6a686b3..8f4f5870b6 100644 --- a/src/vcpkg/base/strings.cpp +++ b/src/vcpkg/base/strings.cpp @@ -24,9 +24,10 @@ namespace vcpkg::Strings::details void append_internal(std::string& into, StringView s) { into.append(s.begin(), s.end()); } } -vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView sv, - void (*cb)(void*, std::string&, StringView), - void* user) +Optional vcpkg::details::api_stable_format_impl(DiagnosticContext& context, + StringView sv, + bool (*cb)(void*, std::string&, StringView), + void* user) { // Transforms similarly to std::format -- "{xyz}" -> f(xyz), "{{" -> "{", "}}" -> "}" @@ -46,7 +47,8 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView { if (p == last) { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } else if (*p == '{') { @@ -60,10 +62,15 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView p = std::find_first_of(p, last, s_brackets, s_brackets + 2); if (p == last || p[0] != '}') { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } // p[0] == '}' - cb(user, out, {seq_start, p}); + if (!cb(user, out, {seq_start, p})) + { + return nullopt; + } + prev = ++p; } } @@ -71,14 +78,16 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView { if (p == last || p[0] != '}') { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } out.push_back('}'); prev = ++p; } } + out.append(prev, last); - return {std::move(out), expected_left_tag}; + return out; } namespace @@ -193,7 +202,7 @@ void Strings::inplace_ascii_to_lowercase(std::string& s) std::string Strings::ascii_to_lowercase(StringView s) { std::string result; - std::transform(s.begin(), s.end(), std::back_inserter(result), tolower_char); + append_ascii_lowercase(result, s); return result; } @@ -204,6 +213,11 @@ std::string Strings::ascii_to_uppercase(StringView s) return result; } +void Strings::append_ascii_lowercase(std::string& target, StringView s) +{ + std::transform(s.begin(), s.end(), std::back_inserter(target), tolower_char); +} + bool Strings::case_insensitive_ascii_starts_with(StringView s, StringView pattern) { if (s.size() < pattern.size()) return false; diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index ffff5d17d6..799542c8e8 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -43,14 +43,6 @@ namespace { using namespace vcpkg; - LocalizedString format_system_error_message(StringLiteral api_name, ExitCodeIntegral error_value) - { - return msg::format_error(msgSystemApiErrorMessage, - msg::system_api = api_name, - msg::exit_code = error_value, - msg::error_msg = std::system_category().message(static_cast(error_value))); - } - static std::atomic_int32_t debug_id_counter{1000}; #if defined(_WIN32) struct CtrlCStateMachine @@ -790,14 +782,15 @@ namespace } }; - ExpectedL windows_create_process(std::int32_t debug_id, - ProcessInfo& process_info, - StringView command_line, - const Optional& working_directory, - const Optional& environment, - BOOL bInheritHandles, - DWORD dwCreationFlags, - STARTUPINFOEXW& startup_info) noexcept + bool windows_create_process(DiagnosticContext& context, + std::int32_t debug_id, + ProcessInfo& process_info, + StringView command_line, + const Optional& working_directory, + const Optional& environment, + BOOL bInheritHandles, + DWORD dwCreationFlags, + STARTUPINFOEXW& startup_info) noexcept { Debug::print(fmt::format("{}: CreateProcessW({})\n", debug_id, command_line)); @@ -840,24 +833,26 @@ namespace &startup_info.StartupInfo, &process_info)) { - return format_system_error_message("CreateProcessW", GetLastError()); + context.report_system_error("CreateProcessW", GetLastError()); + return false; } - return Unit{}; + return true; } // Used to, among other things, control which handles are inherited by child processes. // from https://devblogs.microsoft.com/oldnewthing/20111216-00/?p=8873 struct ProcAttributeList { - ExpectedL create(DWORD dwAttributeCount) + bool create(DiagnosticContext& context, DWORD dwAttributeCount) { Checks::check_exit(VCPKG_LINE_INFO, buffer.empty()); SIZE_T size = 0; if (InitializeProcThreadAttributeList(nullptr, dwAttributeCount, 0, &size) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) { - return format_system_error_message("InitializeProcThreadAttributeList nullptr", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList nullptr", GetLastError()); + return false; } Checks::check_exit(VCPKG_LINE_INFO, size > 0); ASSUME(size > 0); @@ -865,18 +860,21 @@ namespace if (!InitializeProcThreadAttributeList( reinterpret_cast(buffer.data()), dwAttributeCount, 0, &size)) { - return format_system_error_message("InitializeProcThreadAttributeList attribute_list", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList attribute_list", GetLastError()); + return false; } - return Unit{}; + return true; } - ExpectedL update_attribute(DWORD_PTR Attribute, PVOID lpValue, SIZE_T cbSize) + bool update_attribute(DiagnosticContext& context, DWORD_PTR Attribute, PVOID lpValue, SIZE_T cbSize) { if (!UpdateProcThreadAttribute(get(), 0, Attribute, lpValue, cbSize, nullptr, nullptr)) { - return format_system_error_message("InitializeProcThreadAttributeList attribute_list", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList attribute_list", GetLastError()); + return false; } - return Unit{}; + + return true; } LPPROC_THREAD_ATTRIBUTE_LIST get() noexcept { @@ -912,17 +910,18 @@ namespace close_handle_mark_invalid(write_pipe); } - ExpectedL create() + bool create(DiagnosticContext& context) { Checks::check_exit(VCPKG_LINE_INFO, read_pipe == INVALID_HANDLE_VALUE); Checks::check_exit(VCPKG_LINE_INFO, write_pipe == INVALID_HANDLE_VALUE); SECURITY_ATTRIBUTES anonymousSa{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; if (!CreatePipe(&read_pipe, &write_pipe, &anonymousSa, 0)) { - return format_system_error_message("CreatePipe", GetLastError()); + context.report_system_error("CreatePipe", GetLastError()); + return false; } - return Unit{}; + return true; } }; @@ -960,7 +959,7 @@ namespace close_handle_mark_invalid(write_pipe); } - ExpectedL create(std::int32_t debug_id) + bool create(DiagnosticContext& context, std::int32_t debug_id) { Checks::check_exit(VCPKG_LINE_INFO, read_pipe == INVALID_HANDLE_VALUE); Checks::check_exit(VCPKG_LINE_INFO, write_pipe == INVALID_HANDLE_VALUE); @@ -981,17 +980,19 @@ namespace &namedPipeSa); if (write_pipe == INVALID_HANDLE_VALUE) { - return format_system_error_message("CreateNamedPipeW stdin", GetLastError()); + context.report_system_error("CreateNamedPipeW stdin", GetLastError()); + return false; } SECURITY_ATTRIBUTES openSa{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; read_pipe = CreateFileW(pipe_name.c_str(), FILE_GENERIC_READ, 0, &openSa, OPEN_EXISTING, 0, 0); if (read_pipe == INVALID_HANDLE_VALUE) { - return format_system_error_message("CreateFileW stdin", GetLastError()); + context.report_system_error("CreateFileW stdin", GetLastError()); + return false; } - return Unit{}; + return true; } }; @@ -1156,31 +1157,34 @@ namespace } } - ExpectedL create() + bool create(DiagnosticContext& context) { #if defined(__APPLE__) static std::mutex pipe_creation_lock; std::lock_guard lck{pipe_creation_lock}; if (pipe(pipefd)) { - return format_system_error_message("pipe", errno); + context.report_system_error("pipe", errno); + return false; } for (size_t idx = 0; idx < 2; ++idx) { if (fcntl(pipefd[idx], F_SETFD, FD_CLOEXEC)) { - return format_system_error_message("fcntl", errno); + context.report_system_error("fcntl", errno); + return false; } } #else // ^^^ Apple // !Apple vvv if (pipe2(pipefd, O_CLOEXEC)) { - return format_system_error_message("pipe2", errno); + context.report_system_error("pipe2", errno); + return false; } #endif // ^^^ !Apple - return Unit{}; + return true; } }; @@ -1198,15 +1202,16 @@ namespace PosixSpawnFileActions(const PosixSpawnFileActions&) = delete; PosixSpawnFileActions& operator=(const PosixSpawnFileActions&) = delete; - ExpectedL adddup2(int fd, int newfd) + bool adddup2(DiagnosticContext& context, int fd, int newfd) { const int error = posix_spawn_file_actions_adddup2(&actions, fd, newfd); if (error) { - return format_system_error_message("posix_spawn_file_actions_adddup2", error); + context.report_system_error("posix_spawn_file_actions_adddup2", error); + return false; } - return Unit{}; + return true; } }; @@ -1216,7 +1221,7 @@ namespace PosixPid() : pid{-1} { } - ExpectedL wait_for_termination() + Optional wait_for_termination(DiagnosticContext& context) { int exit_code = -1; if (pid != -1) @@ -1225,7 +1230,8 @@ namespace const auto child = waitpid(pid, &status, 0); if (child != pid) { - return format_system_error_message("waitpid", errno); + context.report_system_error("waitpid", errno); + return nullopt; } if (WIFEXITED(status)) @@ -1311,18 +1317,7 @@ namespace vcpkg return new_env; } #endif -} // namespace vcpkg - -namespace -{ - void debug_print_cmd_execute_background_failure(int32_t debug_id, const LocalizedString& error) - { - Debug::print(fmt::format("{}: cmd_execute_background() failed: {}\n", debug_id, error)); - } -} -namespace vcpkg -{ void cmd_execute_background(const Command& cmd_line) { const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); @@ -1334,18 +1329,15 @@ namespace vcpkg startup_info_ex.StartupInfo.cb = sizeof(STARTUPINFOEXW); startup_info_ex.StartupInfo.dwFlags = STARTF_USESHOWWINDOW; startup_info_ex.StartupInfo.wShowWindow = SW_HIDE; - auto process_create = windows_create_process(debug_id, - process_info, - cmd_line.command_line(), - nullopt, - nullopt, - FALSE, - CREATE_NEW_CONSOLE | CREATE_BREAKAWAY_FROM_JOB, - startup_info_ex); - if (!process_create) - { - debug_print_cmd_execute_background_failure(debug_id, process_create.error()); - } + (void)windows_create_process(Debug::g_debugging ? console_diagnostic_context : null_diagnostic_context, + debug_id, + process_info, + cmd_line.command_line(), + nullopt, + nullopt, + FALSE, + CREATE_NEW_CONSOLE | CREATE_BREAKAWAY_FROM_JOB, + startup_info_ex); #else // ^^^ _WIN32 // !_WIN32 pid_t pid; @@ -1366,17 +1358,17 @@ namespace vcpkg argv.emplace_back(nullptr); int error = posix_spawn(&pid, "/bin/sh", nullptr /*file_actions*/, nullptr /*attrp*/, argv.data(), environ); - if (error) + if (error && Debug::g_debugging) { - debug_print_cmd_execute_background_failure(debug_id, format_system_error_message("posix_spawn", errno)); - return; + console_diagnostic_context.report_system_error("posix_spawn", errno); } #endif // ^^^ !_WIN32 } - static ExpectedL cmd_execute_impl(const Command& cmd, - const ProcessLaunchSettings& settings, - const int32_t debug_id) + static Optional cmd_execute_impl(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings, + const int32_t debug_id) { #if defined(_WIN32) STARTUPINFOEXW startup_info_ex; @@ -1386,10 +1378,10 @@ namespace vcpkg startup_info_ex.StartupInfo.wShowWindow = SW_HIDE; ProcAttributeList proc_attribute_list; - auto proc_attribute_list_create = proc_attribute_list.create(1); + auto proc_attribute_list_create = proc_attribute_list.create(context, 1); if (!proc_attribute_list_create) { - return std::move(proc_attribute_list_create).error(); + return nullopt; } constexpr size_t number_of_candidate_handles = 3; @@ -1399,17 +1391,18 @@ namespace vcpkg size_t number_of_handles = std::unique(handles_to_inherit, handles_to_inherit + number_of_candidate_handles) - handles_to_inherit; - auto maybe_error = proc_attribute_list.update_attribute( - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, number_of_handles * sizeof(HANDLE)); - if (!maybe_error.has_value()) + if (!proc_attribute_list.update_attribute( + context, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, number_of_handles * sizeof(HANDLE))) { - return std::move(maybe_error).error(); + return nullopt; } + startup_info_ex.lpAttributeList = proc_attribute_list.get(); SpawnProcessGuard spawn_process_guard; ProcessInfo process_info; - auto process_create = windows_create_process(debug_id, + auto process_create = windows_create_process(context, + debug_id, process_info, cmd.command_line(), settings.working_directory, @@ -1419,11 +1412,12 @@ namespace vcpkg startup_info_ex); if (!process_create) { - return std::move(process_create).error(); + return nullopt; } return process_info.wait(); #else + (void)context; Command real_command_line_builder; if (const auto wd = settings.working_directory.get()) { @@ -1448,46 +1442,50 @@ namespace vcpkg #endif } - ExpectedL cmd_execute(const Command& cmd) + Optional cmd_execute(DiagnosticContext& context, const Command& cmd) { ProcessLaunchSettings default_process_launch_settings; - return cmd_execute(cmd, default_process_launch_settings); + return cmd_execute(context, cmd, default_process_launch_settings); } - ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings) + Optional cmd_execute(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings) { const ElapsedTimer timer; const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); - auto maybe_result = cmd_execute_impl(cmd, settings, debug_id); + auto maybe_exit_code = cmd_execute_impl(context, cmd, settings, debug_id); const auto elapsed = timer.us_64(); g_subprocess_stats += elapsed; - if (auto result = maybe_result.get()) + if (auto exit_code = maybe_exit_code.get()) { - Debug::print(fmt::format("{}: cmd_execute() returned {} after {} us\n", debug_id, *result, elapsed)); + Debug::print(fmt::format("{}: child process returned {} after {} us\n", debug_id, *exit_code, elapsed)); } else { Debug::print( - fmt::format("{}: cmd_execute() returned ({}) after {} us\n", debug_id, maybe_result.error(), elapsed)); + fmt::format("{}: cmd_execute() failed to launch child process after {} us\n", debug_id, elapsed)); } - return maybe_result; + return maybe_exit_code; } - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const std::function& per_line_cb) + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const std::function& per_line_cb) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_stream_lines(cmd, default_redirected_process_launch_settings, per_line_cb); + return cmd_execute_and_stream_lines(context, cmd, default_redirected_process_launch_settings, per_line_cb); } - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& per_line_cb) + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& per_line_cb) { Strings::LinesStream lines; - auto rc = - cmd_execute_and_stream_data(cmd, settings, [&](const StringView sv) { lines.on_data(sv, per_line_cb); }); + auto rc = cmd_execute_and_stream_data( + context, cmd, settings, [&](const StringView sv) { lines.on_data(sv, per_line_cb); }); lines.on_end(per_line_cb); return rc; } @@ -1503,7 +1501,7 @@ namespace std::size_t offset; // Write a hunk of data to `target`. If there is no more input to write, returns `true`. - ExpectedL do_write(int target) + Optional do_write(DiagnosticContext& context, int target) { const auto this_write = input.size() - offset; // Big enough to be big, small enough to avoid implementation limits @@ -1515,7 +1513,8 @@ namespace write(target, static_cast(input.data() + offset), this_write_clamped); if (actually_written < 0) { - return format_system_error_message("write", errno); + context.report_system_error("write", errno); + return nullopt; } offset += actually_written; @@ -1526,10 +1525,11 @@ namespace }; #endif // ^^^ !_WIN32 - ExpectedL cmd_execute_and_stream_data_impl(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb, - uint32_t debug_id) + Optional cmd_execute_and_stream_data_impl(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb, + uint32_t debug_id) { #if defined(_WIN32) std::wstring as_utf16; @@ -1544,7 +1544,8 @@ namespace auto stdin_content_size_raw = stdin_content.size(); if (stdin_content_size_raw > MAXDWORD) { - return format_system_error_message("WriteFileEx", ERROR_INSUFFICIENT_BUFFER); + context.report_system_error("WriteFileEx", ERROR_INSUFFICIENT_BUFFER); + return nullopt; } auto stdin_content_size = static_cast(stdin_content_size_raw); @@ -1567,52 +1568,48 @@ namespace } // Create a pipe for the child process's STDIN. - auto stdin_create = process_info.stdin_pipe.create(debug_id); - if (!stdin_create) + if (!process_info.stdin_pipe.create(context, debug_id)) { - return std::move(stdin_create).error(); + return nullopt; } startup_info_ex.StartupInfo.hStdInput = process_info.stdin_pipe.read_pipe; // Create a pipe for the child process's STDOUT/STDERR. - auto stdout_create = process_info.stdout_pipe.create(); - if (!stdout_create) + if (!process_info.stdout_pipe.create(context)) { - return std::move(stdout_create).error(); + return nullopt; } startup_info_ex.StartupInfo.hStdOutput = process_info.stdout_pipe.write_pipe; startup_info_ex.StartupInfo.hStdError = process_info.stdout_pipe.write_pipe; ProcAttributeList proc_attribute_list; - auto proc_attribute_list_create = proc_attribute_list.create(1); - if (!proc_attribute_list_create) + if (!proc_attribute_list.create(context, 1)) { - return std::move(proc_attribute_list_create).error(); + return nullopt; } HANDLE handles_to_inherit[2] = {startup_info_ex.StartupInfo.hStdOutput, startup_info_ex.StartupInfo.hStdInput}; - auto maybe_error = proc_attribute_list.update_attribute( - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, 2 * sizeof(HANDLE)); - if (!maybe_error.has_value()) + if (!proc_attribute_list.update_attribute( + context, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, 2 * sizeof(HANDLE))) { - return std::move(maybe_error).error(); + return nullopt; } - startup_info_ex.lpAttributeList = proc_attribute_list.get(); - auto process_create = windows_create_process(debug_id, - process_info.proc_info, - cmd.command_line(), - settings.working_directory, - settings.environment, - TRUE, - dwCreationFlags, - startup_info_ex); + startup_info_ex.lpAttributeList = proc_attribute_list.get(); - if (!process_create) + if (!windows_create_process(context, + debug_id, + process_info.proc_info, + cmd.command_line(), + settings.working_directory, + settings.environment, + TRUE, + dwCreationFlags, + startup_info_ex)) { - return std::move(process_create).error(); + return nullopt; } close_handle_mark_invalid(process_info.stdin_pipe.read_pipe); @@ -1675,27 +1672,32 @@ namespace fflush(stdout); AnonymousPipe child_input; + if (!child_input.create(context)) { - auto err = child_input.create(); - if (!err) - { - return std::move(err).error(); - } + return nullopt; } AnonymousPipe child_output; + if (!child_output.create(context)) { - auto err = child_output.create(); - if (!err) - { - return std::move(err).error(); - } + return nullopt; } PosixSpawnFileActions actions; - actions.adddup2(child_input.pipefd[0], 0); - actions.adddup2(child_output.pipefd[1], 1); - actions.adddup2(child_output.pipefd[1], 2); + if (!actions.adddup2(context, child_input.pipefd[0], 0)) + { + return nullopt; + } + + if (!actions.adddup2(context, child_output.pipefd[1], 1)) + { + return nullopt; + } + + if (!actions.adddup2(context, child_output.pipefd[1], 2)) + { + return nullopt; + } std::vector argv_builder; argv_builder.reserve(3); @@ -1716,7 +1718,8 @@ namespace int error = posix_spawn(&pid.pid, "/bin/sh", &actions.actions, nullptr, argv.data(), environ); if (error) { - return format_system_error_message("posix_spawn", error); + context.report_system_error("posix_spawn", error); + return nullopt; } close_mark_invalid(child_input.pipefd[0]); @@ -1732,10 +1735,11 @@ namespace { if (fcntl(child_input.pipefd[1], F_SETFL, O_NONBLOCK)) { - return format_system_error_message("fcntl", errno); + context.report_system_error("fcntl", errno); + return nullopt; } - auto maybe_done = stdin_tracker.do_write(child_input.pipefd[1]); + auto maybe_done = stdin_tracker.do_write(context, child_input.pipefd[1]); bool done = false; if (const auto done_first = maybe_done.get()) { @@ -1747,7 +1751,7 @@ namespace } else { - return std::move(maybe_done).error(); + return nullopt; } if (!done) @@ -1761,7 +1765,8 @@ namespace polls[1].events = POLLIN; if (poll(polls, 2, -1) < 0) { - return format_system_error_message("poll", errno); + context.report_system_error("poll", errno); + return nullopt; } if (polls[0].revents & POLLERR) @@ -1771,7 +1776,7 @@ namespace } else if (polls[0].revents & POLLOUT) { - auto maybe_next_done = stdin_tracker.do_write(child_input.pipefd[1]); + auto maybe_next_done = stdin_tracker.do_write(context, child_input.pipefd[1]); if (const auto next_done = maybe_next_done.get()) { if (*next_done) @@ -1782,7 +1787,7 @@ namespace } else { - return std::move(maybe_next_done).error(); + return nullopt; } } @@ -1791,7 +1796,8 @@ namespace auto read_amount = read(child_output.pipefd[0], buf, sizeof(buf)); if (read_amount < 0) { - return format_system_error_message("read", errno); + context.report_system_error("read", errno); + return nullopt; } // can't be 0 because poll told us otherwise @@ -1823,7 +1829,8 @@ namespace break; } - return format_system_error_message("read", error); + context.report_system_error("read", errno); + return nullopt; } if (read_amount == 0) @@ -1840,27 +1847,29 @@ namespace } } - return pid.wait_for_termination(); + return pid.wait_for_termination(context); #endif /// ^^^ !_WIN32 } } // unnamed namespace namespace vcpkg { - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const std::function& data_cb) + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const std::function& data_cb) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_stream_data(cmd, default_redirected_process_launch_settings, data_cb); + return cmd_execute_and_stream_data(context, cmd, default_redirected_process_launch_settings, data_cb); } - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb) + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb) { const ElapsedTimer timer; const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); - auto maybe_exit_code = cmd_execute_and_stream_data_impl(cmd, settings, data_cb, debug_id); + auto maybe_exit_code = cmd_execute_and_stream_data_impl(context, cmd, settings, data_cb, debug_id); const auto elapsed = timer.us_64(); g_subprocess_stats += elapsed; if (const auto exit_code = maybe_exit_code.get()) @@ -1874,17 +1883,18 @@ namespace vcpkg return maybe_exit_code; } - ExpectedL cmd_execute_and_capture_output(const Command& cmd) + Optional cmd_execute_and_capture_output(DiagnosticContext& context, const Command& cmd) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_capture_output(cmd, default_redirected_process_launch_settings); + return cmd_execute_and_capture_output(context, cmd, default_redirected_process_launch_settings); } - ExpectedL cmd_execute_and_capture_output(const Command& cmd, - const RedirectedProcessLaunchSettings& settings) + Optional cmd_execute_and_capture_output(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings) { std::string output; - return cmd_execute_and_stream_data(cmd, settings, [&](StringView sv) { Strings::append(output, sv); }) + return cmd_execute_and_stream_data(context, cmd, settings, [&](StringView sv) { Strings::append(output, sv); }) .map([&](ExitCodeIntegral exit_code) { return ExitCodeAndOutput{exit_code, std::move(output)}; }); } @@ -1960,4 +1970,23 @@ namespace vcpkg expected_right_tag}; } + std::string* check_zero_exit_code(DiagnosticContext& context, + Optional& maybe_exit, + StringView exe_path) + { + if (auto exit = maybe_exit.get()) + { + if (exit->exit_code == 0) + { + return &exit->output; + } + + context.report( + DiagnosticLine{DiagKind::Error, + exe_path, + msg::format(msgProgramPathReturnedNonzeroExitCode, msg::exit_code = exit->exit_code)}); + } + + return nullptr; + } } // namespace vcpkg diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 50f98f0830..e4b3eb595e 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -381,10 +382,8 @@ namespace struct HTTPPutBinaryProvider : IWriteBinaryProvider { - HTTPPutBinaryProvider(const Filesystem& fs, - std::vector&& urls, - const std::vector& secrets) - : m_fs(fs), m_urls(std::move(urls)), m_secrets(secrets) + HTTPPutBinaryProvider(std::vector&& urls, const std::vector& secrets) + : m_urls(std::move(urls)), m_secrets(secrets) { } @@ -396,11 +395,14 @@ namespace for (auto&& templ : m_urls) { auto url = templ.instantiate_variables(request); - auto maybe_success = put_file(m_fs, url, m_secrets, templ.headers, zip_path); + PrintingDiagnosticContext pdc{msg_sink}; + WarningDiagnosticContext wdc{pdc}; + auto maybe_success = + store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); if (maybe_success) + { count_stored++; - else - msg_sink.println(Color::warning, maybe_success.error()); + } } return count_stored; } @@ -409,7 +411,6 @@ namespace bool needs_zip_file() const override { return true; } private: - const Filesystem& m_fs; std::vector m_urls; std::vector m_secrets; }; @@ -439,20 +440,13 @@ namespace make_temp_archive_path(m_buildtrees, action.spec)); } - auto codes_maybe = download_files(url_paths, m_url_template.headers, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto const& maybe_code = codes_maybe[i]; - if (auto code = maybe_code.get()) - { - if (*code == 200) - { - out_zip_paths[i].emplace(std::move(url_paths[i].second), RemoveWhen::always); - } - } - else + if (codes[i] == 200) { - msg::println_warning(maybe_code.error()); + out_zip_paths[i].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } @@ -465,22 +459,14 @@ namespace urls.push_back(m_url_template.instantiate_variables(BinaryPackageReadInfo{*actions[idx]})); } - auto codes_maybe = url_heads(urls, {}, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto codes = url_heads(wdc, urls, {}, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto result_availability = CacheAvailability::unavailable; - if (const auto code = codes_maybe[i].get()) - { - if (*code == 200) - { - result_availability = CacheAvailability::available; - } - } - - out_status[i] = result_availability; + out_status[i] = codes[i] == 200 ? CacheAvailability::available : CacheAvailability::unavailable; } - for (size_t i = codes_maybe.size(); i < out_status.size(); ++i) + for (size_t i = codes.size(); i < out_status.size(); ++i) { out_status[i] = CacheAvailability::unavailable; } @@ -827,7 +813,9 @@ namespace m_token_header, m_accept_header.to_string(), }; - auto res = invoke_http_request("GET", headers, url); + + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto res = invoke_http_request(wdc, "GET", headers, url); if (auto p = res.get()) { auto maybe_json = Json::parse_object(*p, m_url); @@ -861,21 +849,13 @@ namespace url_indices.push_back(idx); } - const auto codes_maybe = download_files(url_paths, {}, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + const auto codes = download_files_no_cache(wdc, url_paths, {}, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto const& maybe_code = codes_maybe[i]; - if (auto code = maybe_code.get()) + if (codes[i] == 200) { - if (*code == 200) - { - out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), RemoveWhen::always); - } - } - else - { - msg::println(maybe_code.error()); - continue; + out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } @@ -916,7 +896,8 @@ namespace m_token_header, }; - auto res = invoke_http_request("POST", headers, m_url, stringify(payload)); + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto res = invoke_http_request(wdc, "POST", headers, m_url, stringify(payload)); if (auto p = res.get()) { auto maybe_json = Json::parse_object(*p, m_url); @@ -932,7 +913,7 @@ namespace return {}; } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink&) override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { if (!request.zip_path) return 0; @@ -952,8 +933,10 @@ namespace "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", }; - const auto url = m_url + "/" + std::to_string(*cacheId.get()); - if (put_file(m_fs, url, {}, custom_headers, zip_path, "PATCH")) + PrintingDiagnosticContext pdc{msg_sink}; + WarningDiagnosticContext wdc{pdc}; + const auto raw_url = m_url + "/" + std::to_string(*cacheId.get()); + if (store_to_asset_cache(wdc, raw_url, SanitizedUrl{raw_url, {}}, "PATCH", custom_headers, zip_path)) { Json::Object commit; commit.insert("size", std::to_string(cache_size)); @@ -963,15 +946,10 @@ namespace m_token_header, }; - auto res = invoke_http_request("POST", headers, url, stringify(commit)); - if (res) + if (invoke_http_request(wdc, "POST", headers, raw_url, stringify(commit))) { ++upload_count; } - else - { - msg::println(res.error()); - } } } return upload_count; @@ -1058,7 +1036,7 @@ namespace } else { - out_sink.println_warning(res.error()); + msg::println_warning(res.error()); } } } @@ -1117,7 +1095,7 @@ namespace } else { - msg_sink.println_warning(res.error()); + msg_sink.println(warning_prefix().append(std::move(res).error())); } } return upload_count; @@ -1857,16 +1835,19 @@ namespace } bool has_sha = false; bool has_other = false; - api_stable_format(url_template.url_template, [&](std::string&, StringView key) { - if (key == "sha") - { - has_sha = true; - } - else - { - has_other = true; - } - }); + api_stable_format( + null_diagnostic_context, url_template.url_template, [&](std::string&, StringView key) { + if (key == "sha") + { + has_sha = true; + } + else + { + has_other = true; + } + + return true; + }); if (!has_sha) { if (has_other) @@ -2037,30 +2018,38 @@ namespace vcpkg { LocalizedString UrlTemplate::valid() const { + BufferedDiagnosticContext bdc{out_sink}; std::vector invalid_keys; - auto result = api_stable_format(url_template, [&](std::string&, StringView key) { + auto result = api_stable_format(bdc, url_template, [&](std::string&, StringView key) { static constexpr StringLiteral valid_keys[] = {"name", "version", "sha", "triplet"}; if (!Util::Vectors::contains(valid_keys, key)) { invalid_keys.push_back(key.to_string()); } + + return true; }); - if (!result) + + if (!invalid_keys.empty()) { - return std::move(result).error(); + bdc.report_error(msg::format(msgUnknownVariablesInTemplate, + msg::value = url_template, + msg::list = Strings::join(", ", invalid_keys))); + result.clear(); } - if (!invalid_keys.empty()) + + if (result.has_value()) { - return msg::format(msgUnknownVariablesInTemplate, - msg::value = url_template, - msg::list = Strings::join(", ", invalid_keys)); + return {}; } - return {}; + + return LocalizedString::from_raw(std::move(bdc).to_string()); } std::string UrlTemplate::instantiate_variables(const BinaryPackageReadInfo& info) const { - return api_stable_format(url_template, + return api_stable_format(console_diagnostic_context, + url_template, [&](std::string& out, StringView key) { if (key == "version") { @@ -2080,10 +2069,12 @@ namespace vcpkg } else { - Debug::println("Unknown key: ", key); - // We do a input validation while parsing the config - Checks::unreachable(VCPKG_LINE_INFO); + Checks::unreachable( + VCPKG_LINE_INFO, + "used instantiate_variables without checking valid() first"); }; + + return true; }) .value_or_exit(VCPKG_LINE_INFO); } @@ -2263,7 +2254,7 @@ namespace vcpkg if (!s.url_templates_to_put.empty()) { ret.write.push_back( - std::make_unique(fs, std::move(s.url_templates_to_put), s.secrets)); + std::make_unique(std::move(s.url_templates_to_put), s.secrets)); } if (!s.gcs_write_prefixes.empty()) { @@ -2601,14 +2592,15 @@ namespace vcpkg } } -ExpectedL vcpkg::parse_download_configuration(const Optional& arg) +ExpectedL vcpkg::parse_download_configuration(const Optional& arg) { - if (!arg || arg.get()->empty()) return DownloadManagerConfig{}; + AssetCachingSettings result; + if (!arg || arg.get()->empty()) return result; get_global_metrics_collector().track_define(DefineMetric::AssetSource); AssetSourcesState s; - const auto source = Strings::concat("$", EnvironmentVariableXVcpkgAssetSources); + const auto source = format_environment_variable(EnvironmentVariableXVcpkgAssetSources); AssetSourcesParser parser(*arg.get(), source, &s); parser.parse(); if (auto err = parser.get_error()) @@ -2634,27 +2626,22 @@ ExpectedL vcpkg::parse_download_configuration(const Optio .append(msgSeeURL, msg::url = docs::assetcaching_url); } - Optional get_url; if (!s.url_templates_to_get.empty()) { - get_url = std::move(s.url_templates_to_get.back()); + result.m_read_url_template = std::move(s.url_templates_to_get.back()); } - Optional put_url; - std::vector put_headers; + if (!s.azblob_templates_to_put.empty()) { - put_url = std::move(s.azblob_templates_to_put.back()); + result.m_write_url_template = std::move(s.azblob_templates_to_put.back()); auto v = azure_blob_headers(); - put_headers.assign(v.begin(), v.end()); + result.m_write_headers.assign(v.begin(), v.end()); } - return DownloadManagerConfig{std::move(get_url), - std::vector{}, - std::move(put_url), - std::move(put_headers), - std::move(s.secrets), - s.block_origin, - s.script}; + result.m_secrets = std::move(s.secrets); + result.m_block_origin = s.block_origin; + result.m_script = std::move(s.script); + return result; } ExpectedL vcpkg::parse_binary_provider_configs(const std::string& env_string, @@ -2669,7 +2656,9 @@ ExpectedL vcpkg::parse_binary_provider_configs(const st return *err; } - BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES", &s); + // must live until the end of the function due to StringView in BinaryConfigParser + const auto source = format_environment_variable("VCPKG_BINARY_SOURCES"); + BinaryConfigParser env_parser(env_string, source, &s); env_parser.parse(); if (auto err = env_parser.get_error()) { diff --git a/src/vcpkg/commands.bootstrap-standalone.cpp b/src/vcpkg/commands.bootstrap-standalone.cpp index 6663836108..0c0a398273 100644 --- a/src/vcpkg/commands.bootstrap-standalone.cpp +++ b/src/vcpkg/commands.bootstrap-standalone.cpp @@ -27,7 +27,7 @@ namespace vcpkg { (void)args.parse_arguments(CommandBootstrapStandaloneMetadata); - DownloadManager download_manager{{}}; + AssetCachingSettings asset_cache_settings; const auto maybe_vcpkg_root_env = args.vcpkg_root_dir_env.get(); if (!maybe_vcpkg_root_env) { @@ -36,11 +36,17 @@ namespace vcpkg const auto vcpkg_root = fs.almost_canonical(*maybe_vcpkg_root_env, VCPKG_LINE_INFO); fs.create_directories(vcpkg_root, VCPKG_LINE_INFO); - auto tarball = - download_vcpkg_standalone_bundle(download_manager, fs, vcpkg_root).value_or_exit(VCPKG_LINE_INFO); + auto maybe_tarball = + download_vcpkg_standalone_bundle(console_diagnostic_context, asset_cache_settings, fs, vcpkg_root); + auto tarball = maybe_tarball.get(); + if (!tarball) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + fs.remove_all(vcpkg_root / "vcpkg-artifacts", VCPKG_LINE_INFO); - extract_tar(find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), tarball, vcpkg_root); - fs.remove(tarball, VCPKG_LINE_INFO); + extract_tar(find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), *tarball, vcpkg_root); + fs.remove(*tarball, VCPKG_LINE_INFO); Checks::exit_success(VCPKG_LINE_INFO); } } diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 63b344e610..e9e80f5686 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -1062,7 +1062,7 @@ namespace vcpkg size_t error_count = 0; { FileSink file_sink{fs, stdoutlog, Append::YES}; - CombiningSink combo_sink{out_sink, file_sink}; + TeeSink combo_sink{out_sink, file_sink}; error_count = perform_post_build_lint_checks( action.spec, paths, pre_build_info, build_info, scfl.port_directory(), combo_sink); }; diff --git a/src/vcpkg/commands.ci-verify-versions.cpp b/src/vcpkg/commands.ci-verify-versions.cpp index 2fd5191cc4..fc4b4400f1 100644 --- a/src/vcpkg/commands.ci-verify-versions.cpp +++ b/src/vcpkg/commands.ci-verify-versions.cpp @@ -43,14 +43,13 @@ namespace if (!extracted_tree) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append(maybe_extracted_tree.error()) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgWhileValidatingVersion, msg::version = version_entry.version.version) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append(maybe_extracted_tree.error()) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgWhileValidatingVersion, msg::version = version_entry.version.version)); return success; } @@ -70,14 +69,13 @@ namespace // // However including both paths likely helps investigation and there isn't an easy way to replace only that // file path right now - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append(load_result.maybe_scfl.error()) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgWhileValidatingVersion, msg::version = version_entry.version.version) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append(load_result.maybe_scfl.error()) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgWhileValidatingVersion, msg::version = version_entry.version.version)); return success; } @@ -87,50 +85,47 @@ namespace if (version_entry_spec != scfl_spec) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionInDeclarationDoesNotMatch, - msg::git_tree_sha = version_entry.git_tree, - msg::expected = version_entry_spec, - msg::actual = scfl_spec) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionInDeclarationDoesNotMatch, + msg::git_tree_sha = version_entry.git_tree, + msg::expected = version_entry_spec, + msg::actual = scfl_spec)); } if (version_entry.version.scheme != git_tree_version.scheme) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionSchemeMismatch1Old, - msg::version = version_entry.version.version, - msg::expected = get_scheme_name(version_entry.version.scheme), - msg::actual = get_scheme_name(git_tree_version.scheme), - msg::package_name = port_name, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n') - .append_raw(scfl->control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionSchemeMismatch2) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionSchemeMismatch1Old, + msg::version = version_entry.version.version, + msg::expected = get_scheme_name(version_entry.version.scheme), + msg::actual = get_scheme_name(git_tree_version.scheme), + msg::package_name = port_name, + msg::git_tree_sha = version_entry.git_tree) + .append_raw('\n') + .append_raw(scfl->control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionSchemeMismatch2)); } if (success) { - success_sink.print(LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionVerifiedOK, - msg::version_spec = VersionSpec{port_name, version_entry.version.version}, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionVerifiedOK, + msg::version_spec = VersionSpec{port_name, version_entry.version.version}, + msg::git_tree_sha = version_entry.git_tree)); } return success; @@ -156,21 +151,20 @@ namespace if (!entries) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionDatabaseFileMissing) - .append_raw('\n') - .append_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionDatabaseFileMissing2) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionDatabaseFileMissing3, - msg::command_line = fmt::format("vcpkg x-add-version {}", port_name)) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionDatabaseFileMissing) + .append_raw('\n') + .append_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionDatabaseFileMissing2) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionDatabaseFileMissing3, + msg::command_line = fmt::format("vcpkg x-add-version {}", port_name))); return success; } @@ -185,22 +179,21 @@ namespace if (it == versions_end) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionNotFoundInVersionsFile2, - msg::version_spec = VersionSpec{port_name, local_port_version.version}) - .append_raw('\n') - .append_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionNotFoundInVersionsFile3) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionNotFoundInVersionsFile4, - msg::command_line = fmt::format("vcpkg x-add-version {}", port_name)) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionNotFoundInVersionsFile2, + msg::version_spec = VersionSpec{port_name, local_port_version.version}) + .append_raw('\n') + .append_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionNotFoundInVersionsFile3) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionNotFoundInVersionsFile4, + msg::command_line = fmt::format("vcpkg x-add-version {}", port_name))); return success; } @@ -209,7 +202,7 @@ namespace { success = false; // assume the port is correct, so report the error on the version database file - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(versions_database_entry.versions_file_path) .append_raw(": ") @@ -230,58 +223,56 @@ namespace .append_raw('\n') .append_raw(NotePrefix) .append(msgVersionOverwriteVersion, msg::version_spec = local_version_spec) - .append_raw(fmt::format("\nvcpkg x-add-version {} --overwrite-version\n", port_name))); + .append_raw(fmt::format("\nvcpkg x-add-version {} --overwrite-version", port_name))); } if (local_git_tree != version_entry.git_tree) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionShaMismatch1, - msg::version_spec = local_version_spec, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n') - .append_raw(scfl.port_directory()) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionShaMismatch2, msg::git_tree_sha = local_git_tree) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionShaMismatch3, msg::version_spec = local_version_spec) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionShaMismatch4, msg::version_spec = local_version_spec) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {} --overwrite-version\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionShaMismatch1, + msg::version_spec = local_version_spec, + msg::git_tree_sha = version_entry.git_tree) + .append_raw('\n') + .append_raw(scfl.port_directory()) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionShaMismatch2, msg::git_tree_sha = local_git_tree) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionShaMismatch3, msg::version_spec = local_version_spec) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionShaMismatch4, msg::version_spec = local_version_spec) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {} --overwrite-version\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); } if (success) { - success_sink.print(LocalizedString::from_raw(scfl.port_directory()) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionVerifiedOK, - msg::version_spec = local_version_spec, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(scfl.port_directory()) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionVerifiedOK, + msg::version_spec = local_version_spec, + msg::git_tree_sha = version_entry.git_tree)); } return success; @@ -298,67 +289,64 @@ namespace auto maybe_baseline = baseline.find(port_name); if (maybe_baseline == baseline.end()) { - errors_sink.print(Color::error, - LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgBaselineMissing, msg::package_name = port_name) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgAddVersionInstructions, msg::package_name = port_name) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgBaselineMissing, msg::package_name = port_name) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgAddVersionInstructions, msg::package_name = port_name) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); return false; } auto&& baseline_version = maybe_baseline->second; if (baseline_version == local_port_version.version) { - success_sink.print(LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionBaselineMatch, - msg::version_spec = VersionSpec{port_name, local_port_version.version}) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionBaselineMatch, + msg::version_spec = VersionSpec{port_name, local_port_version.version})); return true; } // assume the port is correct, so report the error on the baseline.json file - errors_sink.print(Color::error, - LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionBaselineMismatch, - msg::expected = local_port_version.version, - msg::actual = baseline_version, - msg::package_name = port_name) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgAddVersionInstructions, msg::package_name = port_name) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionBaselineMismatch, + msg::expected = local_port_version.version, + msg::actual = baseline_version, + msg::package_name = port_name) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgAddVersionInstructions, msg::package_name = port_name) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); return false; } @@ -382,16 +370,15 @@ namespace auto this_error = LocalizedString::from_raw(scfl.control_path) .append_raw(": ") .append_raw(ErrorPrefix) - .append(msgDependencyNotInVersionDatabase, msg::package_name = dependency.name) - .append_raw('\n'); + .append(msgDependencyNotInVersionDatabase, msg::package_name = dependency.name); if (feature_name) { - this_error.append_raw(NotePrefix) - .append(msgDependencyInFeature, msg::feature = *feature_name) - .append_raw('\n'); + this_error.append_raw('\n') + .append_raw(NotePrefix) + .append(msgDependencyInFeature, msg::feature = *feature_name); } - errors_sink.print(Color::error, std::move(this_error)); + errors_sink.println(Color::error, std::move(this_error)); return false; } @@ -411,16 +398,15 @@ namespace .append_raw(dependent_versions_db_entry.versions_file_path) .append_raw(": ") .append_raw(NotePrefix) - .append(msgVersionConstraintNotInDatabase2) - .append_raw('\n'); + .append(msgVersionConstraintNotInDatabase2); if (feature_name) { - this_error.append_raw(NotePrefix) - .append(msgDependencyInFeature, msg::feature = *feature_name) - .append_raw('\n'); + this_error.append_raw('\n') + .append_raw(NotePrefix) + .append(msgDependencyInFeature, msg::feature = *feature_name); } - errors_sink.print(Color::error, std::move(this_error)); + errors_sink.println(Color::error, std::move(this_error)); return false; } @@ -463,13 +449,12 @@ namespace if (!override_entries) { success = false; - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(scfl.control_path) .append_raw(": ") .append_raw(ErrorPrefix) - .append(msgVersionOverrideNotInVersionDatabase, msg::package_name = override_.name) - .append_raw('\n')); + .append(msgVersionOverrideNotInVersionDatabase, msg::package_name = override_.name)); continue; } @@ -478,29 +463,27 @@ namespace })) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionOverrideVersionNotInVersionDatabase1, - msg::package_name = override_.name, - msg::version = override_.version) - .append_raw('\n') - .append_raw(override_versions_db_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionOverrideVersionNotInVersionDatabase2) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionOverrideVersionNotInVersionDatabase1, + msg::package_name = override_.name, + msg::version = override_.version) + .append_raw('\n') + .append_raw(override_versions_db_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionOverrideVersionNotInVersionDatabase2)); } } if (success) { - success_sink.print(LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionConstraintOk) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionConstraintOk)); } return success; @@ -568,7 +551,7 @@ namespace vcpkg auto git_tree_it = port_git_tree_map.find(port_name); if (git_tree_it == port_git_tree_map.end()) { - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(scfl.control_path) .append_raw(": ") @@ -587,7 +570,7 @@ namespace vcpkg .append_indent() .append_raw("git add versions\n") .append_indent() - .append_raw(fmt::format("git commit --amend -m \"{}\"\n", + .append_raw(fmt::format("git commit --amend -m \"{}\"", msg::format(msgVersionShaMissing4, msg::package_name = port_name)))); } else diff --git a/src/vcpkg/commands.download.cpp b/src/vcpkg/commands.download.cpp index 101f14880b..8e2ee8f44c 100644 --- a/src/vcpkg/commands.download.cpp +++ b/src/vcpkg/commands.download.cpp @@ -90,8 +90,8 @@ namespace vcpkg void command_download_and_exit(const VcpkgCmdArguments& args, const Filesystem& fs) { auto parsed = args.parse_arguments(CommandDownloadMetadata); - DownloadManager download_manager{ - parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO)}; + auto asset_cache_settings = + parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO); auto file = fs.absolute(parsed.command_arguments[0], VCPKG_LINE_INFO); auto sha = get_sha512_check(parsed); @@ -117,7 +117,12 @@ namespace vcpkg msg::println_error(msgMismatchedFiles); Checks::unreachable(VCPKG_LINE_INFO); } - download_manager.put_file_to_mirror(fs, file, actual_hash).value_or_exit(VCPKG_LINE_INFO); + + if (!store_to_asset_cache(console_diagnostic_context, asset_cache_settings, file, actual_hash)) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + Checks::exit_success(VCPKG_LINE_INFO); } else @@ -137,14 +142,20 @@ namespace vcpkg urls = it_urls->second; } - download_manager.download_file( - fs, - urls, - headers, - file, - sha, - Util::Sets::contains(parsed.switches, SwitchZMachineReadableProgress) ? out_sink : null_sink); - Checks::exit_success(VCPKG_LINE_INFO); + if (download_file_asset_cached( + console_diagnostic_context, + Util::Sets::contains(parsed.switches, SwitchZMachineReadableProgress) ? out_sink : null_sink, + asset_cache_settings, + fs, + urls, + headers, + file, + sha)) + { + Checks::exit_success(VCPKG_LINE_INFO); + } + + Checks::exit_fail(VCPKG_LINE_INFO); } } } // namespace vcpkg diff --git a/src/vcpkg/commands.format-manifest.cpp b/src/vcpkg/commands.format-manifest.cpp index d1199f167f..3bfae50bc4 100644 --- a/src/vcpkg/commands.format-manifest.cpp +++ b/src/vcpkg/commands.format-manifest.cpp @@ -148,7 +148,7 @@ namespace vcpkg else { auto maybe_manifest = - Paragraphs::try_load_project_manifest_text(contents->content, contents->origin, stdout_sink); + Paragraphs::try_load_project_manifest_text(contents->content, contents->origin, out_sink); if (auto manifest = maybe_manifest.get()) { to_write.push_back(ToWrite{contents->content, std::move(*manifest), path, path}); diff --git a/src/vcpkg/commands.help.cpp b/src/vcpkg/commands.help.cpp index 6c29d18ef1..b2206064cf 100644 --- a/src/vcpkg/commands.help.cpp +++ b/src/vcpkg/commands.help.cpp @@ -78,7 +78,7 @@ namespace {"assetcaching", [](const VcpkgPaths&) { msg::println(format_help_topic_asset_caching()); }}, {"binarycaching", [](const VcpkgPaths&) { msg::println(format_help_topic_binary_caching()); }}, {"commands", [](const VcpkgPaths&) { print_full_command_list(); }}, - {"topics", [](const VcpkgPaths&) { msg::print(help_topics()); }}, + {"topics", [](const VcpkgPaths&) { msg::println(help_topics()); }}, {"triplet", [](const VcpkgPaths& paths) { help_topic_valid_triplet(paths.get_triplet_db()); }}, {"versioning", help_topic_versioning}, }; @@ -101,7 +101,6 @@ namespace LocalizedString result; result.append(msgAvailableHelpTopics); result.append_floating_list(1, all_topic_names); - result.append_raw('\n'); return result; } @@ -193,8 +192,8 @@ namespace vcpkg } } - stderr_sink.println_error(msgUnknownTopic, msg::value = topic); - stderr_sink.print(help_topics()); + stderr_sink.println(msg::format_error(msgUnknownTopic, msg::value = topic)); + stderr_sink.println(help_topics()); get_global_metrics_collector().track_string(StringMetric::CommandContext, "unknown"); Checks::exit_fail(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.remove.cpp b/src/vcpkg/commands.remove.cpp index 1b6d43965e..ea54319828 100644 --- a/src/vcpkg/commands.remove.cpp +++ b/src/vcpkg/commands.remove.cpp @@ -68,7 +68,11 @@ namespace vcpkg } else { - msg::println_warning(msgFileNotFound, msg::path = target); + msg::println(Color::warning, + LocalizedString::from_raw(target) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFileNotFound)); } } diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 9fbbe44ce5..ed101d6b81 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -201,8 +201,9 @@ namespace vcpkg bool dependency_graph_success = false; if (snapshot && github_token && github_repository) { + WarningDiagnosticContext wdc{console_diagnostic_context}; dependency_graph_success = submit_github_dependency_graph_snapshot( - args.github_server_url, *github_token, *github_repository, *snapshot); + wdc, args.github_server_url, *github_token, *github_repository, *snapshot); if (dependency_graph_success) { msg::println(msgDependencyGraphSuccess); diff --git a/src/vcpkg/configure-environment.cpp b/src/vcpkg/configure-environment.cpp index f36a3d2e6f..97df6be9fc 100644 --- a/src/vcpkg/configure-environment.cpp +++ b/src/vcpkg/configure-environment.cpp @@ -105,25 +105,46 @@ namespace namespace vcpkg { - ExpectedL download_vcpkg_standalone_bundle(const DownloadManager& download_manager, - const Filesystem& fs, - const Path& download_root) + Optional download_vcpkg_standalone_bundle(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_root) { #if defined(VCPKG_STANDALONE_BUNDLE_SHA) const auto bundle_tarball = download_root / "vcpkg-standalone-bundle-" VCPKG_BASE_VERSION_AS_STRING ".tar.gz"; - msg::println(msgDownloadingVcpkgStandaloneBundle, msg::version = VCPKG_BASE_VERSION_AS_STRING); + context.statusln(msg::format(msgDownloadingVcpkgStandaloneBundle, msg::version = VCPKG_BASE_VERSION_AS_STRING)); const auto bundle_uri = "https://github.com/microsoft/vcpkg-tool/releases/download/" VCPKG_BASE_VERSION_AS_STRING "/vcpkg-standalone-bundle.tar.gz"; - download_manager.download_file( - fs, bundle_uri, {}, bundle_tarball, MACRO_TO_STRING(VCPKG_STANDALONE_BUNDLE_SHA), null_sink); + if (!download_file_asset_cached(context, + null_sink, + asset_cache_settings, + fs, + bundle_uri, + {}, + bundle_tarball, + MACRO_TO_STRING(VCPKG_STANDALONE_BUNDLE_SHA))) + { + return nullopt; + } #else // ^^^ VCPKG_STANDALONE_BUNDLE_SHA / !VCPKG_STANDALONE_BUNDLE_SHA vvv const auto bundle_tarball = download_root / "vcpkg-standalone-bundle-latest.tar.gz"; - msg::println(Color::warning, msgDownloadingVcpkgStandaloneBundleLatest); - fs.remove(bundle_tarball, VCPKG_LINE_INFO); + context.report(DiagnosticLine{DiagKind::Warning, msg::format(msgDownloadingVcpkgStandaloneBundleLatest)}); + std::error_code ec; + fs.remove(bundle_tarball, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "remove", {bundle_tarball})); + return nullopt; + } + const auto bundle_uri = "https://github.com/microsoft/vcpkg-tool/releases/latest/download/vcpkg-standalone-bundle.tar.gz"; - download_manager.download_file(fs, bundle_uri, {}, bundle_tarball, nullopt, null_sink); + if (!download_file_asset_cached( + context, null_sink, asset_cache_settings, fs, bundle_uri, {}, bundle_tarball, nullopt)) + { + return nullopt; + } #endif // ^^^ !VCPKG_STANDALONE_BUNDLE_SHA return bundle_tarball; } @@ -159,11 +180,17 @@ namespace vcpkg fs.remove_all(vcpkg_artifacts_path, VCPKG_LINE_INFO); auto temp = get_exe_path_of_current_process(); temp.replace_filename("vcpkg-artifacts-temp"); - auto tarball = download_vcpkg_standalone_bundle(paths.get_download_manager(), fs, paths.downloads) - .value_or_exit(VCPKG_LINE_INFO); - set_directory_to_archive_contents(fs, paths.get_tool_cache(), null_sink, tarball, temp); + auto maybe_tarball = download_vcpkg_standalone_bundle( + console_diagnostic_context, paths.get_asset_cache_settings(), fs, paths.downloads); + auto tarball = maybe_tarball.get(); + if (!tarball) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + + set_directory_to_archive_contents(fs, paths.get_tool_cache(), null_sink, *tarball, temp); fs.rename_with_retry(temp / "vcpkg-artifacts", vcpkg_artifacts_path, VCPKG_LINE_INFO); - fs.remove(tarball, VCPKG_LINE_INFO); + fs.remove(*tarball, VCPKG_LINE_INFO); fs.remove_all(temp, VCPKG_LINE_INFO); #if defined(VCPKG_STANDALONE_BUNDLE_SHA) fs.write_contents(vcpkg_artifacts_version_path, VCPKG_BASE_VERSION_AS_STRING, VCPKG_LINE_INFO); diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index e9af529c2e..4ac0217ca3 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -415,7 +415,7 @@ namespace vcpkg::Paragraphs std::string{}}; } - return PortLoadResult{try_load_port_manifest_text(manifest_contents, manifest_path, stdout_sink) + return PortLoadResult{try_load_port_manifest_text(manifest_contents, manifest_path, out_sink) .map([&](std::unique_ptr&& scf) { return SourceControlFileAndLocation{ std::move(scf), std::move(manifest_path), port_location.spdx_location}; diff --git a/src/vcpkg/postbuildlint.cpp b/src/vcpkg/postbuildlint.cpp index a11e6131e7..a7b2405729 100644 --- a/src/vcpkg/postbuildlint.cpp +++ b/src/vcpkg/postbuildlint.cpp @@ -42,21 +42,16 @@ namespace vcpkg const Path& relative_dir, const std::vector& relative_paths) { - auto ls = LocalizedString() - .append_raw('\n') - .append_raw(relative_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(kind_prefix) - .append_raw('\n'); + auto ls = + LocalizedString().append_raw(relative_dir).append_raw(": ").append_raw(NotePrefix).append(kind_prefix); for (const Path& package_relative_path : relative_paths) { auto as_generic = package_relative_path; as_generic.make_generic(); - ls.append_raw(NotePrefix).append_raw(as_generic).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append_raw(as_generic); } - msg_sink.print(ls); + msg_sink.println(ls); } // clang-format off @@ -113,12 +108,11 @@ namespace vcpkg const auto include_dir = package_dir / "include"; if (!fs.exists(include_dir, IgnoreErrors{}) || fs.is_empty(include_dir, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingIncludeDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingIncludeDir)); return LintStatus::PROBLEM_DETECTED; } @@ -134,12 +128,11 @@ namespace vcpkg const auto include_dir = package_dir / "include"; if (fs.exists(include_dir, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugIncludeDirInCMakeHelperPort) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugIncludeDirInCMakeHelperPort)); return LintStatus::PROBLEM_DETECTED; } @@ -228,20 +221,18 @@ namespace vcpkg if (!violations.empty()) { Util::sort(violations); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugRestrictedHeaderPaths) - .append_raw('\n')); - msg_sink.print(LocalizedString::from_raw(include_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortBugRestrictedHeaderPathsNote) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugRestrictedHeaderPaths)); + msg_sink.println(LocalizedString::from_raw(include_dir) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortBugRestrictedHeaderPathsNote)); for (auto&& violation : violations) { - msg_sink.print(LocalizedString::from_raw(NotePrefix).append_raw(violation).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append_raw(violation)); } return LintStatus::PROBLEM_DETECTED; @@ -263,14 +254,12 @@ namespace vcpkg if (!files_found.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDuplicateIncludeFiles) - .append_raw('\n')); - msg_sink.print( - LocalizedString::from_raw(NotePrefix).append(msgPortBugDuplicateIncludeFilesFixIt).append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDuplicateIncludeFiles)); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append(msgPortBugDuplicateIncludeFilesFixIt)); return LintStatus::PROBLEM_DETECTED; } @@ -285,12 +274,11 @@ namespace vcpkg const auto debug_share = package_dir / FileDebug / FileShare; if (fs.exists(debug_share, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDebugShareDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDebugShareDir)); return LintStatus::PROBLEM_DETECTED; } @@ -305,12 +293,11 @@ namespace vcpkg { if (!fs.exists(package_dir / FileShare / package_name / FileVcpkgPortConfig, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingCMakeHelperPortFile) - .append_raw("\n")); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingCMakeHelperPortFile)); return LintStatus::PROBLEM_DETECTED; } @@ -332,23 +319,21 @@ namespace vcpkg if (fs.is_regular_file(usage_path_from) && !fs.is_regular_file(usage_path_to)) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingProvidedUsage) - .append_raw('\n')); - msg_sink.print(LocalizedString::from_raw(usage_path_from) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgUsageTextHere) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgUsageInstallInstructions) - .append_raw('\n') - .append_raw(NotePrefix) - .append_raw(STANDARD_INSTALL_USAGE) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingProvidedUsage)); + msg_sink.println(LocalizedString::from_raw(usage_path_from) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgUsageTextHere) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgUsageInstallInstructions) + .append_raw('\n') + .append_raw(NotePrefix) + .append_raw(STANDARD_INSTALL_USAGE)); return LintStatus::PROBLEM_DETECTED; } @@ -382,11 +367,11 @@ namespace vcpkg if (!misplaced_cmake_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedCMakeFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedCMakeFiles)); print_relative_paths( msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, misplaced_cmake_files); return LintStatus::PROBLEM_DETECTED; @@ -404,12 +389,11 @@ namespace vcpkg fs.exists(package_dir / "debug" VCPKG_PREFERRED_SEPARATOR "lib" VCPKG_PREFERRED_SEPARATOR "cmake", IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMergeLibCMakeDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMergeLibCMakeDir)); return LintStatus::PROBLEM_DETECTED; } @@ -448,11 +432,11 @@ namespace vcpkg if (!bad_dlls.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDllInLibDir)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDllInLibDir)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, bad_dlls); return LintStatus::PROBLEM_DETECTED; } @@ -474,22 +458,20 @@ namespace vcpkg { case FileType::regular: return LintStatus::SUCCESS; break; case FileType::directory: - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgCopyrightIsDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgCopyrightIsDir)); return LintStatus::PROBLEM_DETECTED; default: break; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingLicense) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingLicense)); // We only search in the root of each unpacked source archive to reduce false positives auto src_relative_dirs = Util::fmap(fs.get_directories_non_recursive(build_dir / "src", IgnoreErrors{}), @@ -512,14 +494,13 @@ namespace vcpkg auto args = Util::fmap(src_dir_relative_copyright_files, [](StringLiteral copyright_file) { return fmt::format(FMT_COMPILE("\"${{SOURCE_PATH}}/{}\""), copyright_file); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(portfile_cmake) .append_raw(": ") .append_raw(NotePrefix) .append(msgPortBugMissingLicenseFixIt, msg::value = fmt::format(FMT_COMPILE("vcpkg_install_copyright(FILE_LIST {})"), - fmt::join(args, " "))) - .append_raw('\n')); + fmt::join(args, " ")))); } return LintStatus::PROBLEM_DETECTED; @@ -542,10 +523,10 @@ namespace vcpkg if (!build_dir_relative_copyright_files.empty()) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortBugFoundCopyrightFiles)); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortBugFoundCopyrightFiles)); print_relative_paths( msg_sink, msgFilesRelativeToTheBuildDirectoryHere, build_dir, build_dir_relative_copyright_files); } @@ -570,11 +551,11 @@ namespace vcpkg if (!exes.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundExeInBinDir)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundExeInBinDir)); print_relative_paths(msg_sink, msgExecutablesRelativeToThePackageDirectoryHere, package_dir, exes); return LintStatus::PROBLEM_DETECTED; @@ -662,11 +643,11 @@ namespace vcpkg if (!dlls_with_no_exports.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugSetDllsWithoutExports)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugSetDllsWithoutExports)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, dlls_with_no_exports); return LintStatus::PROBLEM_DETECTED; } @@ -696,11 +677,11 @@ namespace vcpkg if (!dlls_with_improper_uwp_bit.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDllAppContainerBitNotSet)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDllAppContainerBitNotSet)); print_relative_paths( msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, dlls_with_improper_uwp_bit); return LintStatus::PROBLEM_DETECTED; @@ -759,28 +740,26 @@ namespace vcpkg std::vector binaries_with_invalid_architecture, MessageSink& msg_sink) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgBuiltWithIncorrectArchitecture, msg::arch = expected_architecture) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgBuiltWithIncorrectArchitecture, msg::arch = expected_architecture)); auto msg = LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n'); + .append(msgBinariesRelativeToThePackageDirectoryHere); for (const FileAndArch& b : binaries_with_invalid_architecture) { - msg.append_raw(NotePrefix) + msg.append_raw('\n') + .append_raw(NotePrefix) .append(msgBinaryWithInvalidArchitecture, msg::path = b.relative_file.generic_u8string(), - msg::arch = b.actual_arch) - .append_raw('\n'); + msg::arch = b.actual_arch); } - msg_sink.print(msg); + msg_sink.println(msg); } static void check_dll_architecture(const std::string& expected_architecture, @@ -879,11 +858,11 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundDllInStaticBuild)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundDllInStaticBuild)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, relative_dlls); return LintStatus::PROBLEM_DETECTED; } @@ -906,7 +885,7 @@ namespace vcpkg { auto as_generic = binary; as_generic.make_generic(); - ls.append_raw(NotePrefix).append_raw(as_generic).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append_raw(as_generic); } } } @@ -924,38 +903,36 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMismatchingNumberOfBinaries) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMismatchingNumberOfBinaries)); LocalizedString ls = LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n'); + .append(msgBinariesRelativeToThePackageDirectoryHere); if (debug_count == 0) { - ls.append_raw(NotePrefix).append(msgPortBugMissingDebugBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugMissingDebugBinaries); } else { - ls.append_raw(NotePrefix).append(msgPortBugFoundDebugBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugFoundDebugBinaries); append_binary_set(ls, relative_debug_binary_sets); } if (release_count == 0) { - ls.append_raw(NotePrefix).append(msgPortBugMissingReleaseBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugMissingReleaseBinaries); } else { - ls.append_raw(NotePrefix).append(msgPortBugFoundReleaseBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugFoundReleaseBinaries); append_binary_set(ls, relative_release_binary_sets); } - msg_sink.print(ls); + msg_sink.println(ls); return LintStatus::PROBLEM_DETECTED; } @@ -966,11 +943,10 @@ namespace vcpkg { if (lib_count == 0 && dll_count != 0) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingImportedLibs) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingImportedLibs)); return LintStatus::PROBLEM_DETECTED; } @@ -998,24 +974,23 @@ namespace vcpkg for (auto&& bad_dir : bad_dirs) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugBinDirExists, msg::path = bad_dir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugBinDirExists, msg::path = bad_dir)); } auto args = Util::fmap(bad_dirs, [](const Path& bad_dir) { return fmt::format(FMT_COMPILE("\"${{CURRENT_PACKAGES_DIR}}/{}\""), bad_dir); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(NotePrefix) .append(msgPortBugRemoveBinDir) .append_raw('\n') .append_raw("if(VCPKG_LIBRARY_LINKAGE STREQUAL \"static\")\n") .append_indent() - .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\nendif()\n"), fmt::join(args, " ")))); + .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\nendif()"), fmt::join(args, " ")))); return bad_dirs.size(); } @@ -1031,24 +1006,23 @@ namespace vcpkg if (!relative_empty_directories.empty()) { Util::sort(relative_empty_directories); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundEmptyDirectories) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundEmptyDirectories)); auto args = Util::fmap(relative_empty_directories, [](const Path& empty_dir) { return fmt::format(FMT_COMPILE("\"${{CURRENT_PACKAGES_DIR}}/{}\""), empty_dir.generic_u8string()); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) .append(msgDirectoriesRelativeToThePackageDirectoryHere) .append_raw('\n') .append_raw(NotePrefix) - .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\n"), fmt::join(args, " ")))); + .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})"), fmt::join(args, " ")))); return LintStatus::PROBLEM_DETECTED; } @@ -1126,18 +1100,18 @@ namespace vcpkg if (!misplaced_pkgconfig_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedPkgConfigFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedPkgConfigFiles)); print_relative_paths(msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, Util::fmap(misplaced_pkgconfig_files, [](const MisplacedFile& mf) -> Path { return mf.relative_path; })); - msg_sink.print(LocalizedString::from_raw(NotePrefix).append(msgPortBugMovePkgConfigFiles).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append(msgPortBugMovePkgConfigFiles)); { auto create_directory_line = LocalizedString::from_raw("file(MAKE_DIRECTORY"); std::vector directories; @@ -1162,8 +1136,8 @@ namespace vcpkg fmt::format(FMT_COMPILE(R"###( "${{CURRENT_PACKAGES_DIR}}/{}")###"), directory)); } - create_directory_line.append_raw(")\n"); - msg_sink.print(create_directory_line); + create_directory_line.append_raw(")"); + msg_sink.println(create_directory_line); } // destroy create_directory_line for (const auto& item : misplaced_pkgconfig_files) @@ -1177,15 +1151,15 @@ namespace vcpkg default: Checks::unreachable(VCPKG_LINE_INFO); } - msg_sink.print(LocalizedString::from_raw(fmt::format( - FMT_COMPILE(R"###(file(RENAME "${{CURRENT_PACKAGES_DIR}}/{}" "${{CURRENT_PACKAGES_DIR}}/{}/{}"))###" - "\n"), + msg_sink.println(LocalizedString::from_raw(fmt::format( + FMT_COMPILE( + R"###(file(RENAME "${{CURRENT_PACKAGES_DIR}}/{}" "${{CURRENT_PACKAGES_DIR}}/{}/{}"))###"), item.relative_path.generic_u8string(), *dir, item.relative_path.filename()))); } - msg_sink.print(LocalizedString::from_raw("vcpkg_fixup_pkgconfig()\n")); + msg_sink.println(LocalizedString::from_raw("vcpkg_fixup_pkgconfig()")); msg_sink.println(msgPortBugRemoveEmptyDirs); return LintStatus::PROBLEM_DETECTED; @@ -1327,32 +1301,28 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugInvalidCrtLinkageHeader) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugInvalidCrtLinkageHeader)); - msg_sink.print(LocalizedString::from_raw(package_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(package_dir) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgBinariesRelativeToThePackageDirectoryHere)); for (auto&& group : groups_of_invalid_crt) { - msg_sink.print(LocalizedString::from_raw(NotePrefix) - .append(msgPortBugInvalidCrtLinkageCrtGroup, msg::expected = to_string(group.first)) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix) + .append(msgPortBugInvalidCrtLinkageCrtGroup, msg::expected = to_string(group.first))); for (auto&& file : group.second) { for (auto&& linkage : file.linkages) { - msg_sink.print(LocalizedString::from_raw(NotePrefix) - .append(msgPortBugInvalidCrtLinkageEntry, - msg::path = file.relative_file.generic_u8string(), - msg::actual = to_string(linkage)) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix) + .append(msgPortBugInvalidCrtLinkageEntry, + msg::path = file.relative_file.generic_u8string(), + msg::actual = to_string(linkage))); } } } @@ -1390,11 +1360,11 @@ namespace vcpkg if (!dlls_with_outdated_crt.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugOutdatedCRT)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugOutdatedCRT)); print_relative_paths( msg_sink, @@ -1437,11 +1407,11 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugKernel32FromXbox)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugKernel32FromXbox)); print_relative_paths( msg_sink, msgDllsRelativeToThePackageDirectoryHere, @@ -1487,11 +1457,11 @@ namespace vcpkg if (!misplaced_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedFiles)); print_relative_paths(msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, misplaced_files); return LintStatus::PROBLEM_DETECTED; } @@ -1586,34 +1556,31 @@ namespace vcpkg } Util::sort(failing_files); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgFilesContainAbsolutePath1) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFilesContainAbsolutePath1)); for (auto&& absolute_path : prohibited_absolute_paths) { - msg_sink.print(LocalizedString::from_raw(NotePrefix).append_raw(absolute_path).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append_raw(absolute_path)); } if (any_pc_file_fails) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgFilesContainAbsolutePathPkgconfigNote) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgFilesContainAbsolutePathPkgconfigNote)); } for (auto&& failing_file : failing_files) { failing_file.make_preferred(); - msg_sink.print(LocalizedString::from_raw(package_dir / failing_file) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgFilesContainAbsolutePath2) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(package_dir / failing_file) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgFilesContainAbsolutePath2)); } return LintStatus::PROBLEM_DETECTED; @@ -1937,12 +1904,11 @@ namespace vcpkg spec, paths, pre_build_info, build_info, port_dir, portfile_cmake, msg_sink); if (error_count != 0) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgFailedPostBuildChecks, msg::count = error_count) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFailedPostBuildChecks, msg::count = error_count)); } return error_count; diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 20c19af394..3ec3bd61f6 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -741,7 +742,7 @@ namespace vcpkg struct ToolCacheImpl final : ToolCache { const Filesystem& fs; - const std::shared_ptr downloader; + AssetCachingSettings asset_cache_settings; const Path downloads; const Path config_path; const Path tools; @@ -751,13 +752,13 @@ namespace vcpkg vcpkg::Lazy> m_tool_data_cache; ToolCacheImpl(const Filesystem& fs, - const std::shared_ptr& downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) : fs(fs) - , downloader(downloader) + , asset_cache_settings(asset_cache_settings) , downloads(std::move(downloads)) , config_path(std::move(config_path)) , tools(std::move(tools)) @@ -799,6 +800,8 @@ namespace vcpkg Path download_tool(const ToolData& tool_data, MessageSink& status_sink) const { + using namespace Hash; + const std::array& version = tool_data.version; const std::string version_as_string = fmt::format("{}.{}.{}", version[0], version[1], version[2]); Checks::msg_check_maybe_upgrade(VCPKG_LINE_INFO, @@ -812,13 +815,38 @@ namespace vcpkg msg::version = version_as_string); const auto download_path = downloads / tool_data.download_subpath; - if (!fs.exists(download_path, IgnoreErrors{})) + const auto hash_result = get_file_hash(console_diagnostic_context, fs, download_path, Algorithm::Sha512); + switch (hash_result.prognosis) { - downloader->download_file(fs, tool_data.url, {}, download_path, tool_data.sha512, null_sink); - } - else - { - verify_downloaded_file_hash(fs, tool_data.url, download_path, tool_data.sha512); + case HashPrognosis::Success: + if (!Strings::case_insensitive_ascii_equals(tool_data.sha512, hash_result.hash)) + { + Checks::msg_exit_with_message(VCPKG_LINE_INFO, + LocalizedString::from_raw(download_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgToolHashMismatch, + msg::tool_name = tool_data.name, + msg::expected = tool_data.sha512, + msg::actual = hash_result.hash)); + } + + break; + case HashPrognosis::FileNotFound: + if (!download_file_asset_cached(console_diagnostic_context, + null_sink, + asset_cache_settings, + fs, + tool_data.url, + {}, + download_path, + tool_data.sha512)) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + break; + case HashPrognosis::OtherError: Checks::exit_fail(VCPKG_LINE_INFO); + default: Checks::unreachable(VCPKG_LINE_INFO); } const auto tool_dir_path = tools / tool_data.tool_dir_subpath; @@ -1095,14 +1123,14 @@ namespace vcpkg } std::unique_ptr get_tool_cache(const Filesystem& fs, - std::shared_ptr downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) { return std::make_unique( - fs, std::move(downloader), downloads, config_path, tools, abiToolVersionHandling); + fs, asset_cache_settings, downloads, config_path, tools, abiToolVersionHandling); } struct ToolDataEntryDeserializer final : Json::IDeserializer diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 06f52c5801..ccfe2f5152 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -324,8 +324,8 @@ namespace , m_ff_settings(args.feature_flag_settings()) , m_manifest_dir(compute_manifest_dir(fs, args, original_cwd)) , m_bundle(bundle) - , m_download_manager(std::make_shared( - parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO))) + , m_asset_cache_settings( + parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO)) , m_builtin_ports(process_output_directory(fs, args.builtin_ports_root_dir.get(), root / "ports")) , m_default_vs_path(args.default_visual_studio_path .map([&fs](const std::string& default_visual_studio_path) { @@ -342,7 +342,7 @@ namespace const FeatureFlagSettings m_ff_settings; const Path m_manifest_dir; const BundleSettings m_bundle; - const std::shared_ptr m_download_manager; + const AssetCachingSettings m_asset_cache_settings; const Path m_builtin_ports; const Path m_default_vs_path; const Path scripts; @@ -572,7 +572,7 @@ namespace vcpkg VCPKG_LINE_INFO)) , m_tool_cache(get_tool_cache( fs, - m_download_manager, + m_asset_cache_settings, downloads, args.tools_data_file.has_value() ? Path{*args.tools_data_file.get()} : scripts / "vcpkg-tools.json", tools, @@ -670,11 +670,6 @@ namespace vcpkg Debug::print("Using vcpkg-root: ", root, '\n'); Debug::print("Using builtin-registry: ", builtin_registry_versions, '\n'); Debug::print("Using downloads-root: ", downloads, '\n'); - m_pimpl->m_download_manager->get_block_origin() - ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") - : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); - m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") - : Debug::println("Asset cache is not configured."); const auto config_path = m_pimpl->m_config_dir / "vcpkg-configuration.json"; auto maybe_manifest_config = config_from_manifest(m_pimpl->m_manifest_doc); @@ -884,7 +879,7 @@ namespace vcpkg } const Filesystem& VcpkgPaths::get_filesystem() const { return m_pimpl->m_fs; } - const DownloadManager& VcpkgPaths::get_download_manager() const { return *m_pimpl->m_download_manager; } + const AssetCachingSettings& VcpkgPaths::get_asset_cache_settings() const { return m_pimpl->m_asset_cache_settings; } const ToolCache& VcpkgPaths::get_tool_cache() const { return *m_pimpl->m_tool_cache; } const Path& VcpkgPaths::get_tool_exe(StringView tool, MessageSink& status_messages) const {