From 43c47e8c8dd7f01d3f8f1273bd24b3657d2ed9f4 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Tue, 24 Oct 2023 16:50:49 -0700 Subject: [PATCH] Use ports-root and registry-versions git repos. (#1241) * Use the git repo of --x-builtin-ports-root and --x-builtin-registry-versions-dir when talking about port commit shas and version tree shas, respectively. I believe this is necessary to make the instructions in https://learn.microsoft.com/vcpkg/produce/publish-to-a-git-registry#4---update-the-versions-database actually work. This also reduces the number of things that assume VCPKG_ROOT is a git repo. (Since it is not when we are shipped in VS or acquired with the one liner) --- .../end-to-end-tests-dir/registries.ps1 | 36 +++++++++---------- .../end-to-end-tests-dir/versions.ps1 | 18 +++++++--- azure-pipelines/end-to-end-tests-prelude.ps1 | 6 ++++ include/vcpkg/vcpkgpaths.h | 1 + src/vcpkg/commands.ci-verify-versions.cpp | 3 +- src/vcpkg/commands.portsdiff.cpp | 6 ++-- src/vcpkg/registries.cpp | 7 ++-- src/vcpkg/vcpkgpaths.cpp | 35 +++++++++--------- 8 files changed, 64 insertions(+), 48 deletions(-) diff --git a/azure-pipelines/end-to-end-tests-dir/registries.ps1 b/azure-pipelines/end-to-end-tests-dir/registries.ps1 index b3134d91a..24cb90a80 100644 --- a/azure-pipelines/end-to-end-tests-dir/registries.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/registries.ps1 @@ -3,17 +3,15 @@ $env:X_VCPKG_REGISTRIES_CACHE = Join-Path $TestingRoot 'registries' New-Item -ItemType Directory -Force $env:X_VCPKG_REGISTRIES_CACHE | Out-Null -$builtinRegistryArgs = $commonArgs + @("--x-builtin-registry-versions-dir=$PSScriptRoot/../e2e-ports/versions") - -Run-Vcpkg install @builtinRegistryArgs 'vcpkg-internal-e2e-test-port' +Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port' Throw-IfNotFailed # We should not look into the versions directory unless we have a baseline, # even if we pass the registries feature flag -Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port' +Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port' Throw-IfNotFailed -Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-cmake' +Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-cmake' Throw-IfFailed Write-Trace "Test git and filesystem registries" @@ -33,9 +31,13 @@ Copy-Item -Recurse ` New-Item ` -Path "$filesystemRegistry/versions" ` -ItemType Directory -Copy-Item ` - -LiteralPath "$PSScriptRoot/../e2e-ports/versions/baseline.json" ` - -Destination "$filesystemRegistry/versions/baseline.json" +Set-Content -Value @" +{ + "default": { + "vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" } + } +} +"@ -LiteralPath "$filesystemRegistry/versions/baseline.json" New-Item ` -Path "$filesystemRegistry/versions/v-" ` -ItemType Directory @@ -62,12 +64,6 @@ $gitRegistryUpstream = (Get-Item $gitRegistryUpstream).FullName Push-Location $gitRegistryUpstream try { - $gitConfigOptions = @( - '-c', 'user.name=Nobody', - '-c', 'user.email=nobody@example.com', - '-c', 'core.autocrlf=false' - ) - $gitMainBranch = 'main' $gitSecondaryBranch = 'secondary' @@ -221,7 +217,7 @@ try New-Item -Path 'vcpkg-configuration.json' -ItemType File ` -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson) - Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' + Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' Throw-IfFailed } finally @@ -267,7 +263,7 @@ try New-Item -Path 'vcpkg-configuration.json' -ItemType File ` -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson) - Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' + Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' Throw-IfFailed } finally @@ -324,7 +320,7 @@ try New-Item -Path 'vcpkg-configuration.json' -ItemType File ` -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson) - Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' --dry-run + Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' --dry-run Throw-IfFailed Require-FileExists $env:X_VCPKG_REGISTRIES_CACHE/git-trees/$vcpkgInternalE2eTestPortGitTree Require-FileExists $env:X_VCPKG_REGISTRIES_CACHE/git-trees/$vcpkgInternalE2eTestPort2GitTree @@ -334,7 +330,7 @@ try # Dry run does not create a lockfile Require-FileNotExists $installRoot/vcpkg/vcpkg-lock.json - Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' + Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' Throw-IfFailed $expectedVcpkgLockJson = "{$(ConvertTo-Json $gitRegistryUpstream):{ @@ -379,7 +375,7 @@ try } New-Item -Path $installRoot/vcpkg/vcpkg-lock.json -ItemType File ` -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgLockJson) - Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' + Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' Throw-IfFailed } finally @@ -407,7 +403,7 @@ try New-Item -Path 'vcpkg.json' -ItemType File ` -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson) - Run-Vcpkg search @builtinRegistryArgs zlib + Run-Vcpkg search @commonArgs zlib Throw-IfFailed } finally diff --git a/azure-pipelines/end-to-end-tests-dir/versions.ps1 b/azure-pipelines/end-to-end-tests-dir/versions.ps1 index 8921ed106..a8c649296 100644 --- a/azure-pipelines/end-to-end-tests-dir/versions.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/versions.ps1 @@ -1,6 +1,18 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 -$versionFilesPath = "$PSScriptRoot/../e2e-ports/version-files" +$versionFilesPathSources = "$PSScriptRoot/../e2e-ports/version-files" +$versionFilesPath = "$TestingRoot/version-files" + +function Refresh-VersionFiles() { + Refresh-TestRoot + Copy-Item -Recurse $versionFilesPathSources $versionFilesPath + git -C $versionFilesPath @gitConfigOptions init + git -C $versionFilesPath @gitConfigOptions add -A + git -C $versionFilesPath @gitConfigOptions commit -m testing + git -C $versionFilesPath fetch https://github.com/vicroms/test-registries +} + +Refresh-VersionFiles # Ensure transitive packages can be used even if they add version constraints $CurrentTest = "transitive constraints without baseline" @@ -8,7 +20,6 @@ Run-Vcpkg install @commonArgs --dry-run ` "--x-builtin-ports-root=$versionFilesPath/transitive-constraints/ports" ` "--x-manifest-root=$versionFilesPath/transitive-constraints" Throw-IfFailed -Refresh-TestRoot # Test verify versions mkdir $VersionFilesRoot | Out-Null @@ -86,11 +97,10 @@ if (($out -notmatch ".*error: Failed to load port because versions are inconsist throw "Expected to fail due to mismatched versions between portfile and the version database" } -git -C "$env:VCPKG_ROOT" fetch https://github.com/vicroms/test-registries foreach ($opt_registries in @("",",registries")) { Write-Trace "testing baselines: $opt_registries" - Refresh-TestRoot + Refresh-VersionFiles $CurrentTest = "without default baseline 2 -- enabling versions should not change behavior" Remove-Item -Recurse $buildtreesRoot/versioning_ -ErrorAction SilentlyContinue Run-Vcpkg @commonArgs "--feature-flags=versions$opt_registries" install ` diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index f8ec6de48..ec656f301 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -19,6 +19,12 @@ $commonArgs = @( $Triplet ) + $directoryArgs +$gitConfigOptions = @( + '-c', 'user.name=Nobody', + '-c', 'user.email=nobody@example.com', + '-c', 'core.autocrlf=false' +) + $Script:CurrentTest = 'unassigned' function Refresh-TestRoot { diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index 79339089c..05c8e34a8 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -89,6 +89,7 @@ namespace vcpkg const Path& downloads; const Path& tools; const Path builtin_registry_versions; + ExpectedL versions_dot_git_dir() const; const Path prefab; const Path buildsystems; const Path buildsystems_msbuild_targets; diff --git a/src/vcpkg/commands.ci-verify-versions.cpp b/src/vcpkg/commands.ci-verify-versions.cpp index acdbb0f9a..51c01bbe2 100644 --- a/src/vcpkg/commands.ci-verify-versions.cpp +++ b/src/vcpkg/commands.ci-verify-versions.cpp @@ -66,7 +66,8 @@ namespace for (StringView control_file : {"CONTROL", "vcpkg.json"}) { auto treeish = Strings::concat(version_entry.git_tree, ':', control_file); - auto maybe_file = paths.git_show(Strings::concat(treeish), paths.root / ".git"); + auto maybe_file = paths.git_show(Strings::concat(treeish), + paths.versions_dot_git_dir().value_or_exit(VCPKG_LINE_INFO)); if (!maybe_file) continue; const auto& file = maybe_file.value_or_exit(VCPKG_LINE_INFO); diff --git a/src/vcpkg/commands.portsdiff.cpp b/src/vcpkg/commands.portsdiff.cpp index ce557ed65..bc888daf5 100644 --- a/src/vcpkg/commands.portsdiff.cpp +++ b/src/vcpkg/commands.portsdiff.cpp @@ -32,9 +32,11 @@ namespace std::vector read_ports_from_commit(const VcpkgPaths& paths, StringView git_commit_id) { auto& fs = paths.get_filesystem(); - const auto dot_git_dir = paths.root / ".git"; + const auto dot_git_dir = fs.try_find_file_recursively_up(paths.builtin_ports_directory().parent_path(), ".git") + .map([](Path&& dot_git_parent) { return std::move(dot_git_parent) / ".git"; }) + .value_or_exit(VCPKG_LINE_INFO); const auto ports_dir_name = paths.builtin_ports_directory().filename(); - const auto temp_checkout_path = paths.root / fmt::format("{}-{}", ports_dir_name, git_commit_id); + const auto temp_checkout_path = paths.buildtrees() / fmt::format("{}-{}", ports_dir_name, git_commit_id); fs.create_directory(temp_checkout_path, IgnoreErrors{}); const auto checkout_this_dir = fmt::format("./{}", ports_dir_name); // Must be relative to the root of the repository diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index 26f464e16..cff0efd53 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -658,7 +658,9 @@ namespace { const auto destination_tmp = destination_parent / "baseline.json.tmp"; auto treeish = Strings::concat(commit_sha, ":versions/baseline.json"); - auto maybe_contents = paths.git_show(treeish, paths.root / ".git"); + auto maybe_contents = + paths.versions_dot_git_dir().then([&](Path&& dot_git) { return paths.git_show(treeish, dot_git); }); + if (auto contents = maybe_contents.get()) { std::error_code ec; @@ -1129,7 +1131,8 @@ namespace } const auto& git_tree = port_versions_soa.git_trees()[it - port_versions.begin()]; - return m_paths.git_checkout_port(port_name, git_tree, m_paths.root / ".git") + return m_paths.versions_dot_git_dir() + .then([&, this](Path&& dot_git) { return m_paths.git_checkout_port(port_name, git_tree, dot_git); }) .map([&git_tree](Path&& p) -> PathAndLocation { return { std::move(p), diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index 96226e168..e39d3639a 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -73,28 +73,19 @@ namespace { std::error_code ec; auto manifest_path = manifest_dir / "vcpkg.json"; - auto manifest_opt = Json::parse_file(fs, manifest_path, ec); - if (ec) + auto maybe_manifest_object = fs.try_read_contents(manifest_path).then([](FileContents&& contents) { + return Json::parse_object(contents.content, contents.origin); + }); + + if (auto manifest_object = maybe_manifest_object.get()) { - Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO, - msg::format(msgFailedToLoadManifest, msg::path = manifest_dir) - .append_raw('\n') - .append_raw(ec.message())); + return ManifestAndPath{std::move(*manifest_object), std::move(manifest_path)}; } - if (!manifest_opt) - { - Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO, - LocalizedString::from_raw(manifest_opt.error()->to_string())); - } - - auto manifest_value = std::move(manifest_opt).value(VCPKG_LINE_INFO).value; - if (!manifest_value.is_object()) - { - msg::println_error(msgFailedToParseNoTopLevelObj, msg::path = manifest_path); - Checks::exit_fail(VCPKG_LINE_INFO); - } - return {std::move(manifest_value).object(VCPKG_LINE_INFO), std::move(manifest_path)}; + Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO, + msg::format(msgFailedToLoadManifest, msg::path = manifest_dir) + .append_raw('\n') + .append(maybe_manifest_object.error())); } static Optional config_from_manifest(const Optional& manifest_doc) @@ -821,6 +812,12 @@ namespace vcpkg } } + ExpectedL VcpkgPaths::versions_dot_git_dir() const + { + return m_pimpl->m_fs.try_find_file_recursively_up(builtin_registry_versions.parent_path(), ".git") + .map([](Path&& dot_git_parent) { return std::move(dot_git_parent) / ".git"; }); + } + std::string VcpkgPaths::get_toolver_diagnostics() const { std::string ret;