Use ports-root and registry-versions git repos. (#1241)

* Use the git repo of --x-builtin-ports-root and --x-builtin-registry-versions-dir when talking about port commit shas and version tree shas, respectively. I believe this is necessary to make the instructions in https://learn.microsoft.com/vcpkg/produce/publish-to-a-git-registry#4---update-the-versions-database actually work. This also reduces the number of things that assume VCPKG_ROOT is a git repo. (Since it is not when we are shipped in VS or acquired with the one liner)
2023-10-24 16:50:49 -07:00 · 2023-10-24 16:50:49 -07:00 · 43c47e8c8d
--- a/azure-pipelines/end-to-end-tests-dir/registries.ps1
+++ b/azure-pipelines/end-to-end-tests-dir/registries.ps1
@ -3,17 +3,15 @@
 $env:X_VCPKG_REGISTRIES_CACHE = Join-Path $TestingRoot 'registries'
 New-Item -ItemType Directory -Force $env:X_VCPKG_REGISTRIES_CACHE | Out-Null

-$builtinRegistryArgs = $commonArgs + @("--x-builtin-registry-versions-dir=$PSScriptRoot/../e2e-ports/versions")
-
-Run-Vcpkg install @builtinRegistryArgs 'vcpkg-internal-e2e-test-port'
+Run-Vcpkg install @commonArgs 'vcpkg-internal-e2e-test-port'
 Throw-IfNotFailed

 # We should not look into the versions directory unless we have a baseline,
 # even if we pass the registries feature flag
-Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
+Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-internal-e2e-test-port'
 Throw-IfNotFailed

-Run-Vcpkg install @builtinRegistryArgs --feature-flags=registries 'vcpkg-cmake'
+Run-Vcpkg install @commonArgs --feature-flags=registries 'vcpkg-cmake'
 Throw-IfFailed

 Write-Trace "Test git and filesystem registries"
@ -33,9 +31,13 @@ Copy-Item -Recurse `
 New-Item `
    -Path "$filesystemRegistry/versions" `
    -ItemType Directory
-Copy-Item `
-    -LiteralPath "$PSScriptRoot/../e2e-ports/versions/baseline.json" `
-    -Destination "$filesystemRegistry/versions/baseline.json"
+Set-Content -Value @"
+{
+    "default": {
+        "vcpkg-internal-e2e-test-port": { "baseline": "1.0.0" }
+    }
+}
+"@ -LiteralPath "$filesystemRegistry/versions/baseline.json"
 New-Item `
    -Path "$filesystemRegistry/versions/v-" `
    -ItemType Directory
@ -62,12 +64,6 @@ $gitRegistryUpstream = (Get-Item $gitRegistryUpstream).FullName
 Push-Location $gitRegistryUpstream
 try
 {
-    $gitConfigOptions = @(
-        '-c', 'user.name=Nobody',
-        '-c', 'user.email=nobody@example.com',
-        '-c', 'core.autocrlf=false'
-    )
-
    $gitMainBranch = 'main'
    $gitSecondaryBranch = 'secondary'

@ -221,7 +217,7 @@ try
    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)

-    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests'
    Throw-IfFailed
 }
 finally
@ -267,7 +263,7 @@ try
    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)

-    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests'
    Throw-IfFailed
 }
 finally
@ -324,7 +320,7 @@ try
    New-Item -Path 'vcpkg-configuration.json' -ItemType File `
        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgConfigurationJson)

-    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests' --dry-run
+    Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests' --dry-run
    Throw-IfFailed
    Require-FileExists $env:X_VCPKG_REGISTRIES_CACHE/git-trees/$vcpkgInternalE2eTestPortGitTree
    Require-FileExists $env:X_VCPKG_REGISTRIES_CACHE/git-trees/$vcpkgInternalE2eTestPort2GitTree
@ -334,7 +330,7 @@ try
    # Dry run does not create a lockfile
    Require-FileNotExists $installRoot/vcpkg/vcpkg-lock.json

-    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests'
    Throw-IfFailed

    $expectedVcpkgLockJson = "{$(ConvertTo-Json $gitRegistryUpstream):{
@ -379,7 +375,7 @@ try
    }
   New-Item -Path $installRoot/vcpkg/vcpkg-lock.json -ItemType File `
        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgLockJson)
-    Run-Vcpkg install @builtinRegistryArgs '--feature-flags=registries,manifests'
+    Run-Vcpkg install @commonArgs '--feature-flags=registries,manifests'
    Throw-IfFailed
 }
 finally
@ -407,7 +403,7 @@ try
    New-Item -Path 'vcpkg.json' -ItemType File `
        -Value (ConvertTo-Json -Depth 5 -InputObject $vcpkgJson)

-    Run-Vcpkg search @builtinRegistryArgs zlib
+    Run-Vcpkg search @commonArgs zlib
    Throw-IfFailed
 }
 finally
--- a/azure-pipelines/end-to-end-tests-dir/versions.ps1
+++ b/azure-pipelines/end-to-end-tests-dir/versions.ps1
@ -1,6 +1,18 @@
 . $PSScriptRoot/../end-to-end-tests-prelude.ps1

-$versionFilesPath = "$PSScriptRoot/../e2e-ports/version-files"
+$versionFilesPathSources = "$PSScriptRoot/../e2e-ports/version-files"
+$versionFilesPath = "$TestingRoot/version-files"
+
+function Refresh-VersionFiles() {
+    Refresh-TestRoot
+    Copy-Item -Recurse $versionFilesPathSources $versionFilesPath
+    git -C $versionFilesPath @gitConfigOptions init
+    git -C $versionFilesPath @gitConfigOptions add -A
+    git -C $versionFilesPath @gitConfigOptions commit -m testing
+    git -C $versionFilesPath fetch https://github.com/vicroms/test-registries
+}
+
+Refresh-VersionFiles

 # Ensure transitive packages can be used even if they add version constraints
 $CurrentTest = "transitive constraints without baseline"
@ -8,7 +20,6 @@ Run-Vcpkg install @commonArgs --dry-run `
    "--x-builtin-ports-root=$versionFilesPath/transitive-constraints/ports" `
    "--x-manifest-root=$versionFilesPath/transitive-constraints"
 Throw-IfFailed
-Refresh-TestRoot

 # Test verify versions
 mkdir $VersionFilesRoot | Out-Null
@ -86,11 +97,10 @@ if (($out -notmatch ".*error: Failed to load port because versions are inconsist
    throw "Expected to fail due to mismatched versions between portfile and the version database"
 }

-git -C "$env:VCPKG_ROOT" fetch https://github.com/vicroms/test-registries
 foreach ($opt_registries in @("",",registries"))
 {
    Write-Trace "testing baselines: $opt_registries"
-    Refresh-TestRoot
+    Refresh-VersionFiles
    $CurrentTest = "without default baseline 2 -- enabling versions should not change behavior"
    Remove-Item -Recurse $buildtreesRoot/versioning_ -ErrorAction SilentlyContinue
    Run-Vcpkg @commonArgs "--feature-flags=versions$opt_registries" install `
--- a/azure-pipelines/end-to-end-tests-prelude.ps1
+++ b/azure-pipelines/end-to-end-tests-prelude.ps1
@ -19,6 +19,12 @@ $commonArgs = @(
    $Triplet
 ) + $directoryArgs

+$gitConfigOptions = @(
+  '-c', 'user.name=Nobody',
+  '-c', 'user.email=nobody@example.com',
+  '-c', 'core.autocrlf=false'
+)
+
 $Script:CurrentTest = 'unassigned'

 function Refresh-TestRoot {
--- a/include/vcpkg/vcpkgpaths.h
+++ b/include/vcpkg/vcpkgpaths.h
@ -89,6 +89,7 @@ namespace vcpkg
        const Path& downloads;
        const Path& tools;
        const Path builtin_registry_versions;
+        ExpectedL<Path> versions_dot_git_dir() const;
        const Path prefab;
        const Path buildsystems;
        const Path buildsystems_msbuild_targets;
--- a/src/vcpkg/commands.ci-verify-versions.cpp
+++ b/src/vcpkg/commands.ci-verify-versions.cpp
@ -66,7 +66,8 @@ namespace
                for (StringView control_file : {"CONTROL", "vcpkg.json"})
                {
                    auto treeish = Strings::concat(version_entry.git_tree, ':', control_file);
-                    auto maybe_file = paths.git_show(Strings::concat(treeish), paths.root / ".git");
+                    auto maybe_file = paths.git_show(Strings::concat(treeish),
+                                                     paths.versions_dot_git_dir().value_or_exit(VCPKG_LINE_INFO));
                    if (!maybe_file) continue;

                    const auto& file = maybe_file.value_or_exit(VCPKG_LINE_INFO);
--- a/src/vcpkg/commands.portsdiff.cpp
+++ b/src/vcpkg/commands.portsdiff.cpp
@ -32,9 +32,11 @@ namespace
    std::vector<VersionSpec> read_ports_from_commit(const VcpkgPaths& paths, StringView git_commit_id)
    {
        auto& fs = paths.get_filesystem();
-        const auto dot_git_dir = paths.root / ".git";
+        const auto dot_git_dir = fs.try_find_file_recursively_up(paths.builtin_ports_directory().parent_path(), ".git")
+                                     .map([](Path&& dot_git_parent) { return std::move(dot_git_parent) / ".git"; })
+                                     .value_or_exit(VCPKG_LINE_INFO);
        const auto ports_dir_name = paths.builtin_ports_directory().filename();
-        const auto temp_checkout_path = paths.root / fmt::format("{}-{}", ports_dir_name, git_commit_id);
+        const auto temp_checkout_path = paths.buildtrees() / fmt::format("{}-{}", ports_dir_name, git_commit_id);
        fs.create_directory(temp_checkout_path, IgnoreErrors{});
        const auto checkout_this_dir =
            fmt::format("./{}", ports_dir_name); // Must be relative to the root of the repository
--- a/src/vcpkg/registries.cpp
+++ b/src/vcpkg/registries.cpp
@ -658,7 +658,9 @@ namespace
        {
            const auto destination_tmp = destination_parent / "baseline.json.tmp";
            auto treeish = Strings::concat(commit_sha, ":versions/baseline.json");
-            auto maybe_contents = paths.git_show(treeish, paths.root / ".git");
+            auto maybe_contents =
+                paths.versions_dot_git_dir().then([&](Path&& dot_git) { return paths.git_show(treeish, dot_git); });
+
            if (auto contents = maybe_contents.get())
            {
                std::error_code ec;
@ -1129,7 +1131,8 @@ namespace
        }

        const auto& git_tree = port_versions_soa.git_trees()[it - port_versions.begin()];
-        return m_paths.git_checkout_port(port_name, git_tree, m_paths.root / ".git")
+        return m_paths.versions_dot_git_dir()
+            .then([&, this](Path&& dot_git) { return m_paths.git_checkout_port(port_name, git_tree, dot_git); })
            .map([&git_tree](Path&& p) -> PathAndLocation {
                return {
                    std::move(p),
--- a/src/vcpkg/vcpkgpaths.cpp
+++ b/src/vcpkg/vcpkgpaths.cpp
@ -73,28 +73,19 @@ namespace
    {
        std::error_code ec;
        auto manifest_path = manifest_dir / "vcpkg.json";
-        auto manifest_opt = Json::parse_file(fs, manifest_path, ec);
-        if (ec)
+        auto maybe_manifest_object = fs.try_read_contents(manifest_path).then([](FileContents&& contents) {
+            return Json::parse_object(contents.content, contents.origin);
+        });
+
+        if (auto manifest_object = maybe_manifest_object.get())
        {
-            Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO,
-                                           msg::format(msgFailedToLoadManifest, msg::path = manifest_dir)
-                                               .append_raw('\n')
-                                               .append_raw(ec.message()));
+            return ManifestAndPath{std::move(*manifest_object), std::move(manifest_path)};
        }

-        if (!manifest_opt)
-        {
-            Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO,
-                                           LocalizedString::from_raw(manifest_opt.error()->to_string()));
-        }
-
-        auto manifest_value = std::move(manifest_opt).value(VCPKG_LINE_INFO).value;
-        if (!manifest_value.is_object())
-        {
-            msg::println_error(msgFailedToParseNoTopLevelObj, msg::path = manifest_path);
-            Checks::exit_fail(VCPKG_LINE_INFO);
-        }
-        return {std::move(manifest_value).object(VCPKG_LINE_INFO), std::move(manifest_path)};
+        Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO,
+                                       msg::format(msgFailedToLoadManifest, msg::path = manifest_dir)
+                                           .append_raw('\n')
+                                           .append(maybe_manifest_object.error()));
    }

    static Optional<ManifestConfiguration> config_from_manifest(const Optional<ManifestAndPath>& manifest_doc)
@ -821,6 +812,12 @@ namespace vcpkg
        }
    }

+    ExpectedL<Path> VcpkgPaths::versions_dot_git_dir() const
+    {
+        return m_pimpl->m_fs.try_find_file_recursively_up(builtin_registry_versions.parent_path(), ".git")
+            .map([](Path&& dot_git_parent) { return std::move(dot_git_parent) / ".git"; });
+    }
+
    std::string VcpkgPaths::get_toolver_diagnostics() const
    {
        std::string ret;