Add x-update-registry command. (#1053)

* Add x-update-registry command.

Docs submitted as https://github.com/microsoft/vcpkg-docs/pull/79

* Actually implement --all.

* Get rid of the _UpdatingRegistryDataFrom$.comment edit.

azure-pipelines/end-to-end-tests-dir/artifacts.ps1

@@ -239,3 +239,15 @@ try {
     Run-Vcpkg deactivate
     Pop-Location
 }
+
+$output = Run-VcpkgAndCaptureOutput x-update-registry microsoft
+Throw-IfFailed
+Test-Match $output "Updating registry data from microsoft"
+
+$output = Run-VcpkgAndCaptureOutput x-update-registry https://aka.ms/vcpkg-ce-default
+Throw-IfFailed
+Test-Match $output "Updating registry data from microsoft"
+
+$output = Run-VcpkgAndCaptureOutput x-update-registry https://example.com
+Throw-IfNotFailed
+Test-Match $output "\[https://example.com/\] could not be updated; it could be malformed\."

include/vcpkg/base/message-data.inc.h

@@ -566,6 +566,15 @@ DECLARE_MESSAGE(CmdUpdateBaselineOptInitial,
                 (),
                 "",
                 "add a `builtin-baseline` to a vcpkg.json that doesn't already have it")
+DECLARE_MESSAGE(CmdUpdateRegistryAll, (), "", "Update all known artifact registries")
+DECLARE_MESSAGE(CmdUpdateRegistryAllExcludesTargets,
+                (),
+                "",
+                "Update registry --all cannot be used with a list of artifact registries")
+DECLARE_MESSAGE(CmdUpdateRegistryAllOrTargets,
+                (),
+                "",
+                "Update registry requires either a list of artifact registry names or URiIs to update, or --all.")
 DECLARE_MESSAGE(CmdUpgradeOptAllowUnsupported,
                 (),
                 "",

include/vcpkg/commands.update-registry.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
+#include <vcpkg/fwd/vcpkgpaths.h>
+
+namespace vcpkg::Commands
+{
+    void command_update_registry_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths);
+}

locales/messages.json

@@ -336,6 +336,9 @@
   "CmdSettingTargetBin": "Path to the binary to analyze",
   "CmdUpdateBaselineOptDryRun": "Print out plan without execution",
   "CmdUpdateBaselineOptInitial": "add a `builtin-baseline` to a vcpkg.json that doesn't already have it",
+  "CmdUpdateRegistryAll": "Update all known artifact registries",
+  "CmdUpdateRegistryAllExcludesTargets": "Update registry --all cannot be used with a list of artifact registries",
+  "CmdUpdateRegistryAllOrTargets": "Update registry requires either a list of artifact registry names or URiIs to update, or --all.",
   "CmdUpgradeOptAllowUnsupported": "Instead of erroring on an unsupported port, continue with a warning.",
   "CmdUpgradeOptNoDryRun": "Actually upgrade",
   "CmdUpgradeOptNoKeepGoing": "Stop installing packages on failure",
@@ -1821,20 +1824,22 @@
   "_Removing$FromProjectManifest.comment": "\n'${p0}' is a parameter of type 'string'\n",
   "unableToFindArtifact$InTheProjectManifest": "unable to find artifact ${p0} in the project manifest",
   "_unableToFindArtifact$InTheProjectManifest.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "Updated$ItContains$MetadataFiles": "Updated ${p0}. It contains ${p1} metadata files.",
+  "_Updated$ItContains$MetadataFiles.comment": "\n'${p0}' is a parameter of type 'string'\n\n'${p1}' is a parameter of type 'string'\n",
+  "UnableToDownload$": "Unable to download ${p0}.",
+  "_UnableToDownload$.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "$CouldNotBeUpdatedItCouldBeMalformed": "${p0} could not be updated; it could be malformed.",
+  "_$CouldNotBeUpdatedItCouldBeMalformed.comment": "\n'${p0}' is a parameter of type 'string'\n",
   "updateTheRegistryFromTheRemote": "update the registry from the remote",
   "ThisDownloadsTheLatestContentsOfTheRegistryFromTheRemoteService": "This downloads the latest contents of the registry from the remote service.",
-  "DownloadingRegistryData": "Downloading registry data",
-  "Updated$RegistryContains$MetadataFiles": "Updated ${p0}. registry contains ${p1} metadata files",
-  "_Updated$RegistryContains$MetadataFiles.comment": "\n'${p0}' is a parameter of type 'string'\n\n'${p1}' is a parameter of type 'string'\n",
-  "UnableToDownloadRegistrySnapshot": "Unable to download registry snapshot",
-  "UnableToFindRegistry$": "Unable to find registry ${p0}",
+  "TheXupdateregistryCommandDownloadsNewRegistryInformationAndThusCannotBeUsedWithLocalRegistriesDidYouMeanXregenerate$": "The x-update-registry command downloads new registry information and thus cannot be used with local registries. Did you mean x-regenerate ${p0}?",
+  "_TheXupdateregistryCommandDownloadsNewRegistryInformationAndThusCannotBeUsedWithLocalRegistriesDidYouMeanXregenerate$.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "UnableToFindRegistry$": "Unable to find registry ${p0}.",
   "_UnableToFindRegistry$.comment": "\n'${p0}' is a parameter of type 'string'\n",
-  "ArtifactRegistryDataIsNotLoaded": "Artifact registry data is not loaded",
-  "AttemptingToUpdateArtifactRegistry": "Attempting to update artifact registry",
-  "UnableToLoadRegistryIndex": "Unable to load registry index",
   "InstantlyActivatesAnArtifactOutsideOfTheProject": "Instantly activates an artifact outside of the project",
   "ThisWillInstantlyActivateAnArtifact": "This will instantly activate an artifact .",
   "NoArtifactsAreBeingAcquired": "No artifacts are being acquired",
+  "UpdateAllKnownArtifactRegistries": "Update all known artifact registries",
   "removesAllFilesInTheLocalCache": "removes all files in the local cache",
   "enablesDebugModeDisplaysInternalMesssagesAboutHow$Works": "enables debug mode, displays internal messsages about how ${p0} works",
   "_enablesDebugModeDisplaysInternalMesssagesAboutHow$Works.comment": "\n'${p0}' is a parameter - it is expecting a value like: \"vcpkg\"",

src/vcpkg/commands.cpp

@@ -41,6 +41,7 @@
 #include <vcpkg/commands.search.h>
 #include <vcpkg/commands.set-installed.h>
 #include <vcpkg/commands.update-baseline.h>
+#include <vcpkg/commands.update-registry.h>
 #include <vcpkg/commands.update.h>
 #include <vcpkg/commands.upgrade.h>
 #include <vcpkg/commands.upload-metrics.h>
@@ -100,6 +101,7 @@ namespace vcpkg::Commands
         {"search", command_search_and_exit},
         {"update", Update::perform_and_exit},
         {"x-update-baseline", command_update_baseline_and_exit},
+        {"x-update-registry", command_update_registry_and_exit},
         {"use", command_use_and_exit},
         {"x-vsinstances", VSInstances::perform_and_exit},
         {"z-ce", command_z_ce_and_exit},

src/vcpkg/commands.update-registry.cpp

@@ -0,0 +1,62 @@
+#include <vcpkg/base/checks.h>
+#include <vcpkg/base/util.h>
+
+#include <vcpkg/commands.update-registry.h>
+#include <vcpkg/configure-environment.h>
+#include <vcpkg/vcpkgcmdarguments.h>
+
+namespace
+{
+    using namespace vcpkg;
+
+    constexpr StringLiteral OPTION_ALL = "all";
+    constexpr std::array<CommandSwitch, 1> UpdateRegistrySwitches{{
+        {OPTION_ALL, []() { return msg::format(msgCmdUpdateRegistryAll); }},
+    }};
+
+    constexpr CommandStructure UpdateRegistryCommandMetadata{
+        []() {
+            return create_example_string("x-update-registry https://example.com")
+                .append_raw("\n")
+                .append(create_example_string("x-update-registry microsoft"));
+        },
+        0,
+        SIZE_MAX,
+        {UpdateRegistrySwitches, {}, {}},
+        nullptr};
+} // unnamed namespace
+
+namespace vcpkg::Commands
+{
+    void command_update_registry_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths)
+    {
+        auto parsed = args.parse_arguments(UpdateRegistryCommandMetadata);
+        const bool all = Util::Sets::contains(parsed.switches, OPTION_ALL);
+        auto&& command_arguments = parsed.command_arguments;
+        if (all)
+        {
+            if (command_arguments.empty())
+            {
+                command_arguments.emplace_back("update");
+                command_arguments.emplace_back("--all");
+            }
+            else
+            {
+                Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCmdUpdateRegistryAllExcludesTargets);
+            }
+        }
+        else
+        {
+            if (command_arguments.empty())
+            {
+                Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgCmdUpdateRegistryAllOrTargets);
+            }
+            else
+            {
+                command_arguments.emplace(command_arguments.begin(), "update");
+            }
+        }
+
+        Checks::exit_with_code(VCPKG_LINE_INFO, run_configure_environment_command(paths, command_arguments));
+    }
+} // vcpkg::Commands

vcpkg-artifacts/cli/commands/update.ts

@@ -2,22 +2,44 @@
 // Licensed under the MIT License.
 
 import { buildRegistryResolver } from '../../artifacts/artifact';
+import { schemeOf } from '../../fs/unified-filesystem';
 import { i } from '../../i18n';
 import { session } from '../../main';
+import { RemoteRegistry } from '../../registries/RemoteRegistry';
 import { Registry } from '../../registries/registries';
 import { RemoteFileUnavailable } from '../../util/exceptions';
 import { Command } from '../command';
-import { CommandLine } from '../command-line';
 import { count } from '../format';
 import { error, log, writeException } from '../styling';
+import { All } from '../switches/all';
 import { Project } from '../switches/project';
 
+async function updateRegistry(registry: Registry, displayName: string) : Promise<boolean> {
+  try {
+    await registry.update(displayName);
+    await registry.load();
+    log(i`Updated ${displayName}. It contains ${count(registry.count)} metadata files.`);
+  } catch (e) {
+    if (e instanceof RemoteFileUnavailable) {
+      log(i`Unable to download ${displayName}.`);
+    } else {
+      log(i`${displayName} could not be updated; it could be malformed.`);
+      writeException(e);
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
 export class UpdateCommand extends Command {
   readonly command = 'update';
   readonly aliases = [];
   seeAlso = [];
   argumentsHelp = [];
   project: Project = new Project(this);
+  all = new All(this);
 
   get summary() {
     return i`update the registry from the remote`;
@@ -32,54 +54,53 @@ export class UpdateCommand extends Command {
   override async run() {
     const resolver = session.globalRegistryResolver.with(
       await buildRegistryResolver(session, (await this.project.manifest)?.metadata.registries));
-    for (const registryName of this.inputs) {
-      const registry = resolver.getRegistryByName(registryName);
-      if (registry) {
-        try {
-          log(i`Downloading registry data`);
-          await registry.update();
-          await registry.load();
-          log(i`Updated ${registryName}. registry contains ${count(registry.count)} metadata files`);
-        } catch (e) {
-          if (e instanceof RemoteFileUnavailable) {
-            log(i`Unable to download registry snapshot`);
+    
+    if (this.all.active) {
+      for (const registryUri of session.registryDatabase.getAllUris()) {
+        if (schemeOf(registryUri) != 'https') { continue; }
+        const parsed = session.fileSystem.parseUri(registryUri);
+        const displayName = resolver.getRegistryDisplayName(parsed);
+        const loaded = resolver.getRegistryByUri(parsed);
+        if (loaded) {
+          if (!await updateRegistry(loaded, displayName)) {
             return false;
           }
-          writeException(e);
-          return false;
         }
-      } else {
-        error(i`Unable to find registry ${registryName}`);
       }
     }
 
-    return true;
-  }
+    for (const registryInput of this.inputs) {
+      const registryByName = resolver.getRegistryByName(registryInput);
+      if (registryByName) {
+        // if it matched a name, it's a name
+        if (!await updateRegistry(registryByName, registryInput)) {
+          return false;
+        }
 
-  static async update(registry: Registry) {
-    log(i`Artifact registry data is not loaded`);
-    log(i`Attempting to update artifact registry`);
-    const update = new UpdateCommand(new CommandLine([]));
+        continue;
+      }
+      
+      const scheme = schemeOf(registryInput);
+      switch (scheme) {
+        case 'https':
+          const registryInputAsUri = session.fileSystem.parseUri(registryInput);
+          const registryByUri = resolver.getRegistryByUri(registryInputAsUri)
+            ?? new RemoteRegistry(session, registryInputAsUri);
+          if (!await updateRegistry(registryByUri, resolver.getRegistryDisplayName(registryInputAsUri))) {
+            return false;
+          }
 
-    let success = true;
-    try {
-      success = await update.run();
-    } catch (e) {
-      writeException(e);
-      success = false;
-    }
-    if (!success) {
-      error(i`Unable to load registry index`);
-      return false;
-    }
-    try {
-      await registry.load();
-    } catch (e) {
-      writeException(e);
-      // it just doesn't want to load.
-      error(i`Unable to load registry index`);
+          continue;
+
+        case 'file':
+          error(i`The x-update-registry command downloads new registry information and thus cannot be used with local registries. Did you mean x-regenerate ${registryInput}?`);
+          return false;
+      }
+
+      error(i`Unable to find registry ${registryInput}.`);
       return false;
     }
+
     return true;
   }
 }

vcpkg-artifacts/cli/switches/all.ts

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { i } from '../../i18n';
+import { Switch } from '../switch';
+
+export class All extends Switch {
+  switch = 'all';
+  get help() {
+    return [
+      i`Update all known artifact registries`
+    ];
+  }
+}

vcpkg-artifacts/locales/messages.json

@@ -274,20 +274,22 @@
   "_Removing$FromProjectManifest.comment": "\n'${p0}' is a parameter of type 'string'\n",
   "unableToFindArtifact$InTheProjectManifest": "unable to find artifact ${p0} in the project manifest",
   "_unableToFindArtifact$InTheProjectManifest.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "Updated$ItContains$MetadataFiles": "Updated ${p0}. It contains ${p1} metadata files.",
+  "_Updated$ItContains$MetadataFiles.comment": "\n'${p0}' is a parameter of type 'string'\n\n'${p1}' is a parameter of type 'string'\n",
+  "UnableToDownload$": "Unable to download ${p0}.",
+  "_UnableToDownload$.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "$CouldNotBeUpdatedItCouldBeMalformed": "${p0} could not be updated; it could be malformed.",
+  "_$CouldNotBeUpdatedItCouldBeMalformed.comment": "\n'${p0}' is a parameter of type 'string'\n",
   "updateTheRegistryFromTheRemote": "update the registry from the remote",
   "ThisDownloadsTheLatestContentsOfTheRegistryFromTheRemoteService": "This downloads the latest contents of the registry from the remote service.",
-  "DownloadingRegistryData": "Downloading registry data",
-  "Updated$RegistryContains$MetadataFiles": "Updated ${p0}. registry contains ${p1} metadata files",
-  "_Updated$RegistryContains$MetadataFiles.comment": "\n'${p0}' is a parameter of type 'string'\n\n'${p1}' is a parameter of type 'string'\n",
-  "UnableToDownloadRegistrySnapshot": "Unable to download registry snapshot",
-  "UnableToFindRegistry$": "Unable to find registry ${p0}",
+  "TheXupdateregistryCommandDownloadsNewRegistryInformationAndThusCannotBeUsedWithLocalRegistriesDidYouMeanXregenerate$": "The x-update-registry command downloads new registry information and thus cannot be used with local registries. Did you mean x-regenerate ${p0}?",
+  "_TheXupdateregistryCommandDownloadsNewRegistryInformationAndThusCannotBeUsedWithLocalRegistriesDidYouMeanXregenerate$.comment": "\n'${p0}' is a parameter of type 'string'\n",
+  "UnableToFindRegistry$": "Unable to find registry ${p0}.",
   "_UnableToFindRegistry$.comment": "\n'${p0}' is a parameter of type 'string'\n",
-  "ArtifactRegistryDataIsNotLoaded": "Artifact registry data is not loaded",
-  "AttemptingToUpdateArtifactRegistry": "Attempting to update artifact registry",
-  "UnableToLoadRegistryIndex": "Unable to load registry index",
   "InstantlyActivatesAnArtifactOutsideOfTheProject": "Instantly activates an artifact outside of the project",
   "ThisWillInstantlyActivateAnArtifact": "This will instantly activate an artifact .",
   "NoArtifactsAreBeingAcquired": "No artifacts are being acquired",
+  "UpdateAllKnownArtifactRegistries": "Update all known artifact registries",
   "removesAllFilesInTheLocalCache": "removes all files in the local cache",
   "enablesDebugModeDisplaysInternalMesssagesAboutHow$Works": "enables debug mode, displays internal messsages about how ${p0} works",
   "_enablesDebugModeDisplaysInternalMesssagesAboutHow$Works.comment": "\n'${p0}' is a parameter - it is expecting a value like: \"vcpkg\"",

vcpkg-artifacts/registries/ArtifactRegistry.ts

@@ -38,7 +38,7 @@ export abstract class ArtifactRegistry implements Registry {
     this.#loaded = loaded;
   }
 
-  abstract update(): Promise<void>;
+  abstract update(displayName?: string): Promise<void>;
 
   async regenerate(normalize?: boolean): Promise<void> {
     // reset the index to blank.

vcpkg-artifacts/registries/LocalRegistry.ts

@@ -24,7 +24,7 @@ export class LocalRegistry extends ArtifactRegistry {
     this.installationFolder = session.installFolder.join(this.localName);
   }
 
-  update(): Promise<void> {
+  update(displayName?: string): Promise<void> {
     return this.regenerate();
   }
 

vcpkg-artifacts/registries/RemoteRegistry.ts

@@ -11,8 +11,8 @@ import { i } from '../i18n';
 import { Session } from '../session';
 import { isGithubRepo } from '../util/checks';
 import { Uri } from '../util/uri';
-import { ArtifactIndex } from './artifact-index';
 import { ArtifactRegistry } from './ArtifactRegistry';
+import { ArtifactIndex } from './artifact-index';
 import { Index } from './indexer';
 
 export class RemoteRegistry extends ArtifactRegistry {
@@ -76,8 +76,10 @@ export class RemoteRegistry extends ArtifactRegistry {
     }
   }
 
-  async update() {
-    this.session.channels.message(i`Updating registry data from ${this.location.toString()}`);
+  async update(displayName?: string) {
+    const displayNameStr = displayName ?? this.location.toString();
+
+    this.session.channels.message(i`Updating registry data from ${displayNameStr}`);
 
     let locations = [this.location];
 
@@ -86,7 +88,7 @@ export class RemoteRegistry extends ArtifactRegistry {
       locations = [this.location.join('archive/refs/heads/main.zip'), this.location.join('archive/refs/heads/master.zip')];
     }
 
-    const file = await acquireArtifactFile(this.session, locations, `${this.safeName}-registry.zip`, {});
+    const file = await acquireArtifactFile(this.session, locations, `${this.safeName}-registry.zip`, {}, {force: true});
     if (await file.exists()) {
       await unpackZip(this.session, file, this.cacheFolder, {}, { strip: -1 });
       await file.delete();

vcpkg-artifacts/registries/registries.ts

@@ -28,7 +28,7 @@ export interface Registry extends ArtifactSearchable {
 
   load(force?: boolean): Promise<void>;
   save(): Promise<void>;
-  update(): Promise<void>;
+  update(displayName?: string): Promise<void>;
   regenerate(normalize?: boolean): Promise<void>;
 }
 
@@ -103,6 +103,10 @@ export class RegistryDatabase {
     await loaded.load();
     return loaded;
   }
+
+  getAllUris() {
+    return Array.from(this.#uriToRegistry.keys());
+  }
 }
 
 // When a registry resolver is used to map a URI back to some form of for-display-purposes-only name.

vcpkg-artifacts/test/core/registry-resolver-tests.ts

@@ -1,10 +1,10 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { strict } from 'assert';
 import { Artifact } from '../../artifacts/artifact';
 import { Registry, RegistryDatabase, RegistryResolver, SearchCriteria } from '../../registries/registries';
 import { Uri } from '../../util/uri';
-import { strict } from 'assert';
 import { SuiteLocal } from './SuiteLocal';
 
 class FakeRegistry implements Registry {
@@ -19,7 +19,7 @@ class FakeRegistry implements Registry {
 
   load(force?: boolean): Promise<void> { return Promise.resolve(); }
   save(): Promise<void> { return Promise.resolve(); }
-  update(): Promise<void> { return Promise.resolve(); }
+  update(displayName?: string): Promise<void> { return Promise.resolve(); }
   regenerate(normalize?: boolean): Promise<void> { return Promise.resolve(); }
 }