Add ToC Generation script and wire up to docindex.yml (#19915)

* Add ToC Generation script and wire up to docindex.yml for testing

Add Docs-ToC.ps1

Remove eng/scripts/docs from .gitignore but keep filtering other docs/ folders

Also checkout packages-legacy.json in sparse checkout

Also check out docs-ref-mapping/

Use -toc-test branch for demo

Review feedback

Review feedback

Remove unnecessary comment

Undo working changes

Add extension point to update the ToC before output

Undo changes to demo set-daily-docs-branch-name.yml

Check for FileMetadata before accessing it

Add ability to artbitarily nest packages in the 'Other' service.

Add comments from review feedback

Add plugin nodes

* Revert unnecessary changes to eng/common

* Review feedback

* Remove automatic ToC generation from main

* Set-StrictMode

* Remove Release-DocsMsDocs.ps1

* Use original syntax

.gitignore

@@ -127,6 +127,7 @@ Thumbs.db
 
 # docs #
 docs/
+!eng/scripts/docs
 main.conf.json
 child.conf.json
 

eng/pipelines/docindex.yml

@@ -20,7 +20,9 @@ jobs:
           Paths:
             - ci-configs/packages-latest.json
             - ci-configs/packages-preview.json
+            - ci-configs/packages-legacy.json
             - metadata/
+            - docs-ref-mapping/
           Repositories:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
@@ -72,6 +74,15 @@ jobs:
           arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding for Daily branch
 
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsToc.ps1
+          arguments: >-
+            -DocRepoLocation $(DocRepoLocation)
+            -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml
+        displayName: Generate ToC
+
       - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
         parameters:
           BaseRepoBranch: $(DailyDocsBranchName)

eng/scripts/Language-Settings.ps1

@@ -6,6 +6,8 @@ $packagePattern = "*.tgz"
 $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/js-packages.csv"
 $BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=javascript%2F&delimiter=%2F"
 
+. "$PSScriptRoot/docs/Docs-ToC.ps1"
+
 function Confirm-NodeInstallation
 {
   if (!(Get-Command npm -ErrorAction SilentlyContinue))

eng/scripts/docs/Docs-ToC.ps1

@@ -0,0 +1,89 @@
+function Get-javascript-OnboardedDocsMsPackages($DocRepoLocation) {
+  $packageOnboardingFiles = @(
+    "$DocRepoLocation/ci-configs/packages-latest.json",
+    "$DocRepoLocation/ci-configs/packages-preview.json",
+    "$DocRepoLocation/ci-configs/packages-legacy.json")
+
+  $onboardedPackages = @{}
+  foreach ($file in $packageOnboardingFiles) {
+    $onboardingSpec = ConvertFrom-Json (Get-Content $file -Raw)
+    foreach ($spec in $onboardingSpec.npm_package_sources) {
+      $packageName = $spec.name
+      if ($packageName.LastIndexOf('@') -gt 0) {
+        # Package has an '@' symbol deliminting the end of the package name
+        $packageName = $packageName.Substring(0, $packageName.LastIndexOf('@'))
+      }
+      $onboardedPackages[$packageName] = $null
+    }
+  }
+
+  return $onboardedPackages
+}
+
+function Get-javascript-DocsMsTocData($packageMetadata, $docRepoLocation) {
+  # Fallback to get package-level readme name if metadata file info does not exist
+  $packageLevelReadmeName = $packageMetadata.Package.Replace('@azure/', '').Replace('@azure-tools/', '').Replace('azure-', '');
+
+  # Fallback to get package-level readme name if metadata file info does not exist
+  if ($packageMetadata.Package.StartsWith('@azure-rest/')) {
+    $packageLevelReadmeName = "$($packageMetadata.Package.Replace('@azure-rest/', ''))-rest"
+  }
+
+  # If there is a metadata json for the package use the DocsMsReadmeName from
+  # the metadata function
+  if ($packageMetadata.PSObject.Members.Name -contains "FileMetadata") {
+    $readmeMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageMetadata.FileMetadata
+    $packageLevelReadmeName = $readmeMetadata.DocsMsReadMeName
+  }
+
+
+  $packageTocHeader = $packageMetadata.Package
+  if ($clientPackage.DisplayName) {
+    $packageTocHeader = $clientPackage.DisplayName
+  }
+  $output = [PSCustomObject]@{
+    PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
+    PackageTocHeader       = $packageTocHeader
+    TocChildren            = @($clientPackage.Package)
+  }
+
+  return $output
+}
+
+function Get-javascript-DocsMsTocChildrenForManagementPackages($packageMetadata, $docRepoLocation) {
+  return @($packageMetadata.Package)
+}
+
+function Get-javascript-UpdatedDocsMsToc($toc) {
+  $services = $toc[0].items
+  for ($i = 0; $i -lt $services.Count; $i++) {
+
+    # Add "Plugin" docs to Identity. Packages associated with these entries do
+    # not build successfully in the docs CI system becaues they export nothing
+    # that the docs CI system can document. This ensures that the readme pages
+    # are documented properly even if their packages are not onboarded.
+    if ($services[$i].name -eq 'Identity') {
+      $services[$i].items += [PSCustomObject]@{
+        name  = "Plugins";
+        items = @(
+          [PSCustomObject]@{
+            name            = "Token Cache Persistence";
+            href            = "~/docs-ref-services/{moniker}/identity-cache-persistence-readme.md";
+            landingPageType = "Service";
+          },
+          [PSCustomObject]@{
+            name            = "VSCode Authentication";
+            href            = "~/docs-ref-services/{moniker}/identity-vscode-readme.md";
+            landingPageType = "Service";
+          }
+        )
+      }
+    }
+  }
+
+  # PowerShell outputs a single object if the output is an array with only one
+  # object. The preceeding comma ensures that the output remains an array for
+  # appropriate export formatting. Other formatting (e.g. `@($toc)`) does not
+  # produce useful outputs.
+  return , $toc
+}