(docs): update build-docs to deploy v2 artifacts (#22934)

Update to deployment and docs generation: adjustments to deploy v2 artifact instead of main artifacts. Note: while the update aims to deploy artifacts from the latest release branches instead of main, the actual website build is still done on the main branch. This will prevent the pipeline from deploying docs which were generated with an outdated version of the website. Notable changes: deploy-docs: - shifted most of the stages from build-docs to deploy-docs - build-docs now runs check_branch_version -> upload-json -> trigger deploy build-docs: - shifted check_branch_version to the top since its output is required to properly in the upload json step upload-json-step: - added a new upload step which uploads artifacts with the name "latest-v*.tar.gz" - takes in new parameters from check_branch_version to ensure it only uploads with this name when the pipeline is running on the latest release branch latestVersions: - since upload-json-step requires majorVersion, I figured it was best to include it as an output variable for the function as it seems more robust compared to relying on a string parsing script in the pipeline itself download-apis: - update such that it only downloads "latest-v*.tar.gz" instead of the main branch artifacts
2024-11-18 15:30:43 -08:00 · 2024-11-18 15:30:43 -08:00 · e8d8c654e1
--- a/build-tools/packages/build-cli/src/commands/check/latestVersions.ts
+++ b/build-tools/packages/build-cli/src/commands/check/latestVersions.ts
@ -62,7 +62,10 @@ export default class LatestVersionsCommand extends BaseCommand<typeof LatestVers
 					this.log(
 						`Version ${versionInput.version} is the latest version for major version ${majorVersion}`,
 					);
-					this.log(`##vso[task.setvariable variable=shouldDeploy;isOutput=true]true`);
+					this.log(`##vso[task.setvariable variable=shouldDeploy;isoutput=true]true`);
+					this.log(
+						`##vso[task.setvariable variable=majorVersion;isoutput=true]${majorVersion}`,
+					);
 					return;
 				}

@ -70,7 +73,8 @@ export default class LatestVersionsCommand extends BaseCommand<typeof LatestVers
 				this.log(
 					`##[warning]skipping deployment stage. input version ${versionInput.version} does not match the latest version ${v.version}`,
 				);
-				this.log(`##vso[task.setvariable variable=shouldDeploy;isOutput=true]false`);
+				this.log(`##vso[task.setvariable variable=shouldDeploy;isoutput=true]false`);
+				this.log(`##vso[task.setvariable variable=majorVersion;isoutput=true]${majorVersion}`);
 				return;
 			}
 		}
@ -79,6 +83,9 @@ export default class LatestVersionsCommand extends BaseCommand<typeof LatestVers
 		this.log(
 			`##[warning]No major version found corresponding to input version ${versionInput.version}`,
 		);
-		this.log(`##vso[task.setvariable variable=shouldDeploy;isOutput=true]false`);
+		this.log(`##vso[task.setvariable variable=shouldDeploy;isoutput=true]false`);
+		this.log(
+			`##vso[task.setvariable variable=majorVersion;isoutput=true]${inputMajorVersion}`,
+		);
 	}
 }
--- a/docs/infra/download-doc-models.mjs
+++ b/docs/infra/download-doc-models.mjs
@ -23,7 +23,7 @@ const versions = ["1", "2"];
 const artifactUrlBase = "https://storage.fluidframework.com/api-extractor-json/latest";
 const artifacts = {
 	1: `${artifactUrlBase}-v1.tar.gz`,
-	2: `${artifactUrlBase}.tar.gz`,
+	2: `${artifactUrlBase}-v2.tar.gz`,
 };

 const docModelsDirectory = path.resolve(dirname, "..", ".doc-models");
--- a/tools/pipelines/deploy-website.yml
+++ b/tools/pipelines/deploy-website.yml
@ -1,19 +1,15 @@
 # Copyright (c) Microsoft Corporation and contributors. All rights reserved.
 # Licensed under the MIT License.

-# build-docs pipeline
-# This pipeline builds the main branch docs as well as previous major release branch docs each time the client packages are built.
-# Note that the method for getting the api json for main differs from previous versions. For main, the pipeline bundles
-# the published artifacts from other pipeline resources and publishes an artifact to the pipeline which can be later downloaded.
-# However, for previous version branches (in this case: 1.0) cannot run on the pipeline to generate resources. Thus, api-extractor
-# was run on 1.0 locally and manually published to the azure storage as "lts-1.0.tar.gz" which is later downloaded using
-# the AzureCLI@2 task.
-# For now, the only previous version which is manually uploaded is lts. However, for any previous versions which cannot run on the
-# pipeline, we assume that the api json content would be uploaded to the azure storage in the format: latest-${{version}}.tar.gz
+# deploy-website pipeline
+# This pipeline builds the Fluid Framework documentation website and deploys it to Azure Static Web Apps.
+# The api-model artifacts are downloaded from the azure blob storage and used to generate the markdown files.
+# The site build artifacts are published as a pipeline artifact and then deployed to Azure Static Web Apps.

 name: $(Build.BuildId)

 parameters:
+  # override deployment condition to skip or force deployment
  - name: deployOverride
    displayName: Deployment Override (default = based on branch)
    type: string
@ -22,6 +18,7 @@ parameters:
      - default
      - skip
      - force
+  # determines deployment token to be used for Azure Static Web Apps
  - name: deployEnvironment
    displayName: Static web app environment to deploy to
    type: string
@ -29,6 +26,7 @@ parameters:
    values:
      - new
      - old
+  # determines whether to retain guardian assets
  - name: guardianAssetRetentionOverride
    displayName: Guardian Asset Retention Override (default = based on branch)
    type: string
@ -37,6 +35,7 @@ parameters:
      - default
      - skip
      - force
+  # determines whether to publish guardian baseline files
  - name: publishGuardianBaselines
    displayName: Publish Guardian Baseline Files
    type: boolean
@ -47,27 +46,10 @@ variables:
  - group: storage-vars
  - name: repoToTrigger
    value: microsoft/FluidFramework
-  - name: latestPipeline
-    value: ${{ or(
-      eq(variables['Build.SourceBranchName'], 'main'),
-      eq(variables['Build.SourceBranchName'], 'pl-test')
-      )}}
-  - name: n1Branch
-    value: ${{ join('/refs/heads/release/', variables['N1_BRANCH']) }}
-  - name: n1Pipeline
-    value: ${{ eq(variables['Build.SourceBranchName'], variables['N1_BRANCH']) }}
-  - name: releasePipeline
-    value: ${{ eq(variables['Build.SourceBranchName'], variables['RELEASE_BRANCH']) }}
-  - name: validRun
-    value: ${{ or(variables.releasePipeline, variables.n1Pipeline, variables.latestPipeline) }}
-  - name: Packaging.EnableSBOMSigning
-    value: true
-  - name: isMain
-    value: ${{ eq(variables['Build.SourceBranchName'], 'main') }}
  - name: shouldRetainGuardianAssets
    value: ${{ or(
      eq(parameters.guardianAssetRetentionOverride, 'force'),
-      and(eq(variables.isMain, true), eq(parameters.guardianAssetRetentionOverride, 'default'))
+      and(eq(parameters.guardianAssetRetentionOverride, 'default'))
      )}}
  - name: deploymentToken
    ${{ if eq( parameters['deployEnvironment'], 'new' ) }}:
@ -80,22 +62,15 @@ variables:
  - name: pnpmStorePath
    value: $(Pipeline.Workspace)/.pnpm-store

-# TODO: Move trigger to pipeline designated for artifact model generation. Currently,
-# the website docs builds and deploys on each commit. However, inconsistencies between
-# the release branches and main is causing the website to change unexpectedly we can
-# move the release/* trigger after separating the deploy step to another pipeline
-# no PR triggers
 trigger:
  branches:
    include:
-    - release/*
    - main
 pr: none

 stages:
 - stage: build
  displayName: 'Build website'
-  dependsOn: [] # run in parallel
  jobs:
    - job: debug_variables
      displayName: Show Variables
@ -110,9 +85,6 @@ stages:
            echo MAIN_BRANCH_VERSION: $(MAIN_BRANCH_VERSION)
            echo N1_VERSION: $(N1_VERSION)
            echo HUGO_PARAMS_APPINSIGHTKEY=$(HUGO_PARAMS_APPINSIGHTKEY)
-            echo releasePipeline ${{ variables.releasePipeline }}
-            echo latestPipeline ${{ variables.latestPipeline }}
-            echo n1Pipeline ${{ variables.n1Pipeline }}
            echo repoToTrigger ${{ variables.repoToTrigger }}
            echo shouldRetainGuardianAssets ${{ variables.shouldRetainGuardianAssets }}
            echo publishGuardianBaselines ${{ variables.publishGuardianBaselines }}
@ -131,20 +103,6 @@ stages:
            scanType: Register
            alertWarningLevel: High

-    - deployment: upload_json
-      displayName: 'Combine api-extractor JSON'
-      dependsOn: [] # run in parallel
-      environment: 'fluid-docs-env'
-      pool: Large-eastus2
-      strategy:
-        runOnce:
-          deploy:
-            steps:
-            - template: templates/upload-json-steps.yml
-              parameters:
-                STORAGE_ACCOUNT: $(STORAGE_ACCOUNT)
-                uploadAsLatest: ${{ variables.isMain }}
-
    - job: build_site
      displayName: 'Build website'
      dependsOn: upload_json
@ -306,48 +264,16 @@ stages:
            workingDir: $(Build.SourcesDirectory)/docs
            customCommand: 'run ci:check-links'

- stage: check_branch_version
-  displayName: 'Check Version Deployment Condition'
-  pool: Small-eastus2
-  jobs:
-    - job: check_branch_version
-      displayName: 'Check Version Deployment Condition'
-      steps:
-        - template: /tools/pipelines/templates/include-install-build-tools.yml
-          parameters:
-            buildDirectory: $(Build.SourcesDirectory)
-        - task: Bash@3
-          name: SetVersion
-          displayName: 'Set Build Version'
-          env:
-            VERSION_BUILDNUMBER: $(Build.BuildNumber)
-          inputs:
-            targetType: 'inline'
-            workingDirectory: $(Build.SourcesDirectory)
-            script: |
-              # Generate the build version. Sets the environment variables version, codeVersion, and isLatest.
-              flub generate buildVersion
-        - task: Bash@3
-          name: SetShouldDeploy
-          displayName: 'Check Version Deployment Condition'
-          inputs:
-            targetType: 'inline'
-            workingDirectory: $(Build.SourcesDirectory)
-            script: |
-              # Check if the version is the latest minor of its corresponding major version series
-              # Sets variable shouldDeploy to true if the version is the latest minor and false otherwise
-              flub check latestVersions $(SetVersion.version) client
-
 - stage: deploy
  displayName: 'Deploy website'
-  pool: Small-eastus2
-  dependsOn: ['build', 'guardian', 'check_branch_version']
+  pool: Small
+  dependsOn: ['build', 'guardian']
  variables:
    deployOverrideVar: ${{ parameters.deployOverride }}
  condition: and(
    not(eq(variables['deployOverrideVar'], 'skip')),
    or(
-      eq(dependencies.check_branch_version.outputs['check_branch_version.SetShouldDeploy.shouldDeploy'], 'true'),
+      eq(variables['deployOverrideVar'], 'default'),
      eq(variables['deployOverrideVar'], 'force')
    ))
  jobs:
--- a/tools/pipelines/publish-api-model-artifact.yml
+++ b/tools/pipelines/publish-api-model-artifact.yml
@ -0,0 +1,177 @@
+# Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+# Licensed under the MIT License.
+
+# publish-api-model-artifact pipeline
+# This pipeline downloads the api-extractor artifacts from build pipelines and uploads them to the azure blob storage.
+# After the artifacts are uploaded, the pipeline triggers the deploy-website pipeline to deploy the website.
+# By default, artifacts are only published from the latest minor release branch of each major version series.
+
+name: $(Build.BuildId)
+
+parameters:
+  # override deployment condition to skip or force deployment
+  - name: deployOverride
+    displayName: Deployment Override (default = based on branch)
+    type: string
+    default: default
+    values:
+      - default
+      - skip
+      - force
+  # determines deployment token to be used for Azure Static Web Apps
+  - name: deployEnvironment
+    displayName: Static web app environment to deploy to
+    type: string
+    default: new
+    values:
+      - new
+      - old
+  # determines whether to retain guardian assets
+  - name: guardianAssetRetentionOverride
+    displayName: Guardian Asset Retention Override (default = based on branch)
+    type: string
+    default: default
+    values:
+      - default
+      - skip
+      - force
+  # determines whether to publish guardian baseline files
+  - name: publishGuardianBaselines
+    displayName: Publish Guardian Baseline Files
+    type: boolean
+    default: false
+
+variables:
+  - group: doc-versions
+  - group: storage-vars
+  - name: repoToTrigger
+    value: microsoft/FluidFramework
+  - name: latestPipeline
+    value: ${{ or(
+      eq(variables['Build.SourceBranchName'], 'main'),
+      eq(variables['Build.SourceBranchName'], 'pl-test')
+      )}}
+  - name: n1Branch
+    value: ${{ join('/refs/heads/release/', variables['N1_BRANCH']) }}
+  - name: n1Pipeline
+    value: ${{ eq(variables['Build.SourceBranchName'], variables['N1_BRANCH']) }}
+  - name: releasePipeline
+    value: ${{ eq(variables['Build.SourceBranchName'], variables['RELEASE_BRANCH']) }}
+  - name: validRun
+    value: ${{ or(variables.releasePipeline, variables.n1Pipeline, variables.latestPipeline) }}
+  - name: Packaging.EnableSBOMSigning
+    value: true
+  - name: isMain
+    value: ${{ eq(variables['Build.SourceBranchName'], 'main') }}
+  - name: shouldRetainGuardianAssets
+    value: ${{ or(
+      eq(parameters.guardianAssetRetentionOverride, 'force'),
+      and(eq(parameters.guardianAssetRetentionOverride, 'default'))
+      )}}
+  - name: deploymentToken
+    ${{ if eq( parameters['deployEnvironment'], 'new' ) }}:
+      value: "$(FLUID_WEBSITE_TORUS_API_TOKEN)"
+    ${{ if eq( parameters['deployEnvironment'], 'old') }}:
+      value: "$(AZURE_STATIC_WEB_APPS_API_TOKEN)"
+  # skip injected CG detection as we manually trigger it in a parallel job
+  - name: skipComponentGovernanceDetection
+    value: true
+  - name: pnpmStorePath
+    value: $(Pipeline.Workspace)/.pnpm-store
+
+trigger:
+  branches:
+    include:
+    - release/**
+pr: none
+
+stages:
+# Check release version of branch, and determine if the branch should be deployed
+# If the branch is the latest minor version of its major version series, it should be deployed
+# also, the artifact will be uploaded as latest-v*.tar.gz where * is the major version
+- stage: check_branch_version
+  displayName: 'Check Version Deployment Condition'
+  pool: Small
+  jobs:
+    - job: check_branch_version
+      displayName: 'Check Version Deployment Condition'
+      steps:
+        - checkout: self
+          submodules: false
+          clean: true
+
+        - template: /tools/pipelines/templates/include-install-build-tools.yml
+          parameters:
+            buildDirectory: $(Build.SourcesDirectory)
+        - task: Bash@3
+          name: SetVersion
+          displayName: 'Set Build Version'
+          env:
+            VERSION_BUILDNUMBER: $(Build.BuildNumber)
+          inputs:
+            targetType: 'inline'
+            workingDirectory: $(Build.SourcesDirectory)
+            script: |
+              # Generate the build version. Sets the environment variables version, codeVersion, and isLatest.
+              flub generate buildVersion
+        - task: Bash@3
+          name: SetShouldDeploy
+          displayName: 'Check Version Deployment Condition'
+          inputs:
+            targetType: 'inline'
+            workingDirectory: $(Build.SourcesDirectory)
+            script: |
+              # Check if the version is the latest minor of its corresponding major version series
+              # Sets variable shouldDeploy to true if the version is the latest minor and false otherwise
+              # Sets variable majorVersion to the major version extracted from $(SetVersion.version)
+              flub check latestVersions $(SetVersion.version) client
+
+    - deployment: upload_json
+      displayName: 'Combine api-extractor JSON'
+      dependsOn: [] # run in parallel
+      environment: 'fluid-docs-env'
+      pool: Large
+      variables:
+        uploadAsLatestRelease: $[ stageDependencies.check_branch_version.outputs['check_branch_version.SetShouldDeploy.shouldDeploy'] ]
+        majorVersion: $[ stageDependencies.check_branch_version.outputs['check_branch_version.SetVersion.majorVersion'] ]
+      strategy:
+        runOnce:
+          deploy:
+            steps:
+            - template: templates/upload-json-steps.yml
+              parameters:
+                STORAGE_ACCOUNT: $(STORAGE_ACCOUNT)
+                uploadAsLatest: ${{ variables.isMain }}
+                uploadAsLatestRelease: $(uploadAsLatestRelease)
+                majorVersion: $(majorVersion)
+
+# Runs TriggerBuild@4 to trigger the deploy-website pipeline
+# this stage runs depending on the check_branch_version stage and deployOverride parameter
+# the trigger is configured such that deploy-website runs using main branch resources
+# this ensures that the generated website is up-to-date with the latest changes
+- stage: deploy docs
+  displayName: 'Deploy website'
+  pool: Small
+  dependsOn: ['build', 'guardian', 'check_branch_version']
+  jobs:
+    - job: deploy_site
+      displayName: 'Deploy website'
+      variables:
+        shouldDeploy: $[eq(stageDependencies.check_branch_version.check_branch_version.outputs['SetShouldDeploy.shouldDeploy'], 'true')]
+      steps:
+        - checkout: self
+          submodules: false
+          clean: true
+        - task: TriggerBuild@4
+          condition: eq(variables['shouldDeploy'], 'true')
+          inputs:
+            definitionIsInCurrentTeamProject: True
+            buildDefinition: 'deploy-website'
+            useSameSourceVersion: False
+            useSameBranch: False
+            branchToUse: main
+            templateParameters: 'deployOverride:${{ parameters.deployOverride }}, deployEnvironment:${{ parameters.deployEnvironment }}, guardianAssetRetentionOverride:${{ parameters.guardianAssetRetentionOverride }}, publishGuardianBaselines:${{ parameters.publishGuardianBaselines }}'
+            storeInEnvironmentVariable: True
+            authenticationMethod: 'OAuth Token'
+            password: '$(System.AccessToken)'
+          displayName: Run deploy-website pipeline
--- a/tools/pipelines/templates/upload-json-steps.yml
+++ b/tools/pipelines/templates/upload-json-steps.yml
@ -2,15 +2,28 @@
 # Licensed under the MIT License.

 # upload-json-steps pipeline
+# This pipeline downloads the api-extractor artifacts from build pipelines and publishes the combined JSON files as a pipeline artifact.
+# The combined JSON files are also uploaded to the azure blob storage in the format `latest-v*.tar.gz`. Where * is the major version number.

 parameters:
 - name: STORAGE_ACCOUNT
  type: string

+# Determines if artifact should be published as latest.tar.gz (for main branch)
 - name: uploadAsLatest
  type: boolean
  default: false

+# Determines if artifact should be published as latest-v*.tar.gz (for release branches)
+- name: uploadAsLatestRelease
+  type: boolean
+  default: false
+
+# Major version to upload as latest-v*.tar.gz
+- name: majorVersion
+  type: string
+  default: none
+
 steps:
 - checkout: none
  clean: true
@ -102,3 +115,14 @@ steps:
      scriptLocation: inlineScript
      inlineScript: |
        az storage blob upload -f '$(Pipeline.Workspace)/$(Build.SourceVersion).tar.gz' -c 'api-extractor-json' -n latest.tar.gz --account-name ${{ parameters.STORAGE_ACCOUNT }} --auth-mode login --overwrite --verbose
+
+- ${{ if eq(parameters.uploadAsLatestRelease, true) }}:
+  - task: AzureCLI@2
+    displayName: 'Upload JSON as latest-v*.tar.gz'
+    continueOnError: true
+    inputs:
+      azureSubscription: 'fluid-docs'
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        az storage blob upload -f '$(Pipeline.Workspace)/$(Build.SourceVersion).tar.gz' -c 'api-extractor-json' -n latest-v${{ parameters.majorVersion }}.tar.gz --account-name ${{ parameters.STORAGE_ACCOUNT }} --auth-mode login --overwrite --verbose