Added daily branch ci-config updates, and queue the python docs ci (#20896)

eng/dailydocsconfig.json

@@ -0,0 +1,7 @@
+{
+    "target_repo": {
+        "url": "https://github.com/MicrosoftDocs/azure-docs-sdk-python",
+        "branch": "%%DailyDocsBranchName%%"
+    },
+    "run_post_toc_script": false 
+}

eng/pipelines/docindex.yml

@@ -40,7 +40,7 @@ jobs:
           filePath: eng/common/scripts/Update-DocsMsPackages.ps1
           arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding
-
+        condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
       # Push changes to docs repo
       - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
         parameters:
@@ -53,4 +53,43 @@ jobs:
           CommitMsg: "Update docs CI configuration"
           TargetRepoName: $(DocRepoName)
           TargetRepoOwner: $(DocRepoOwner)
-          WorkingDirectory: $(DocRepoLocation)
+          WorkingDirectory: $(DocRepoLocation)
+
+      # Prepare daily docs CI
+      - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml
+        parameters:
+          DailyBranchVariableName: DailyDocsBranchName
+      - pwsh: |
+          $ErrorActionPreference = "Continue"
+          git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null
+          $LASTEXITCODE = 0 # This ignores any error from git checkout
+          git status
+        displayName: Checkout daily branch if it exists
+        workingDirectory: $(DocRepoLocation)
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackages.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation) -PackageSourceOverride "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
+        displayName: Update Docs Onboarding for Daily branch
+      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+        parameters:
+          BaseRepoBranch: $(DailyDocsBranchName)
+          BaseRepoOwner: $(DocRepoOwner)
+          CommitMsg: "Update targeting packages based on release metadata. (Daily docs)"
+          TargetRepoName: $(DocRepoName)
+          TargetRepoOwner: $(DocRepoOwner)
+          WorkingDirectory: $(DocRepoLocation)
+          ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts
+
+      - task: PowerShell@2
+        displayName: Queue Docs CI build
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Queue-Pipeline.ps1
+          arguments: >
+            -Organization "apidrop"
+            -Project "Content%20CI"
+            -DefinitionId 3453
+            -Base64EncodedAuthToken "$(azuresdk-apidrop-devops-queue-build-pat)"
+            -BuildParametersJson (@{ params = (Get-Content ./eng/dailydocsconfig.json -Raw) -replace '%%DailyDocsBranchName%%', "$(DailyDocsBranchName)" } | ConvertTo-Json)

eng/scripts/Language-Settings.ps1

@@ -172,12 +172,25 @@ function ValidatePackage($packageName, $packageVersion, $workingDirectory) {
   # possible
   # https://github.com/Azure/azure-sdk-for-python/issues/20109
   try {
-    Write-Host "pip install $packageExpression --no-cache-dir --target $installTargetFolder"
-    $pipInstallOutput = pip `
-      install `
-      $packageExpression `
-      --no-cache-dir `
-      --target $installTargetFolder 2>&1
+    $pipInstallOutput = ""
+    $extraIndexUrl = " --extra-index-url=$PackageSourceOverride"
+    if ($PackageSourceOverride) {
+      Write-Host "pip install $packageExpression --no-cache-dir --target $installTargetFolder --extra-index-url=$PackageSourceOverride"
+      $pipInstallOutput = pip `
+        install `
+        $packageExpression `
+        --no-cache-dir `
+        --target $installTargetFolder `
+        --extra-index-url=$PackageSourceOverride 2>&1
+    }
+    else {
+      Write-Host "pip install $packageExpression --no-cache-dir --target $installTargetFolder"
+      $pipInstallOutput = pip `
+        install `
+        $packageExpression `
+        --no-cache-dir `
+        --target $installTargetFolder 2>&1
+    }
     if ($LASTEXITCODE -ne 0) {
       LogWarning "pip install failed for $packageExpression"
       Write-Host $pipInstallOutput
@@ -231,7 +244,6 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
   $outputPackages = @()
   foreach ($package in $packageConfig.packages) {
     $packageName = $package.package_info.name
-
     if (!$packageName) { 
       Write-Host "Keeping package with no name: $($package.package_info)"
       $outputPackages += $package
@@ -301,6 +313,15 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
         -Value $packageVersion `
         -PassThru `
         -Force 
+      if ($PackageSourceOverride) {
+        $package.package_info = Add-Member `
+          -InputObject $package.package_info `
+          -MemberType NoteProperty `
+          -Name 'extra_index_url' `
+          -Value $PackageSourceOverride `
+          -PassThru `
+          -Force 
+      }
     }
 
     Write-Host "Keeping tracked package: $packageName."
@@ -334,23 +355,35 @@ function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
     if ($Mode -eq 'preview') {
       $packageVersion = "==$($package.VersionPreview)"
     }
-
     if (!(ValidatePackage -packageName $packageName -packageVersion $packageVersion -workingDirectory $installValidationFolder)) {
       LogWarning "Package is not valid: $packageName. Cannot onboard."
       continue
     }
 
     Write-Host "Add new package from metadata: $packageName"
-    $package = [ordered]@{
+    if ($PackageSourceOverride) {
+      $package = [ordered]@{
         package_info = [ordered]@{
           name = $packageName;
           install_type = 'pypi';
           prefer_source_distribution = 'true';
           version = $packageVersion;
+          extra_index_url = $PackageSourceOverride
         };
         exclude_path = @("test*","example*","sample*","doc*");
+      }
+    }
+    else {
+      $package = [ordered]@{
+          package_info = [ordered]@{
+            name = $packageName;
+            install_type = 'pypi';
+            prefer_source_distribution = 'true';
+            version = $packageVersion;
+          };
+          exclude_path = @("test*","example*","sample*","doc*");
+      }
     }
-
     $outputPackages += $package
   }