API compatibility validation improvements, DotNet (#6481)

* Drop task 'Verify publish compat results success' * Load all DLLs for compare * Add test compares * Cleanup * Get contract from nuget.org instead of SDK_Dotnet_V4_org * Add ApiCompatBaseline.txt * Add Show API compat issue suppression list * Tweaks
2022-10-04 15:11:10 -07:00 · 2022-10-04 15:11:10 -07:00 · 2c2aadf3b0
--- a/ApiCompatBaseline.txt
+++ b/ApiCompatBaseline.txt
@ -0,0 +1,4 @@
+# These types are no longer supported in Microsoft.Bot.Builder.Azure:
+TypesMustExist : Type 'Microsoft.Bot.Builder.Azure.CosmosDbCustomClientOptions' does not exist in the implementation but it does exist in the contract.
+TypesMustExist : Type 'Microsoft.Bot.Builder.Azure.CosmosDbStorage' does not exist in the implementation but it does exist in the contract.
+TypesMustExist : Type 'Microsoft.Bot.Builder.Azure.CosmosDbStorageOptions' does not exist in the implementation but it does exist in the contract.
--- a/build/yaml/botbuilder-dotnet-ci.yml
+++ b/build/yaml/botbuilder-dotnet-ci.yml
@ -15,8 +15,8 @@ trigger: none # ci trigger is set in ADO
 pr: none # pr trigger is set in ADO

 variables:
-  ApiCompatVersion: 4.6.3
-  BotBuilderDll: Microsoft.Bot.Builder.AI.Luis,Microsoft.Bot.Builder.AI.QnA,Microsoft.Bot.Builder.ApplicationInsights,Microsoft.Bot.Builder.Azure,Microsoft.Bot.Builder.Dialogs,Microsoft.Bot.Builder.Integration.ApplicationInsights.Core,Microsoft.Bot.Builder.Integration.AspNet.Core,Microsoft.Bot.Builder.TemplateManager,Microsoft.Bot.Builder.Testing,Microsoft.Bot.Builder,Microsoft.Bot.Configuration,Microsoft.Bot.Connector,Microsoft.Bot.Schema,Microsoft.Bot.Streaming
+  ApiContractVersion: 4.6.3
+  PackagesToValidate: Microsoft.Bot.Builder.AI.Luis,Microsoft.Bot.Builder.AI.QnA,Microsoft.Bot.Builder.ApplicationInsights,Microsoft.Bot.Builder.Azure,Microsoft.Bot.Builder.Dialogs,Microsoft.Bot.Builder.Integration.ApplicationInsights.Core,Microsoft.Bot.Builder.Integration.AspNet.Core,Microsoft.Bot.Builder.TemplateManager,Microsoft.Bot.Builder.Testing,Microsoft.Bot.Builder,Microsoft.Bot.Configuration,Microsoft.Bot.Connector,Microsoft.Bot.Schema,Microsoft.Bot.Streaming
  BuildConfiguration: Debug-Windows
  BuildPlatform: any cpu
  IsBuildServer: true # This is consumed by tests\Microsoft.Bot.Builder.Dialogs.Declarative.Tests\SchemaTestsFixture.cs.
@ -84,13 +84,47 @@ stages:
  jobs:
  - job: generate_multiconfig_var
    steps:
+    # Download contract .dlls from nuget.org.
+    - powershell: |
+        $PackageNames = "$(PackagesToValidate)";
+        $ApiContractVersion = "$(ApiContractVersion)";
+        $TempContractInstallDirectory = ".\TempContractInstallDir";
+        $OutputDirectory = ".\ContractDlls";
+
+        Write-Host "`nDownloading packages version $ApiContractVersion.";
+
+        New-Item -ItemType directory -Path $OutputDirectory -Force | Out-Null;
+
+        $Names = $PackageNames.Split(',');
+
+        foreach ($Name in $Names) {
+          "---- $Name ------------------";
+          nuget install $Name -Version $ApiContractVersion -OutputDirectory $TempContractInstallDirectory -DirectDownload -NonInteractive;
+
+          if ($LASTEXITCODE -eq 0) {
+            Copy-Item "$TempContractInstallDirectory\$Name.$ApiContractVersion\lib\netstandard2.0\$Name.dll" $OutputDirectory;
+          }
+          else {
+            Write-Host "##vso[task.complete result=Failed;]";
+          }
+        }
+      displayName: 'Download Contract DLLs to ContractDlls folder'
+      continueOnError: true
+
+    - task: PublishPipelineArtifact@0
+      inputs:
+        artifactName: 'ContractDlls'
+        targetPath: ContractDlls
+      displayName: 'Push to ContractDlls in Artifacts'
+      continueOnError: true
+
    - powershell: |
       $multiconfig = '{';
       if ("$(Build.Reason)" -in ('Schedule', 'Manual')) {
-         $env:BotBuilderDll.Split(",") | ForEach {
+         $env:PackagesToValidate.Split(",") | ForEach {
           $library = $_.Trim()
           $threadName = $library -replace "Microsoft.", "";
-           $multiconfig += "'" + $threadName + "':{'BotBuilderDll':'" + $library + "'}, ";
+           $multiconfig += "'" + $threadName + "':{'PackageName':'" + $library + "'}, ";
         }
       }
       else {
@ -117,7 +151,7 @@ stages:
           if ($changedLibrary.Length -gt 0) {
             Write-Host $changedLibrary
             $threadName = $changedLibrary.Split(".")[-1];
-             $multiconfig += "'" + $threadName + "':{'BotBuilderDll':'" + $changedLibrary + "'}, ";
+             $multiconfig += "'" + $threadName + "':{'PackageName':'" + $changedLibrary + "'}, ";
           }
          }
       }
--- a/build/yaml/ci-api-validation-steps.yml
+++ b/build/yaml/ci-api-validation-steps.yml
@ -1,45 +1,47 @@
 steps:
- task: DownloadPipelineArtifact@1
-  displayName: 'Download BotBuilderDLLs artifact'
+- task: DownloadPipelineArtifact@2
+  displayName: 'Download BotBuilderDLLs from Artifacts'
  inputs:
    artifactName: 'BotBuilderDLLs-Debug-Windows-netcoreapp31'
-    targetPath: '$(System.ArtifactsDirectory)/Artifacts'
+    targetPath: '$(System.ArtifactsDirectory)/OutputDlls'

- task: NuGetCommand@2
-  displayName: 'NuGet Install of Compatability Target Package: $(BotBuilderDll)'
+- task: DownloadPipelineArtifact@2
+  displayName: 'Download ContractDlls from Artifacts'
  inputs:
-    command: custom
-    arguments: 'install $(BotBuilderDll) -Version $(ApiCompatVersion) -OutputDirectory $(System.DefaultWorkingDirectory)\DownloadedNuGet'
+    artifactName: 'ContractDlls'
+    targetPath: '$(System.ArtifactsDirectory)/ContractDlls'

- task: CmdLine@1
-  displayName: 'Run dir'
-  inputs:
-    filename: dir
-    arguments: '..\*.* /s'
-  enabled: false
+- powershell: |
+    Write-Host "The following API compatibility issues are suppressed:";
+    Get-Content "ApiCompatBaseline.txt";
+  displayName: 'Show API compat issue suppressions in ApiCompatBaseline.txt'
+  continueOnError: true

 - task: SOUTHWORKS.binaries-comparer.custom-build-release-task.binaries-comparer@0
  displayName: 'Compare Binaries'
  inputs:
-    contractsRootFolder: 'DownloadedNuGet\$(BotBuilderDll).$(ApiCompatVersion)\lib\netstandard2.0'
-    contractsFileName: '$(BotBuilderDll).dll'
-    implFolder: '$(System.ArtifactsDirectory)/Artifacts'
+    contractsRootFolder: '$(System.ArtifactsDirectory)/ContractDlls'
+    contractsFileName: '$(PackageName).dll'
+    implFolder: '$(System.ArtifactsDirectory)/OutputDlls'
    failOnIssue: false
    resolveFx: false
    generateLog: true
-    outputFilename: '$(BotBuilderDll).$(ApiCompatVersion).CompatResults.txt'
+    outputFilename: '$(PackageName).$(ApiContractVersion).CompatResults.txt'
    outputFolder: '$(Build.ArtifactStagingDirectory)'
-    useBaseline: false
+    useBaseline: true
+    baselineFile: ApiCompatBaseline.txt
+  continueOnError: false

 - powershell: |
-    $filePath = "$(Build.ArtifactStagingDirectory)\$(BotBuilderDll).$(ApiCompatVersion).CompatResults.txt"
-    $nugetLink = "compared against [version $(ApiCompatVersion)](https://www.nuget.org/packages/$(BotBuilderDll)/$(ApiCompatVersion)).";
+    $filePath = "$(Build.ArtifactStagingDirectory)\$(PackageName).$(ApiContractVersion).CompatResults.txt"
+    $nugetLink = "compared against [version $(ApiContractVersion)](https://www.nuget.org/packages/$(PackageName)/$(ApiContractVersion)).";
    Write-Host "Compatibility Check:"; 

    if (-not (Test-Path $filePath)) {
-      $content = "The binary compatibility report for library '$(BotBuilderDll)' wasn't generated. This may have happened because the NuGet library '$(BotBuilderDll)' for version '$(ApiCompatVersion)' was unavailable or a connectivity issue."
-      New-Item -Path '$(Build.ArtifactStagingDirectory)' -Name '$(BotBuilderDll).$(ApiCompatVersion).CompatResults.txt' -ItemType "file" -Value $content
+      $content = "The binary compatibility report for library '$(PackageName)' wasn't generated. This may have happened because the NuGet library '$(PackageName)' for version '$(ApiContractVersion)' was unavailable or a connectivity issue."
+      New-Item -Path '$(Build.ArtifactStagingDirectory)' -Name '$(PackageName).$(ApiContractVersion).CompatResults.txt' -ItemType "file" -Value $content
      $content;
+      Write-Host "##vso[task.complete result=Failed;]";
      return;
    }

@ -102,7 +104,7 @@ steps:
        Write-Host "##vso[task.complete result=Failed;]";
      }
      else {
-        $newFile = ":heavy_check_mark: No Binary Compatibility issues for **$(BotBuilderDll)** $nugetLink";
+        $newFile = ":heavy_check_mark: No Binary Compatibility issues for **$(PackageName)** $nugetLink";
      }

      $baseline = $newFile;
@ -111,17 +113,17 @@ steps:
      $baseline;
    }
  displayName: 'Compatibility Check'
+  continueOnError: false
  condition: succeededOrFailed()

 - task: PublishBuildArtifacts@1
  displayName: 'Publish Compat Results artifact'
  inputs:
-    ArtifactName: '$(BotBuilderDll).$(ApiCompatVersion).CompatResults'
+    ArtifactName: '$(PackageName).$(ApiContractVersion).CompatResults'
  condition: succeededOrFailed()

 - script: |
-   cd ..
-   dir /s
+   dir .. /s
  displayName: 'Dir workspace'
  continueOnError: true
  condition: succeededOrFailed()
--- a/build/yaml/ci-post-to-github-steps.yml
+++ b/build/yaml/ci-post-to-github-steps.yml
@ -5,89 +5,34 @@ steps:
    downloadType: specific
    itemPattern: '**\*.txt'
    downloadPath: '$(System.ArtifactsDirectory)\ApiCompat'
-  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True'))
+
+- task: CopyFiles@2
+  displayName: 'Copy results for publish to Artifacts'
+  inputs:
+    SourceFolder: '$(System.ArtifactsDirectory)\ApiCompat'
+    Contents: '**\*.txt'
+    TargetFolder: '$(System.ArtifactsDirectory)\ApiCompatibilityResults'
+    flattenFolders: true
+
+- task: PublishPipelineArtifact@1
+  inputs:
+    artifactName: 'ApiCompatibilityResults'
+    targetPath: '$(System.ArtifactsDirectory)\ApiCompatibilityResults'
+  displayName: 'Publish compat results to Artifacts'
+  continueOnError: true

 - task: SOUTHWORKS.github-pr-comment.custom-publish-comment-task.github-pr-comment@0
-  displayName: 'Publish compat results to Github'
+  displayName: 'Publish compat results to GitHub'
  inputs:
    userToken: '$(GitHubCommentApiKey)'
    bodyFilePath: '$(System.ArtifactsDirectory)\ApiCompat'
    getSubFolders: true
-  # Skip forks, as they can't get credentials.
-  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True'))
-
- powershell: |
-    Start-Sleep -Seconds 30
-  displayName: 'Wait for last task log to finish writing'
-  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True'))
-
- powershell: |
-    # This task compensates for the 'Publish compat results...' task above which can fail silently.
-    # Check task log for a "success" string in the current pipeline run.
-    # If not succeeded, fail this task.
-    # 
-    # Note: The target task needs time to finish writing its log. Hence the wait task above.
-    # Calls the Azure DevOps REST API.
-    # Enable OAuth token access in the pipeline agent job for $(System.Accesstoken) to populate.
-
-    $taskToCheck = 'Publish compat results to Github';
-    $successStringToCheckFor = 'status: 201';
-    
-    $collectionUri = "$env:SYSTEM_COLLECTIONURI";  # e.g. 'https://fuselabs.visualstudio.com'
-    $teamProjectId = "$env:SYSTEM_TEAMPROJECTID";  # e.g. '86659c66-c9df-418a-a371-7de7aed35064' = SDK_v4
-    
-    # Get the current build ID.
-    $buildId = "$env:BUILD_BUILDID";
-    Write-Host 'Build ID = ' $buildId;
-    
-    # Get the log containers for the run.
-    $uri = "$collectionUri/$teamProjectId/_apis/build/builds/$buildId/logs";
-    
-    $token = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$(System.AccessToken)"));
-    $header = @{authorization = "Basic $token"};
-    
-    $runLogContainers = Invoke-RestMethod "$uri" -Method Get -ContentType "application/json" -Headers $header;
-    $containersCount = $runLogContainers.count;
-
-    # Get the log from each log container.
-    Write-Host 'Checking the logs:';
-    $found = $false;
-    foreach ($container in $runLogContainers.value) {
-      # Search only the last 3 task logs.
-      if ($container.id -gt $containersCount - 3) {
-        $container.id;
-        $uri = $container.url;
-        $uri;
-        $log = Invoke-RestMethod "$uri" -Method Get -ContentType "application/json" -Headers $header;
-      
-        # Search for the success string.
-        if ($log.Contains($taskToCheck)) {
-          $log;
-          if ($log.Contains($successStringToCheckFor)) {
-            $found = $true;
-            $mess = 'String "' + $successStringToCheckFor + '" found in log #' + $container.id;
-            Write-Host $mess;
-          }
-        } else {
-          ($log -split '\r?\n')[0] + '...';  # Print first line
-        }
-      }
-    }
-   
-    # If not found, throw an error.
-    if (!$found) {
-      Write-Host;
-      $mess = "$taskToCheck failed. Is there a PR associated with this build? String '" + $successStringToCheckFor + "' not found in the log.";
-      throw $mess;
-    }
-  displayName: 'Verify publish compat results success'
-  ignoreLASTEXITCODE: true
-  continueOnError: true
+    keepCommentHistory: false
+  # Skip for forks, as secret tokens are not available to them.
  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True'))

 - script: |
-   cd ..
-   dir /s
+   dir .. /s
  displayName: 'Dir workspace'
  continueOnError: true
  condition: succeededOrFailed()
--- a/build/yaml/ci-test-steps.yml
+++ b/build/yaml/ci-test-steps.yml
@ -75,7 +75,7 @@ steps:
   
   $buildTarget = $env:BuildConfiguration.Split("-")[0];
   
-   $env:BotBuilderDll.Split(",") | ForEach {
+   $env:PackagesToValidate.Split(",") | ForEach {
       $library = $_.Trim()
       Write-Host $library
   
@ -92,8 +92,7 @@ steps:
  continueOnError: true

 - script: |
-   cd ..
-   dir *.* /s
+   dir .. /s
  displayName: 'Dir workspace'
  continueOnError: true
  condition: succeededOrFailed()