Run scenario tests in PCS pipeline (#3917)

Co-authored-by: Přemek Vysoký <premek.vysoky@microsoft.com>

.vault-config/product-construction-dev.yaml

@@ -25,3 +25,10 @@ secrets:
         location: engkeyvault
         name: BotAccount-dotnet-bot
       gitHubBotAccountName: dotnet-bot
+
+  github:
+    type: github-app-secret
+    parameters:
+      hasPrivateKey: true
+      hasWebhookSecret: false
+      hasOAuthSecret: true

azure-pipelines-product-construction-service.yml

@@ -1,6 +1,3 @@
-# Changes the format of the Build.BuildNumber variable
-name: $(Date:yyyyMMdd)$(Rev:r)
-
 trigger:
   batch: true
   branches:
@@ -21,6 +18,14 @@ variables:
   value: product-construction-service.api
 - name: diffFolder
   value: $(Build.ArtifactStagingDirectory)/diff
+- name: _TeamName
+  value: DotNetCore
+- name: _PublishUsingPipelines
+  value: true
+- name: _DotNetArtifactsCategory
+  value: .NETCore
+- name: _SignType
+  value: test
 - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}:
   - name: subscriptionId
     value: e6b5f9f5-0ca4-4351-879b-014d78400ec2
@@ -133,3 +138,55 @@ stages:
         - publish: $(diffFolder)
           displayName: Upload snapshot diff
           artifact: DeploymentDiff
+
+  - job: BuildAndPublish
+    displayName: Build and Publish Repo
+    pool:
+      name: NetCore1ESPool-Internal
+      demands: ImageOverride -equals 1es-windows-2019
+
+    variables:
+      - name: _BuildConfig
+        value: Release
+      - name: _BuildArgs
+        value: >
+          /p:DotNetSignType=$(_SignType)
+          /p:TeamName=$(_TeamName)
+          /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+          /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
+          /p:OfficialBuildId=$(BUILD.BUILDNUMBER)"
+
+    steps:
+    - checkout: self
+
+    - template: /eng/templates/steps/build.yml
+      parameters:
+        configuration: $(_BuildConfig)
+        buildArgs: $(_BuildArgs)
+
+    - publish: $(Build.SourcesDirectory)\artifacts\bin\ProductConstructionService.ScenarioTests\$(_BuildConfig)\net8.0\publish
+      artifact: ProductConstructionService.ScenarioTests
+
+- stage: TestPCS
+  displayName: Run E2E Product Construction Service Tests
+  dependsOn: 
+  - DeployPCS
+  
+  jobs:
+  - template: /eng/templates/jobs/e2e-pcs-tests.yml
+    parameters:
+      name: scenarioTests_GitHub
+      displayName: GitHub tests
+      testFilter: 'TestCategory=GitHub'
+
+  - template: /eng/templates/jobs/e2e-pcs-tests.yml
+    parameters:
+      name: scenarioTests_AzDO
+      displayName: AzDO tests
+      testFilter: 'TestCategory=AzDO'
+
+  - template: /eng/templates/jobs/e2e-pcs-tests.yml
+    parameters:
+      name: scenarioTests_Other
+      displayName: Other tests
+      testFilter: 'TestCategory!=GitHub&TestCategory!=AzDO'

eng/templates/jobs/e2e-pcs-tests.yml

@@ -0,0 +1,111 @@
+parameters:
+- name: name
+  type: string
+- name: displayName
+  type: string
+- name: testFilter
+  type: string
+
+jobs:
+- job: ${{ parameters.name }}
+  displayName: ${{ parameters.displayName }}
+  timeoutInMinutes: 60
+  pool:
+      name: NetCore1ESPool-Internal
+      demands: ImageOverride -equals 1es-windows-2019
+  variables:
+    # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets
+    # Required for MaestroAppClientId, MaestroStagingAppClientId
+    - group: Publish-Build-Assets
+    - group: MaestroInt KeyVault
+    - name: PcsTestEndpoint
+      value: https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io
+    - name: ScenarioTestSubscription
+      value: "Darc: Maestro Staging"
+    - name: MaestroAppId
+      value: $(MaestroStagingAppClientId)
+  steps:
+  - download: current
+    displayName: Download Darc
+    artifact: PackageArtifacts
+    patterns: Microsoft.DotNet.Darc.*
+
+  - download: current
+    displayName: Download ScenarioTets
+    artifact: ProductConstructionService.ScenarioTests
+
+  - task: NuGetToolInstaller@1
+    displayName: Use NuGet
+    inputs:
+      versionSpec: 5.3.x
+
+  - powershell: |
+      . .\eng\common\tools.ps1
+      InitializeDotNetCli -install:$true
+      .\.dotnet\dotnet workload install aspire
+    displayName: Install .NET and Aspire Workload
+
+  - powershell: .\eng\common\build.ps1 -restore
+    displayName: Install .NET
+
+  - powershell: |
+      mkdir darc
+      .\.dotnet\dotnet tool install Microsoft.DotNet.Darc --prerelease --tool-path .\darc --add-source $(Pipeline.Workspace)\PackageArtifacts
+    displayName: Install Darc
+
+  - task: AzureCLI@2
+    name: GetAuthInfo
+    displayName: Get auth information
+    inputs:
+      azureSubscription: ${{ variables.ScenarioTestSubscription }}
+      addSpnToEnvironment: true
+      scriptType: ps
+      scriptLocation: inlineScript
+      inlineScript: |
+        # Fetch token used for scenario tests
+        $token = (az account get-access-token --resource "$(MaestroAppId)" | ConvertFrom-Json).accessToken
+        echo "##vso[task.setvariable variable=Token;isOutput=true;isSecret=true]$token"
+
+        # Set variables with auth info for tests below
+        echo "##vso[task.setvariable variable=ServicePrincipalId;isOutput=true]$env:servicePrincipalId"
+        echo "##vso[task.setvariable variable=FederatedToken;isOutput=true;isSecret=true]$env:idToken"
+        echo "##vso[task.setvariable variable=TenantId;isOutput=true]$env:tenantId"
+
+  - template: /eng/common/templates-official/steps/get-federated-access-token.yml
+    parameters:
+      federatedServiceConnection: "ArcadeServicesInternal"
+      outputVariableName: "AzdoToken"
+
+  - task: DotNetCoreCLI@2
+    displayName: Run E2E tests
+    inputs:
+      command: custom
+      projects: |
+        $(Pipeline.Workspace)/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.dll
+      custom: test
+      arguments: >
+        --filter "TestCategory=PostDeployment&${{ parameters.testFilter }}"
+        --no-build
+        --logger "trx;LogFilePrefix=TestResults-"
+        --parallel
+        --
+        "RunConfiguration.ResultsDirectory=$(Build.ArtifactStagingDirectory)\TestResults"
+        RunConfiguration.MapCpuCount=4
+    env:
+      PCS_BASEURI: ${{ variables.PcsTestEndpoint }}
+      PCS_TOKEN: $(GetAuthInfo.Token)
+      GITHUB_TOKEN: $(maestro-scenario-test-github-token)
+      AZDO_TOKEN: $(AzdoToken)
+      DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts
+      DARC_DIR: $(Build.SourcesDirectory)\darc
+      DARC_IS_CI: true
+
+  - task: PublishTestResults@2
+    displayName: Publish Core Test Results
+    condition: succeededOrFailed()
+    inputs:
+      testRunner: VSTest
+      testResultsFiles: '**/TestResults-*'
+      searchFolder: $(Build.ArtifactStagingDirectory)\TestResults
+      testRunTitle: ${{ parameters.displayName }}
+      mergeTestResults: true

eng/templates/steps/docker-build.yml

@@ -9,7 +9,7 @@ steps:
 - powershell: |
     Write-Host "Dev branch suffix is ${{ parameters.devBranchSuffix }}"
     $shortSha = "$(Build.SourceVersion)".Substring(0,10)
-    $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha${{ parameters.devBranchSuffix }}"
+    $newDockerTag = "$(Build.BuildNumber)-$(System.JobAttempt)-$shortSha${{ parameters.devBranchSuffix }}".Replace(".", "")
     Write-Host "##vso[task.setvariable variable=newDockerImageTag;isOutput=true]$newDockerTag"
     Write-Host "set newDockerImageTag to $newDockerTag"
   displayName: Generate docker image tag