Persist cluster configs in AZP (#426)

* update cluster configs for more version testing
2019-10-29 14:12:44 -07:00 · 2019-10-29 14:12:44 -07:00 · d0a9343e75
--- a/.pipelines/Dockerfile
+++ b/.pipelines/Dockerfile
@ -1,5 +1,5 @@
 FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg
+RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg jq
 RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
 RUN sudo dpkg -i packages-microsoft-prod.deb
 RUN add-apt-repository ppa:longsleep/golang-backports 
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@ -1,7 +1,7 @@
 parameters:
  name: ""
  pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-  clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+  clusterDefinition: ""
  clusterDefinitionCniTypeKey: ""
  clusterDefinitionCniBuildOS: ""
  clusterDefinitionCniBuildExt: ""
@ -18,13 +18,15 @@ jobs:
      GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
      GOBIN: "$(GOPATH)/bin" # Go binaries path
      modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
+      acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
      Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
      CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]

    steps:
      - template: e2e-step-template.yaml
        parameters:
-          clusterDefinitionUrl: ${{ parameters.clusterDefinitionUrl }}
+          name: ${{ parameters.name }}
+          clusterDefinition: ${{ parameters.clusterDefinition }}
          clusterDefinitionCniTypeKey: ${{ parameters.clusterDefinitionCniTypeKey }}
          clusterDefinitionCniBuildOS: ${{ parameters.clusterDefinitionCniBuildOS }}
          clusterDefinitionCniBuildExt: ${{ parameters.clusterDefinitionCniBuildExt }}
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@ -1,6 +1,6 @@
 parameters:
  name: ""
-  clusterDefinitionUrl: ""
+  clusterDefinition: ""
  clusterDefinitionCniTypeKey: ""
  clusterDefinitionCniBuildOS: ""
  clusterDefinitionCniBuildExt: ""
@ -19,16 +19,24 @@ steps:
    name: "GoEnv"
    displayName: "Set up the Go environment"

+  - task: DownloadPipelineArtifact@2
+    inputs:
+      buildType: current
+      artifactName: clusterdefinitions
+      targetPath: "$(modulePath)"
+      #buildType: 'current' # Options: current, specific
+      #project: # Required when buildType == Specific
+
  - bash: |
-      rm -f clusterDefinition.json*
-      wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
+      ls -lah
      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
      export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
      echo CNI type is $CNI_TYPE
-      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" clusterDefinition.json
-      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
      echo "Running E2E tests against a cluster built with the following API model:"
-      cat ./clusterDefinition.json
+      cat '${{ parameters.clusterDefinition }}'
+      cp ${{ parameters.clusterDefinition }} clusterDefinition.json
      curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
@ -38,7 +46,7 @@ steps:
    workingDirectory: "$(modulePath)"

  - bash: |
-      export CLUSTER_DEFINITION=./clusterDefinition.json 
+      export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
      export ORCHESTRATOR=kubernetes 
      export CREATE_VNET=false 
      export TIMEOUT=10m 
@ -50,6 +58,16 @@ steps:
      export REGIONS=$(AKS_ENGINE_REGION) 
      export IS_JENKINS=false 
      make test-kubernetes
+      mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
+      cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
+      cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
    name: DeployAKSEngine
    displayName: Deploy AKS-Engine
    workingDirectory: "$(modulePath)"
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: "kube-${{ parameters.name }}"
+      pathtoPublish: "$(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}"
+    displayName: "Save cluster configs"
+    condition: always()
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@ -80,6 +80,7 @@ stages:
            displayName: "Install Go dependencies"

          - script: |
+              echo Building in $(pwd)
              echo Build tag is $(EnvironmentalVariables.Tag)
              export GOOS=linux
              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
@ -103,7 +104,6 @@ stages:
                } | { read xs; exit $xs; } 
              } 4>&1
            workingDirectory: "$(modulePath)"
-            failOnStderr: true
            name: "Test"
            displayName: "Run Tests"

@ -111,6 +111,8 @@ stages:
              bash <(curl -s https://codecov.io/bash)
              gocov convert coverage.out > coverage.json
              gocov-xml < coverage.json > coverage.xml
+              echo listing cluster definitions
+              ls $(modulePath)/test/e2e/kubernetes/*
            workingDirectory: "$(modulePath)"
            name: "Coverage"
            displayName: "Generate Coverage Reports"
@ -140,6 +142,9 @@ stages:
              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
            condition: succeeded()

+          - publish: $(modulePath)/test/e2e/kubernetes/
+            artifact: clusterdefinitions
+
          - task: AzureCLI@1
            inputs:
              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
@ -153,22 +158,41 @@ stages:
      - template: e2e-job-template.yaml
        parameters:
          name: "ubuntu_16_04_linux_e2e"
-          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniLinux1604.json"
          clusterDefinitionCniTypeKey: "azureCNIURLLinux"
          clusterDefinitionCniBuildOS: "linux"
          clusterDefinitionCniBuildExt: ".tgz"

      - template: e2e-job-template.yaml
        parameters:
-          name: "windows_e2e"
-          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+          name: "ubuntu_18_04_linux_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniLinux1804.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+          clusterDefinitionCniBuildOS: "linux"
+          clusterDefinitionCniBuildExt: ".tgz"
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "windows_18_09_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniWindows1809.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+          clusterDefinitionCniBuildOS: "windows"
+          clusterDefinitionCniBuildExt: ".zip"
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "windows_19_03_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniWindows1903.json"
          clusterDefinitionCniTypeKey: "azureCNIURLWindows"
          clusterDefinitionCniBuildOS: "windows"
          clusterDefinitionCniBuildExt: ".zip"

  - stage: cleanup
+    condition: always()
    jobs:
      - job: delete_remote_artifacts
        pool:
@ -178,7 +202,7 @@ stages:
          image: containernetworking/pipeline-ci:1.0.4
        variables:
          Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
-
+        condition: always()
        steps:
          - checkout: none
          - task: AzureCLI@1
@ -190,3 +214,4 @@ stages:
                az storage container delete -n acn-$CommitHash --account-name $(STORAGE_ACCOUNT_NAME)
              workingDirectory: "$(modulePath)"
            displayName: Cleanup remote Azure storage container
+            condition: always()
--- a/test/e2e/kubernetes/cniLinux.json
+++ b/test/e2e/kubernetes/cniLinux.json
@ -1,42 +0,0 @@
-{
-  "apiVersion":"vlabs",
-  "properties":{
-     "orchestratorProfile":{
-        "orchestratorType":"Kubernetes",
-        "orchestratorRelease":"1.14",
-        "kubernetesConfig":{
-           "networkPlugin":"azure",
-           "networkPolicy":"azure",
-           "azureCNIVersion":"",
-           "azureCNIURLLinux":""
-        }
-     },
-     "masterProfile":{
-        "count":1,
-        "dnsPrefix":"cniLinux",
-        "vmSize":"Standard_D2_v2"
-     },
-     "agentPoolProfiles":[
-        {
-           "name":"agentpool1",
-           "count":3,
-           "vmSize":"Standard_D2_v2",
-           "availabilityProfile":"AvailabilitySet"
-        }
-     ],
-     "linuxProfile":{
-        "adminUsername":"azureuser",
-        "ssh":{
-           "publicKeys":[
-              {
-                 "keyData":""
-              }
-           ]
-        }
-     },
-     "servicePrincipalProfile":{
-        "clientId":"",
-        "secret":""
-     }
-  }
-}
--- a/test/e2e/kubernetes/cniLinux1604.json
+++ b/test/e2e/kubernetes/cniLinux1604.json
@ -0,0 +1,44 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLLinux": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniLinux",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "agentpool1",
+            "count": 3,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Linux",
+            "distro": "aks-ubuntu-16.04"
+         }
+      ],
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}
--- a/test/e2e/kubernetes/cniLinux18.json
+++ b/test/e2e/kubernetes/cniLinux18.json
@ -1,44 +0,0 @@
-{
-  "apiVersion":"vlabs",
-  "properties":{
-     "orchestratorProfile":{
-        "orchestratorType":"Kubernetes",
-        "orchestratorRelease":"1.14",
-        "kubernetesConfig":{
-           "networkPlugin":"azure",
-           "networkPolicy":"azure",
-           "azureCNIVersion":"",
-           "azureCNIURLLinux":""
-        }
-     },
-     "masterProfile":{
-        "count":1,
-        "dnsPrefix":"cniLinux",
-        "vmSize":"Standard_D2_v2"
-     },
-     "agentPoolProfiles":[
-        {
-           "name":"agentpool1",
-           "count":3,
-           "vmSize":"Standard_D2_v2",
-	       "osType": "Linux",
-	       "distro": "aks-ubuntu-18.04", 
-           "availabilityProfile":"AvailabilitySet"
-        }
-     ],
-     "linuxProfile":{
-        "adminUsername":"azureuser",
-        "ssh":{
-           "publicKeys":[
-              {
-                 "keyData":""
-              }
-           ]
-        }
-     },
-     "servicePrincipalProfile":{
-        "clientId":"",
-        "secret":""
-     }
-  }
-}
--- a/test/e2e/kubernetes/cniLinux1804.json
+++ b/test/e2e/kubernetes/cniLinux1804.json
@ -0,0 +1,44 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLLinux": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniLinux",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "agentpool1",
+            "count": 3,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Linux",
+            "distro": "aks-ubuntu-18.04"
+         }
+      ],
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}
--- a/test/e2e/kubernetes/cniWindows.json
+++ b/test/e2e/kubernetes/cniWindows.json
@ -1,54 +0,0 @@
-{
-   "apiVersion":"vlabs",
-   "properties":{
-      "orchestratorProfile":{
-         "orchestratorType":"Kubernetes",
-         "orchestratorRelease":"1.14",
-         "kubernetesConfig":{
-            "networkPlugin":"azure",
-            "networkPolicy":"azure",
-            "azureCNIVersion":"",
-            "azureCNIURLWindows":""
-         }
-      },
-      "masterProfile":{
-         "count":1,
-         "dnsPrefix":"cniWindows",
-         "vmSize":"Standard_D2_v2"
-      },
-      "agentPoolProfiles":[
-         {
-            "name":"windowspool2",
-            "count":2,
-            "vmSize":"Standard_D2_v2",
-            "availabilityProfile":"AvailabilitySet",
-            "osType":"Windows",
-            "extensions":[
-               {
-                  "name":"windows-patches"
-               }
-            ]
-         }
-      ],
-      "windowsProfile":{
-         "adminUsername":"azureuser",
-         "adminPassword":"azureTest@!",
-         "enableAutomaticUpdates":false,
-         "sshEnabled":true
-      },
-      "linuxProfile":{
-         "adminUsername":"azureuser",
-         "ssh":{
-            "publicKeys":[
-               {
-                  "keyData":""
-               }
-            ]
-         }
-      },
-      "servicePrincipalProfile":{
-         "clientId":"",
-         "secret":""
-      }
-   }
-}
--- a/test/e2e/kubernetes/cniWindows1809.json
+++ b/test/e2e/kubernetes/cniWindows1809.json
@ -0,0 +1,58 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLWindows": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniWindows",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "windowspool2",
+            "count": 2,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Windows",
+            "extensions": [
+               {
+                  "name": "windows-patches"
+               }
+            ]
+         }
+      ],
+      "windowsProfile": {
+         "adminUsername": "azureuser",
+         "adminPassword": "azureTest@!",
+         "enableAutomaticUpdates": false,
+         "sshEnabled": true,
+         "windowsPublisher": "MicrosoftWindowsServer",
+         "windowsOffer": "WindowsServer",
+         "windowsSku": "Datacenter-Core-1809-with-Containers-smalldisk",
+         "imageVersion": "1809.0.20190603"
+      },
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}
--- a/test/e2e/kubernetes/cniWindows1903.json
+++ b/test/e2e/kubernetes/cniWindows1903.json
@ -0,0 +1,58 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLWindows": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniWindows",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "windowspool2",
+            "count": 2,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Windows",
+            "extensions": [
+               {
+                  "name": "windows-patches"
+               }
+            ]
+         }
+      ],
+      "windowsProfile": {
+         "adminUsername": "azureuser",
+         "adminPassword": "azureTest@!",
+         "enableAutomaticUpdates": false,
+         "sshEnabled": true,
+         "windowsPublisher": "MicrosoftWindowsServer",
+         "windowsOffer": "WindowsServer",
+         "windowsSku": "Datacenter-Core-1903-with-Containers-smalldisk",
+         "imageVersion": "1903.0.20190603"
+      },
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}