pr: none
trigger:
  tags:
    include:
    - dropgz/*
    - azure-ipam/*
    - v*

stages:
  - stage: setup
    displayName: Setup
    jobs:
      - job: env
        displayName: Setup
        pool:
          name: "$(BUILD_POOL_NAME_DEFAULT)"
        steps:
          - script: |
              echo "Setting up environment"
              go version
              echo "##vso[task.setvariable variable=commitID;isOutput=true]$(echo $(make revision)-$(date "+%d%H%M"))"
              echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)"
              echo "##vso[task.setvariable variable=cnsVersion;isOutput=true]$(CNS_VERSION)"
            name: "SetEnvVars"
            displayName: "Set Environment Variables"
            condition: always()

  - stage: build_images
    dependsOn: setup
    displayName: "Build Images"
    jobs:
      - job: containerize_amd64
        displayName: Build Images
        pool:
          name: "$(BUILD_POOL_NAME_LINUX_AMD64)"
        strategy:
          matrix:
            azure_ipam_linux_amd64:
              arch: amd64
              name: azure-ipam
              os: linux
            azure_ipam_windows_amd64:
              arch: amd64
              name: azure-ipam
              os: windows
            cni_linux_amd64:
              arch: amd64
              name: cni
              os: linux
            cni_windows_amd64:
              arch: amd64
              name: cni
              os: windows
            ipv6_hp_bpf_linux_amd64:
              arch: amd64
              name: ipv6-hp-bpf
              os: linux
            npm_linux_amd64:
              arch: amd64
              name: npm
              os: linux
            npm_windows_amd64:
              arch: amd64
              name: npm
              os: windows
        steps:
          - template: ../containers/container-template.yaml
            parameters:
              arch: $(arch)
              name: $(name)
              os: $(os)
      - job: containerize_linux_arm64
        displayName: Build Images
        pool:
          name: "$(BUILD_POOL_NAME_LINUX_ARM64)"
        strategy:
          matrix:
            azure_ipam_linux_arm64:
              arch: arm64
              name: azure-ipam
              os: linux
            cni_linux_arm64:
              arch: arm64
              name: cni
              os: linux
            ipv6_hp_bpf_linux_arm64:
              arch: arm64
              name: ipv6-hp-bpf
              os: linux
            npm_linux_arm64:
              arch: arm64
              name: npm
              os: linux
        steps:
          - template: ../containers/container-template.yaml
            parameters:
              arch: $(arch)
              name: $(name)
              os: $(os)

  - stage: binaries
    displayName: Build Binaries
    dependsOn: setup
    pool:
      name: $(BUILD_POOL_NAME_DEFAULT)
      demands:
        - agent.os -equals Linux
        - Role -equals Build
    jobs:
      - job:
        displayName: "Build k8s NPM Linux Test Suite Binary"
        steps:
          - bash: |
              git clone https://github.com/kubernetes/kubernetes.git --depth=1
              cd kubernetes
              export PATH=$PATH:/usr/local/go/bin/
              make WHAT=test/e2e/e2e.test
            displayName: "Build Kubernetes e2e.test"
          - publish: $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64
            artifact: npm_k8s
      - job:
        displayName: "Build k8s NPM Windows Test Suite Binary"
        steps:
          - bash: |
              git clone https://github.com/kubernetes/kubernetes.git --depth=1
              cd kubernetes
              export PATH=$PATH:/usr/local/go/bin/
              make WHAT=test/e2e/e2e.test
            displayName: "Build Kubernetes e2e.test"
          - publish: $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64
            artifact: npm_k8s_windows

  - stage: publish
    displayName: Publish Multiarch Manifests
    dependsOn:
      - build_images
    jobs:
      - job: manifest
        displayName: Compile Manifests
        pool:
          name: "$(BUILD_POOL_NAME_DEFAULT)"
        strategy:
          matrix:
            azure-ipam:
              name: azure-ipam
              platforms: linux/amd64 linux/arm64 windows/amd64
            cni:
              name: cni
              platforms: linux/amd64 linux/arm64 windows/amd64
            ipv6-hp-bpf:
              name: ipv6-hp-bpf
              platforms: linux/amd64 linux/arm64
            npm:
              name: npm
              platforms: linux/amd64 linux/arm64 windows/amd64
        steps:
          - template: ../containers/manifest-template.yaml
            parameters:
              name: $(name)
              os_versions: $(os_versions)
              platforms: $(platforms)

## Windows E2E
  - template: singletenancy/cniv1-template.yaml
    parameters:
      name: win22_cniv1
      clusterType: windows-cniv1-up
      clusterName: "win22-cniv1"
      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
      nodeCountWin: ${NODE_COUNT_WIN}
      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
      vmSizeWin: ${VM_SIZE_WIN}
      arch: amd64
      os: windows
      os_version: 'ltsc2022'
      osSkuWin: 'Windows2022'
      scaleup: ${SCALEUP_WIN}
      iterations: ${ITERATIONS_WIN}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: windows_podsubnet
      clusterType: swift-byocni-up
      clusterName: w22-podsub
      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
      nodeCountWin: ${NODE_COUNT_WIN}
      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
      vmSizeWin: ${VM_SIZE_WIN}
      arch: amd64
      os: windows
      os_version: 'ltsc2022'
      osSkuWin: 'Windows2022'
      scaleup: ${SCALEUP_WIN}
      iterations: ${ITERATIONS_WIN}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: windows_overlay
      clusterType: overlay-byocni-up
      clusterName: w22-over
      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
      nodeCountWin: ${NODE_COUNT_WIN}
      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
      vmSizeWin: ${VM_SIZE_WIN}
      arch: amd64
      os: windows
      os_version: 'ltsc2022'
      osSkuWin: 'Windows2022'
      scaleup: ${SCALEUP_WIN}
      iterations: ${ITERATIONS_WIN}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: windows19_overlay
      clusterType: overlay-byocni-up
      clusterName: w19-amd-ov
      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
      nodeCountWin: ${NODE_COUNT_WIN}
      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
      vmSizeWin: ${VM_SIZE_WIN}
      arch: amd64
      os: windows
      os_version: 'ltsc2019'
      osSkuWin: 'Windows2019'
      scaleup: ${SCALEUP_WIN}
      iterations: ${ITERATIONS_WIN}

## Linux E2E
  - template: singletenancy/cniv1-template.yaml
    parameters:
      name: linux_cniv1
      clusterType: linux-cniv1-up
      clusterName: "linux-cniv1"
      nodeCount: ${NODE_COUNT_LINUX}
      vmSize: ${VM_SIZE}
      os: linux
      arch: amd64
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: linux_podsubnet
      clusterType: swift-byocni-up
      clusterName: "linux-podsub"
      nodeCount: ${NODE_COUNT_LINUX}
      vmSize: ${VM_SIZE}
      arch: amd64
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: linux_overlay
      clusterType: overlay-byocni-up
      clusterName: "linux-over"
      nodeCount: ${NODE_COUNT_LINUX}
      vmSize: ${VM_SIZE}
      arch: amd64
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: mariner_linux_overlay
      clusterType: overlay-byocni-up
      clusterName: "mariner-over"
      nodeCount: ${NODE_COUNT_LINUX}
      vmSize:  ${VM_SIZE}
      arch: amd64
      osSKU: AzureLinux
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: arm_linux_overlay
      clusterType: overlay-byocni-up
      clusterName: "arm-over"
      nodeCount: ${NODE_COUNT_LINUX}
      vmSize: Standard_D8ps_v5
      arch: arm64
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

  - template: singletenancy/cniv2-template.yaml
    parameters:
      name: rdma_linux_overlay
      clusterType: overlay-byocni-up
      clusterName: "rdma-over"
      nodeCount: 2
      vmSize: Standard_HC44-16rs
      arch: amd64
      scaleup: ${SCALEUP_LINUX}
      iterations: ${ITERATIONS_LINUX}

## Cilium E2E
  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay
      clusterType: overlay-byocni-nokubeproxy-up
      clusterName: "cilium-over"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: ${VM_SIZE_CILIUM}
      arch: amd64
      cni: "cilium"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay_hubble
      clusterType: overlay-byocni-nokubeproxy-up
      clusterName: "cil-over-hub"
      hubbleEnabled: true
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: ${VM_SIZE_CILIUM}
      arch: amd64
      cni: "cilium"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay_mariner
      clusterType: overlay-byocni-nokubeproxy-up
      clusterName: "cil-over-mar"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: ${VM_SIZE_CILIUM}
      arch: amd64
      osSKU: AzureLinux
      cni: "cilium"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay_arm
      clusterType: overlay-byocni-nokubeproxy-up
      clusterName: "cil-over-arm"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: Standard_D8ps_v5
      arch: arm64
      cni: "cilium"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay_rdma
      clusterType: overlay-byocni-nokubeproxy-up
      clusterName: "cil-over-rdma"
      nodeCount: 2
      vmSize: Standard_HC44-16rs
      arch: amd64
      cni: "cilium"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_overlay_ds
      clusterType: dualstack-byocni-nokubeproxy-up
      clusterName: "cil-ds-ov"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: ${VM_SIZE_CILIUM}
      arch: amd64
      dualstackVersion: ${CILIUM_DUALSTACK_VERSION}
      cni: "cilium_dualstack"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_ds_arm
      clusterType: dualstack-byocni-nokubeproxy-up
      clusterName: "cil-ds-arm"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: Standard_D8ps_v5
      arch: arm64
      dualstackVersion: ${CILIUM_DUALSTACK_VERSION}
      cni: "cilium_dualstack"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_ds_mariner
      clusterType: dualstack-byocni-nokubeproxy-up
      clusterName: "cil-ds-mar"
      nodeCount: ${NODE_COUNT_CILIUM}
      vmSize: ${VM_SIZE_CILIUM}
      arch: amd64
      osSKU: AzureLinux
      dualstackVersion: ${CILIUM_DUALSTACK_VERSION}
      cni: "cilium_dualstack"

  - template: cilium/cilium-overlay-load-test-template.yaml
    parameters:
      name: cilium_ds_rdma
      clusterType: dualstack-byocni-nokubeproxy-up
      clusterName: "cil-ds-rdma"
      nodeCount: 2
      vmSize: Standard_HC44-16rs
      arch: amd64
      dualstackVersion: ${CILIUM_DUALSTACK_VERSION}
      cni: "cilium_dualstack"

  - stage: delete_resources
    displayName: "Delete Resources"
    pool:
      name: "$(BUILD_POOL_NAME_DEFAULT)"
    condition: always()
    dependsOn:
      - cilium_overlay
      - cilium_overlay_hubble
      - cilium_overlay_mariner
      - cilium_overlay_arm
      - cilium_overlay_rdma
      - cilium_overlay_ds
      - cilium_ds_arm
      - cilium_ds_mariner
      - cilium_ds_rdma
      - win22_cniv1_HNS
      - linux_cniv1
      - linux_podsubnet
      - linux_overlay
      - mariner_linux_overlay
      - arm_linux_overlay
      - rdma_linux_overlay
      - windows_podsubnet_HNS
      - windows_overlay_HNS
      - windows19_overlay_HNS
      - setup
    variables:
      commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
    jobs:
      - job: delete
        displayName: Delete Cluster
        pool:
          name: "$(BUILD_POOL_NAME_DEFAULT)"
        strategy:
          matrix:
            cilium_overlay:
              name: cilium_overlay
              clusterName: "cilium-over"
            cilium_overlay_hubble:
               name: cilium_overlay_hubble
               clusterName: "cil-over-hub"
            cilium_overlay_mariner:
              name: cilium_overlay_mariner
              clusterName: "cil-over-mar"
            cilium_overlay_arm:
              name: cilium_overlay_arm
              clusterName: "cil-over-arm"
            cilium_overlay_rdma:
              name: cilium_overlay_rdma
              clusterName: "cil-over-rdma"
            cilium_overlay_ds:
              name: cilium_overlay_ds
              clusterName: "cil-ds-ov"
            cilium_ds_arm:
              name: cilium_ds_arm
              clusterName: "cil-ds-arm"
            cilium_ds_mariner:
              name: cilium_ds_mariner
              clusterName: "cil-ds-mar"
            cilium_ds_rdma:
              name: cilium_ds_rdma
              clusterName: "cil-ds-rdma"
            win22-cniv1:
              name: win22-cniv1
              clusterName: "win22-cniv1"
            linux_cniv1:
              name: linux_cniv1
              clusterName: "linux-cniv1"
            linux_podsubnet:
              name: linux_podsubnet
              clusterName: "linux-podsub"
            linux_overlay:
              name: linux_overlay
              clusterName: "linux-over"
            mariner_linux_overlay:
              name: mariner_linux_overlay
              clusterName: "mariner-over"
            arm_linux_overlay:
              name: arm_linux_overlay
              clusterName: "arm-over"
            rdma_linux_overlay:
              name: rdma_linux_overlay
              clusterName: "rdma-over"
            win-cniv2-podsubnet:
              name: windows_podsubnet
              clusterName: w22-podsub
            win-cniv2-overlay:
              name: windows_overlay
              clusterName: w22-over
            windows19_overlay:
              name: windows19_overlay
              clusterName: w19-amd-ov
        steps:
          - task: AzureCLI@2
            inputs:
              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
              scriptLocation: "inlineScript"
              scriptType: "bash"
              addSpnToEnvironment: true
              inlineScript: |
                set -x

                if [[ ${clusterName} =~ 'rdma' ]]; then
                  region=${LOCATION_RDMA}
                elif [[ ${clusterName} =~ 'arm' ]]; then
                  region=${LOCATION_ARM64}
                else
                  region=${LOCATION_AMD64}
                fi

                if [ "$(DELETE_RESOURCES)" ]
                then
                  echo "Deleting Cluster and resource group"
                  make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=$(clusterName)-$(commitID)
                  make -C ./hack/aks azcfg AZCLI=az REGION=${region}
                  make -C ./hack/aks down AZCLI=az REGION=${region} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=$(clusterName)-$(commitID)
                  echo "Cluster and resources down"
                else
                  echo "Deletion of resources is False"
                fi
            name: "CleanUpCluster"
            displayName: "Cleanup cluster - $(name)"