ci: Add k8s conformance tests to Cilium CI/CD (#2348)

* ci: add k8s conformance

* ci: remove kubetest2 k8se2e

* chore: remove test comments

* ci: add k8se2e to nightly pipeline

.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml

@@ -161,6 +161,17 @@ stages:
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
             displayName: "Run Cilium Connectivity Tests"
+      - template: ../k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          cni: cilium
+          dependsOn: cni_tests
+          datapath: true
+          dns: true
+          portforward: true
+          service: true
       - job: failedE2ELogs
         displayName: "Failure Logs"
         dependsOn:
@@ -169,6 +180,7 @@ stages:
           - restart_nodes
           - restart_cns
           - cni_tests
+          - cni_${{ parameters.os }}
         condition: failed()
         steps:
           - template: ../../templates/log-template.yaml

.pipelines/cni/cilium/nightly-release-test.yml

@@ -91,10 +91,24 @@ stages:
               name: "cilium_nightly"
               clusterName: ciliumnightly-$(commitID)
               testHubble: true
+
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ciliumnightly-$(commitID)
+          os: linux
+          cni: cilium
+          dependsOn: cilium_nightly
+          datapath: true
+          dns: true
+          portforward: true
+          service: true
+
       - job: logs
         displayName: "Failure Logs"
         dependsOn:
           - cilium_nightly
+          - cni_linux
         condition: failed()
         steps:
           - template: ../../templates/log-template.yaml

.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

@@ -3,6 +3,7 @@ parameters:
   os: ""
   dependsOn: ""
   sub: ""
+  cni: cni
 
 
 jobs:
@@ -63,7 +64,7 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
-      - ${{ if eq(parameters.service, true) }}:
+      - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cni') ) }}:
         - template: ../k8s-e2e/k8s-e2e-step-template.yaml
           parameters:
             testName: Service Conformance
@@ -73,6 +74,16 @@ jobs:
             os: ${{ parameters.os }}
             processes: 8
             attempts: 3
+      - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cilium') ) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Service Conformance|Cilium
+            name: service
+            ginkgoFocus: 'Services.*\[Conformance\].*'
+            ginkgoSkip: 'should serve endpoints on same port and different protocols' # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
       - ${{ if eq(parameters.hostport, true) }}:
         - template: ../k8s-e2e/k8s-e2e-step-template.yaml
           parameters:

.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml

@@ -6,6 +6,7 @@ parameters:
   vmSize: ""
   k8sVersion: ""
   dependsOn: ""
+  os: "linux"
 
 stages:
   - stage: ${{ parameters.clusterName }}
@@ -57,10 +58,23 @@ stages:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
 
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          cni: cilium
+          dependsOn: ${{ parameters.name }}
+          datapath: true
+          dns: true
+          portforward: true
+          service: true
+
       - job: failedE2ELogs
         displayName: "Failure Logs"
         dependsOn:
           - ${{ parameters.name }}
+          - cni_${{ parameters.os }}
         condition: failed()
         steps:
           - template: ../../templates/log-template.yaml

.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml

@@ -73,20 +73,6 @@ steps:
     name: "installCiliumCLI"
     displayName: "Install Cilium CLI"
 
-  - script: |
-      echo "install kubetest2 and gsutils"
-      go get github.com/onsi/ginkgo/ginkgo
-      go get github.com/onsi/gomega/...
-      go install github.com/onsi/ginkgo/ginkgo@latest
-      go install sigs.k8s.io/kubetest2@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
-      wget https://storage.googleapis.com/pub/gsutil.tar.gz
-      tar xfz gsutil.tar.gz
-      sudo mv gsutil /usr/local/bin
-    name: "installKubetest"
-    displayName: "Set up Conformance Tests"
-
   - script: |
       echo "Start Azilium E2E Tests on Overlay Cluster"
       if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]
@@ -129,16 +115,6 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
-  - script: |
-      echo "Run Service Conformance E2E"
-      export PATH=${PATH}:/usr/local/bin/gsutil
-      KUBECONFIG=~/.kube/config kubetest2 noop \
-        --test ginkgo -- \
-        --focus-regex "Services.*\[Conformance\].*" \
-        --skip-regex "should serve endpoints on same port and different protocols" # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
-    name: "servicesConformance"
-    displayName: "Run Services Conformance Tests"
-
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status

.pipelines/singletenancy/cilium/cilium-e2e-job-template.yaml

@@ -6,6 +6,7 @@ parameters:
   vmSize: ""
   k8sVersion: ""
   dependsOn: ""
+  os: "linux"
 
 stages:
   - stage: ${{ parameters.clusterName }}
@@ -60,10 +61,23 @@ stages:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
 
+      - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          cni: cilium
+          dependsOn: ${{ parameters.name }}
+          datapath: true
+          dns: true
+          portforward: true
+          service: true
+
       - job: failedE2ELogs
         displayName: "Failure Logs"
         dependsOn:
           - ${{ parameters.name }}
+          - cni_${{ parameters.os }}
         condition: failed()
         steps:
           - template: ../../templates/log-template.yaml

.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml

@@ -69,20 +69,6 @@ steps:
     name: "installCiliumCLI"
     displayName: "Install Cilium CLI"
 
-  - script: |
-      echo "install kubetest2 and gsutils"
-      go get github.com/onsi/ginkgo/ginkgo
-      go get github.com/onsi/gomega/...
-      go install github.com/onsi/ginkgo/ginkgo@latest
-      go install sigs.k8s.io/kubetest2@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
-      go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
-      wget https://storage.googleapis.com/pub/gsutil.tar.gz
-      tar xfz gsutil.tar.gz
-      sudo mv gsutil /usr/local/bin
-    name: "installKubetest"
-    displayName: "Set up Conformance Tests"
-
   - script: |
       echo "Start Azilium E2E Tests"
       kubectl get po -owide -A
@@ -120,16 +106,6 @@ steps:
     name: "CiliumStatus"
     displayName: "Cilium Status"
 
-  - script: |
-      echo "Run Service Conformance E2E"
-      export PATH=${PATH}:/usr/local/bin/gsutil
-      KUBECONFIG=~/.kube/config kubetest2 noop \
-        --test ginkgo -- \
-        --focus-regex "Services.*\[Conformance\].*" \
-        --skip-regex "should serve endpoints on same port and different protocols" # Cilium does not support this feature. For more info on test: https://github.com/kubernetes/kubernetes/blame/e602e9e03cd744c23dde9fee09396812dd7bdd93/test/conformance/testdata/conformance.yaml#L1780-L1788
-    name: "servicesConformance"
-    displayName: "Run Services Conformance Tests"
-
   - script: |
       echo "Run Cilium Connectivity Tests"
       cilium status