fix: Service Account Mitigation for CNS on k8s Windows 2022 (#1367)

* Service Account Mitigation for CNS on k8s Windows 2022 * pick up Neha's bug fix * addressing comments * add node selector back Co-authored-by: Jaeryn <tsun.chu@microsoft.com>
2022-05-20 10:03:10 -07:00 · 2022-05-20 10:03:10 -07:00 · 7219bb2dd9
--- a/cni/network/network_windows.go
+++ b/cni/network/network_windows.go
@ -37,8 +37,8 @@ var (
 * Issue link: https://github.com/kubernetes/kubernetes/issues/57253
 */
 func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string,
-	nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
-
+	nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig,
+) (*cniTypesCurr.Result, error) {
 	epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId)
 	if epInfo == nil {
 		return nil, nil
@ -141,13 +141,13 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) {
 }

 func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResult, nwCfg *cni.NetworkConfig) (string, error) {
+	determineWinVer()
 	// For singletenancy, the network name is simply the nwCfg.Name
 	if !nwCfg.MultiTenancy {
 		return nwCfg.Name, nil
 	}

 	// in multitenancy case, the network name will be in the state file or can be built from cnsResponse
-	determineWinVer()
 	if len(strings.TrimSpace(netNs)) == 0 {
 		return "", fmt.Errorf("NetNs cannot be empty")
 	}
--- a/cns/azure-cns-windows.yaml
+++ b/cns/azure-cns-windows.yaml
@ -1,3 +1,11 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: azure-cns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@ -13,30 +21,31 @@ spec:
    metadata:
      labels:
        k8s-app: azure-cns-win
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+        prometheus.io/port: "10092"
    spec:
      securityContext:
        windowsOptions:
          hostProcess: true
          runAsUserName: "NT AUTHORITY\\SYSTEM"
-      hostNetwork: true
      containers:
        - name: cns-container
-          image: acnpublic.azurecr.io/azure-cns-win:v1.4.15-1-g1b238a3f9
+          image: acnpublic.azurecr.io/azure-cns:v1.4.26-9-gc40fb852
          imagePullPolicy: IfNotPresent
-          command:
-            ["%CONTAINER_SANDBOX_MOUNT_POINT%/usr/local/bin/azure-cns.exe"]
-          args:
-            [
-              "-cns-url",
-              "tcp://$(CNSIpAddress):$(CNSPort)",
-              "-log-target",
-              "$(CNSLogTarget)",
-              "-log-location",
-              "$(CNSLogDirectory)",
-              "-store-file-path",
-              "$(CNSStoreFilePath)",
-              "-config-path",
-              "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)",
+          securityContext:
+            privileged: true
+          command: ["powershell.exe"]
+          args: 
+            [ 
+              '.\setkubeconfigpath.ps1', ";",
+              'powershell.exe', '.\azure-cns.exe',
+              '-c', "tcp://$(CNSIpAddress):$(CNSPort)",
+              '-t', "$(CNSLogTarget)",
+              '-o', "$(CNSLogDir)",
+              '-storefilepath', "$(CNSStoreFilePath)",
+              '-config-path', "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)",
+              '--kubeconfig', '.\kubeconfig',
            ]
          volumeMounts:
            - name: log
@ -45,6 +54,9 @@ spec:
              mountPath: etc/azure-cns
          ports:
            - containerPort: 10090
+              name: api
+            - containerPort: 10092
+              name: metrics
          env:
            - name: CNSIpAddress
              value: "127.0.0.1"
@ -52,19 +64,18 @@ spec:
              value: "10090"
            - name: CNSLogTarget
              value: "stdoutfile"
-            - name: CNSLogDirectory
-              value: "C:\\k"
+            - name: CNSLogDir
+              value: /k
            - name: CNSStoreFilePath
-              value: "C:\\k\\"
+              value: /k/
            - name: CNS_CONFIGURATION_PATH
-              value: "etc\\azure-cns\\cns_config.json"
+              value: etc/azure-cns/cns_config.json
            - name: NODENAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
-                  fieldPath: spec.nodeName
-            - name: KUBECONFIG
-              value: "C:\\k\\config"
+                  fieldPath: spec.nodeName 
+      hostNetwork: true
      volumes:
        - name: log
          hostPath:
@ -75,6 +86,7 @@ spec:
            name: cns-config
      nodeSelector:
        kubernetes.io/os: windows
+      serviceAccountName: azure-cns
 ---
 apiVersion: v1
 kind: ConfigMap
--- a/cns/kubeconfigtemplate.yaml
+++ b/cns/kubeconfigtemplate.yaml
@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: kubernetes
+  cluster:
+    certificate-authority-data: <ca>
+    <server>
+contexts:
+- name: azure-cns-win@kubernetes
+  context:
+    cluster: kubernetes
+    namespace: kube-system
+    user: azure-cns-win
+current-context: azure-cns-win@kubernetes
+users:
+- name: azure-cns-win
+  user:
+    token: <token>
--- a/cns/service/main.go
+++ b/cns/service/main.go
@ -910,11 +910,12 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn

 	// build default clientset.
 	kubeConfig, err := ctrl.GetConfig()
-	kubeConfig.UserAgent = fmt.Sprintf("azure-cns-%s", version)
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to get kubeconfig for request controller: %v", err)
-		return err
+		return errors.Wrap(err, "failed to get kubeconfig")
 	}
+	kubeConfig.UserAgent = fmt.Sprintf("azure-cns-%s", version)
+
 	clientset, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to build clientset")
--- a/cns/singletenantcontroller/reconciler.go
+++ b/cns/singletenantcontroller/reconciler.go
@ -88,7 +88,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
 			req, err = CreateNCRequestFromDynamicNC(nnc.Status.NetworkContainers[i])
 			// in dynamic, we will also push this NNC to the IPAM Pool Monitor when we're done.
 			listenersToNotify = append(listenersToNotify, r.ipampoolmonitorcli)
-
 		}

 		if err != nil {
--- a/cns/windows.Dockerfile
+++ b/cns/windows.Dockerfile
@ -5,18 +5,20 @@ ARG VERSION
 ARG CNS_AI_PATH
 ARG CNS_AI_ID

-WORKDIR /usr/local/src/cns
+WORKDIR /usr/src/cns
+RUN mkdir /usr/bin/

 # Copy the source
 COPY . .

 # Build cns
-RUN $Env:CGO_ENABLED=0; go build -v -o /usr/local/bin/azure-cns.exe -ldflags """-X main.version=${env:VERSION} -X ${env:CNS_AI_PATH}=${env:CNS_AI_ID}""" -gcflags="-dwarflocationlists=true" ./cns/service
+RUN $Env:CGO_ENABLED=0; go build -v -o /usr/bin/azure-cns.exe -ldflags """-X main.version=${env:VERSION} -X ${env:CNS_AI_PATH}=${env:CNS_AI_ID}""" -gcflags="-dwarflocationlists=true" ./cns/service

 # Copy into final image
 FROM mcr.microsoft.com/windows/servercore:ltsc2022
-COPY --from=builder /usr/local/bin/azure-cns.exe \
-    /usr/local/bin/azure-cns.exe
+COPY --from=builder /usr/src/cns/cns/kubeconfigtemplate.yaml kubeconfigtemplate.yaml
+COPY --from=builder /usr/src/cns/npm/examples/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1
+COPY --from=builder /usr/bin/azure-cns.exe azure-cns.exe

-ENTRYPOINT ["/usr/local/bin/azure-cns.exe"]
+ENTRYPOINT ["azure-cns.exe"]
 EXPOSE 10090