Initial check in of Build Out Traefik for Flux Deployment

2022-07-27 18:00:12 -05:00 · 2022-07-27 18:00:12 -05:00 · d51f4e960b
--- a/.github/workflows/CD-Build-Traefik.yml
+++ b/.github/workflows/CD-Build-Traefik.yml
@ -1,24 +1,158 @@
-name: "CD Build for Traefik"
+# Populates your CD repo for flux or argo deployment of traefik

-on: 
+name: CD Build - Traefik
+
+on:
  workflow_dispatch:
-      inputs:
-        test:
-          description: "testing"
-          default: "cset"
-          required: true
+    inputs:
+      ENVIRONMENT:
+        description: 'A GitHub Environment to pull action secrets from'
+        required: true
+        default: 'cset'
+        type: string
+      RG:
+        description: 'The Resource Group where your resources are deployed'
+        required: true
+        type: string
+      ACRNAME:
+        description: 'The Name of the ACR resource'
+        required: true
+        type: string
+      AKVNAME:
+        description: 'The Name of the AKV resource'
+        required: true
+        type: string
+      MINAME:
+        description: 'The Name of the Managed Identity for traefik to access secrets in the AKV'
+        required: true
+        type: string
+
+
+    secrets:
+      AZURE_CLIENT_ID:
+        required: true
+      AZURE_TENANT_ID:
+        required: true
+      AZURE_SUBSCRIPTION_ID:
+        required: true
+
+env:
+  event_sha: +refs/pull/${{ github.event.issue.number }}/merge
+
+permissions:
+      id-token: write
+      contents: write
+
 jobs:
-  Explore-GitHub-Actions:
+  App_Init:
    runs-on: ubuntu-latest
+    environment: ${{ inputs.ENVIRONMENT }}
+    env:
+      RG: "${{ inputs.RG }}"
+      ACRNAME: "${{ inputs.ACRNAME}}"
+      AKVNAME: "${{ inputs.AKVNAME}}"
+      MINAME: "${{ inputs.MINAME}}"
+
    steps:
-      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
-      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
-      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
-      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
-      - name: List files in the repository
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+      - name: Job parameter inspection
        run: |
-          ls ${{ github.workspace }}
-      - run: echo "🍏 This job's status is ${{ job.status }}."
+          echo "RG is ${{ inputs.RG }}"
+          echo "ACR name is ${{ inputs.ACRNAME }}"
+          echo "AKV name is ${{ inputs.AKVNAME }}"
+          echo "MI name is ${{ inputs.MINAME }}"
+
+      - name: Azure Login
+        uses: Azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Push to a Azure Container Registry (ACR)
+        run: |
+          az acr import --source docker.io/library/traefik:v2.5.3 -n ${{ inputs.ACRNAME }} --force
+
+      - name: Pull identity information
+        run: |
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID=$(az identity show -n $MINAME -g $RG --query id -o tsv)" >> $GITHUB_ENV
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID=$(az identity show -n $MINAME -g $RG --query principalId -o tsv)" >> $GITHUB_ENV
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID= ${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID }}"
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID= ${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID }}"
+
+      - name: Create Working Directory
+        run: |
+          mkdir tmpworking
+          echo "TMPWORKDIR=$(pwd)/tmpworking" >> $GITHUB_ENV
+
+      - name: Copy Templates to Working Directory
+        run: |
+          cp  shared-services/templates/a0008/azureidentity.yaml  ${{ env.TMPWORKDIR }}/
+          cp  shared-services/templates/a0008/secretproviderclass.yaml ${{ env.TMPWORKDIR }}/
+          cp  shared-services/templates/a0008/traefik.yaml  ${{ env.TMPWORKDIR }}/
+          ls -al ${{ env.TMPWORKDIR }}
+
+      - name: Dump variables used to populate yaml files
+        run: |
+          echo "TMPWORKDIR=${{ env.TMPWORKDIR }}"
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID=${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID }}"
+          echo "TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID=${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID }}"
+          echo "KEYVAULT_NAME_AKS_BASELINE=${{ env.KEYVAULT_NAME_AKS_BASELINE }}"
+          echo "TENANTID_AZURERBAC_AKS_BASELINE=${{ env.TENANTID_AZURERBAC_AKS_BASELINE }}"
+          echo "ACR_NAME_AKS_BASELINE=${{ env.ACR_NAME_AKS_BASELINE }}"
+
+
+      - name: Populate variables in yaml files
+        run: |
+          escaped_var=$( echo ${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID }} | sed 's;/;\\/;g')
+          echo "Escaped TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID for sed usage: $escaped_var"          
+          sed -i "s/\${TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID}/$escaped_var/g" ${{ env.TMPWORKDIR }}/azureidentity.yaml
+          sed -i "s/\${TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID}/${{ env.TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID }}/g" ${{ env.TMPWORKDIR }}/azureidentity.yaml
+          sed -i "s/\${KEYVAULT_NAME_AKS_BASELINE}/${{ inputs.AKVNAME }}/g" ${{ env.TMPWORKDIR }}/secretproviderclass.yaml
+          sed -i "s/\${TENANTID_AZURERBAC_AKS_BASELINE}/${{ secrets.AZURE_TENANT_ID }}/g" ${{ env.TMPWORKDIR }}/secretproviderclass.yaml
+          sed -i "s/\${ACR_NAME_AKS_BASELINE}/${{ inputs.ACRNAME }}/g" ${{ env.TMPWORKDIR }}/traefik.yaml
+
+      - name: Move yaml files to cluster mainifest directory and remove temp working dir
+        run: |
+          cp ${{ env.TMPWORKDIR }}/*.yaml shared-services/cluster-manifests/a0008/
+          rm -rf ${{ env.TMPWORKDIR }}
+
+      - name: Check AzureIdentity yaml
+        run: |
+          echo "Traefiks AzureIdentity yaml file:"
+          cat shared-services/cluster-manifests/a0008/azureidentity.yaml
+
+      - name: Check SecretProviderClass yaml
+        run: |
+          echo "Traefiks secretproviderclass yaml file:"
+          cat shared-services/cluster-manifests/a0008/secretproviderclass.yaml
+
+      - name: Check Traefiks deployment yaml
+        run: |
+          echo "Traefiks deployment yaml file:"
+          cat shared-services/cluster-manifests/a0008/traefik.yaml
+
+      - name: Commit files
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          if [[ "$(git status)" != *"nothing to commit"* ]];then
+             git add .
+             git commit -a -m "Add Traefik manifest files"
+          else
+             echo "No changes to traefik yaml files, therefore nothing to commit"
+          fi
+
+      - name: Dump github ref
+        run: |
+          echo "Preparing to push files to ${{ github.ref}}"
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}
+
--- a/shared-services/templates/a0008/azureidentity.yaml
+++ b/shared-services/templates/a0008/azureidentity.yaml
@ -0,0 +1,18 @@
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentity
+metadata:
+  name: podmi-ingress-controller-identity
+  namespace: a0008
+spec:
+  type: 0
+  resourceID: ${TRAEFIK_USER_ASSIGNED_IDENTITY_RESOURCE_ID}
+  clientID: ${TRAEFIK_USER_ASSIGNED_IDENTITY_CLIENT_ID}
+---
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentityBinding
+metadata:
+  name: podmi-ingress-controller-binding
+  namespace: a0008
+spec:
+  azureIdentity: podmi-ingress-controller-identity
+  selector: podmi-ingress-controller
--- a/shared-services/templates/a0008/secretproviderclass.yaml
+++ b/shared-services/templates/a0008/secretproviderclass.yaml
@ -0,0 +1,22 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: aks-ingress-tls-secret-csi-akv
+  namespace: a0008
+spec:
+  provider: azure
+  parameters:
+    usePodIdentity: "true"
+    useVMManagedIdentity: "false"
+    keyvaultName: ${KEYVAULT_NAME_AKS_BASELINE}
+    objects:  |
+      array:
+        - |
+          objectName: traefik-ingress-internal-aks-ingress-tls
+          objectAlias: tls.crt
+          objectType: cert
+        - |
+          objectName: traefik-ingress-internal-aks-ingress-tls
+          objectAlias: tls.key
+          objectType: secret
+    tenantId: ${TENANTID_AZURERBAC_AKS_BASELINE}
--- a/shared-services/templates/a0008/traefik.yaml
+++ b/shared-services/templates/a0008/traefik.yaml
@ -0,0 +1,305 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: traefik-ingress-controller
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: traefik-ingress-controller
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+      - networking.k8s.io
+    resources:
+      - ingresses
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - traefik.containo.us
+    resources:
+      - middlewares
+      - middlewaretcps
+      - ingressroutes
+      - traefikservices
+      - ingressroutetcps
+      - ingressrouteudps
+      - tlsoptions
+      - tlsstores
+      - serverstransports
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: traefik-watch-workloads
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: traefik-ingress-controller
+subjects:
+- kind: ServiceAccount
+  name: traefik-ingress-controller
+  namespace: a0008
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: traefik-ingress-config
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+data:
+  traefik.toml: |
+    [metrics]
+      [metrics.prometheus]
+        entryPoint = "metrics"
+        addEntryPointsLabels = true
+        addServicesLabels = true
+    [accessLog]
+      filePath = "/data/access.log"
+      bufferingSize = 100
+    [global]
+      # prevent Traefik from checking newer versions in production
+      checknewversion = false
+      # prevent Traefik from collecting and sending stats from production
+      sendanonymoususage = false
+    [log]
+      level = "ERROR"
+      format = "json"
+    [api]
+      dashboard = false
+    [providers]
+      # Configuration reload frequency:
+      #   * duration that Traefik waits for, after a configuration reload, before taking into account any new configuration refresh event
+      #   * the most recent one is taken into account, and all the previous others are dropped.
+      providersThrottleDuration = 10
+      [providers.file]
+        filename = "/config/traefik.toml"
+        watch = true
+      # Traefik provider that supports the native Kubernetes Ingress specification
+      # and derives the corresponding dynamic configuration from it. https://kubernetes.io/docs/concepts/services-networking/ingress/
+      [providers.kubernetesingress]
+        ingressClass = "traefik-internal"
+        namespaces = ["a0008"]
+        [providers.kubernetesIngress.ingressEndpoint]
+          publishedService = "a0008/traefik-ingress-service"
+    # Enable gzip compression
+    [http.middlewares]
+      [http.middlewares.gzip-compress.compress]
+      [http.middlewares.app-gateway-snet.ipWhiteList]
+        sourceRange = ["10.240.5.0/24"]
+    [entryPoints]
+      [entryPoints.metrics]
+        address = ":8082"
+      [entryPoints.traefik]
+        address = ":9000"
+      [entryPoints.websecure]
+        address = ":8443"
+        [entryPoints.websecure.forwardedHeaders]
+          trustedIPs = ["10.240.5.0/24"]
+        [entryPoints.websecure.http.tls]
+          options = "default"
+    [ping]
+      entryPoint = "traefik"
+    [tls]
+      # without duplicating this cert config and with SNI enabled, Traefik won't
+      # find the certificates for your host. It may be a Traefik's issue.
+      [[tls.certificates]]
+        certFile = "/certs/tls.crt"
+        keyFile = "/certs/tls.key"
+        stores = ["default"]
+      [tls.stores]
+        [tls.stores.default]
+          [tls.stores.default.defaultCertificate]
+            # without specifying in here your certs, Traefik will create its own
+            # certificate
+            certFile = "/certs/tls.crt"
+            keyFile = "/certs/tls.key"
+      [tls.options.default]
+        minVersion = "VersionTLS12"
+        sniStrict = true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-ingress-service
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+  annotations:
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+    service.beta.kubernetes.io/azure-load-balancer-internal-subnet: "snet-clusteringressservices"
+spec:
+  type: LoadBalancer
+  loadBalancerIP: 10.240.4.4
+  externalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+  ports:
+  - port: 443
+    name: "https"
+    targetPort: "websecure"
+    protocol: "TCP"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: traefik-ingress-controller
+  namespace: a0008
+  labels:
+    app.kubernetes.io/name: traefik-ingress-ilb
+    app.kubernetes.io/instance: traefik-ingress-ilb
+    aadpodidbinding: podmi-ingress-controller
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: traefik-ingress-ilb
+      app.kubernetes.io/instance: traefik-ingress-ilb
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8082"
+      labels:
+        app.kubernetes.io/name: traefik-ingress-ilb
+        app.kubernetes.io/instance: traefik-ingress-ilb
+        aadpodidbinding: podmi-ingress-controller
+    spec:
+      hostNetwork: false
+      serviceAccountName: traefik-ingress-controller
+      terminationGracePeriodSeconds: 60
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app.kubernetes.io/name
+                operator: In
+                values:
+                - traefik-ingress-ilb
+            topologyKey: "kubernetes.io/hostname"
+      containers:
+        # PRODUCTION READINESS CHANGE REQUIRED
+        # This image should be sourced from a non-public container registry, such as the
+        # one deployed along side of this reference implementation.
+        # az acr import --source docker.io/library/traefik:v2.5.3 -n <your-acr-instance-name>
+        # and then set this to
+        # image: <your-acr-instance-name>.azurecr.io/library/traefik:v2.5.3
+        # in order to use the public image, replace the image setting with the following line
+        # - image: docker.io/library/traefik:v2.5.3
+      - image: ${ACR_NAME_AKS_BASELINE}.azurecr.io/library/traefik:v2.5.3
+        name: traefik-ingress-controller
+        resources:
+          requests:
+            cpu: 100m
+            memory: 64Mi
+          limits:
+            cpu: 200m
+            memory: 128Mi
+        readinessProbe:
+          httpGet:
+            path: /ping
+            port: "traefik"
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 2
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: "traefik"
+          failureThreshold: 3
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 2
+        ports:
+        - name: "traefik"
+          containerPort: 9000
+          protocol: TCP
+        - name: "websecure"
+          containerPort: 8443
+          protocol: TCP
+        - name: "metrics"
+          containerPort: 8082
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsGroup: 65532
+          runAsNonRoot: true
+          runAsUser: 65532
+        volumeMounts:
+          - name: data
+            mountPath: /data
+          - name: config
+            mountPath: /config
+            readOnly: true
+          - name: ssl-csi
+            mountPath: /certs
+            readOnly: true
+        args:
+        - --configfile=/config/traefik.toml
+      volumes:
+        - name: config
+          configMap:
+            name: traefik-ingress-config
+        - name: ssl-csi
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: aks-ingress-tls-secret-csi-akv
+        - name: data
+          emptyDir: {}
+      nodeSelector:
+       agentpool: npuser01