From 9d9bb4113478a6ae1d726054a87954fdf15512ee Mon Sep 17 00:00:00 2001 From: Gordon Byers Date: Thu, 18 Nov 2021 13:55:56 +0000 Subject: [PATCH] Implementing GitHub reusable workflows (#121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactored the Java app smoke tests to reusable workflows Conditionally applying Azure Key Vault Access Policies regression Targeting JavaAppV3 Chart in minihelm Optionally uninstalls the java app before attempting install. (Good for when AGIC knots up) Using loops with short sleeps to wait for DNS/Certificate issue/App ready, rather than random length sleeps Changed Network Plugin preset to Azure from Calico (and added a Playwright test to cover 😎 ) A few spelling mistakes GitHub pages publish helper workflow Added spell checker to stop typo's slipping through Move release tag to env variable since it was used in 2 places and caused a problem with the v3.0.0-preview release Added doc for the regression workflow to the contribution guide Quite a few spelling mistakes, throughout various files A few stray markdown whitespace warnings taken care of that the VSCode docs plugin cared about --- .github/workflows/AppDeploy_JavaApp.yml | 347 ++++++++++ .github/workflows/ByoVnetCI.yml | 591 ++---------------- .github/workflows/Test_ReusableWorkflows.yml | 72 +++ .github/workflows/ghpages.yml | 19 +- CONTRIBUTING.md | 31 +- README.md | 17 +- SECURITY.md | 2 +- cspell.json | 48 +- .../helper-test-default-networkpolicy.spec.js | 13 + helper/src/components/addonsTab.js | 6 +- helper/src/components/appsTab.js | 8 +- helper/src/components/deployTab.js | 20 +- helper/src/components/networkTab.js | 2 +- helper/src/components/portalnav.js | 20 +- helper/src/config.json | 10 +- 15 files changed, 633 insertions(+), 573 deletions(-) create mode 100644 .github/workflows/AppDeploy_JavaApp.yml create mode 100644 .github/workflows/Test_ReusableWorkflows.yml create mode 100644 helper/.playwrighttests/helper-test-default-networkpolicy.spec.js diff --git a/.github/workflows/AppDeploy_JavaApp.yml b/.github/workflows/AppDeploy_JavaApp.yml new file mode 100644 index 00000000..a7be2a74 --- /dev/null +++ b/.github/workflows/AppDeploy_JavaApp.yml @@ -0,0 +1,347 @@ +on: + workflow_call: + inputs: + RG: + required: true + type: string + AKSNAME: + required: true + type: string + DNSDOMAIN: + required: true + type: string + DNSRG: + required: true + type: string + DNSRECORDNAME: + required: true + type: string + AKVNAME: + required: true + type: string + AGNAME: + required: true + type: string + APPNAME: + default: "openjdk-kvssl" + required: true + type: string + FRONTENDCERTTYPE: + default: "certmanager-staging" + required: true + type: string + #type: choice + ##options: + #- certmanager-staging + #- certmanager-production + #- appgw-selfsigned + MINIHELMBRANCH: + default: "main" + required: false + type: string + CERTMANAGERVERSION: + default: "v1.5.3" + required: false + type: string + UNINSTALLAFTERVERIFY: + description: 'Uninstall app after tests [yes|no]' + type: boolean + default: true + secrets: + AZURE_CREDENTIALS: + required: true + +jobs: + App_Create: + runs-on: ubuntu-latest + concurrency: ${{ inputs.AGNAME}} #AppGateway doesn't like concurrent updates + steps: + - uses: actions/checkout@v2 + + - name: Job parameter inspection + run: | + echo "RG is ${{ inputs.RG }}" + echo "AKS name is ${{ inputs.AKSNAME }}" + echo "FrontendCert type is ${{ inputs.FRONTENDCERTTYPE }}" + + - name: Azure Login + uses: Azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + enable-AzPSSession: false + environment: azurecloud + allow-no-subscriptions: false + + - name: Create self signed certs in Key Vault + env: + CERTNAMES: '["${{ inputs.APPNAME }}","${{ inputs.APPNAME }}-fe"]' + run: | + RG='${{ inputs.RG }}' + AKVNAME='${{ inputs.AKVNAME}}' + + echo $CERTNAMES | jq -r '.[]' | while read cert; do + echo "creating $cert"; + COMMON_NAME=$cert; + az keyvault certificate create --vault-name $AKVNAME -n $COMMON_NAME -p "$(az keyvault certificate get-default-policy | sed -e s/CN=CLIGetDefaultPolicy/CN=${COMMON_NAME}/g )"; + done + + sleep 2m + + - name: Create Root Cert references in AppGw + env: + CERTNAMES: '["${{ inputs.APPNAME }}"]' + id: AppGwRootCert + run: | + AGNAME='${{ inputs.AGNAME}}' + RG='${{ inputs.RG }}' + KVNAME='${{ inputs.AKVNAME}}' + + echo $CERTNAMES | jq -r '.[]' | while read cert; do + echo "getting id for $cert"; + versionedSecretId=$(az keyvault certificate show -n $cert --vault-name $KVNAME --query "sid" -o tsv); + unversionedSecretId=$(echo $versionedSecretId | cut -d'/' -f-5) # remove the version from the url; + echo $unversionedSecretId; + rootcertcmd="az network application-gateway root-cert create --gateway-name $AGNAME -g $RG -n $cert --keyvault-secret $unversionedSecretId"; + $rootcertcmd || $rootcertcmd --verbose + done + + - name: Create FrontEnd Cert references in AppGw + if: inputs.FRONTENDCERTTYPE == 'appgw-selfsigned' + env: + CERTNAMES: '["${{ inputs.APPNAME }}-fe"]' + run: | + AGNAME='${{ inputs.AGNAME}}' + RG='${{ inputs.RG }}' + KVNAME='${{ inputs.AKVNAME}}' + + echo $CERTNAMES | jq -r '.[]' | while read cert; do + echo "getting id for $cert"; + versionedSecretId=$(az keyvault certificate show -n $cert --vault-name $KVNAME --query "sid" -o tsv); + unversionedSecretId=$(echo $versionedSecretId | cut -d'/' -f-5) # remove the version from the url; + echo $unversionedSecretId; + fecertcmd="az network application-gateway ssl-cert create -n $cert --gateway-name $AGNAME -g $RG --key-vault-secret-id $unversionedSecretId"; + $fecertcmd || $fecertcmd --verbose + done + + - name: Confirm certificates registered in AppGw + run: | + AGNAME='${{ inputs.AGNAME}}' + RG='${{ inputs.RG }}' + + az network application-gateway ssl-cert list -g $RG --gateway-name $AGNAME --query "[].name" + az network application-gateway root-cert list -g $RG --gateway-name $AGNAME --query "[].name" + + - name: AKS Connect + env: + RG: "${{ inputs.RG }}" + AKSNAME: "${{ inputs.AKSNAME }}" + run: | + az aks get-credentials -n $AKSNAME -g $RG --admin --overwrite-existing + + - name: Kubectl + run: | + #sudo az aks install-cli + kubectl version + + - name: Observe Existing DNSConfig + env: + DNSDOMAIN: "${{ inputs.DNSDOMAIN }}" + DNSRG: "${{ inputs.DNSRG }}" + RECORDNAME: "${{ inputs.DNSRECORDNAME }}" + run: | + az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]" + + - name: Grab some Helm charts to install + run: | + BRANCH='${{ inputs.MINIHELMBRANCH }}' + echo "$BRANCH" + + if [ -z "$BRANCH" ] + then + BRANCH='main' + fi + + git clone -b $BRANCH https://github.com/Gordonby/minihelm.git helmcharts + + - name: Check if App Helm Chart already installed + env: + APPNAME: "${{ inputs.APPNAME }}" + run: | + APPINSTALLED=$(helm list --filter ^$APPNAME$ --short) + if [ -z "$APPINSTALLED" ] + then + echo "App not installed" + else + echo "App already installed, forcing uninstall" + helm list --short + helm uninstall $APPNAME + sleep 1m + fi + + - name: Install Cert Manager + if: startsWith(inputs.FRONTENDCERTTYPE, 'certmanager' ) + env: + MANIFESTTESTURL: "https://github.com/jetstack/cert-manager/releases/download/${{ inputs.CERTMANAGERVERSION }}/cert-manager.yaml" + run: | + kubectl apply -f $MANIFESTTESTURL + sleep 1m + + - name: Install Cert Manager ClusterIssuer + if: startsWith(inputs.FRONTENDCERTTYPE, 'certmanager' ) + env: + EMAILAD: "gdogg@microsoft.com" + run: | + echo "Email Address for Lets Encrypt: $EMAILAD" + helm upgrade --install smokecertissuer ./helmcharts/samples/certmanagerissuer --set email=$EMAILAD + sleep 1m + + - name: Verify Cert Manager ClusterIssuer + if: startsWith(inputs.FRONTENDCERTTYPE, 'certmanager' ) + env: + EMAILAD: "gdogg@microsoft.com" + run: | + case "${{ inputs.FRONTENDCERTTYPE }}" in + "certmanager-staging") CERTSOURCE="letsEncrypt"; LEISSUER="letsencrypt-staging" ;; + "certmanager-production") CERTSOURCE="letsEncrypt" LEISSUER="letsencrypt-production" ;; + esac + + kubectl describe clusterissuer $LEISSUER + + - name: Add a azure-config secret for use with externaldns + env: + DNSRESOURCEGROUP: "${{ inputs.DNSRG }}" + RG: "${{ inputs.RG }}" + AKSNAME: "${{ inputs.AKSNAME }}" + run: | + echo "AKS $AKSNAME" + echo $DNSRESOURCEGROUP + + KubeletId=$(az aks show -n $AKSNAME -g $RG --query "identityProfile.kubeletidentity.clientId" -o tsv) + TenantId=$(az account show --query tenantId -o tsv) + SubscriptionId=$(az account show --query id -o tsv) + + JSONSECRETPATH="azure.json" + cat<$JSONSECRETPATH + { + "userAssignedIdentityID": "$KubeletId", + "tenantId": "$TenantId", + "useManagedIdentityExtension": true, + "subscriptionId": "$SubscriptionId", + "resourceGroup": "$DNSRESOURCEGROUP" + } + EOF + + kubectl create secret generic azure-config-file --dry-run=client -o yaml --from-file=azure.json | kubectl apply -f - + + - name: Add external dns config + env: + DOMAINFILTER: "${{ inputs.DNSDOMAIN }}" + run: | + echo $DOMAINFILTER + helm upgrade --install externaldns ./helmcharts/samples/externaldns --set externaldns.domainfilter="$DOMAINFILTER" + + - name: Install the Java sample app + env: + NAMESP: "default" + APPNAME: "${{ inputs.APPNAME }}" #Using backend cert common name + DOMAINSUFFIX: "${{ inputs.DNSDOMAIN }}" + FRONTENDCERTNAME: "${{ inputs.APPNAME }}-fe" + + run: | + # REF: https://github.com/khowling/e2e-tls-java-aks + + AKSNAME='${{ inputs.AKSNAME}}' + RG='${{ inputs.RG }}' + KVNAME='${{ inputs.AKVNAME}}' + KVNAMELOWER=${KVNAME,,} + APPNAME='${{ inputs.APPNAME }}' + KVTENANT=$(az account show --query tenantId -o tsv) + AGNAME='${{ inputs.AGNAME}}' + AKSNAME='${{ inputs.AKSNAME}}' + DNSNAME=${APPNAME}.${DOMAINSUFFIX} + + case "${{ inputs.FRONTENDCERTTYPE }}" in + "appgw-selfsigned") CERTSOURCE="appGw"; LEISSUER="" ;; + "certmanager-staging") CERTSOURCE="letsEncrypt"; LEISSUER="letsencrypt-staging" ;; + "certmanager-production") CERTSOURCE="letsEncrypt" LEISSUER="letsencrypt-production" ;; + esac + + echo 'Get the identity created from the KeyVaultSecret Addon' + export CSISECRET_CLIENTID=$(az aks show -g $RG --name $AKSNAME --query addonProfiles.azureKeyvaultSecretsProvider.identity.clientId -o tsv) + echo $CSISECRET_CLIENTID + + helm upgrade --install $APPNAME ./helmcharts/samples/javatlsappv3 --set nameOverride="${APPNAME}",frontendCertificateSource="${CERTSOURCE}",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}",appgw.frontendCertificateName="${APPNAME}-fe",appgw.rootCertificateName="${APPNAME}",letsEncrypt.issuer="${LEISSUER}",letsEncrypt.secretname="${APPNAME}-tls" --dry-run + helm upgrade --install $APPNAME ./helmcharts/samples/javatlsappv3 --set nameOverride="${APPNAME}",frontendCertificateSource="${CERTSOURCE}",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}",appgw.frontendCertificateName="${APPNAME}-fe",appgw.rootCertificateName="${APPNAME}",letsEncrypt.issuer="${LEISSUER},letsEncrypt.secretname="${APPNAME}-tls"" + + - name: Wait for External DNS config + env: + DNSDOMAIN: "${{ inputs.DNSDOMAIN }}" + DNSRG: "${{ inputs.DNSRG }}" + RECORDNAME: "${{ inputs.DNSRECORDNAME }}" + run: | + counter=1 + while [ $counter -le 10 ] + do + dnsrecords=$(az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]") + + if [ "$dnsrecords" != "[]" ] + then + echo "DNS Records have been created [$counter] [$(date +"%T")]" + echo $dnsrecords + break 2 + fi + + echo "ExternalDNS has not created Azure DNS Records yet [$counter] [$(date +"%T")]" + + ((counter++)) + sleep 10s + done + + - name: Wait for Cert Manager to issue certificate + if: startsWith(inputs.FRONTENDCERTTYPE, 'certmanager' ) + env: + SECRETNAME: "${{ inputs.APPNAME }}-tls" + run: | + counter=1 + while [ $counter -le 10 ] + do + kubectl get po + + secret=$(kubectl get secret $SECRETNAME --ignore-not-found) + kubectl get secret $SECRETNAME --ignore-not-found + + if [ ! -z "$secret" ] + then + echo "Secret $SECRETNAME has been created" + break 2 + fi + + ((counter++)) + sleep 10s + done + + - name: Verify Java App is reachable + env: + NAMESP: "default" + APPNAME: "${{ inputs.APPNAME }}" + URLTOHIT: "https://${{ inputs.APPNAME }}.${{ inputs.DNSDOMAIN }}" + run: | + echo "curl $URLTOHIT [$(date +"%T")]" + curlcommand="curl --connect-timeout 2 --retry 25 --retry-delay 20 --no-keepalive --no-tcp-nodelay -X GET --insecure --write-out %{http_code} --silent --fail --output /dev/null $URLTOHIT -v --trace-time" + respcode=$($curlcommand || sleep 1m; $curlcommand) + + echo $respcode + + curl --insecure $URLTOHIT + + - name: Verify Frontend Certificate + env: + URLTOHIT: "https://${{ inputs.APPNAME }}.${{ inputs.DNSDOMAIN }}" + run: | + curl --insecure -vvI $URLTOHIT 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' + + - name: Cleanup app + if: inputs.UNINSTALLAFTERVERIFY + env: + APPNAME: "${{ inputs.APPNAME }}" + run: helm uninstall $APPNAME diff --git a/.github/workflows/ByoVnetCI.yml b/.github/workflows/ByoVnetCI.yml index 7c41c23e..cb78ea4e 100644 --- a/.github/workflows/ByoVnetCI.yml +++ b/.github/workflows/ByoVnetCI.yml @@ -47,6 +47,7 @@ env: ParamFilePath: ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json" RESNAME: "Byo" DEPNAME: "Dep${{ github.run_number }}" + KVACCESS: "IAM" jobs: Well_Architected: @@ -84,6 +85,7 @@ jobs: echo "Resource name is ${{ env.RESNAME }}" echo "Deployment name is ${{ env.DEPNAME }}" echo "Ref is ${{ github.ref }}" + echo "Ref name is ${{GITHUB.REF_NAME}}" echo "EventTrigger name is ${{github.event_name}}" echo "PR contains bug : ${{contains(github.event.pull_request.labels.*.name, 'bug')}}" echo "PR labels : ${{github.event.pull_request.labels.*.name}}" @@ -474,499 +476,46 @@ jobs: az role assignment list --assignee $agicobjid } - - CertificateConfig: - runs-on: ubuntu-latest - needs: [Deploy] - environment: azurecirgs - steps: - - uses: actions/checkout@v2 + + SmokeTest_JavaApp-certmgr: + needs: [Deploy,SmokeTest_SimpleApp] + uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@0.3.1 + with: + RG: Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows + AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} + DNSDOMAIN: azdemo.co.uk + DNSRG: domainssl + DNSRECORDNAME: openjdk-demo + AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}} + AGNAME: ${{ needs.Deploy.outputs.AGNAME}} + APPNAME: openjdk-demo + FRONTENDCERTTYPE: certmanager-staging + MINIHELMBRANCH: ${{ github.event.inputs.minihelmbranch }} + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} - - name: Azure Login - uses: Azure/login@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - enable-AzPSSession: false - environment: azurecloud - allow-no-subscriptions: false - - - name: Add KV access policy for managing certificates as this CI action user - run: | - CiSpId='${{ secrets.AZURE_CREDENTIALS_OBJID }}' - AKVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - - az keyvault set-policy --name $AKVNAME --object-id $CiSpId --secret-permissions get list --certificate-permissions get list delete create import update managecontacts getissuers listissuers setissuers deleteissuers manageissuers recover purge backup restore - - - name: Create self signed certs in Key Vault - env: - CERTNAMES: '["openjdk-demo","openjdk-demo-service","openjdk-kvssl","openjdk-kvssl-fe","aspnetapp"]' - run: | - RG='${{ env.RG }}' - AKVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - - echo $CERTNAMES - - jq --version - - echo $CERTNAMES | jq -r - - echo $CERTNAMES | jq -r '.[]' | while read cert; do - echo "creating $cert"; - COMMON_NAME=$cert; - az keyvault certificate create --vault-name $AKVNAME -n $COMMON_NAME -p "$(az keyvault certificate get-default-policy | sed -e s/CN=CLIGetDefaultPolicy/CN=${COMMON_NAME}/g )"; - done - - sleep 2m - - - name: Create Root Cert references in AppGW - env: - CERTNAMES: '["openjdk-demo","openjdk-demo-service","openjdk-kvssl"]' - id: AppGwRootCert - run: | - AGNAME='${{ needs.Deploy.outputs.AGNAME}}' - RG='${{ env.RG }}' - KVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - - echo $CERTNAMES | jq -r '.[]' | while read cert; do - echo "getting id for $cert"; - versionedSecretId=$(az keyvault certificate show -n $cert --vault-name $KVNAME --query "sid" -o tsv); - unversionedSecretId=$(echo $versionedSecretId | cut -d'/' -f-5) # remove the version from the url; - echo $unversionedSecretId; - rootcertcmd="az network application-gateway root-cert create --gateway-name $AGNAME -g $RG -n $cert --keyvault-secret $unversionedSecretId"; - $rootcertcmd || $rootcertcmd --verbose - done - - - name: Create FrontEnd Cert references in AppGW - env: - CERTNAMES: '["aspnetapp","openjdk-kvssl-fe"]' - run: | - AGNAME='${{ needs.Deploy.outputs.AGNAME}}' - RG='${{ env.RG }}' - KVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - - echo $CERTNAMES | jq -r '.[]' | while read cert; do - echo "getting id for $cert"; - versionedSecretId=$(az keyvault certificate show -n $cert --vault-name $KVNAME --query "sid" -o tsv); - unversionedSecretId=$(echo $versionedSecretId | cut -d'/' -f-5) # remove the version from the url; - echo $unversionedSecretId; - fecertcmd="az network application-gateway ssl-cert create -n $cert --gateway-name $AGNAME -g $RG --key-vault-secret-id $unversionedSecretId"; - $fecertcmd || $fecertcmd --verbose - done - - - name: Confirm certificates registered in AppGw - if: github.event.inputs.doVerifySteps != 'no' - run: | - AGNAME='${{ needs.Deploy.outputs.AGNAME}}' - RG='${{ env.RG }}' - - az network application-gateway ssl-cert list -g $RG --gateway-name $AGNAME --query "[].name" - az network application-gateway root-cert list -g $RG --gateway-name $AGNAME --query "[].name" - - SmokeTest_PrivateJavaTLSApp: - runs-on: ubuntu-latest - needs: [Deploy, CertificateConfig, SmokeTest_SimpleApp] - environment: azurecirgs - steps: - - uses: actions/checkout@v2 - - - name: Param check - if: github.event.inputs.doVerifySteps != 'no' - run: | - RG='${{ env.RG }}' - echo "RG is: $RG" - - echo "Param file path is: ${{ env.ParamFilePath }}" - echo "Resource name is ${{ env.RESNAME }}" - echo "Deployment name is ${{ env.DEPNAME }}" - echo "AKS name is ${{ needs.Deploy.outputs.AKSNAME}}" - echo "ACR name is ${{ needs.Deploy.outputs.ACRNAME}}" - - - name: Azure Login - uses: Azure/login@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - enable-AzPSSession: true - environment: azurecloud - allow-no-subscriptions: false - - - name: AKS Connect - run: | - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - az aks get-credentials -n $AKSNAME -g $RG --admin --overwrite-existing - - - name: Kubectl - if: github.event.inputs.doDebugSteps != 'no' - run: | - #sudo az aks install-cli - kubectl version - - - name: Observe Existing DNSConfig - if: github.event.inputs.doDebugSteps != 'no' - env: - DNSDOMAIN: "azdemo.co.uk" - DNSRG: "domainssl" - RECORDNAME: "openjdk-demo" - run: | - az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]" - - - name: Grab some Helm charts to install - run: | - BRANCH='${{ github.event.inputs.minihelmbranch }}' - echo "$BRANCH" - - if [ -z "$BRANCH" ] - then - BRANCH='main' - fi - - git clone -b $BRANCH https://github.com/Gordonby/minihelm.git helmcharts - - - name: Add a azure-config secret for use with externaldns - env: - DNSRESOURCEGROUP: "domainssl" - run: | - RG='${{ env.RG }}' - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - - echo "AKS $AKSNAME" - echo $DNSRESOURCEGROUP - - KubeletId=$(az aks show -n $AKSNAME -g $RG --query "identityProfile.kubeletidentity.clientId" -o tsv) - TenantId=$(az account show --query tenantId -o tsv) - SubscriptionId=$(az account show --query id -o tsv) - - JSONSECRETPATH="azure.json" - cat<$JSONSECRETPATH - { - "userAssignedIdentityID": "$KubeletId", - "tenantId": "$TenantId", - "useManagedIdentityExtension": true, - "subscriptionId": "$SubscriptionId", - "resourceGroup": "$DNSRESOURCEGROUP" - } - EOF - - kubectl create secret generic azure-config-file --dry-run=client -o yaml --from-file=azure.json | kubectl apply -f - - - - name: Add external dns config - env: - DOMAINFILTER: "azdemo.co.uk" - run: | - echo $DOMAINFILTER - helm upgrade --install externaldns ./helmcharts/samples/externaldns --set externaldns.domainfilter="$DOMAINFILTER" - - - name: Check DNS config - if: github.event.inputs.doVerifySteps != 'no' - env: - DNSDOMAIN: "azdemo.co.uk" - DNSRG: "domainssl" - RECORDNAME: "openjdk-demo" - run: | - az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]" - - - name: Install the Java sample app - env: - NAMESP: "default" - APPNAME: "openjdk-kvssl" #Using backend cert common name - DOMAINSUFFIX: "azdemo.co.uk" - FRONTENDCERTNAME: "openjdk-kvssl-fe" - - run: | - # REF: https://github.com/khowling/e2e-tls-java-aks - - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - RG='${{ env.RG }}' - KVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - KVNAMELOWER=${KVNAME,,} - APPNAME='${{ env.APPNAME }}' - KVTENANT=$(az account show --query tenantId -o tsv) - AGNAME='${{ needs.Deploy.outputs.AGNAME}}' - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - DNSNAME=${APPNAME}.${DOMAINSUFFIX} - - echo 'Get the identity created from the KeyVaultSecret Addon' - export CSISECRET_CLIENTID=$(az aks show -g $RG --name $AKSNAME --query addonProfiles.azureKeyvaultSecretsProvider.identity.clientId -o tsv) - echo $CSISECRET_CLIENTID - - helm upgrade --install $APPNAME ./helmcharts/samples/javatlsappv2 --set nameOverride="${APPNAME}",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}",appgw.frontendCertificateName="${FRONTENDCERTNAME}",appgw.rootCertificateName="${APPNAME}" --dry-run - helm upgrade --install $APPNAME ./helmcharts/samples/javatlsappv2 --set nameOverride="${APPNAME}",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}",appgw.frontendCertificateName="${FRONTENDCERTNAME}",appgw.rootCertificateName="${APPNAME}" + SmokeTest_JavaApp-appgw: + needs: [Deploy,SmokeTest_JavaApp-certmgr] + uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@0.3.1 + with: + RG: Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows + AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} + DNSDOMAIN: azdemo.co.uk + DNSRG: domainssl + DNSRECORDNAME: openjdk-kvssl + AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}} + AGNAME: ${{ needs.Deploy.outputs.AGNAME}} + APPNAME: openjdk-kvssl + FRONTENDCERTTYPE: appgw-selfsigned + MINIHELMBRANCH: ${{ github.event.inputs.minihelmbranch }} + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} - - name: Verify Java App is reachable - env: - NAMESP: "default" - APPNAME: "openjdk-kvssl" - - URLTOHIT: "https://openjdk-kvssl.azdemo.co.uk" - run: | - sleep 3m #Give everything a chance to work - - INGNAME="${APPNAME}-ingress" - - kubectl get po -n $NAMESP - kubectl get svc -n $NAMESP - kubectl get ingress -n $NAMESP - - echo "Getting public ip" - pubId=$(kubectl get ing -n $NAMESP $INGNAME -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo "Public ip : $pubId" - echo "URL to hit: $URLTOHIT" - - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - - #TODO: This is going to need to be rewritten. #loopshambles - #SSH'ing on the runner shows this works, sometimes it just takes ages - #Needs to be investigated. - if [ "$respcode" != "200" ]; - then - echo "going to need to wait longer i guess?" - sleep 3m - - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - - if [ "$respcode" != "200" ]; - then - echo "going to need to wait EVEN longer i guess? (wtf)" - sleep 3m - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - if [ "$respcode" != "200" ]; - then - echo "Non 200 response code from app - Raising error" - exit 1 - fi - fi - fi - - curl --insecure $URLTOHIT - - - name: Verify Frontend Certificate - if: github.event.inputs.doVerifySteps != 'no' - env: - URLTOHIT: "https://openjdk-demo.azdemo.co.uk" - run: | - curl --insecure -vvI $URLTOHIT 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' - - - name: Cleanup app - if: github.event.inputs.doSmokeTestUninstall == 'yes' - env: - APPNAME: "openjdk-kvssl" - run: helm uninstall $APPNAME - - SmokeTest_PublicJavaTLSApp: - runs-on: ubuntu-latest - needs: [Deploy, CertificateConfig, SmokeTest_SimpleApp, SmokeTest_PrivateJavaTLSApp] - environment: azurecirgs - steps: - - uses: actions/checkout@v2 - - - name: Param check - if: github.event.inputs.doDebugSteps != 'no' - run: | - RG='${{ env.RG }}' - echo "RG is: $RG" - - echo "Param file path is: ${{ env.ParamFilePath }}" - echo "Resource name is ${{ env.RESNAME }}" - echo "Deployment name is ${{ env.DEPNAME }}" - echo "AKS name is ${{ needs.Deploy.outputs.AKSNAME}}" - echo "ACR name is ${{ needs.Deploy.outputs.ACRNAME}}" - - - name: Azure Login - uses: Azure/login@v1 - with: - creds: ${{ secrets.AZURE_CREDENTIALS }} - enable-AzPSSession: true - environment: azurecloud - allow-no-subscriptions: false - - - name: AKS Connect - run: | - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - az aks get-credentials -n $AKSNAME -g $RG --admin --overwrite-existing - - - name: Kubectl - if: github.event.inputs.doDebugSteps != 'no' - run: | - #sudo az aks install-cli - kubectl version - - - name: Observe Existing DNSConfig - if: github.event.inputs.doDebugSteps != 'no' - env: - DNSDOMAIN: "azdemo.co.uk" - DNSRG: "domainssl" - RECORDNAME: "openjdk-demo" - run: | - az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]" - - - name: Grab some Helm charts to install - run: | - git clone https://github.com/Gordonby/minihelm.git helmcharts - - - name: Add a azure-config secret for use with externaldns - env: - DNSRESOURCEGROUP: "domainssl" - run: | - RG='${{ env.RG }}' - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - - echo "AKS $AKSNAME" - echo $DNSRESOURCEGROUP - - KubeletId=$(az aks show -n $AKSNAME -g $RG --query "identityProfile.kubeletidentity.clientId" -o tsv) - TenantId=$(az account show --query tenantId -o tsv) - SubscriptionId=$(az account show --query id -o tsv) - - JSONSECRETPATH="azure.json" - cat<$JSONSECRETPATH - { - "userAssignedIdentityID": "$KubeletId", - "tenantId": "$TenantId", - "useManagedIdentityExtension": true, - "subscriptionId": "$SubscriptionId", - "resourceGroup": "$DNSRESOURCEGROUP" - } - EOF - - kubectl create secret generic azure-config-file --dry-run=client -o yaml --from-file=azure.json | kubectl apply -f - - - - name: Add external dns config - env: - DOMAINFILTER: "azdemo.co.uk" - run: | - echo $DOMAINFILTER - helm upgrade --install externaldns ./helmcharts/samples/externaldns --set externaldns.domainfilter="$DOMAINFILTER" - - - name: Check DNS config - if: github.event.inputs.doVerifySteps != 'no' - env: - DNSDOMAIN: "azdemo.co.uk" - DNSRG: "domainssl" - RECORDNAME: "openjdk-demo" - run: | - az network dns record-set list -g $DNSRG -z $DNSDOMAIN --query "[?name=='$RECORDNAME'][{type:type,fqdn:fqdn,aRecords:aRecords,txtRecords:txtRecords}]" - - - name: Install Cert Manager - env: - MANIFESTTESTURL: "https://github.com/jetstack/cert-manager/releases/download/v1.5.3/cert-manager.yaml" - run: | - kubectl apply -f $MANIFESTTESTURL - sleep 1m - - - name: Install ClusterIssuer - env: - EMAILAD: "gdogg@microsoft.com" - run: | - echo "Email Address for Lets Encrypt: $EMAILAD" - helm upgrade --install smokecertissuer ./helmcharts/samples/certmanagerissuer --set email=$EMAILAD - - - name: Verify ClusterIssuer - if: github.event.inputs.doVerifySteps != 'no' - env: - EMAILAD: "gdogg@microsoft.com" - run: | - sleep 1m - kubectl describe clusterissuer letsencrypt-staging - - - name: Install the Java sample app - env: - NAMESP: "default" - CERTNAME: "openjdk-demo" - DOMAINSUFFIX: "azdemo.co.uk" - DOMAINPREFIX: "openjdk-demo" - - run: | - # REF: https://github.com/khowling/e2e-tls-java-aks - - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - RG='${{ env.RG }}' - KVNAME='${{ needs.Deploy.outputs.AKVNAME}}' - KVNAMELOWER=${KVNAME,,} - CERTNAME='${{ env.CERTNAME }}' - KVTENANT=$(az account show --query tenantId -o tsv) - AGNAME='${{ needs.Deploy.outputs.AGNAME}}' - AKSNAME='${{ needs.Deploy.outputs.AKSNAME}}' - DNSNAME=${DOMAINPREFIX}.${DOMAINSUFFIX} - - echo 'Get the identity created from the KeyVaultSecret Addon' - export CSISECRET_CLIENTID=$(az aks show -g $RG --name $AKSNAME --query addonProfiles.azureKeyvaultSecretsProvider.identity.clientId -o tsv) - echo $CSISECRET_CLIENTID - - helm upgrade --install e2esmokeapp ./helmcharts/samples/javatlsapp --set nameOverride="openjdk-demo",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}" --dry-run - helm upgrade --install e2esmokeapp ./helmcharts/samples/javatlsapp --set nameOverride="openjdk-demo",csisecrets.vaultname="${KVNAMELOWER}",csisecrets.tenantId="${KVTENANT}",csisecrets.clientId="${CSISECRET_CLIENTID}",dnsname="${DNSNAME}" - - - name: Verify LetsEncrypt Certificate logs - if: github.event.inputs.doVerifySteps != 'no' - run: | - sleep 1m - kubectl describe cert - - - name: Verify Java App is reachable - env: - NAMESP: "default" - INGNAME: "openjdk-demo-ingress" - URLTOHIT: "https://openjdk-demo.azdemo.co.uk" - run: | - sleep 2m #Give everything a chance to work - - kubectl get po -n $NAMESP - kubectl get svc -n $NAMESP - kubectl get ingress -n $NAMESP - - echo "Getting public ip" - pubId=$(kubectl get ing -n $NAMESP $INGNAME -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo "Public ip : $pubId" - echo "URL to hit: $URLTOHIT" - - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - - #TODO: This is going to need to be rewritten. #loopshambles - #SSH'ing on the runner shows this works, sometimes it just takes ages - #Needs to be investigated. - if [ "$respcode" != "200" ]; - then - echo "going to need to wait longer i guess?" - sleep 3m - - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - - if [ "$respcode" != "200" ]; - then - echo "going to need to wait EVEN longer i guess? (wtf)" - sleep 3m - respcode=$(curl -X GET --insecure --write-out %{http_code} --silent --output /dev/null $URLTOHIT) - echo $respcode - if [ "$respcode" != "200" ]; - then - echo "Non 200 response code from app - Raising error" - exit 1 - fi - fi - fi - - curl --insecure $URLTOHIT - - - name: Verify Frontend Certificate - if: github.event.inputs.doVerifySteps != 'no' - env: - URLTOHIT: "https://openjdk-demo.azdemo.co.uk" - run: | - curl --insecure -vvI $URLTOHIT 2>&1 | awk 'BEGIN { cert=0 } /^\* SSL connection/ { cert=1 } /^\*/ { if (cert) print }' - - - name: Cleanup app - if: github.event.inputs.doSmokeTestUninstall == 'yes' - env: - APPNAME: "e2esmokeapp" - run: helm uninstall $APPNAME - SmokeTest_SimpleApp: runs-on: ubuntu-latest needs: [Deploy] environment: azurecirgs + concurrency: ${{ needs.Deploy.outputs.AGNAME}} steps: - uses: actions/checkout@v2 @@ -1019,45 +568,36 @@ jobs: env: NAMESP: "default" run: | - sleep 2m #Give everything a chance to work - kubectl get po -n $NAMESP kubectl get svc -n $NAMESP kubectl get ingress -n $NAMESP - echo "Getting public ip" - pubId=$(kubectl get ing -n $NAMESP azure-vote-ing -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo "Public ip : $pubId" + counter=1 + pubIp="" + while [ $counter -le 20 ] && [ "$pubIp" == "" ] + do + sleep 10s + + echo "Getting public ip [$counter] [$(date +"%T")]" + pubIp=$(kubectl get ing -n $NAMESP azure-vote-ing -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo "Public ip : $pubIp" + done - respcode=$(curl -o /dev/null -s -w "%{http_code}\n" $pubId) - echo $respcode + counter=1 + while [ $counter -le 20 ] && [ "$respcode" != "200" ] + do + echo "curl $pubIp [$counter]" + respcode=$(curl -o /dev/null -s -w "%{http_code}\n" $pubIp) + echo $respcode + if [ "$respcode" != "200" ]; + then + echo "going to wait another 30s [$counter] [$(date +"%T")]" + ((counter++)) + sleep 30s + fi + done - #TODO: This is going to need to be rewritten. #loopshambles - #SSH'ing on the runner shows this works, sometimes it just takes ages - #Needs to be investigated. - if [ "$respcode" != "200" ]; - then - echo "going to need to wait longer i guess?" - sleep 3m - - respcode=$(curl -o /dev/null -s -w "%{http_code}\n" $pubId) - echo $respcode - - if [ "$respcode" != "200" ]; - then - echo "going to need to wait EVEN longer i guess? (wtf)" - sleep 6m - respcode=$(curl -o /dev/null -s -w "%{http_code}\n" $pubId) - echo $respcode - if [ "$respcode" != "200" ]; - then - echo "Non 200 response code from app - Raising error" - exit 1 - fi - fi - fi - - curl $pubId + curl $pubIp - name: Remove App from cluster env: @@ -1172,8 +712,8 @@ jobs: CaptureAgicLogs: runs-on: ubuntu-latest - needs: [Deploy, SmokeTest_SimpleApp, SmokeTest_PublicJavaTLSApp] - if: always() && ${{ needs.Deploy.outputs.AGNAME != '' }} + needs: [Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw, SmokeTest_SimpleApp] + if: ${{ always() && needs.Deploy.outputs.AGNAME != '' }} steps: - name: Azure Login uses: Azure/login@v1 @@ -1194,6 +734,11 @@ jobs: #sudo az aks install-cli kubectl version + - name: Kubectl get events + if: github.event.inputs.doDebugSteps != 'no' + run: | + kubectl get events + - name: Describe AGIC Pod env: NAMESP: "kube-system" @@ -1216,7 +761,7 @@ jobs: Infra_Destroy: runs-on: ubuntu-latest if: github.event_name == 'schedule' - needs: [Validation, Deploy, InfraTests, DeployVerify, CertificateConfig, SmokeTest_SimpleApp, SmokeTest_PublicJavaTLSApp, Well_Architected, CaptureAgicLogs] + needs: [Validation, Deploy, InfraTests, DeployVerify, SmokeTest_SimpleApp, Well_Architected, CaptureAgicLogs] steps: - name: Azure Login uses: Azure/login@v1 diff --git a/.github/workflows/Test_ReusableWorkflows.yml b/.github/workflows/Test_ReusableWorkflows.yml new file mode 100644 index 00000000..b77aa065 --- /dev/null +++ b/.github/workflows/Test_ReusableWorkflows.yml @@ -0,0 +1,72 @@ +# Workflow is for testing reusable workflows to expedite the developer inner loop. + +name: Test-Java App Deploy + +on: + workflow_dispatch: + + #pull_request: + # branches: [main] + # paths: + # - ".github/workflows/Test_ReusableWorkflows.yml" + +concurrency: ci-${{ github.ref }} + +env: + RG: "Automation-Actions-AksDeployCI" + +jobs: + deploy-java-app-certmgr: + uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@0.3.1 + with: + RG: Automation-Actions-AksDeployCI #$RG + AKSNAME: aks-Byo + DNSDOMAIN: azdemo.co.uk + DNSRG: domainssl + DNSRECORDNAME: openjdk-demo + AKVNAME: kv-Byo + AGNAME: agw-Byo + APPNAME: openjdk-demo + FRONTENDCERTTYPE: certmanager-staging + MINIHELMBRANCH: main + UNINSTALLAFTERVERIFY: false + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + + deploy-java-app-certmgr2: + needs: [deploy-java-app-certmgr, deploy-java-app-kvssl] + uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@0.3.1 + with: + RG: Automation-Actions-AksDeployCI #$RG + AKSNAME: aks-Byo + DNSDOMAIN: azdemo.co.uk + DNSRG: domainssl + DNSRECORDNAME: openjdk-demo2 + AKVNAME: kv-Byo + AGNAME: agw-Byo + APPNAME: openjdk-demo2 + FRONTENDCERTTYPE: certmanager-staging + MINIHELMBRANCH: main + UNINSTALLAFTERVERIFY: false + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + + deploy-java-app-kvssl: + needs: [deploy-java-app-certmgr] + #uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@gb-privatelink-cicd + uses: azure/aks-construction/.github/workflows/AppDeploy_JavaApp.yml@0.3.1 + with: + RG: Automation-Actions-AksDeployCI #$RG + AKSNAME: aks-Byo + DNSDOMAIN: azdemo.co.uk + DNSRG: domainssl + DNSRECORDNAME: openjdk-kvssl + AKVNAME: kv-Byo + AGNAME: agw-Byo + APPNAME: openjdk-kvssl + FRONTENDCERTTYPE: appgw-selfsigned + MINIHELMBRANCH: main + UNINSTALLAFTERVERIFY: false + + secrets: + AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} diff --git a/.github/workflows/ghpages.yml b/.github/workflows/ghpages.yml index e3e8e0ac..ebf0aad4 100644 --- a/.github/workflows/ghpages.yml +++ b/.github/workflows/ghpages.yml @@ -18,17 +18,32 @@ on: workflow_dispatch: +env: + templateRelease: 0.3.0-preview + jobs: + Validation: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + + - uses: streetsidesoftware/cspell-action@v1.3.4 + name: Spell Check + if: ${{github.event_name!='workflow_dispatch'}} + with: + config: './cspell.json' + inline: warning + root: '.' + files: "**/*.{ts,js,md}" #You need to respecify this setting - even though it's in the cspell.json config :( + incremental_files_only: false + strict: true #setting to false allows the build to continue even if spelling mistakes are detected - name: Build and start node app run: | cd helper npm install - REACT_APP_K8S_VERSION="1.20.9" REACT_APP_AZ_TEMPLATE_ARG="--template-uri https://github.com/Azure/Aks-Construction/releases/download/0.3.0-preview/main.json" npm run build + REACT_APP_K8S_VERSION="1.20.9" REACT_APP_AZ_TEMPLATE_ARG="--template-uri https://github.com/Azure/Aks-Construction/releases/download/${{env.templateRelease}}/main.json" npm run build npm run start& - name: Playwright - Install w/ OS dependencies @@ -96,7 +111,7 @@ jobs: run: | cd helper npm install - REACT_APP_APPINSIGHTS_KEY=${{ secrets.REACT_APP_APPINSIGHTS_KEY}} REACT_APP_K8S_VERSION="1.20.9" REACT_APP_AZ_TEMPLATE_ARG="--template-uri https://github.com/Azure/Aks-Construction/releases/download/0.3.0-preview/main.json" npm run build + REACT_APP_APPINSIGHTS_KEY=${{ secrets.REACT_APP_APPINSIGHTS_KEY}} REACT_APP_K8S_VERSION="1.20.9" REACT_APP_AZ_TEMPLATE_ARG="--template-uri https://github.com/Azure/Aks-Construction/releases/download/${{env.templateRelease}}/main.json" npm run build - name: Deploy to GitHub Pages uses: crazy-max/ghaction-github-pages@v2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbfbc78d..5fea05fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,11 @@ A few important factoids to consume about the Repo, before you contribute. +## Opportunities to contribute + +Start by looking through the active issues for [low hanging fruit](https://github.com/Azure/Aks-Construction/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). +Another area that will help you get more familiar with the project is by running the Helper Web App locally and writing some new [Playwright web tests](helper/.playwrighttests) to make our web publishing/testing process more robust. + ## Action Workflows Various workflows run on Push / PR / Schedule. @@ -11,13 +16,14 @@ Various workflows run on Push / PR / Schedule. | Bicep Build | Every Push | `Quality` To run the bicep linter upon changes to the bicep files | | Greetings | Issue / PR | `Community` Greeting new contributors to the repo | | Stale bot | Issue / PR | `Tidy` Marks old issues as stale | -| Labeler | PR | `Tidy` Adds relevant labels to PR's based on files changed | +| Labeller | PR | `Tidy` Adds relevant labels to PR's based on files changed | | Publish Helper | PR | `Quality` Tests changes to the UI work | | Publish Helper | Push `main` | Publishes the UI to GitHub Pages | | Check Markdown | PR | `Quality` Checks markdown files for spelling mistakes | | Infra CI - Private Cluster | Push / PR / Schedule | `Quality` Low maturity IaC deployment example. Tests the most secure/private parameter config | | Infra CI - Byo Vnet Cluster | Push / PR / Schedule | `Quality` High maturity IaC deployment example. Tests a typical production grade parameter config | | Infra CI - Starter Cluster | Push / PR / Schedule | `Quality` Low maturity IaC deployment example. Tests a sandbox grade parameter config | +| InfraCI - Regression Validation | Push / PR / Schedule | `Quality` Validates multiple parameter files against the bicep code to cover regression scenarios | | App CI | Manual | `Quality` Application deployment sample showing different application deployment practices and automation capabilities | ### Enforced PR Checks @@ -30,7 +36,7 @@ Each has a *Validate job*, that is required to pass before merging to main. PR's For the *most part* we try to use feature branches to PR to Main -``` +```text ┌─────────────────┐ ┌───────────────┐ │ │ │ │ │ Feature Branch ├────────►│ Main │ @@ -43,11 +49,11 @@ Branch Policies require the Validation stage of our GitHub Action Workflows to s ### The Develop Branch -Where there have been significant changes and we want the full gammit of CI testing to be run on real Azure Infrastucture - then the Develop branch is used. -It gives us the nice warm fuzzy feeling before merging into Main. +Where there have been significant changes and we want the full gamut of CI testing to be run on real Azure Infrastructure - then the Develop branch is used. +It gives us the nice warm fuzzy feeling before merging into Main. We anticipate the use of the Develop branch is temporary. -``` +```text ┌─────────────────┐ ┌─────────────┐ ┌────────────┐ │ │ │ │ │ │ │ Feature Branch ├────────►│ Develop ├──────►│ Main │ @@ -66,6 +72,16 @@ We anticipate the use of the Develop branch is temporary. Releases are used to capture a tested release (all stages, not just Validation), where there are significant new features or bugfixes. The release does not include CI Action files, just the Bicep code. +## The Wizard Web App + +The [configuration experience](https://azure.github.io/Aks-Construction/) is hosted in GitHub pages. It's a static web app, written in NodeJS using FluentUI. + +### Playwright tests + +Playwright is used to help verify that the app works properly, you can use Playwright in your local dev experience (see Codespaces below), but crucially it's also leveraged as part of the publish process. If the tests don't pass, then the app will not publish. The `fragile` keyword should be used in any tests where you're learning how they work and run. Once the test is of sufficient quality to be considered a core test, the `fragile` keyword is removed. + +We're trying to ensure that PR's that contain Web UI changes have appropriate Playwright tests that use `data-testid` for navigating the dom. + ## Dev Container / Codespaces A dev container is present in the repo which makes dev and testing of the UI Helper component much easier. @@ -83,6 +99,7 @@ npm start ``` Running the playwright tests after starting the Wizard web app + ```bash #Open a new terminal window cd helper @@ -90,3 +107,7 @@ cd helper #npx playwright install-deps chromium npx playwright test --browser chromium .playwrighttests/ --reporter list ``` + +## Issues + +Issues that are inactive are marked as stale and then closed pretty aggressively. We'll periodically look through the stale issues to see if any genuine issues have snuck their way through. diff --git a/README.md b/README.md index 233c2730..d8e898f1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Building Kubernetes clusters can be hard work! The AKS Bicep Accelerator focuses This project unifies guidance provided by the [AKS Secure Baseline](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/containers/aks/secure-baseline-aks), [Well Architected Framework](https://docs.microsoft.com/en-us/azure/architecture/framework/), [Cloud Adoption Framework](https://azure.microsoft.com/en-gb/cloud-adoption-framework/) and [Enterprise-Scale](https://github.com/Azure/Enterprise-Scale) by providing tangible artifacts to deploy Azure resources from CLI or CI/CD systems. -The AKS Bicep Accelerator is part of the official [Enterprise Scale for AKS](https://github.com/Azure/enterprise-scale-for-aks) architectural approach. To read more about this project and how the AKS Bicep Accelerator fits with Enterprise Scale and the AKS Secure Baseline, look [here](referencearchs.md). +The AKS Bicep Accelerator is part of the official [Enterprise Scale for AKS](https://github.com/Azure/enterprise-scale-for-aks) architectural approach. To read more about this project and how the AKS Bicep Accelerator fits with Enterprise Scale and the AKS Secure Baseline, look [here](referencearchs.md). ## The 3 Components @@ -14,7 +14,7 @@ This projects focus is split equally over 3 areas, configuration, modular templa ### Wizard experience -To help guide your AKS configuration, use the [Deployment Helper](https://azure.github.io/Aks-Construction/), which will provide a set of parameters and scripts to make deployment simple. It uses several preset configurations to guide configuration decisions. +To help guide your AKS configuration, use the [Deployment Helper](https://azure.github.io/Aks-Construction/), which will provide a set of parameters and scripts to make deployment simple. It uses several preset configurations to guide configuration decisions. The deployment helper provides links to the official Microsoft documentation to help provide additional context for each feature. [![preview screenshot of the helper wizard](helper_preview_es.png)](https://azure.github.io/Aks-Construction/) @@ -27,7 +27,7 @@ Releases are used to version the bicep code files, they can be leveraged directl ### DevOps - GitHub Actions -A number of [GitHub actions](https://github.com/Azure/Aks-Construction/tree/main/.github/workflows) are used in the repo that run on push/pr/schedules. These can be copied into your own repo and customised for your CI/CD pipeline. A robust deployment pipeline is essential when coordinating the deployment of multiple Azure services that work together, additionally there is configuration that cannot be set in the template and that needs to be automated (and tested) consistently. +A number of [GitHub actions](https://github.com/Azure/Aks-Construction/tree/main/.github/workflows) are used in the repo that run on push/pr/schedules. These can be copied into your own repo and customised for your CI/CD pipeline. A robust deployment pipeline is essential when coordinating the deployment of multiple Azure services that work together, additionally there is configuration that cannot be set in the template and that needs to be automated (and tested) consistently. ![preview screenshot of the helper wizard](docassets/ghactionworkflow.jpg) CI Name | Actions Workflow | Parameter file | CI Status | Notes @@ -44,7 +44,7 @@ For a more in depth look at the GitHub Actions used in this project, which steps If this is the first time you're using the project, follow these steps. -1. Use the [Deployment Helper](https://azure.github.io/Aks-Construction/) to guide your AKS configuration. +1. Use the [Deployment Helper](https://azure.github.io/Aks-Construction/) to guide your AKS configuration. 1. Run the commands in the *Provision Environment* tab to create your AKS Environment in your Azure subscription 1. Run the commands in the *Post Configuration* tab to complete your implementation 1. [Connect to your AKS Cluster](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough#connect-to-the-cluster), and deploy your applications as you see fit. @@ -53,7 +53,7 @@ If this is the first time you're using the project, follow these steps. If you're looking to use this project as part of your deployments, follow these steps. -1. Use the [Deployment Helper](https://azure.github.io/Aks-Construction/) to guide your AKS configuration. +1. Use the [Deployment Helper](https://azure.github.io/Aks-Construction/) to guide your AKS configuration. 1. Capture the parameters on the *Template Parameters File* tab to a file - this is your configuration 1. Check the *Post Configuration* tab for any commands and save them to a file 1. Grab the [latest release](https://github.com/Azure/Aks-Construction/releases) of the bicep code @@ -64,7 +64,8 @@ If you're looking to use this project as part of your deployments, follow these ## Project Principals The guiding principal we have with this project is to focus on the the *downstream use* of the project (see [releases](https://github.com/Azure/Aks-Construction/releases)). As such, these are our specific practices. -1. Deploy all components through a single, modular, itempotent bicep template Converge on a single bicep template, which can easily be consumed as a module + +1. Deploy all components through a single, modular, idempotent bicep template Converge on a single bicep template, which can easily be consumed as a module 2. Provide best-practice defaults, then use parameters for different environment deployments 3. Minimise "manual" steps for ease of automation 4. Maintain quality through validation & CI/CD pipelines that also serve as working samples/docs @@ -88,8 +89,8 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio ## Trademarks -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft -trademarks or logos is subject to and must follow +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/SECURITY.md b/SECURITY.md index f7b89984..1f9d3d44 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ Instead, please report them to the Microsoft Security Response Center (MSRC) at If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). -You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: diff --git a/cspell.json b/cspell.json index e5e078a1..109e03eb 100644 --- a/cspell.json +++ b/cspell.json @@ -1,61 +1,107 @@ { "version": "0.2", + "language": "en-GB", + "files": [ + "**/*.{ts,js,md}" + ], "ignorePaths": [], "dictionaryDefinitions": [], "dictionaries": [], "words": [ "aadgroupids", + "accv", + "ACRNAME", "adminprincipleid", "AGIC", + "aksvnetparams", + "allok", "apiips", "apisecurity", "appgw", "APPINSIGHTS", + "APPNAME", "autoscale", "Autoscaler", + "azurecr", "azurepolicy", "Burstable", "BYOVNET", "carvals", + "Cleanup", "CNCF", + "Codespaces", + "Consolas", "csisecret", + "csisecrets", + "currenttab", + "currt", + "currv", "Daemonset", "demoapp", + "deploycmd", "dnszones", "entscale", "ersub", "ESLZ", "fluentui", + "ghpages", "gitops", "Grafana", "ilbsub", + "Inconsolata", + "Jumpboxes", + "keyvault", + "khowling", + "kube", "kubectl", "kubeletidentity", "kubenet", "Kubernetes", + "KVNAME", "letsencrypt", "linksrc", + "managedclusters", + "messg", + "Microservices", "monospace", "mounttime", + "msrc", + "netpolicy", "noopener", "noreferrer", "notopmargin", "omsagent", "pagename", + "pivotkey", + "playwrighttests", "podid", "podscale", "portalnav", "pwsh", "routable", + "serviceparams", + "snuck", "stackstyle", + "systempool", + "templating", "testid", + "urlname", + "valres", + "vaultname", "VNET", "vnetonly", "vnetprivateend", "Wautoscale", "Wsku" ], - "ignoreWords": [], + "ignoreWords": [ + "sconfiguration", + "bulets" + ], + "ignoreRegExpList": [ + "\\((.*)\\)", //Markdown links + "```[a-z]*\n[\\s\\S]*?\n```" //Markdown code blocks. h/t https://coderwall.com/p/r6b4xg/regex-to-match-github-s-markdown-code-blocks, + ], "import": [], "enableFiletypes": [ "!css", diff --git a/helper/.playwrighttests/helper-test-default-networkpolicy.spec.js b/helper/.playwrighttests/helper-test-default-networkpolicy.spec.js new file mode 100644 index 00000000..c0e1786b --- /dev/null +++ b/helper/.playwrighttests/helper-test-default-networkpolicy.spec.js @@ -0,0 +1,13 @@ +const { test, expect } = require('@playwright/test'); + +test('networkpolicy-test-defaul-is-azure', async ({ page }) => { + + await page.goto('http://localhost:3000/Aks-Construction'); + + // Click the 3rd Tab in the portal Navigation Pivot (addons) + await page.click('[data-testid="portalnav-Pivot"] > button:nth-child(3)'); + + // Expect azure network policy to be checked! + expect (await page.isChecked('[data-testid="addons-netpolicy-azure"]')).toBeTruthy() + +}); diff --git a/helper/src/components/addonsTab.js b/helper/src/components/addonsTab.js index 05a57fdb..094b16a7 100644 --- a/helper/src/components/addonsTab.js +++ b/helper/src/components/addonsTab.js @@ -84,9 +84,9 @@ export default function ({ tabValues, updateFn, invalidArray }) { styles={{ root: { marginLeft: '50px' } }} selectedKey={addons.networkPolicy} options={[ - { key: 'none', text: 'No restrictions, all PODs can access each other' }, - { key: 'calico', text: 'Use Network Policy addon with Calico to implemented intra-cluster traffic restrictions (driven from "NetworkPolicy" objects)' }, - { key: 'azure', text: 'Use Network Policy addon with Azure provider to implemented intra-cluster traffic restrictions (driven from "NetworkPolicy" objects)' } + { "data-testid":'addons-netpolicy-none', key: 'none', text: 'No restrictions, all PODs can access each other' }, + { "data-testid":'addons-netpolicy-calico', key: 'calico', text: 'Use Network Policy addon with Calico to implemented intra-cluster traffic restrictions (driven from "NetworkPolicy" objects)' }, + { "data-testid":'addons-netpolicy-azure', key: 'azure', text: 'Use Network Policy addon with Azure provider to implemented intra-cluster traffic restrictions (driven from "NetworkPolicy" objects)' } ]} onChange={(ev, { key }) => updateFn("networkPolicy", key)} diff --git a/helper/src/components/appsTab.js b/helper/src/components/appsTab.js index 026844c8..7a015d56 100644 --- a/helper/src/components/appsTab.js +++ b/helper/src/components/appsTab.js @@ -49,13 +49,13 @@ ${cluster.apisecurity === "private" ? `"` : ``} - Please ensure you deployment meets all the requirements below to run the demo app successfully + Please ensure your deployment meets all the requirements below in order to run the demo app successfully - + @@ -68,7 +68,7 @@ ${cluster.apisecurity === "private" ? `"` : ``}
TBC
 - + @@ -77,7 +77,7 @@ ${cluster.apisecurity === "private" ? `"` : ``}
TBC
 - + diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js index dd5965f4..7f86ce07 100644 --- a/helper/src/components/deployTab.js +++ b/helper/src/components/deployTab.js @@ -13,7 +13,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(net.vnetAddressPrefix !== defaults.net.vnetAddressPrefix && { vnetAddressPrefix: net.vnetAddressPrefix }), ...(net.vnetAksSubnetAddressPrefix !== defaults.net.vnetAksSubnetAddressPrefix && { vnetAksSubnetAddressPrefix: net.vnetAksSubnetAddressPrefix }) } - const sericeparams = { + const serviceparams = { ...(net.serviceCidr !== defaults.net.serviceCidr && { serviceCidr: net.serviceCidr }), ...(net.dnsServiceIP !== defaults.net.dnsServiceIP && { dnsServiceIP: net.dnsServiceIP }) } @@ -26,8 +26,8 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(cluster.vmSize !== "default" && { agentVMSize: cluster.vmSize }), ...(cluster.autoscale && { agentCountMax: cluster.maxCount }), ...(cluster.osDiskType === "Managed" && { osDiskType: cluster.osDiskType, ...(cluster.osDiskSizeGB > 0 && { osDiskSizeGB: cluster.osDiskSizeGB }) }), - ...(net.vnet_opt === "custom" && { custom_vnet: true, ...sericeparams, ...aksvnetparams }), - ...(net.vnet_opt === "byo" && { byoAKSSubnetId: net.byoAKSSubnetId, ...sericeparams }), + ...(net.vnet_opt === "custom" && { custom_vnet: true, ...serviceparams, ...aksvnetparams }), + ...(net.vnet_opt === "byo" && { byoAKSSubnetId: net.byoAKSSubnetId, ...serviceparams }), ...(net.vnet_opt === "byo" && addons.ingress === 'appgw' && { byoAGWSubnetId: net.byoAGWSubnetId }), ...(cluster.enable_aad && { enable_aad: true, ...(cluster.enableAzureRBAC === false && cluster.aad_tenant_id && { aad_tenant_id: cluster.aad_tenant_id }) }), ...(cluster.enable_aad && cluster.enableAzureRBAC && { enableAzureRBAC: true, ...(deploy.clusterAdminRole && { adminprincipleid: "$(az ad signed-in-user show --query objectId --out tsv)" }) }), @@ -96,8 +96,8 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, `az deployment group create -g ${deploy.rg} ${process.env.REACT_APP_AZ_TEMPLATE_ARG} --parameters` + params2CLI(finalParams) const param_file = JSON.stringify(params2file(finalParams), null, 2).replaceAll('\\\\\\', '').replaceAll('\\\\\\', '') - const promethous_namespace = 'monitoring' - const promethous_helm_release_name = 'monitoring' + const prometheus_namespace = 'monitoring' + const prometheus_helm_release_name = 'monitoring' const nginx_namespace = 'ingress-basic' const nginx_helm_release_name = 'nginx-ingress' @@ -131,8 +131,8 @@ az aks get-credentials -g ${deploy.rg} -n ${aks} ` : "" (addons.monitor === 'oss' ? `\n\n# Install kube-prometheus-stack helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update -kubectl create namespace ${promethous_namespace} -helm install ${promethous_helm_release_name} prometheus-community/kube-prometheus-stack --namespace ${promethous_namespace}` : '') + +kubectl create namespace ${prometheus_namespace} +helm install ${prometheus_helm_release_name} prometheus-community/kube-prometheus-stack --namespace ${prometheus_namespace}` : '') + // Nginx Ingress Controller (addons.ingress === 'nginx' ? `\n\n# Create a namespace for your ingress resources kubectl create namespace ${nginx_namespace} @@ -150,8 +150,8 @@ helm install ${nginx_helm_release_name} ingress-nginx/ingress-nginx \\ (addons.monitor === 'oss' ? ` --set controller.metrics.enabled=true \\ --set controller.metrics.serviceMonitor.enabled=true \\ - --set controller.metrics.serviceMonitor.namespace=${promethous_namespace} \\ - --set controller.metrics.serviceMonitor.additionalLabels.release=${promethous_helm_release_name} \\ + --set controller.metrics.serviceMonitor.namespace=${prometheus_namespace} \\ + --set controller.metrics.serviceMonitor.additionalLabels.release=${prometheus_helm_release_name} \\ ` : '') + ` --namespace ${nginx_namespace}` : '') + (addons.ingress === 'contour' ? `\n\n# Install Contour Ingress Controller @@ -175,7 +175,7 @@ kubectl create secret generic azure-config-file --from-file=azure.json=/dev/stdi } EOF ${cluster.apisecurity === "private" ? `"` : ``} -# external-dns manifest (for clusters with RBAC) replacing {{domain-filter}} and {{proviers}} values +# external-dns manifest (for clusters with RBAC) replacing {{domain-filter}} and {{providers}} values curl https://raw.githubusercontent.com/Azure/Aks-Construction/main/helper/config/external-dns.yml | sed -e "s|{{domain-filter}}|${addons.dnsZoneId.split('/')[8]}|g" -e "s|{{provider}}|${addons.dnsZoneId.split('/')[7] === 'privateDnsZones' ? 'azure-private-dns' : 'azure'}|g" >/tmp/aks-ext-dns.yml ${cluster.apisecurity === "private" ? `az aks command invoke -g ${deploy.rg} -n ${aks} --command "kubectl apply -f ./aks-ext-dns.yml" --file /tmp/aks-ext-dns.yml` diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index aac8b9f1..cdfc40fd 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -253,7 +253,7 @@ function CustomVNET({ net, addons, updateFn }) { - updateFn("acrAgentPoolSubnetAddressPrefix", val)} value={net.vnetprivateend && addons.registry !== "none" && addons.acrPrivatePool ? net.acrAgentPoolSubnetAddressPrefix : "No Agen Pool requested"} /> + updateFn("acrAgentPoolSubnetAddressPrefix", val)} value={net.vnetprivateend && addons.registry !== "none" && addons.acrPrivatePool ? net.acrAgentPoolSubnetAddressPrefix : "No Agent Pool requested"} /> diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index 1be218ec..f6377842 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -93,12 +93,12 @@ export default function PortalNav({ config }) { const [tabValues, setTabValues] = useState(() => { const clusterName = `az-k8s-${(Math.floor(Math.random() * 900000) + 100000).toString(36)}` - // Apply selected presets to tab vlues + // Apply selected presets to tab values const tabApplySections = Object.keys(selected.values).reduce((acc,curr) => updateTabValues (acc, sections, curr, selected.values[curr]) , defaults) - // Apply dynamic presets to tab vlues + // Apply dynamic presets to tab values const dynamicApplySections = { ...tabApplySections, deploy: { @@ -108,7 +108,7 @@ export default function PortalNav({ config }) { ...(process.env.REACT_APP_K8S_VERSION && { kubernetesVersion: process.env.REACT_APP_K8S_VERSION }) } } - // Apply url params to tab vlues + // Apply url params to tab values const urlApplySections = Object.keys(dynamicApplySections).reduce((acct, currt) => { return { ...acct, @@ -154,25 +154,25 @@ export default function PortalNav({ config }) { function updateSelected(sectionKey, cardKey) { - setUrlParams((currrentUrlParams) => { + setUrlParams((currentUrlParams) => { if (selected.entScale !== entScale) { console.log (`User changed entScale switch, and selected a new card, need to unselect old cards`) defaultOps.forEach(element => { - currrentUrlParams.delete(element.key) + currentUrlParams.delete(element.key) }) entScaleOps.forEach(element => { - currrentUrlParams.delete(element.key) + currentUrlParams.delete(element.key) }) } if (entScale) { - currrentUrlParams.set('entScale', 1) + currentUrlParams.set('entScale', 1) } else { - currrentUrlParams.delete('entScale') + currentUrlParams.delete('entScale') } - currrentUrlParams.set(sectionKey,cardKey) - return currrentUrlParams + currentUrlParams.set(sectionKey,cardKey) + return currentUrlParams }) diff --git a/helper/src/config.json b/helper/src/config.json index 6c63a866..3cdb2358 100644 --- a/helper/src/config.json +++ b/helper/src/config.json @@ -4,7 +4,7 @@ "cluster": "Cluster Details", "addons": "Addon Details", "net": "Networking Details", - "app": "Sameple Apps" + "app": "Sample Apps" }, "defaults": { "deploy": { @@ -305,7 +305,7 @@ "apisecurity": "whitelist" }, "addons": { - "networkPolicy": "calico", + "networkPolicy": "azure", "registry": [ { "page": "addons", @@ -401,7 +401,7 @@ "apisecurity": "private" }, "addons": { - "networkPolicy": "calico", + "networkPolicy": "azure", "registry": [ { "page": "addons", @@ -581,7 +581,7 @@ "upgradeChannel": "none" }, "addons": { - "networkPolicy": "calico", + "networkPolicy": "azure", "registry": "Premium", "azurepolicy": "audit", "ingress": "appgw", @@ -662,7 +662,7 @@ "upgradeChannel": "none" }, "addons": { - "networkPolicy": "calico", + "networkPolicy": "azure", "registry": "Premium", "azurepolicy": "audit", "ingress": "appgw",