PR review update

2022-06-29 21:12:00 -07:00 · 2022-06-29 21:12:00 -07:00 · aba3b41c8c
--- a/deploy/README.md
+++ b/deploy/README.md
@ -427,7 +427,7 @@ To run the pipeline, open the Synapse Studio for the Synapse workspace that you
 | Prefix | This is the Storage container name created in [Running the pipeline section](#running-the-pipeline) that hosts the Raw data|
 | StorageAccountName | Name of the Storage Account in <environmentCode>-data-rg resource group that hosts the Raw data |
 | AOI | Area of Interest over which the AI Model is run |
-| AksManagementRestApiURL | AKS Management Rest API Endpoint URL where Azure Synapse makes request calls to send kubectl commands to |
+| AksManagementRestApiURL | AKS Management Rest API Endpoint URL where Azure Synapse makes request calls to send kubectl commands to. Refer to [doc](https://docs.microsoft.com/en-us/rest/api/aks/managed-clusters/run-command). |
 | PersistentVolumeClaim | Persistent Volume Claim Name used for the AI-Model execution Kubernetes pod. This is preconfigured during setup and configuration, and can be found from Azure portal (provisioned AKS-> 'Storage'-> 'Persistent volume claims'). |

 - Once the parameters are entered, click ok to submit and kick off the pipeline.
--- a/deploy/configure.sh
+++ b/deploy/configure.sh
@ -17,7 +17,7 @@ SYNAPSE_WORKSPACE_RG=${6:-${SYNAPSE_WORKSPACE_RG:-"${ENV_CODE}-pipeline-rg"}}
 SYNAPSE_WORKSPACE=${7:-${SYNAPSE_WORKSPACE:-"${ENV_CODE}-pipeline-syn-ws"}}
 SYNAPSE_POOL=${8:-${SYNAPSE_POOL}}
 SYNAPSE_STORAGE_ACCOUNT=${9:-${SYNAPSE_STORAGE_ACCOUNT}}
-
+DATA_RESOURCE_GROUP=${10:-${DATA_RESOURCE_GROUP:-"${ENV_CODE}-data-rg"}}

 if [[ "$AI_MODEL_INFRA_TYPE" != "batch-account" ]] && [[ "$AI_MODEL_INFRA_TYPE" != "aks" ]]; then
  echo "Invalid value for AI_MODEL_INFRA_TYPE! Supported values are 'aks' and 'batch-account'."
@ -76,92 +76,52 @@ if [[ "$AI_MODEL_INFRA_TYPE" == "batch-account" ]]; then
    fi
 elif [[ "$AI_MODEL_INFRA_TYPE" == "aks" ]]; then
    echo "Selected AI model processing infra-type: AKS!!!"
-    DATA_RESOURCE_GROUP="${ENV_CODE}-data-rg"
    AKS_NAMESPACE=vision
    PV_SUFFIX=fileshare
    PV_NAME="${ENV_CODE}-${AKS_NAMESPACE}-${PV_SUFFIX}"

    RAW_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw'].name" -o tsv -g $DATA_RESOURCE_GROUP)
    RAW_STORAGE_KEY=$(az storage account keys list --resource-group $DATA_RESOURCE_GROUP --account-name $RAW_STORAGE_ACCT --query "[0].value" -o tsv)
-    VISION_FILE_SHARE_NAME=$(az storage share list --account-key $RAW_STORAGE_KEY --account-name $RAW_STORAGE_ACCT --query "[].name" -o tsv)
+    FILE_SHARE_NAME=$(az storage share list --account-key $RAW_STORAGE_KEY --account-name $RAW_STORAGE_ACCT --query "[].name" -o tsv)

    AKS_CLUSTER_NAME=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].name" -otsv)
+    counter=0
    while [[ ${AKS_CLUSTER_NAME} == '' ]];
    do
+        if [[ $counter -gt 4 ]]; then
+            echo "Failed to get AKS cluster name"
+            break
+        fi
        sleep 60
        AKS_CLUSTER_NAME=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].name" -otsv)
+        counter=$((counter+1))
    done
    # force to provision aks-command namespace
    az aks command invoke -g ${ENV_CODE}-orc-rg -n ${AKS_CLUSTER_NAME} -c "kubectl get ns"
    az aks get-credentials --resource-group ${ENV_CODE}-orc-rg --name ${AKS_CLUSTER_NAME} --context ${AKS_CLUSTER_NAME} --overwrite-existing
    kubectl config set-context ${AKS_CLUSTER_NAME}

-    cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: $AKS_NAMESPACE
-EOF
-
-    cat <<EOF | kubectl -n $AKS_NAMESPACE apply -f -
-apiVersion: v1
-kind: Secret
-metadata:
-  name: azure-secret
-  namespace: $AKS_NAMESPACE
-type: Opaque
-stringData:
-  azurestorageaccountname: ${RAW_STORAGE_ACCT}
-  azurestorageaccountkey: ${RAW_STORAGE_KEY}
-EOF
-
-    cat <<EOF | kubectl -n $AKS_NAMESPACE apply -f -
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: ${PV_NAME}
-spec:
-  capacity:
-    storage: 5Gi
-  accessModes:
-    - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
-  storageClassName: azurefile-csi
-  csi:
-    driver: file.csi.azure.com
-    readOnly: false
-    volumeHandle: ${PV_NAME}
-    volumeAttributes:
-      resourceGroup: ${DATA_RESOURCE_GROUP}
-      shareName: ${VISION_FILE_SHARE_NAME}
-    nodeStageSecretRef:
-      name: azure-secret
-      namespace: ${AKS_NAMESPACE}
-  mountOptions:
-    - dir_mode=0777
-    - file_mode=0777
-    - uid=0
-    - gid=0
-    - mfsymlinks
-    - cache=strict
-    - nosharesock
-    - nobrl
-EOF
-
-    cat << EOF | kubectl -n $AKS_NAMESPACE apply -f -
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: ${PV_NAME}
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: azurefile-csi
-  volumeName: ${PV_NAME}
-  resources:
-    requests:
-      storage: 5Gi
-EOF
+    # create vison namespace
+    AKS_NAMESPACE=$AKS_NAMESPACE \
+      envsubst < "$PRJ_ROOT/deploy/kube_yaml/namespace.yaml" | kubectl apply -f -
+    
+    # create storage secret
+    AKS_NAMESPACE=$AKS_NAMESPACE \
+    RAW_STORAGE_ACCT=$RAW_STORAGE_ACCT \
+    RAW_STORAGE_KEY=$RAW_STORAGE_KEY \
+      envsubst < "$PRJ_ROOT/deploy/kube_yaml/storage_secret.yaml" | kubectl -n $AKS_NAMESPACE apply -f -
+    
+    # create pv
+    AKS_NAMESPACE=$AKS_NAMESPACE \
+    PV_NAME=$PV_NAME \
+    DATA_RESOURCE_GROUP=$DATA_RESOURCE_GROUP \
+    FILE_SHARE_NAME=$FILE_SHARE_NAME \
+      envsubst < "$PRJ_ROOT/deploy/kube_yaml/pv.yaml" | kubectl -n $AKS_NAMESPACE apply -f -
+    
+    # create pvc
+    AKS_NAMESPACE=$AKS_NAMESPACE \
+    PV_NAME=$PV_NAME \
+      envsubst < "$PRJ_ROOT/deploy/kube_yaml/pvc.yaml" | kubectl -n $AKS_NAMESPACE apply -f -
 fi # end of "$AI_MODEL_INFRA_TYPE" == "aks"

 echo "configuration completed!"
--- a/deploy/infra/groups/orchestration.bicep
+++ b/deploy/infra/groups/orchestration.bicep
@ -339,6 +339,7 @@ module attachACRtoAKS '../modules/aks-attach-acr.bicep' = if(deployAksCluster) {
  }
  dependsOn: [
    acr
+    aksCluster
  ]
 }

@ -360,7 +361,6 @@ module aksInvokerRoleDef '../modules/custom.roledef.bicep' = if(deployAksCluster
 module aksCustomRoleAssignment '../modules/aks-invoker-role-assignment.bicep' = if(deployAksCluster) {
  name: 'custom-role-assignment-for-${aksClusterNameVar}'
  params: {
-    principalId: synapseMIPrincipalId
    aksClusterName: aksClusterNameVar
    customRoleDefId: deployAksCluster?aksInvokerRoleDef.outputs.Id :''
  }
@ -377,27 +377,21 @@ module functionAppStorageAccount '../modules/storage.bicep' = if(deployAksCluste
    location: location
    storeType: 'fapp-storage'
  }
-  dependsOn: [
-    aksCluster
-  ]
 }

 module functionAppHostPlan '../modules/asp.bicep' = if(deployAksCluster) {
  name: '${namingPrefix}-asp'
  params: {
-    aspName: '${namingPrefix}-asp'
-    aspKind: 'linux'
-    aspReserved: true
-    mewCount: 1
+    name: '${namingPrefix}-asp'
+    kind: 'linux'
+    reserved: true
+    maximumElasticWorkerCount: 1
    skuTier: 'Dynamic'
    skuSize: 'Y1'
    skuName: 'Y1'
    location: location
    environmentName: environmentTag
  }
-  dependsOn: [
-    aksCluster
-  ]
 }

 module functionApp '../modules/functionapp.bicep' = if(deployAksCluster) {
@ -417,38 +411,30 @@ module functionApp '../modules/functionapp.bicep' = if(deployAksCluster) {
      linuxFxVersion: 'Python|3.9'     
    }
  }
-  dependsOn: [
-    aksCluster
-  ]
 }

 module base64EncodedZipContentFunction '../modules/function.bicep' = if(deployAksCluster) {
  name: '${namingPrefix}-base64fapp'
  params: {
-    functionAppName: functionAppNameVar
-    functionName: 'base64EncodedZipContent'
-    functionFiles : {
+    appName: functionAppNameVar
+    name: 'base64EncodedZipContent'
+    files : {
      '__init__.py': loadTextContent('gen_base64_encoded_content.py')
    }
-    functionLanguage: 'python'
+    language: 'python'
  }
  dependsOn:[
    functionApp
  ]
 }

-module base64EncodedZipContentFunctionKey '../modules/akv.secrets.bicep' = if(deployAksCluster) {
+module base64EncodedZipContentFunctionKey '../modules/function.key.to.keyvault.bicep' = if(deployAksCluster) {
  name: '${namingPrefix}-base64fapp-fkey'
-  scope: resourceGroup(pipelineResourceGroupName)
  params: {
    environmentName: environmentTag
-    keyVaultName: '${environmentCode}-pipeline-kv'
-    secretName: 'GenBase64EncondingFunctionKey' 
-    secretValue: deployAksCluster?base64EncodedZipContentFunction.outputs.functionkey:''
+    keyVaultName: pipelineLinkedSvcKeyVaultName
+    keyVaultResourceGroup: pipelineResourceGroupName
+    functionSecretName: 'GenBase64EncondingFunctionKey'
+    functionId: deployAksCluster?base64EncodedZipContentFunction.outputs.id:''
  }
 }
-
-
-
-
-
--- a/deploy/infra/modules/aks-attach-acr.bicep
+++ b/deploy/infra/modules/aks-attach-acr.bicep
@ -4,7 +4,7 @@
 param kubeletIdentityId string
 param acrName string
 param roleAssignmentId string = guid(kubeletIdentityId, kubeletIdentityId, acrName)
-
+//this RolId maps to AcrPull role
 var acrPullRoleId = '/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d'

 resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = {
@ -19,4 +19,3 @@ resource acrPullRole 'Microsoft.Authorization/roleAssignments@2020-10-01-preview
    roleDefinitionId: acrPullRoleId
  }
 }
-
--- a/deploy/infra/modules/aks-cluster.bicep
+++ b/deploy/infra/modules/aks-cluster.bicep
@ -49,8 +49,6 @@ resource aks 'Microsoft.ContainerService/managedClusters@2022-01-01' = {
  }
 }

-
 output Id string = aks.id
 output principalId string = aks.identity.principalId
 output kubeletIdentityId string = aks.properties.identityProfile.kubeletidentity.objectId
-
--- a/deploy/infra/modules/aks-invoker-role-assignment.bicep
+++ b/deploy/infra/modules/aks-invoker-role-assignment.bicep
@ -1,10 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.

-param principalId string
 param aksClusterName string
 param customRoleDefId string
-param roleAssignmentId string = guid(principalId, aksClusterName, customRoleDefId)
+param roleAssignmentId string = guid(aksClusterName, customRoleDefId)

 resource aks 'Microsoft.ContainerService/managedClusters@2021-10-01' existing = {
  name: aksClusterName
@ -14,12 +13,9 @@ resource aksInvokerRoleAssignment 'Microsoft.Authorization/roleAssignments@2020-
  name: roleAssignmentId
  scope: aks
  properties: {
-    principalId: principalId
+    principalId: aks.identity.principalId
    roleDefinitionId: customRoleDefId
  }
 }

 output Id string = aksInvokerRoleAssignment.id
-
-
-
--- a/deploy/infra/modules/asp.bicep
+++ b/deploy/infra/modules/asp.bicep
@ -1,7 +1,7 @@
-param aspName string
-param aspKind string
-param aspReserved bool
-param mewCount int
+param name string
+param kind string
+param reserved bool
+param maximumElasticWorkerCount int
 param skuTier string
 param skuSize string
 param skuName string
@ -9,12 +9,12 @@ param location string = resourceGroup().location
 param environmentName string

 resource hostingPlan 'Microsoft.Web/serverfarms@2021-01-15' = {
-  name: aspName
+  name: name
  location: location 
-  kind: aspKind
+  kind: kind
  properties: {
-    maximumElasticWorkerCount: mewCount
-    reserved: aspReserved 
+    maximumElasticWorkerCount: maximumElasticWorkerCount
+    reserved: reserved 
  }
  sku: {
    name: skuName 
@ -26,6 +26,4 @@ resource hostingPlan 'Microsoft.Web/serverfarms@2021-01-15' = {
  }
 }

-output id string = hostingPlan.id
-
- 
+output id string = hostingPlan.id 
--- a/deploy/infra/modules/function.bicep
+++ b/deploy/infra/modules/function.bicep
@ -1,11 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.

-param functionAppName string
-param functionName string
-param functionFiles object = {}
-param functionLanguage string
-param functionInputBinding object = {
+param appName string
+param name string
+param files object = {}
+param language string
+param inputBinding object = {
  name: 'req'
  type: 'httpTrigger'
  direction: 'in'
@ -21,20 +21,19 @@ param functionOutputBinding object = {
 }

 resource function 'Microsoft.Web/sites/functions@2021-03-01' = {
-  name: '${functionAppName}/${functionName}'
+  name: '${appName}/${name}'
  properties: {
    config: {
      disabled: false
      bindings: [
-        functionInputBinding
+        inputBinding
        functionOutputBinding
      ]
    }
-    files: functionFiles
-    language: functionLanguage
+    files: files
+    language: language
  }
 }

 output id string = function.id
-#disable-next-line outputs-should-not-contain-secrets // we do not output to terminal. calling bicep will take this & store in keyvault
-output functionkey string = listKeys(function.id, '2021-03-01').default
+output fullName string = function.name
--- a/deploy/infra/modules/function.key.to.keyvault.bicep
+++ b/deploy/infra/modules/function.key.to.keyvault.bicep
@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param environmentName string
+param functionId string
+param keyVaultName string
+param functionSecretName string
+param keyVaultResourceGroup string
+param utcValue string = utcNow()
+
+module functionKeySecret './akv.secrets.bicep' = {
+  name: 'function-key-${utcValue}'
+  scope: resourceGroup(keyVaultResourceGroup)
+  params: {
+    environmentName: environmentName
+    keyVaultName: keyVaultName
+    secretName: functionSecretName
+    secretValue: listKeys(functionId, '2021-03-01').default
+  }
+}
+
--- a/deploy/kube_yaml/namespace.yaml
+++ b/deploy/kube_yaml/namespace.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: $AKS_NAMESPACE
--- a/deploy/kube_yaml/pv.yaml
+++ b/deploy/kube_yaml/pv.yaml
@ -0,0 +1,30 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ${PV_NAME}
+spec:
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: azurefile-csi
+  csi:
+    driver: file.csi.azure.com
+    readOnly: false
+    volumeHandle: ${PV_NAME}
+    volumeAttributes:
+      resourceGroup: ${DATA_RESOURCE_GROUP}
+      shareName: ${FILE_SHARE_NAME}
+    nodeStageSecretRef:
+      name: azure-secret
+      namespace: ${AKS_NAMESPACE}
+  mountOptions:
+    - dir_mode=0777
+    - file_mode=0777
+    - uid=0
+    - gid=0
+    - mfsymlinks
+    - cache=strict
+    - nosharesock
+    - nobrl
--- a/deploy/kube_yaml/pvc.yaml
+++ b/deploy/kube_yaml/pvc.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${PV_NAME}
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: azurefile-csi
+  volumeName: ${PV_NAME}
+  resources:
+    requests:
+      storage: 5Gi
--- a/deploy/kube_yaml/storage_secret.yaml
+++ b/deploy/kube_yaml/storage_secret.yaml
@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-secret
+  namespace: $AKS_NAMESPACE
+type: Opaque
+stringData:
+  azurestorageaccountname: ${RAW_STORAGE_ACCT}
+  azurestorageaccountkey: ${RAW_STORAGE_KEY}
--- a/deploy/package.py
+++ b/deploy/package.py
@ -18,9 +18,9 @@ parser.add_argument('--synapse_storage_account_name', type=str, required=True, h
 parser.add_argument('--synapse_pool_name', type=str, required=True, help='Name of the Synapse pool in the Synapse workspace to use as default')
 parser.add_argument('--synapse_workspace_id', type=str, required=True, help='Id for the Synapse workspace')
 parser.add_argument('--synapse_workspace', type=str, required=True, help='Synapse pool name')
-parser.add_argument('--batch_storage_account_name', type=str, required=True, help='Name of the Batch Storage Account')
-parser.add_argument('--batch_account', type=str, required=True, help="Batch Account name")
-parser.add_argument('--batch_pool_name', type=str, required=True, help="Batch Pool name")
+parser.add_argument('--batch_storage_account_name', type=str, required=False, help='Name of the Batch Storage Account')
+parser.add_argument('--batch_account', type=str, required=False, help="Batch Account name")
+parser.add_argument('--batch_pool_name', type=str, required=False, help="Batch Pool name")
 parser.add_argument('--linked_key_vault', type=str, required=True, help="Key Vault to be added as Linked Service")
 parser.add_argument('--location', type=str, required=False, help="Batch Account Location")
 parser.add_argument('--pipeline_name', type=str, required=True, help="Name of the pipeline to package")
--- a/deploy/package.sh
+++ b/deploy/package.sh
@ -103,22 +103,33 @@ if [[ "$AI_MODEL_INFRA_TYPE" == "batch-account" ]]; then
 elif [[ "$AI_MODEL_INFRA_TYPE" == "aks" ]]; then
    echo "Selected AI model processing infra-type: AKS!!!"
    AKS_ID=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].id" -otsv)
+    counter=0
    while [[ ${AKS_ID} == '' ]];
    do
+        if [[ $counter -gt 4 ]]; then
+            echo "Failed to get AKS ID"
+            break
+        fi
        sleep 60
        AKS_ID=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].id" -otsv)
+        counter=$((counter+1))
    done

    PERSISTENT_VOLUME_CLAIM="${ENV_CODE}-vision-fileshare"
    AKS_MANAGEMENT_REST_URL="https://management.azure.com${AKS_ID}/runCommand?api-version=2022-02-01"
    BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST=$(az functionapp list -g ${ENV_CODE}-orc-rg \
        --query "[?tags.type && tags.type == 'functionapp'].hostNames[0]" | jq -r '.[0]')
-
+    counter=0
    while [[ ${BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST} == '' ]];
    do
+        if [[ $counter -gt 4 ]]; then
+            echo "Failed to get functionapp host"
+            break
+        fi
        sleep 60
        BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST=$(az functionapp list -g ${ENV_CODE}-orc-rg \
        --query "[?tags.type && tags.type == 'functionapp'].hostNames[0]" | jq -r '.[0]')
+        counter=$((counter+1))
    done
    BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_URL="https://${BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST}"
    PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py \