aks-based data-pipeline support

2022-06-29 11:34:06 -07:00 · 2022-06-29 11:34:06 -07:00 · 9a84ad5763
--- a/deploy/README.md
+++ b/deploy/README.md
@ -409,7 +409,7 @@ To run the pipeline, open the Synapse Studio for the Synapse workspace that you

 - Open the `E2E Custom Vision Model Flow` and click on debug button

- When presented with the parameters, fill out the values. Below table provide the details on that each parameter represents.
+- When presented with the parameters, fill out the values. Below table provide the details on that each parameter represents if batch-account (by default) is provisioned in infrastructure setup.

 | parameter | description |
 |--|--|
@ -420,6 +420,16 @@ To run the pipeline, open the Synapse Studio for the Synapse workspace that you
 | BatchJobName | Job name within the Batch Account in <environmentCode>-orc-rg resource group that runs the AI Model |
 | BatchLocation | Location of the Batch Account in <environmentCode>-orc-rg resource group that runs the AI Model |

+- In case AKS (Azure Kubernetes Service) is provisioned, parameters and their descriptions are:
+
+| parameter | description |
+|--|--|
+| Prefix | This is the Storage container name created in [Running the pipeline section](#running-the-pipeline) that hosts the Raw data|
+| StorageAccountName | Name of the Storage Account in <environmentCode>-data-rg resource group that hosts the Raw data |
+| AOI | Area of Interest over which the AI Model is run |
+| AksManagementRestApiURL | AKS Management Rest API Endpoint URL where Azure Synapse makes request calls to send kubectl commands to |
+| PersistentVolumeClaim | Persistent Volume Claim Name used for the AI-Model execution Kubernetes pod. This is preconfigured during setup and configuration, and can be found from Azure portal (provisioned AKS-> 'Storage'-> 'Persistent volume claims'). |
+
 - Once the parameters are entered, click ok to submit and kick off the pipeline.

 - Wait for the pipeline to complete.
--- a/deploy/configure.sh
+++ b/deploy/configure.sh
@ -47,7 +47,7 @@ if [[ -n $SYNAPSE_STORAGE_ACCOUNT ]]; then
    # create a container to upload the spark job python files
    az storage container create --name "spark-jobs" --account-name ${SYNAPSE_STORAGE_ACCOUNT}
    # uploads the spark job python files
-    az storage blob upload-batch --destination "spark-jobs" --account-name ${SYNAPSE_STORAGE_ACCOUNT} --source "${PRJ_ROOT}/src/transforms/spark-jobs"
+    az storage blob upload-batch --destination "spark-jobs" --account-name ${SYNAPSE_STORAGE_ACCOUNT} --source "${PRJ_ROOT}/src/transforms/spark-jobs" --overwrite
 fi

 if [[ "$AI_MODEL_INFRA_TYPE" == "batch-account" ]]; then
@ -72,10 +72,96 @@ if [[ "$AI_MODEL_INFRA_TYPE" == "batch-account" ]]; then
    then
        az batch account login --name ${BATCH_ACCOUNT_NAME} --resource-group ${BATCH_ACCOUNT_RG_NAME}
        # create batch job for custom vision model
-        az batch job create --id ${BATCH_ACCOUNT_POOL_NAME} --pool-id ${BATCH_ACCOUNT_POOL_NAME} --account-name ${BATCH_ACCOUNT_NAME} --account-key ${BATCH_ACCOUNT_KEY}
+        az batch job create --id ${BATCH_ACCOUNT_POOL_NAME} --pool-id ${BATCH_ACCOUNT_POOL_NAME} --account-name ${BATCH_ACCOUNT_NAME} --account-key ${BATCH_ACCOUNT_KEY} || true
    fi
 elif [[ "$AI_MODEL_INFRA_TYPE" == "aks" ]]; then
    echo "Selected AI model processing infra-type: AKS!!!"
-fi
+    DATA_RESOURCE_GROUP="${ENV_CODE}-data-rg"
+    AKS_NAMESPACE=vision
+    PV_SUFFIX=fileshare
+    PV_NAME="${ENV_CODE}-${AKS_NAMESPACE}-${PV_SUFFIX}"
+
+    RAW_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'raw'].name" -o tsv -g $DATA_RESOURCE_GROUP)
+    RAW_STORAGE_KEY=$(az storage account keys list --resource-group $DATA_RESOURCE_GROUP --account-name $RAW_STORAGE_ACCT --query "[0].value" -o tsv)
+    VISION_FILE_SHARE_NAME=$(az storage share list --account-key $RAW_STORAGE_KEY --account-name $RAW_STORAGE_ACCT --query "[].name" -o tsv)
+
+    AKS_CLUSTER_NAME=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].name" -otsv)
+    while [[ ${AKS_CLUSTER_NAME} == '' ]];
+    do
+        sleep 60
+        AKS_CLUSTER_NAME=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].name" -otsv)
+    done
+    # force to provision aks-command namespace
+    az aks command invoke -g ${ENV_CODE}-orc-rg -n ${AKS_CLUSTER_NAME} -c "kubectl get ns"
+    az aks get-credentials --resource-group ${ENV_CODE}-orc-rg --name ${AKS_CLUSTER_NAME} --context ${AKS_CLUSTER_NAME} --overwrite-existing
+    kubectl config set-context ${AKS_CLUSTER_NAME}
+
+    cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: $AKS_NAMESPACE
+EOF
+
+    cat <<EOF | kubectl -n $AKS_NAMESPACE apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-secret
+  namespace: $AKS_NAMESPACE
+type: Opaque
+stringData:
+  azurestorageaccountname: ${RAW_STORAGE_ACCT}
+  azurestorageaccountkey: ${RAW_STORAGE_KEY}
+EOF
+
+    cat <<EOF | kubectl -n $AKS_NAMESPACE apply -f -
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: ${PV_NAME}
+spec:
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: azurefile-csi
+  csi:
+    driver: file.csi.azure.com
+    readOnly: false
+    volumeHandle: ${PV_NAME}
+    volumeAttributes:
+      resourceGroup: ${DATA_RESOURCE_GROUP}
+      shareName: ${VISION_FILE_SHARE_NAME}
+    nodeStageSecretRef:
+      name: azure-secret
+      namespace: ${AKS_NAMESPACE}
+  mountOptions:
+    - dir_mode=0777
+    - file_mode=0777
+    - uid=0
+    - gid=0
+    - mfsymlinks
+    - cache=strict
+    - nosharesock
+    - nobrl
+EOF
+
+    cat << EOF | kubectl -n $AKS_NAMESPACE apply -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${PV_NAME}
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: azurefile-csi
+  volumeName: ${PV_NAME}
+  resources:
+    requests:
+      storage: 5Gi
+EOF
+fi # end of "$AI_MODEL_INFRA_TYPE" == "aks"

 echo "configuration completed!"
--- a/deploy/infra/groups/gen_base64_encoded_content.py
+++ b/deploy/infra/groups/gen_base64_encoded_content.py
@ -0,0 +1,65 @@
+import logging
+import azure.functions as func
+import base64
+from zipfile import ZipFile
+import json
+
+test_data = '''
+{
+    "apiVersion": "batch/v1",
+    "kind": "Job",
+    "metadata": {
+      "name": "aoi-cv-task",
+      "namespace": "vision",
+      "labels": {
+        "app": "busybox"
+      }
+    },
+    "spec": {
+      "ttlSecondsAfterFinished": 10,
+      "template": {
+        "spec": {
+          "containers": [
+            {
+              "name": "busybox",
+              "image": "busybox",
+              "args": [
+                "bin/sh",
+                "-c",
+                "echo test;sleep 30; exit 0"
+              ]
+            }
+          ],
+          "restartPolicy": "Never"
+        }
+      }
+    }
+}
+'''
+
+def create_job_file(content='', output_file_name='run.json'):
+    with open(output_file_name, 'w') as output:
+        output.write(content)
+
+def create_zip_job_file(zip_file_name='run.zip', lst_files=['run.json']):
+    with ZipFile(zip_file_name, 'w') as zipObj:
+        for file in lst_files:
+            zipObj.write(file)
+
+def gen_base64_encoded_content(zip_file_name='run.zip'):
+    with open(zip_file_name, 'rb') as input_zip_file:
+        base64content = base64.b64encode(input_zip_file.read()).decode('ascii')
+    return base64content
+
+def main(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info('Python HTTP trigger function processed a request.')
+    json_body = req.get_json()
+    body_in_str = json.dumps(json_body)
+    filename = 'run.json'
+    if (json_body.get("metadata") is not None) and (json_body.get("metadata").get("name") is not None):
+        filename = json_body["metadata"]["name"] + '.json'
+        logging.info("filename set to {filename}")
+    create_job_file(body_in_str, filename)
+    create_zip_job_file('run.zip', [filename])
+    base64_encoded_content = gen_base64_encoded_content()
+    return func.HttpResponse(f"{base64_encoded_content}")
--- a/deploy/infra/groups/monitoring.bicep
+++ b/deploy/infra/groups/monitoring.bicep
@ -33,3 +33,5 @@ module appinsights '../modules/appinsights.bicep' = {
 }

 output workspaceId string = workspace.outputs.workspaceId
+output appinsightsId string = appinsights.outputs.Id
+output appInsightsInstrumentationKey string = appinsights.outputs.instrumentationKey
--- a/deploy/infra/groups/orchestration.bicep
+++ b/deploy/infra/groups/orchestration.bicep
@ -8,7 +8,6 @@ param projectName string
 param location string

 param synapseMIPrincipalId string
-
 // Guid to role definitions to be used during role
 // assignments including the below roles definitions:
 // Contributor
@ -23,6 +22,10 @@ param batchAccountName string = ''
 param batchAccountAutoStorageAccountName string = ''
 param acrName string = ''
 param uamiName string = ''
+param aksClusterName string = ''
+param aksVmSize string = 'Standard_D2_v5'
+param functionStorageAccountName string = ''
+param functionAppName string = ''

 param pipelineResourceGroupName string
 param pipelineLinkedSvcKeyVaultName string
@ -89,7 +92,6 @@ param batchAccountCpuOnlyPoolImageReferenceSku string = '20-04-lts'
 param batchAccountCpuOnlyPoolImageReferenceVersion string = 'latest'
 param batchAccountCpuOnlyPoolStartTaskCommandLine string = '/bin/bash -c "apt-get update && apt-get install -y python3-pip && pip install requests && pip install azure-storage-blob && pip install pandas"'

-
 param batchLogsDiagCategories array = [
  'allLogs'
 ]
@ -97,7 +99,7 @@ param batchMetricsDiagCategories array = [
  'AllMetrics'
 ]
 param logAnalyticsWorkspaceId string
-
+param appInsightsInstrumentationKey string
 // Parameters with default values for ACR
 param acrSku string = 'Standard'

@ -109,6 +111,9 @@ var keyvaultNameVar = empty(keyvaultName) ? '${namingPrefix}-kv' : keyvaultName
 var batchAccountNameVar = empty(batchAccountName) ? '${environmentCode}${projectName}batchact' : batchAccountName
 var batchAccountAutoStorageAccountNameVar = empty(batchAccountAutoStorageAccountName) ? 'batchacc${nameSuffix}' : batchAccountAutoStorageAccountName
 var acrNameVar = empty(acrName) ? '${environmentCode}${projectName}acr' : acrName
+var aksClusterNameVar = empty(aksClusterName) ? '${environmentCode}${projectName}aks' : aksClusterName
+var functionStorageAccountNameVar = empty(functionStorageAccountName) ? 'funxacc${nameSuffix}' : functionStorageAccountName
+var functionAppNameVar = empty(functionAppName) ? '${namingPrefix}-fapp' : functionAppName

 var deployBatchAccount = deployAiModelInfra && (aiModelInfraType=='batch-account')
 var deployAksCluster = deployAiModelInfra && (aiModelInfraType=='aks')
@ -311,3 +316,139 @@ module batchDiagnosticSettings '../modules/batch-diagnostic-settings.bicep' = if
    batchAccount
  ]
 }
+
+module aksCluster '../modules/aks-cluster.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-aks'
+  params: {
+    environmentName: environmentTag
+    clusterName: aksClusterNameVar
+    location: location
+    logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceId
+    vmSize: aksVmSize
+  }
+  dependsOn: [
+    acr
+  ]
+}
+
+module attachACRtoAKS '../modules/aks-attach-acr.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-attachACRtoAKS'
+  params: {
+    kubeletIdentityId: deployAksCluster?aksCluster.outputs.kubeletIdentityId:''
+    acrName:  acrNameVar
+  }
+  dependsOn: [
+    acr
+  ]
+}
+
+module aksInvokerRoleDef '../modules/custom.roledef.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-AKSInvokerCustomRole'
+  params: {
+    actions: [
+      'Microsoft.ContainerService/managedClusters/runcommand/action'
+      'Microsoft.ContainerService/managedclusters/commandResults/read'
+    ]
+    roleDescription: 'Can invoke and read runcommand to/from AKS'
+    roleName: 'custom-role-for-${aksClusterNameVar}'
+  }
+  dependsOn: [
+    aksCluster
+  ]
+}
+
+module aksCustomRoleAssignment '../modules/aks-invoker-role-assignment.bicep' = if(deployAksCluster) {
+  name: 'custom-role-assignment-for-${aksClusterNameVar}'
+  params: {
+    principalId: synapseMIPrincipalId
+    aksClusterName: aksClusterNameVar
+    customRoleDefId: deployAksCluster?aksInvokerRoleDef.outputs.Id :''
+  }
+  dependsOn: [
+    aksCluster
+  ]
+}
+
+module functionAppStorageAccount '../modules/storage.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-functionapp-storage'
+  params: {
+    storageAccountName: functionStorageAccountNameVar
+    environmentName: environmentTag
+    location: location
+    storeType: 'fapp-storage'
+  }
+  dependsOn: [
+    aksCluster
+  ]
+}
+
+module functionAppHostPlan '../modules/asp.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-asp'
+  params: {
+    aspName: '${namingPrefix}-asp'
+    aspKind: 'linux'
+    aspReserved: true
+    mewCount: 1
+    skuTier: 'Dynamic'
+    skuSize: 'Y1'
+    skuName: 'Y1'
+    location: location
+    environmentName: environmentTag
+  }
+  dependsOn: [
+    aksCluster
+  ]
+}
+
+module functionApp '../modules/functionapp.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-fapp'
+  params: {
+    functionAppName: functionAppNameVar
+    functionName: 'base64EncodedZipContent'
+    location: location
+    serverFarmId: deployAksCluster?functionAppHostPlan.outputs.id:''
+    appInsightsInstrumentationKey: appInsightsInstrumentationKey
+    functionRuntime: 'python'
+    storageAccountName: functionStorageAccountNameVar
+    storageAccountKey: deployAksCluster?functionAppStorageAccount.outputs.primaryKey:''
+    environmentName: environmentTag
+    extendedSiteConfig : {
+      use32BitWorkerProcess: false
+      linuxFxVersion: 'Python|3.9'     
+    }
+  }
+  dependsOn: [
+    aksCluster
+  ]
+}
+
+module base64EncodedZipContentFunction '../modules/function.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-base64fapp'
+  params: {
+    functionAppName: functionAppNameVar
+    functionName: 'base64EncodedZipContent'
+    functionFiles : {
+      '__init__.py': loadTextContent('gen_base64_encoded_content.py')
+    }
+    functionLanguage: 'python'
+  }
+  dependsOn:[
+    functionApp
+  ]
+}
+
+module base64EncodedZipContentFunctionKey '../modules/akv.secrets.bicep' = if(deployAksCluster) {
+  name: '${namingPrefix}-base64fapp-fkey'
+  scope: resourceGroup(pipelineResourceGroupName)
+  params: {
+    environmentName: environmentTag
+    keyVaultName: '${environmentCode}-pipeline-kv'
+    secretName: 'GenBase64EncondingFunctionKey' 
+    secretValue: deployAksCluster?base64EncodedZipContentFunction.outputs.functionkey:''
+  }
+}
+
+
+
+
+
--- a/deploy/infra/main.bicep
+++ b/deploy/infra/main.bicep
@ -176,6 +176,7 @@ module orchestrationModule 'groups/orchestration.bicep' = {
    environmentCode: environmentCode
    environmentTag: environment
    logAnalyticsWorkspaceId: monitorModule.outputs.workspaceId
+    appInsightsInstrumentationKey: monitorModule.outputs.appInsightsInstrumentationKey
    mountAccountKey: dataModule.outputs.rawStoragePrimaryKey
    deployAiModelInfra: deployAiModelInfra
    aiModelInfraType: aiModelInfraType
--- a/deploy/infra/modules/aci.bicep
+++ b/deploy/infra/modules/aci.bicep
@ -14,7 +14,7 @@ param port int = 80
 param cpuCores int = 1

@description('The amount of memory to allocate to the container in gigabytes.')
-param memoryInGb int = 4
+param memoryInGb int = 2

 param userManagedIdentityId string

--- a/deploy/infra/modules/aks-attach-acr.bicep
+++ b/deploy/infra/modules/aks-attach-acr.bicep
@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param kubeletIdentityId string
+param acrName string
+param roleAssignmentId string = guid(kubeletIdentityId, kubeletIdentityId, acrName)
+
+var acrPullRoleId = '/providers/Microsoft.Authorization/roleDefinitions/7f951dda-4ed3-4680-a7ca-43fe172d538d'
+
+resource acr 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = {
+  name: acrName
+}
+
+resource acrPullRole 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = {
+  name: roleAssignmentId
+  scope: acr
+  properties: {
+    principalId: kubeletIdentityId
+    roleDefinitionId: acrPullRoleId
+  }
+}
+
--- a/deploy/infra/modules/aks-cluster.bicep
+++ b/deploy/infra/modules/aks-cluster.bicep
@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param environmentName string
+param location string = resourceGroup().location 
+param clusterName string
+param nodeCount int = 3
+param vmSize string = 'Standard_D2s_v4'
+param networkPlugin string = 'azure'
+param networkMode string = 'transparent'
+param logAnalyticsWorkspaceResourceID string
+
+resource aks 'Microsoft.ContainerService/managedClusters@2022-01-01' = {
+  name: clusterName
+  location: location
+  tags: {
+    environment: environmentName
+    type: 'k8s'
+  }
+  identity: {
+    type: 'SystemAssigned'
+  }
+  properties: {
+    dnsPrefix: clusterName
+    enableRBAC: true
+    agentPoolProfiles: [
+      {
+        name: 'default'
+        count: nodeCount
+        vmSize: vmSize
+        mode: 'System'
+        nodeLabels: {
+          App : 'default'
+        }
+      }
+    ]
+    addonProfiles: {
+      omsagent: {
+        enabled: true
+        config: {
+          logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceResourceID
+        }
+      }
+    }
+    networkProfile: {
+      networkPlugin: networkPlugin
+      networkMode: networkMode
+    }
+  }
+}
+
+
+output Id string = aks.id
+output principalId string = aks.identity.principalId
+output kubeletIdentityId string = aks.properties.identityProfile.kubeletidentity.objectId
+
--- a/deploy/infra/modules/aks-invoker-role-assignment.bicep
+++ b/deploy/infra/modules/aks-invoker-role-assignment.bicep
@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param principalId string
+param aksClusterName string
+param customRoleDefId string
+param roleAssignmentId string = guid(principalId, aksClusterName, customRoleDefId)
+
+resource aks 'Microsoft.ContainerService/managedClusters@2021-10-01' existing = {
+  name: aksClusterName
+}
+
+resource aksInvokerRoleAssignment 'Microsoft.Authorization/roleAssignments@2020-10-01-preview' = {
+  name: roleAssignmentId
+  scope: aks
+  properties: {
+    principalId: principalId
+    roleDefinitionId: customRoleDefId
+  }
+}
+
+output Id string = aksInvokerRoleAssignment.id
+
+
+
--- a/deploy/infra/modules/appinsights.bicep
+++ b/deploy/infra/modules/appinsights.bicep
@ -20,3 +20,6 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02-preview'
    WorkspaceResourceId: workspaceId
  }
 }
+
+output Id string = applicationInsights.id
+output instrumentationKey string = applicationInsights.properties.InstrumentationKey
--- a/deploy/infra/modules/asp.bicep
+++ b/deploy/infra/modules/asp.bicep
@ -0,0 +1,31 @@
+param aspName string
+param aspKind string
+param aspReserved bool
+param mewCount int
+param skuTier string
+param skuSize string
+param skuName string
+param location string = resourceGroup().location
+param environmentName string
+
+resource hostingPlan 'Microsoft.Web/serverfarms@2021-01-15' = {
+  name: aspName
+  location: location 
+  kind: aspKind
+  properties: {
+    maximumElasticWorkerCount: mewCount
+    reserved: aspReserved 
+  }
+  sku: {
+    name: skuName 
+    tier: skuTier
+    size: skuSize
+  }
+  tags: {
+    environment: environmentName
+  }
+}
+
+output id string = hostingPlan.id
+
+ 
--- a/deploy/infra/modules/custom.roledef.bicep
+++ b/deploy/infra/modules/custom.roledef.bicep
@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+@description('Array of actions for the roleDefinition')
+param actions array = []
+
+@description('Array of notActions for the roleDefinition')
+param notActions array = []
+
+@description('Friendly name of the role definition')
+param roleName string
+
+@description('Detailed description of the role definition')
+param roleDescription string
+
+var roleDefName_var = guid(roleName)
+
+resource roleDefName 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' = {
+  name: roleDefName_var
+  properties: {
+    roleName: roleName
+    description: roleDescription
+    type: 'customRole'
+    permissions: [
+      {
+        actions: actions
+        notActions: notActions
+      }
+    ]
+    assignableScopes: [
+      resourceGroup().id
+    ]
+  }
+}
+
+output Id string = roleDefName.id
--- a/deploy/infra/modules/function.bicep
+++ b/deploy/infra/modules/function.bicep
@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param functionAppName string
+param functionName string
+param functionFiles object = {}
+param functionLanguage string
+param functionInputBinding object = {
+  name: 'req'
+  type: 'httpTrigger'
+  direction: 'in'
+  authLevel: 'function'
+  methods: [
+    'post'
+  ]  
+}
+param functionOutputBinding object = {
+  name: '$return'
+  type: 'http'
+  direction: 'out'
+}
+
+resource function 'Microsoft.Web/sites/functions@2021-03-01' = {
+  name: '${functionAppName}/${functionName}'
+  properties: {
+    config: {
+      disabled: false
+      bindings: [
+        functionInputBinding
+        functionOutputBinding
+      ]
+    }
+    files: functionFiles
+    language: functionLanguage
+  }
+}
+
+output id string = function.id
+#disable-next-line outputs-should-not-contain-secrets // we do not output to terminal. calling bicep will take this & store in keyvault
+output functionkey string = listKeys(function.id, '2021-03-01').default
--- a/deploy/infra/modules/functionapp.bicep
+++ b/deploy/infra/modules/functionapp.bicep
@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+param functionAppName string
+param functionName string
+param serverFarmId string
+param location string = resourceGroup().location
+param appInsightsInstrumentationKey string 
+param storageAccountName string
+param storageAccountKey string
+param functionRuntime string
+param environmentName string
+param extendedSiteConfig object = {}
+param extendedAppSettings object = {}
+
+var defaultAppSetting = {
+  AzureWebJobsStorage                      : 'DefaultEndpointsProtocol=https;AccountName=${storageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccountKey}'
+  WEBSITE_CONTENTSHARE                     : toLower('${functionName}-content')
+  WEBSITE_CONTENTAZUREFILECONNECTIONSTRING : 'DefaultEndpointsProtocol=https;AccountName=${storageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccountKey}'
+  APPINSIGHTS_INSTRUMENTATIONKEY           : appInsightsInstrumentationKey
+  APPLICATIONINSIGHTS_CONNECTION_STRING    : 'InstrumentationKey=${appInsightsInstrumentationKey}'
+  FUNCTIONS_WORKER_RUNTIME                 : functionRuntime
+  FUNCTIONS_EXTENSION_VERSION              : '~4'
+}
+
+var defaultSiteConfig = {
+  cors: {
+    allowedOrigins: [
+      'https://portal.azure.com'
+    ]
+  }
+}
+
+resource functionApp 'Microsoft.Web/sites@2021-03-01' = {
+  name: functionAppName
+  location: location
+  tags : {
+    type: 'functionapp'
+    environment: environmentName
+  }
+  kind: 'functionapp'
+  properties: {
+    serverFarmId: serverFarmId
+    siteConfig: union(defaultSiteConfig, extendedSiteConfig)
+    httpsOnly: true
+  }
+}
+
+resource functionAppSettings 'Microsoft.Web/sites/config@2021-03-01' = {
+  name: 'appsettings'
+  parent: functionApp
+  properties: union(defaultAppSetting, extendedAppSettings)
+}
+output id string = functionApp.id
--- a/deploy/package.py
+++ b/deploy/package.py
@ -22,10 +22,22 @@ parser.add_argument('--batch_storage_account_name', type=str, required=True, hel
 parser.add_argument('--batch_account', type=str, required=True, help="Batch Account name")
 parser.add_argument('--batch_pool_name', type=str, required=True, help="Batch Pool name")
 parser.add_argument('--linked_key_vault', type=str, required=True, help="Key Vault to be added as Linked Service")
-parser.add_argument('--location', type=str, required=True, help="Batch Account Location")
+parser.add_argument('--location', type=str, required=False, help="Batch Account Location")
 parser.add_argument('--pipeline_name', type=str, required=True, help="Name of the pipeline to package")
-parser.add_argument('--pg_db_username', type=str, required=False, help="Username to login to postgres db", default='')
-parser.add_argument('--pg_db_server_name', type=str, required=False, help="Server name to login to postgres db", default='')
+parser.add_argument('--pg_db_username', type=str, required=False, help="Username to login to postgres db")
+parser.add_argument('--pg_db_server_name', type=str, required=False, help="Server name to login to postgres db")
+parser.add_argument('--persistent_volume_claim',
+    type=str, required=False,
+    default='__env_code__-vision-fileshare',
+    help="persistent volume claim object name set up in AKS")
+parser.add_argument('--aks_management_rest_url',
+    type=str, required=False,
+    default='https://management.azure.com/subscriptions/__subscription__/resourceGroups/__env_code__-orc-rg/providers/Microsoft.ContainerService/managedClusters/__env_code__-aks2/runCommand?api-version=2022-02-01',
+    help="AKS management rest URL")
+parser.add_argument('--base64encodedzipcontent_functionapp_url',
+    type=str, required=False,
+    help="functionapp url for base64encodedzipcontent"
+)

 #Parse Args
 args = parser.parse_args()
@ -82,6 +94,15 @@ def package(pipeline_name: str, tokens_map: dict, modes='batch-account'):
                                shutil.move(
                                    os.path.join(package_manifest_folder, src),
                                    os.path.join(package_manifest_folder, dest))
+                                modified_jdata = {}
+                                with open(os.path.join(package_manifest_folder, dest), 'r') as fp:
+                                    modified_jdata = json.load(fp)
+                                    if modified_jdata.get('name') is not None:
+                                        path_splitted = dest.split('/')
+                                        name_only_with_extension = path_splitted[-1].split('.')[0]
+                                        modified_jdata['name'] = name_only_with_extension
+                                with open(os.path.join(package_manifest_folder, dest), 'w') as fp:
+                                    json.dump(modified_jdata, fp, indent=4)
                        if instruction == 'removePropertyAtPath':
                            for fileToModify in package_manifest['removePropertyAtPath']:
                                
@ -105,8 +126,6 @@ def package(pipeline_name: str, tokens_map: dict, modes='batch-account'):
                                    file = open(os.path.join(os.getcwd(), pipeline_name, fileToModify['file']), 'w')
                                    json.dump(data, file, indent=10)

-
-        
        # finally clean up .package folder before zipping it
        shutil.rmtree(package_folder_path + "/.package")
    
@ -142,21 +161,38 @@ def package(pipeline_name: str, tokens_map: dict, modes='batch-account'):
    
 if __name__ == "__main__":

-    # list of tokens and their values to be replaced
-    tokens_map = {
-        '__raw_data_storage_account__': args.raw_storage_account_name,
-        '__batch_storage_account__': args.batch_storage_account_name,
-        '__batch_account__': args.batch_account,
-        '__batch_pool_name__': args.batch_pool_name,
-        '__linked_key_vault__': args.linked_key_vault,
-        '__synapse_storage_account__': args.synapse_storage_account_name,
-        '__synapse_pool_name__': args.synapse_pool_name,
-        '__synapse_workspace_id__':args.synapse_workspace_id,
-        '__synapse_workspace__':args.synapse_workspace,
-        '__location__': args.location,
-        '__pg_db_username__': args.pg_db_username,
-        '__pg_db_server_name__': args.pg_db_server_name
-    }
-
+    if args.modes.find('batch-account') > -1:
+        # list of tokens and their values to be replaced
+        tokens_map = {
+            '__raw_data_storage_account__': args.raw_storage_account_name,
+            '__batch_storage_account__': args.batch_storage_account_name,
+            '__batch_account__': args.batch_account,
+            '__batch_pool_name__': args.batch_pool_name,
+            '__linked_key_vault__': args.linked_key_vault,
+            '__synapse_storage_account__': args.synapse_storage_account_name,
+            '__synapse_pool_name__': args.synapse_pool_name,
+            '__synapse_workspace_id__':args.synapse_workspace_id,
+            '__synapse_workspace__':args.synapse_workspace,
+            '__location__': args.location,
+            '__pg_db_username__': args.pg_db_username,
+            '__pg_db_server_name__': args.pg_db_server_name
+        }
+    elif args.modes.find('aks') > -1: 
+        # list of tokens and their values to be replaced for aks based pipeline
+        tokens_map = {
+            '__raw_data_storage_account__': args.raw_storage_account_name,
+            '__persistent_volume_claim__': args.persistent_volume_claim,
+            '__aks_management_rest_url__': args.aks_management_rest_url,
+            '__base64encodedzipcontent_functionapp_url__': args.base64encodedzipcontent_functionapp_url,
+            '__linked_key_vault__': args.linked_key_vault,
+            '__synapse_storage_account__': args.synapse_storage_account_name,
+            '__synapse_pool_name__': args.synapse_pool_name,
+            '__synapse_workspace_id__':args.synapse_workspace_id,
+            '__synapse_workspace__':args.synapse_workspace,
+            '__pg_db_username__': args.pg_db_username,
+            '__pg_db_server_name__': args.pg_db_server_name
+        }
+    else:
+        raise ValueError('args.modes should include at least either batch-account or aks.')
    # invoke package method
    package(args.pipeline_name, tokens_map, args.modes)
--- a/deploy/package.sh
+++ b/deploy/package.sh
@ -102,6 +102,37 @@ if [[ "$AI_MODEL_INFRA_TYPE" == "batch-account" ]]; then

 elif [[ "$AI_MODEL_INFRA_TYPE" == "aks" ]]; then
    echo "Selected AI model processing infra-type: AKS!!!"
+    AKS_ID=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].id" -otsv)
+    while [[ ${AKS_ID} == '' ]];
+    do
+        sleep 60
+        AKS_ID=$(az aks list -g ${ENV_CODE}-orc-rg --query "[?tags.type && tags.type == 'k8s'].id" -otsv)
+    done
+
+    PERSISTENT_VOLUME_CLAIM="${ENV_CODE}-vision-fileshare"
+    AKS_MANAGEMENT_REST_URL="https://management.azure.com${AKS_ID}/runCommand?api-version=2022-02-01"
+    BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST=$(az functionapp list -g ${ENV_CODE}-orc-rg \
+        --query "[?tags.type && tags.type == 'functionapp'].hostNames[0]" | jq -r '.[0]')
+
+    while [[ ${BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST} == '' ]];
+    do
+        sleep 60
+        BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST=$(az functionapp list -g ${ENV_CODE}-orc-rg \
+        --query "[?tags.type && tags.type == 'functionapp'].hostNames[0]" | jq -r '.[0]')
+    done
+    BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_URL="https://${BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_HOST}"
+    PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py \
+        --raw_storage_account_name $RAW_STORAGE_ACCOUNT_NAME \
+        --synapse_storage_account_name $SYNAPSE_STORAGE_ACCOUNT_NAME \
+        --modes $MODE \
+        --persistent_volume_claim $PERSISTENT_VOLUME_CLAIM \
+        --aks_management_rest_url $AKS_MANAGEMENT_REST_URL \
+        --base64encodedzipcontent_functionapp_url $BASE64ENCODEDZIPCONTENT_FUNCTIONAPP_URL \
+        --linked_key_vault $KEY_VAULT_NAME \
+        --synapse_pool_name $SYNAPSE_POOL \
+        --pipeline_name $PIPELINE_NAME \
+        --synapse_workspace $SYNAPSE_WORKSPACE_NAME \
+        --synapse_workspace_id $SYNAPSE_WORKSPACE_ID"
 fi

 if [[ $DEPLOY_PGSQL == "true" ]]; then
--- a/src/workflow/custom-vision-model-v2/.package/manifest.json
+++ b/src/workflow/custom-vision-model-v2/.package/manifest.json
@ -1,14 +1,20 @@
 {
    "modes": {
        "batch-account" : {
-            "exclude" : []
+            "exclude" : [
+                "../pipeline/Custom Vision Object Detection v2_aks.json",
+                "../pipeline/E2E Custom Vision Model Flow_aks.json",
+                "../linkedService/GenBase64Encoding.json"
+            ]
        },
        "aks": {
            "exclude" : [
                "../pipeline/Custom Vision Object Detection v2.json",
                "../pipeline/E2E Custom Vision Model Flow.json"
            ],
-            "rename" : {}
+            "rename" : {
+                "../pipeline/E2E Custom Vision Model Flow_aks.json": "../pipeline/E2E Custom Vision Model Flow.json"
+            }
        },
        "no-postgres": {
            "removePropertyAtPath" : [ {
--- a/src/workflow/custom-vision-model-v2/linkedService/GenBase64Encoding.json
+++ b/src/workflow/custom-vision-model-v2/linkedService/GenBase64Encoding.json
@ -0,0 +1,20 @@
+{
+    "name": "GenBase64Encoding",
+    "properties": {
+        "annotations": [],
+        "type": "AzureFunction",
+        "typeProperties": {
+            "functionAppUrl": "__base64encodedzipcontent_functionapp_url__",
+            "functionKey": {
+                "type": "AzureKeyVaultSecret",
+                "store": {
+                    "referenceName": "AOI Pipeline Key Vault",
+                    "type": "LinkedServiceReference"
+                },
+                "secretName": "GenBase64EncondingFunctionKey"
+            },
+            "authentication": "Anonymous"
+        }
+    },
+    "type": "Microsoft.Synapse/workspaces/linkedservices"
+}
--- a/src/workflow/custom-vision-model-v2/pipeline/Custom
+++ b/src/workflow/custom-vision-model-v2/pipeline/Custom
@ -0,0 +1,909 @@
+{
+    "name": "Custom Vision Object Detection v2_aks",
+    "properties": {
+        "activities": [
+            {
+                "name": "Pool Geolocation",
+                "type": "SparkJob",
+                "dependsOn": [
+                    {
+                        "activity": "Copy Xml",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Json",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "sparkJob": {
+                        "referenceName": "Pool Geolocation",
+                        "type": "SparkJobDefinitionReference"
+                    },
+                    "file": "abfss://spark-jobs@__synapse_storage_account__.dfs.core.windows.net/pool_geolocation/src/main.py",
+                    "args": [
+                        "--storage_account_name",
+                        "@pipeline().parameters.StorageAccountName",
+                        "--storage_container",
+                        "@pipeline().parameters.Prefix",
+                        "--src_folder_name",
+                        "detections",
+                        "--key_vault_name",
+                        "__linked_key_vault__",
+                        "--storage_account_key_secret_name",
+                        "GeospatialStorageAccountKey",
+                        "--linked_service_name",
+                        "AOI Pipeline Key Vault"
+                    ],
+                    "targetBigDataPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    },
+                    "executorSize": "Medium",
+                    "conf": {
+                        "spark.dynamicAllocation.minExecutors": 2,
+                        "spark.dynamicAllocation.maxExecutors": 3
+                    },
+                    "driverSize": "Medium",
+                    "numExecutors": 2
+                }
+            },
+            {
+                "name": "Copy Json",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Wait for Custom Vision",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureFileStorageReadSettings",
+                            "recursive": true,
+                            "wildcardFolderPath": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'], '/json')",
+                                "type": "Expression"
+                            },
+                            "wildcardFileName": "*.json",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureBlobFSWriteSettings"
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "gls",
+                        "type": "DatasetReference"
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "gld",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "DestinationFolderPath": "detections",
+                            "DestinationContainerName": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Copy Xml",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Wait for Custom Vision",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureFileStorageReadSettings",
+                            "recursive": true,
+                            "wildcardFolderPath": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'], '/other')",
+                                "type": "Expression"
+                            },
+                            "wildcardFileName": "*.xml",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureBlobFSWriteSettings"
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "gls",
+                        "type": "DatasetReference"
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "gld",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "DestinationFolderPath": "detections",
+                            "DestinationContainerName": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Read Spec Document",
+                "type": "ExecuteDataFlow",
+                "dependsOn": [],
+                "policy": {
+                    "timeout": "1.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "dataflow": {
+                        "referenceName": "ReadSpecDocumentFlow",
+                        "type": "DataFlowReference",
+                        "datasetParameters": {
+                            "source": {
+                                "filename": "custom_vision_object_detection.json",
+                                "folderpath": "config",
+                                "containername": {
+                                    "value": "@pipeline().parameters.Prefix",
+                                    "type": "Expression"
+                                }
+                            }
+                        }
+                    },
+                    "compute": {
+                        "coreCount": 8,
+                        "computeType": "General"
+                    },
+                    "traceLevel": "None",
+                    "cacheSinks": {
+                        "firstRowOnly": true
+                    }
+                }
+            },
+            {
+                "name": "Copy Tiles",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureBlobFSReadSettings",
+                            "maxConcurrentConnections": 15,
+                            "recursive": true,
+                            "wildcardFolderPath": "tiles",
+                            "wildcardFileName": "*.png",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureFileStorageWriteSettings",
+                            "maxConcurrentConnections": 15
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "run_container",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "container_name": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "folder_path": "tiles"
+                        }
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Copy Config",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureBlobFSReadSettings",
+                            "maxConcurrentConnections": 15,
+                            "recursive": true,
+                            "wildcardFolderPath": "config",
+                            "wildcardFileName": {
+                                "value": "@activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName']",
+                                "type": "Expression"
+                            },
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureFileStorageWriteSettings",
+                            "maxConcurrentConnections": 15
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "run_container",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "container_name": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "folder_path": "config"
+                        }
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix)",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Copy Georeference Xml",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureBlobFSReadSettings",
+                            "maxConcurrentConnections": 15,
+                            "recursive": true,
+                            "wildcardFolderPath": "convert",
+                            "wildcardFileName": "*.aux.xml",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureFileStorageWriteSettings",
+                            "maxConcurrentConnections": 15
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "run_container",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "container_name": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "folder_path": "convert"
+                        }
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Create Results Directory with placeholder",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureBlobFSReadSettings",
+                            "maxConcurrentConnections": 15,
+                            "recursive": true,
+                            "wildcardFolderPath": "config",
+                            "wildcardFileName": {
+                                "value": "@activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName']",
+                                "type": "Expression"
+                            },
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureFileStorageWriteSettings",
+                            "maxConcurrentConnections": 15
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "run_container",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "container_name": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "folder_path": "config"
+                        }
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Create Logs Directory with placeholder_copy1",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureBlobFSReadSettings",
+                            "maxConcurrentConnections": 15,
+                            "recursive": true,
+                            "wildcardFolderPath": "config",
+                            "wildcardFileName": {
+                                "value": "@activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName']",
+                                "type": "Expression"
+                            },
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureFileStorageWriteSettings",
+                            "maxConcurrentConnections": 15
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "run_container",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "container_name": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "folder_path": "config"
+                        }
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['logsDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Delete Results Directory placeholder",
+                "type": "Delete",
+                "dependsOn": [
+                    {
+                        "activity": "Create Results Directory with placeholder",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "dataset": {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    },
+                    "logStorageSettings": {
+                        "linkedServiceName": {
+                            "referenceName": "AOI Geospatial v2",
+                            "type": "LinkedServiceReference"
+                        }
+                    },
+                    "enableLogging": true,
+                    "storeSettings": {
+                        "type": "AzureFileStorageReadSettings",
+                        "recursive": true,
+                        "wildcardFileName": "*.*",
+                        "enablePartitionDiscovery": false
+                    }
+                }
+            },
+            {
+                "name": "Delete Logs Directory placeholder",
+                "type": "Delete",
+                "dependsOn": [
+                    {
+                        "activity": "Create Logs Directory with placeholder_copy1",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "dataset": {
+                        "referenceName": "run_fileshare",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "folder_path": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['logsDirectory'])",
+                                "type": "Expression"
+                            }
+                        }
+                    },
+                    "logStorageSettings": {
+                        "linkedServiceName": {
+                            "referenceName": "AOI Geospatial v2",
+                            "type": "LinkedServiceReference"
+                        }
+                    },
+                    "enableLogging": true,
+                    "storeSettings": {
+                        "type": "AzureFileStorageReadSettings",
+                        "recursive": true,
+                        "wildcardFileName": "*.*",
+                        "enablePartitionDiscovery": false
+                    }
+                }
+            },
+            {
+                "name": "Gen SpecZipBase64Encoding",
+                "type": "AzureFunctionActivity",
+                "dependsOn": [
+                    {
+                        "activity": "Delete Results Directory placeholder",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Delete Logs Directory placeholder",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Tiles",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Config",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Georeference Xml",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "functionName": "base64EncodedZipContent",
+                    "method": "POST",
+                    "headers": {
+                        "Content-Type": "application/json"
+                    },
+                    "body": {
+                        "value": "@json(concat('{\n    \"apiVersion\": \"batch/v1\",\n    \"kind\": \"Job\",\n    \"metadata\": {\n      \"name\": \"aoi-cv-task\",\n      \"namespace\": \"vision\",\n      \"labels\": {\n        \"run_id\": \"',pipeline().RunId, '\",\n      }\n    },\n    \"spec\": {\n      \"ttlSecondsAfterFinished\": 5,\n      \"template\": {\n        \"spec\": {\n          \"containers\": [\n            {\n              \"name\": \"aoi-cv-task-xyz\",\n              \"image\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['algImageName'] ,'\",\n              \"env\": [\n                {\n                  \"name\": \"APP_INPUT_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'],'\"\n                },\n                {\n                  \"name\": \"APP_OUTPUT_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'],'\"\n                },\n                {\n                  \"name\": \"APP_CONFIG_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName'],'\"\n                }\n              ],\n              \"volumeMounts\": [\n                {\n                  \"name\": \"azure\",\n                  \"mountPath\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'], '\"\n                }\n              ]\n            }\n          ],\n          \"volumes\": [\n            {\n              \"name\": \"azure\",\n              \"persistentVolumeClaim\": {\n                \"claimName\": \"', pipeline().parameters.PersistentVolumeClaim, '\"\n              }\n            }\n          ],\n          \"automountServiceAccountToken\": false,\n          \"restartPolicy\": \"Never\"\n        }\n      }\n    }\n}'))",
+                        "type": "Expression"
+                    }
+                },
+                "linkedServiceName": {
+                    "referenceName": "GenBase64Encoding",
+                    "type": "LinkedServiceReference"
+                }
+            },
+            {
+                "name": "Set SpecZipBase64Encoded",
+                "type": "SetVariable",
+                "dependsOn": [
+                    {
+                        "activity": "Gen SpecZipBase64Encoding",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "variableName": "KubeCmdSpecInBase64Encoded",
+                    "value": {
+                        "value": "@activity('Gen SpecZipBase64Encoding').output.Response",
+                        "type": "Expression"
+                    }
+                }
+            },
+            {
+                "name": "Invoke Workload in AKS",
+                "type": "WebActivity",
+                "dependsOn": [
+                    {
+                        "activity": "Set SpecZipBase64Encoded",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 3,
+                    "retryIntervalInSeconds": 120,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "url": {
+                        "value": "@pipeline().parameters.AksManagementRestApiURL",
+                        "type": "Expression"
+                    },
+                    "connectVia": {
+                        "referenceName": "AutoResolveIntegrationRuntime",
+                        "type": "IntegrationRuntimeReference"
+                    },
+                    "method": "POST",
+                    "body": {
+                        "value": "@json(concat(\n'{',\n'\"command\" : \"', 'kubectl apply -f aoi-cv-task.json', '\",',\n'\"context\" : \"', variables('KubeCmdSpecInBase64Encoded'), '\"',\n'}'\n))",
+                        "type": "Expression"
+                    },
+                    "authentication": {
+                        "type": "MSI",
+                        "resource": "https://management.core.windows.net"
+                    }
+                }
+            },
+            {
+                "name": "Wait for Custom Vision",
+                "type": "Until",
+                "dependsOn": [
+                    {
+                        "activity": "Invoke Workload in AKS",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "expression": {
+                        "value": "@contains(variables('KubeJobStatusCheckResult'),'No resources found')",
+                        "type": "Expression"
+                    },
+                    "activities": [
+                        {
+                            "name": "Check Job Status",
+                            "type": "WebActivity",
+                            "dependsOn": [
+                                {
+                                    "activity": "Wait",
+                                    "dependencyConditions": [
+                                        "Succeeded"
+                                    ]
+                                }
+                            ],
+                            "policy": {
+                                "timeout": "7.00:00:00",
+                                "retry": 5,
+                                "retryIntervalInSeconds": 60,
+                                "secureOutput": false,
+                                "secureInput": false
+                            },
+                            "userProperties": [],
+                            "typeProperties": {
+                                "url": {
+                                    "value": "@pipeline().parameters.AksManagementRestApiURL",
+                                    "type": "Expression"
+                                },
+                                "connectVia": {
+                                    "referenceName": "AutoResolveIntegrationRuntime",
+                                    "type": "IntegrationRuntimeReference"
+                                },
+                                "method": "POST",
+                                "body": {
+                                    "value": "@concat(\n'{',\n'\"command\" : \"', 'kubectl -n vision get jobs --selector run_id=', pipeline().RunId ,' \"',\n'}'\n)",
+                                    "type": "Expression"
+                                },
+                                "authentication": {
+                                    "type": "MSI",
+                                    "resource": "https://management.core.windows.net"
+                                }
+                            }
+                        },
+                        {
+                            "name": "Set JobStatus",
+                            "type": "SetVariable",
+                            "dependsOn": [
+                                {
+                                    "activity": "Check Job Status",
+                                    "dependencyConditions": [
+                                        "Succeeded"
+                                    ]
+                                }
+                            ],
+                            "userProperties": [],
+                            "typeProperties": {
+                                "variableName": "KubeJobStatusCheckResult",
+                                "value": {
+                                    "value": "@activity('Check Job Status').output['properties']['logs']\n\n",
+                                    "type": "Expression"
+                                }
+                            }
+                        },
+                        {
+                            "name": "Wait",
+                            "type": "Wait",
+                            "dependsOn": [],
+                            "userProperties": [],
+                            "typeProperties": {
+                                "waitTimeInSeconds": 15
+                            }
+                        }
+                    ],
+                    "timeout": "3.00:00:00"
+                }
+            }
+        ],
+        "parameters": {
+            "Prefix": {
+                "type": "string"
+            },
+            "StorageAccountName": {
+                "type": "string"
+            },
+            "AksManagementRestApiURL": {
+                "type": "string",
+                "defaultValue": "__aks_management_rest_url__"
+            },
+            "PersistentVolumeClaim": {
+                "type": "string",
+                "defaultValue": "__persistent_volume_claim__"
+            }
+        },
+        "variables": {
+            "KubeCmdSpecInBase64Encoded": {
+                "type": "String"
+            },
+            "KubeJobStatusCheckResult": {
+                "type": "String"
+            }
+        },
+        "annotations": [],
+        "lastPublishTime": "2022-03-06T05:52:44Z"
+    },
+    "type": "Microsoft.Synapse/workspaces/pipelines"
+}
--- a/src/workflow/custom-vision-model-v2/pipeline/E2E
+++ b/src/workflow/custom-vision-model-v2/pipeline/E2E
@ -0,0 +1,189 @@
+{
+    "name": "E2E Custom Vision Model Flow_aks",
+    "properties": {
+        "activities": [
+            {
+                "name": "Transforms",
+                "type": "ExecutePipeline",
+                "dependsOn": [],
+                "userProperties": [],
+                "typeProperties": {
+                    "pipeline": {
+                        "referenceName": "Custom Vision Model Transforms v2",
+                        "type": "PipelineReference"
+                    },
+                    "waitOnCompletion": true,
+                    "parameters": {
+                        "Prefix": {
+                            "value": "@pipeline().parameters.Prefix",
+                            "type": "Expression"
+                        },
+                        "StorageAccountName": {
+                            "value": "@pipeline().parameters.StorageAccountName",
+                            "type": "Expression"
+                        },
+                        "AOI": {
+                            "value": "@pipeline().parameters.AOI",
+                            "type": "Expression"
+                        }
+                    }
+                }
+            },
+            {
+                "name": "Custom Vision Object Detection with AKS",
+                "type": "ExecutePipeline",
+                "dependsOn": [
+                    {
+                        "activity": "Transforms",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "pipeline": {
+                        "referenceName": "Custom Vision Object Detection v2_aks",
+                        "type": "PipelineReference"
+                    },
+                    "waitOnCompletion": true,
+                    "parameters": {
+                        "Prefix": {
+                            "value": "@pipeline().parameters.Prefix",
+                            "type": "Expression"
+                        },
+                        "StorageAccountName": {
+                            "value": "@pipeline().parameters.StorageAccountName",
+                            "type": "Expression"
+                        },
+                        "AksManagementRestApiURL": {
+                            "value": "@pipeline().parameters.AksManagementRestApiURL",
+                            "type": "Expression"
+                        },
+                        "PersistentVolumeClaim": {
+                            "value": "@pipeline().parameters.PersistentVolumeClaim",
+                            "type": "Expression"
+                        }
+                    }
+                }
+            },
+            {
+                "name": "Insert CV Object Detection Data Into DB",
+                "type": "SynapseNotebook",
+                "dependsOn": [
+                    {
+                        "activity": "Custom Vision Object Detection with AKS",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 3,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "notebook": {
+                        "referenceName": "Insert CV Object Detection Data Into DB Notebook Job",
+                        "type": "NotebookReference"
+                    },
+                    "parameters": {
+                        "storage_account_name": {
+                            "value": {
+                                "value": "@pipeline().parameters.StorageAccountName",
+                                "type": "Expression"
+                            },
+                            "type": "string"
+                        },
+                        "container_name": {
+                            "value": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            },
+                            "type": "string"
+                        },
+                        "folder_path": {
+                            "value": "pool-geolocation",
+                            "type": "string"
+                        },
+                        "key_vault_name": {
+                            "value": "__linked_key_vault__",
+                            "type": "string"
+                        },
+                        "storage_account_key_secret_name": {
+                            "value": "GeospatialStorageAccountKey",
+                            "type": "string"
+                        },
+                        "linked_service_name": {
+                            "value": "AOI Pipeline Key Vault",
+                            "type": "string"
+                        },
+                        "db_password_secret_name": {
+                            "value": "PostgresAdminPassword",
+                            "type": "string"
+                        },
+                        "db_username": {
+                            "value": "__pg_db_username__",
+                            "type": "string"
+                        },
+                        "db_host": {
+                            "value": "__pg_db_server_name__",
+                            "type": "string"
+                        },
+                        "db_name": {
+                            "value": "postgres",
+                            "type": "string"
+                        },
+                        "db_port": {
+                            "value": "5432",
+                            "type": "int"
+                        },
+                        "ssl_root_path": {
+                            "value": "/opt/src/BaltimoreCyberTrustRoot.crt.pem",
+                            "type": "string"
+                        }
+                    },
+                    "snapshot": true,
+                    "sparkPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    }
+                }
+            }
+        ],
+        "parameters": {
+            "Prefix": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "StorageAccountName": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "AOI": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "AksManagementRestApiURL": {
+                "type": "string",
+                "defaultValue": "__aks_management_rest_url__"
+            },
+            "PersistentVolumeClaim": {
+                "type": "string",
+                "defaultValue": "__persistent_volume_claim__"
+            }
+        },
+        "variables": {
+            "Storage_Account_Conn_String": {
+                "type": "String"
+            }
+        },
+        "annotations": [],
+        "lastPublishTime": "2022-03-06T05:42:39Z"
+    },
+    "type": "Microsoft.Synapse/workspaces/pipelines"
+}
--- a/src/workflow/custom-vision-model/.package/manifest.json
+++ b/src/workflow/custom-vision-model/.package/manifest.json
@ -0,0 +1,20 @@
+{
+    "ai_model_infra_type": {
+        "batch-account" : {
+            "exclude" : [
+                "../pipeline/Custom Vision Object Detection v2_aks.json",
+                "../pipeline/E2E Custom Vision Model Flow_aks.json",
+                "../linkedService/GenBase64Encoding.json"
+            ]
+        },
+        "aks": {
+            "exclude" : [
+                "../pipeline/Custom Vision Object Detection v2.json",
+                "../pipeline/E2E Custom Vision Model Flow.json"
+            ],
+            "rename" : {
+                "../pipeline/E2E Custom Vision Model Flow_aks.json": "../pipeline/E2E Custom Vision Model Flow.json"
+            }
+        }
+    }
+}
--- a/src/workflow/custom-vision-model/linkedService/GenBase64Encoding.json
+++ b/src/workflow/custom-vision-model/linkedService/GenBase64Encoding.json
@ -0,0 +1,20 @@
+{
+    "name": "GenBase64Encoding",
+    "properties": {
+        "annotations": [],
+        "type": "AzureFunction",
+        "typeProperties": {
+            "functionAppUrl": "__base64encodedzipcontent_functionapp_url__",
+            "functionKey": {
+                "type": "AzureKeyVaultSecret",
+                "store": {
+                    "referenceName": "AOI Pipeline Key Vault",
+                    "type": "LinkedServiceReference"
+                },
+                "secretName": "GenBase64EncondingFunctionKey"
+            },
+            "authentication": "Anonymous"
+        }
+    },
+    "type": "Microsoft.Synapse/workspaces/linkedservices"
+}
--- a/src/workflow/custom-vision-model/pipeline/Custom
+++ b/src/workflow/custom-vision-model/pipeline/Custom
@ -0,0 +1,646 @@
+{
+    "name": "Custom Vision Object Detection v2_aks",
+    "properties": {
+        "activities": [
+            {
+                "name": "Pool Geolocation",
+                "type": "SparkJob",
+                "dependsOn": [
+                    {
+                        "activity": "Copy Xml",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Json",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "sparkJob": {
+                        "referenceName": "Pool Geolocation",
+                        "type": "SparkJobDefinitionReference"
+                    },
+                    "file": "abfss://spark-jobs@__synapse_storage_account__.dfs.core.windows.net/pool_geolocation/src/main.py",
+                    "args": [
+                        "--storage_account_name",
+                        "@pipeline().parameters.StorageAccountName",
+                        "--storage_container",
+                        "@pipeline().parameters.Prefix",
+                        "--src_folder_name",
+                        "detections",
+                        "--key_vault_name",
+                        "__linked_key_vault__",
+                        "--storage_account_key_secret_name",
+                        "GeospatialStorageAccountKey",
+                        "--linked_service_name",
+                        "AOI Pipeline Key Vault",
+                        "--config_file_name",
+                        "config-pool-geolocation.json"
+                    ],
+                    "targetBigDataPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    },
+                    "executorSize": "Medium",
+                    "conf": {
+                        "spark.dynamicAllocation.minExecutors": 2,
+                        "spark.dynamicAllocation.maxExecutors": 3
+                    },
+                    "driverSize": "Medium",
+                    "numExecutors": 2
+                }
+            },
+            {
+                "name": "Copy Tiles",
+                "type": "SparkJob",
+                "dependsOn": [
+                    {
+                        "activity": "Read Spec Document",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "sparkJob": {
+                        "referenceName": "Copy noop",
+                        "type": "SparkJobDefinitionReference"
+                    },
+                    "file": "abfss://spark-jobs@__synapse_storage_account__.dfs.core.windows.net/copy_noop/src/main.py",
+                    "args": [
+                        "--storage_account_name",
+                        "@pipeline().parameters.StorageAccountName",
+                        "--src_container",
+                        "@pipeline().parameters.Prefix",
+                        "--src_folder",
+                        "tiles",
+                        "--key_vault_name",
+                        "__linked_key_vault__",
+                        "--storage_account_key_secret_name",
+                        "GeospatialStorageAccountKey",
+                        "--linked_service_name",
+                        "AOI Pipeline Key Vault",
+                        "--dst_fileshare",
+                        "volume-a",
+                        "--dst_folder",
+                        "@concat(pipeline().parameters.Prefix,'/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'])",
+                        "--folders_to_create",
+                        "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'])",
+                        "--folders_to_create",
+                        "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'])",
+                        "--folders_to_create",
+                        "@concat(pipeline().parameters.Prefix,'/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['logsDirectory'])"
+                    ],
+                    "targetBigDataPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    },
+                    "executorSize": "Medium",
+                    "conf": {
+                        "spark.dynamicAllocation.minExecutors": 2,
+                        "spark.dynamicAllocation.maxExecutors": 3
+                    },
+                    "driverSize": "Medium",
+                    "numExecutors": 2
+                }
+            },
+            {
+                "name": "Copy Config file",
+                "type": "SparkJob",
+                "dependsOn": [
+                    {
+                        "activity": "Copy Tiles",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "sparkJob": {
+                        "referenceName": "Copy noop",
+                        "type": "SparkJobDefinitionReference"
+                    },
+                    "file": "abfss://spark-jobs@__synapse_storage_account__.dfs.core.windows.net/copy_noop/src/main.py",
+                    "args": [
+                        "--storage_account_name",
+                        "@pipeline().parameters.StorageAccountName",
+                        "--src_container",
+                        "@pipeline().parameters.Prefix",
+                        "--src_folder",
+                        "@concat('config/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName'])",
+                        "--key_vault_name",
+                        "__linked_key_vault__",
+                        "--storage_account_key_secret_name",
+                        "GeospatialStorageAccountKey",
+                        "--linked_service_name",
+                        "AOI Pipeline Key Vault",
+                        "--dst_fileshare",
+                        "volume-a",
+                        "--dst_folder",
+                        "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName'])"
+                    ],
+                    "targetBigDataPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    },
+                    "executorSize": "Medium",
+                    "conf": {
+                        "spark.dynamicAllocation.minExecutors": 2,
+                        "spark.dynamicAllocation.maxExecutors": 3
+                    },
+                    "driverSize": "Medium",
+                    "numExecutors": 2
+                }
+            },
+            {
+                "name": "Copy Json",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Until1",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureFileStorageReadSettings",
+                            "recursive": true,
+                            "wildcardFolderPath": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'], '/json')",
+                                "type": "Expression"
+                            },
+                            "wildcardFileName": "*.json",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureBlobFSWriteSettings"
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "gls",
+                        "type": "DatasetReference"
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "gld",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "DestinationFolderPath": "detections",
+                            "DestinationContainerName": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Copy Xml",
+                "type": "Copy",
+                "dependsOn": [
+                    {
+                        "activity": "Until1",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "source": {
+                        "type": "BinarySource",
+                        "storeSettings": {
+                            "type": "AzureFileStorageReadSettings",
+                            "recursive": true,
+                            "wildcardFolderPath": {
+                                "value": "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'], '/other')",
+                                "type": "Expression"
+                            },
+                            "wildcardFileName": "*.xml",
+                            "deleteFilesAfterCompletion": false
+                        },
+                        "formatSettings": {
+                            "type": "BinaryReadSettings"
+                        }
+                    },
+                    "sink": {
+                        "type": "BinarySink",
+                        "storeSettings": {
+                            "type": "AzureBlobFSWriteSettings"
+                        }
+                    },
+                    "enableStaging": false
+                },
+                "inputs": [
+                    {
+                        "referenceName": "gls",
+                        "type": "DatasetReference"
+                    }
+                ],
+                "outputs": [
+                    {
+                        "referenceName": "gld",
+                        "type": "DatasetReference",
+                        "parameters": {
+                            "DestinationFolderPath": "detections",
+                            "DestinationContainerName": {
+                                "value": "@pipeline().parameters.Prefix",
+                                "type": "Expression"
+                            }
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "Read Spec Document",
+                "type": "ExecuteDataFlow",
+                "dependsOn": [],
+                "policy": {
+                    "timeout": "1.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "dataflow": {
+                        "referenceName": "ReadSpecDocumentFlow",
+                        "type": "DataFlowReference",
+                        "datasetParameters": {
+                            "source": {
+                                "filename": "custom_vision_object_detection.json",
+                                "folderpath": "config",
+                                "containername": {
+                                    "value": "@pipeline().parameters.Prefix",
+                                    "type": "Expression"
+                                }
+                            }
+                        }
+                    },
+                    "compute": {
+                        "coreCount": 8,
+                        "computeType": "General"
+                    },
+                    "traceLevel": "None",
+                    "cacheSinks": {
+                        "firstRowOnly": true
+                    }
+                }
+            },
+            {
+                "name": "Copy Xml From Convert Transform",
+                "type": "SparkJob",
+                "dependsOn": [
+                    {
+                        "activity": "Copy Tiles",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "sparkJob": {
+                        "referenceName": "Copy noop",
+                        "type": "SparkJobDefinitionReference"
+                    },
+                    "file": "abfss://spark-jobs@__synapse_storage_account__.dfs.core.windows.net/copy_noop/src/main.py",
+                    "args": [
+                        "--storage_account_name",
+                        "@pipeline().parameters.StorageAccountName",
+                        "--src_container",
+                        "@pipeline().parameters.Prefix",
+                        "--src_folder",
+                        "convert/output.png.aux.xml",
+                        "--key_vault_name",
+                        "__linked_key_vault__",
+                        "--storage_account_key_secret_name",
+                        "GeospatialStorageAccountKey",
+                        "--linked_service_name",
+                        "AOI Pipeline Key Vault",
+                        "--dst_fileshare",
+                        "volume-a",
+                        "--dst_folder",
+                        "@concat(pipeline().parameters.Prefix, '/', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'], '/output.png.aux.xml')"
+                    ],
+                    "targetBigDataPool": {
+                        "referenceName": "__synapse_pool_name__",
+                        "type": "BigDataPoolReference"
+                    },
+                    "executorSize": "Medium",
+                    "conf": {
+                        "spark.dynamicAllocation.minExecutors": 2,
+                        "spark.dynamicAllocation.maxExecutors": 3
+                    },
+                    "driverSize": "Medium",
+                    "numExecutors": 2
+                }
+            },
+            {
+                "name": "Gen Kubectl Task Cmd",
+                "type": "SetVariable",
+                "dependsOn": [
+                    {
+                        "activity": "Copy Config file",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    },
+                    {
+                        "activity": "Copy Xml From Convert Transform",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "variableName": "KubeTaskJob",
+                    "value": {
+                        "value": "@concat('{\n    \"apiVersion\": \"batch/v1\",\n    \"kind\": \"Job\",\n    \"metadata\": {\n      \"name\": \"aoi-cv-task\",\n      \"namespace\": \"vision\",\n      \"labels\": {\n        \"run_id\": \"',pipeline().RunId, '\",\n      }\n    },\n    \"spec\": {\n      \"ttlSecondsAfterFinished\": 5,\n      \"template\": {\n        \"spec\": {\n          \"containers\": [\n            {\n              \"name\": \"aoi-cv-task-xyz\",\n              \"image\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['algImageName'] ,'\",\n              \"env\": [\n                {\n                  \"name\": \"APP_INPUT_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['submissionDirectory'],'\"\n                },\n                {\n                  \"name\": \"APP_OUTPUT_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['resultsDirectory'],'\"\n                },\n                {\n                  \"name\": \"APP_CONFIG_DIR\",\n                  \"value\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'] , '/',pipeline().parameters.Prefix,'/' , activity('Read Spec Document').output['runStatus'].output.sink.value[0]['contextFileName'],'\"\n                }\n              ],\n              \"volumeMounts\": [\n                {\n                  \"name\": \"azure\",\n                  \"mountPath\": \"', activity('Read Spec Document').output['runStatus'].output.sink.value[0]['mountedDirectory'], '\"\n                }\n              ]\n            }\n          ],\n          \"volumes\": [\n            {\n              \"name\": \"azure\",\n              \"persistentVolumeClaim\": {\n                \"claimName\": \"', pipeline().parameters.PersistentVolumeClaim, '\"\n              }\n            }\n          ],\n          \"automountServiceAccountToken\": false,\n          \"restartPolicy\": \"Never\"\n        }\n      }\n    }\n}')",
+                        "type": "Expression"
+                    }
+                }
+            },
+            {
+                "name": "Gen SpecZipBase64Encoding",
+                "type": "AzureFunctionActivity",
+                "dependsOn": [
+                    {
+                        "activity": "Gen Kubectl Task Cmd",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 5,
+                    "retryIntervalInSeconds": 60,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "functionName": "base64EncodedZipContent",
+                    "method": "POST",
+                    "headers": {
+                        "Content-Type": "application/json"
+                    },
+                    "body": {
+                        "value": "@json(variables('KubeTaskJob'))",
+                        "type": "Expression"
+                    }
+                },
+                "linkedServiceName": {
+                    "referenceName": "GenBase64Encoding",
+                    "type": "LinkedServiceReference"
+                }
+            },
+            {
+                "name": "Set SpecZipBase64Encoded",
+                "type": "SetVariable",
+                "dependsOn": [
+                    {
+                        "activity": "Gen SpecZipBase64Encoding",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "variableName": "KubeCmdSpecInBase64Encoded",
+                    "value": {
+                        "value": "@activity('Gen SpecZipBase64Encoding').output.Response",
+                        "type": "Expression"
+                    }
+                }
+            },
+            {
+                "name": "Invoke Workload in AKS",
+                "type": "WebActivity",
+                "dependsOn": [
+                    {
+                        "activity": "Set SpecZipBase64Encoded",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "policy": {
+                    "timeout": "7.00:00:00",
+                    "retry": 3,
+                    "retryIntervalInSeconds": 120,
+                    "secureOutput": false,
+                    "secureInput": false
+                },
+                "userProperties": [],
+                "typeProperties": {
+                    "url": "@pipeline().parameters.AksManagementRestApiURL",
+                    "connectVia": {
+                        "referenceName": "AutoResolveIntegrationRuntime",
+                        "type": "IntegrationRuntimeReference"
+                    },
+                    "method": "POST",
+                    "body": {
+                        "value": "@json(concat(\n'{',\n'\"command\" : \"', 'kubectl apply -f aoi-cv-task.json', '\",',\n'\"context\" : \"', variables('KubeCmdSpecInBase64Encoded'), '\"',\n'}'\n))",
+                        "type": "Expression"
+                    },
+                    "authentication": {
+                        "type": "MSI",
+                        "resource": "https://management.core.windows.net"
+                    }
+                }
+            },
+            {
+                "name": "Until1",
+                "type": "Until",
+                "dependsOn": [
+                    {
+                        "activity": "Invoke Workload in AKS",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "expression": {
+                        "value": "@contains(variables('KubeJobStatusCheckResult'),'No resources found')",
+                        "type": "Expression"
+                    },
+                    "activities": [
+                        {
+                            "name": "Check Job Status",
+                            "type": "WebActivity",
+                            "dependsOn": [
+                                {
+                                    "activity": "Wait",
+                                    "dependencyConditions": [
+                                        "Succeeded"
+                                    ]
+                                }
+                            ],
+                            "policy": {
+                                "timeout": "7.00:00:00",
+                                "retry": 5,
+                                "retryIntervalInSeconds": 60,
+                                "secureOutput": false,
+                                "secureInput": false
+                            },
+                            "userProperties": [],
+                            "typeProperties": {
+                                "url": "@pipeline().parameters.AksManagementRestApiURL",
+                                "connectVia": {
+                                    "referenceName": "AutoResolveIntegrationRuntime",
+                                    "type": "IntegrationRuntimeReference"
+                                },
+                                "method": "POST",
+                                "body": {
+                                    "value": "@concat(\n'{',\n'\"command\" : \"', 'kubectl -n vision get jobs --selector run_id=', pipeline().RunId ,' \"',\n'}'\n)",
+                                    "type": "Expression"
+                                },
+                                "authentication": {
+                                    "type": "MSI",
+                                    "resource": "https://management.core.windows.net"
+                                }
+                            }
+                        },
+                        {
+                            "name": "Set JobStatus",
+                            "type": "SetVariable",
+                            "dependsOn": [
+                                {
+                                    "activity": "Check Job Status",
+                                    "dependencyConditions": [
+                                        "Succeeded"
+                                    ]
+                                }
+                            ],
+                            "userProperties": [],
+                            "typeProperties": {
+                                "variableName": "KubeJobStatusCheckResult",
+                                "value": {
+                                    "value": "@activity('Check Job Status').output['properties']['logs']\n\n",
+                                    "type": "Expression"
+                                }
+                            }
+                        },
+                        {
+                            "name": "Wait",
+                            "type": "Wait",
+                            "dependsOn": [],
+                            "userProperties": [],
+                            "typeProperties": {
+                                "waitTimeInSeconds": 15
+                            }
+                        }
+                    ],
+                    "timeout": "3.00:00:00"
+                }
+            }
+        ],
+        "parameters": {
+            "Prefix": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "StorageAccountName": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "StorageAccountKey": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "AksManagementRestApiURL": {
+                "type": "string",
+                "defaultValue": "__aks_management_rest_url__"
+            },
+            "PersistentVolumeClaim": {
+                "type": "string",
+                "defaultValue": "__persistent_volume_claim__"
+            }
+        },
+        "variables": {
+            "KubeTaskJob": {
+                "type": "String"
+            },
+            "KubeCmdSpecInBase64Encoded": {
+                "type": "String"
+            },
+            "KubeJobStatusCheckResult": {
+                "type": "String"
+            }
+        },
+        "annotations": [],
+        "lastPublishTime": "2022-03-06T05:52:44Z"
+    },
+    "type": "Microsoft.Synapse/workspaces/pipelines"
+}
--- a/src/workflow/custom-vision-model/pipeline/E2E
+++ b/src/workflow/custom-vision-model/pipeline/E2E
@ -0,0 +1,106 @@
+{
+    "name": "E2E Custom Vision Model Flow_aks",
+    "properties": {
+        "activities": [
+            {
+                "name": "Transforms",
+                "type": "ExecutePipeline",
+                "dependsOn": [],
+                "userProperties": [],
+                "typeProperties": {
+                    "pipeline": {
+                        "referenceName": "Custom Vision Model Transforms v2",
+                        "type": "PipelineReference"
+                    },
+                    "waitOnCompletion": true,
+                    "parameters": {
+                        "Prefix": {
+                            "value": "@pipeline().parameters.Prefix",
+                            "type": "Expression"
+                        },
+                        "StorageAccountName": {
+                            "value": "@pipeline().parameters.StorageAccountName",
+                            "type": "Expression"
+                        },
+                        "StorageAccountKey": {
+                            "value": "@pipeline().parameters.StorageAccountKey",
+                            "type": "Expression"
+                        }
+                    }
+                }
+            },
+            {
+                "name": "Custom Vision Object Detection with AKS",
+                "type": "ExecutePipeline",
+                "dependsOn": [
+                    {
+                        "activity": "Transforms",
+                        "dependencyConditions": [
+                            "Succeeded"
+                        ]
+                    }
+                ],
+                "userProperties": [],
+                "typeProperties": {
+                    "pipeline": {
+                        "referenceName": "Custom Vision Object Detection v2_aks",
+                        "type": "PipelineReference"
+                    },
+                    "waitOnCompletion": true,
+                    "parameters": {
+                        "Prefix": {
+                            "value": "@pipeline().parameters.Prefix",
+                            "type": "Expression"
+                        },
+                        "StorageAccountName": {
+                            "value": "@pipeline().parameters.StorageAccountName",
+                            "type": "Expression"
+                        },
+                        "StorageAccountKey": {
+                            "value": "@pipeline().parameters.StorageAccountKey",
+                            "type": "Expression"
+                        },
+                        "AksManagementRestApiURL": {
+                            "value": "@pipeline().parameters.AksManagementRestApiURL",
+                            "type": "Expression"
+                        },
+                        "PersistentVolumeClaim": {
+                            "value": "@pipeline().parameters.PersistentVolumeClaim",
+                            "type": "Expression"
+                        }
+                    }
+                }
+            }
+        ],
+        "parameters": {
+            "Prefix": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "StorageAccountName": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "StorageAccountKey": {
+                "type": "string",
+                "defaultValue": ""
+            },
+            "AksManagementRestApiURL": {
+                "type": "string",
+                "defaultValue": "__aks_management_rest_url__"
+            },
+            "PersistentVolumeClaim": {
+                "type": "string",
+                "defaultValue": "__persistent_volume_claim__"
+            }
+        },
+        "variables": {
+            "Storage_Account_Conn_String": {
+                "type": "String"
+            }
+        },
+        "annotations": [],
+        "lastPublishTime": "2022-03-06T05:42:39Z"
+    },
+    "type": "Microsoft.Synapse/workspaces/pipelines"
+}