chore: Databricks task order

azure-pipelines/iac/templates/stage.deploy.yml

@@ -80,18 +80,6 @@ stages:
             env:
               AzureDevOpsPAT: $(System.AccessToken)
           
-          - task: AzurePowerShell@4
-            displayName: 'Publish Databricks Secrets'
-            inputs:
-              azureSubscription: ${{ parameters.azureServiceConnection }}
-              scriptType: filePath
-              scriptPath: infrastructure-as-code/scripts/DatabricksSecrets.ps1
-              scriptArguments: >
-                -Environment ${{ parameters.environment }}
-                -DeploymentOutputFile "armOutput.json"
-              azurePowerShellVersion: 'OtherVersion'
-              preferredAzurePowerShellVersion: $(azPowershellVersion)
-
   - deployment: deploy_dbw_clusters
     displayName: 'Deploy DBW Clusters to ${{ parameters.environment }}'
     condition: succeeded()
@@ -109,6 +97,18 @@ stages:
           - template: step.install-databricks-cli.yml
             parameters:
               azureServiceConnection: ${{ parameters.azureServiceConnection }}
+          
+          - task: AzurePowerShell@4
+            displayName: 'Publish Databricks Secrets'
+            inputs:
+              azureSubscription: ${{ parameters.azureServiceConnection }}
+              scriptType: filePath
+              scriptPath: infrastructure-as-code/scripts/DatabricksSecrets.ps1
+              scriptArguments: >
+                -Environment ${{ parameters.environment }}
+              azurePowerShellVersion: 'OtherVersion'
+              preferredAzurePowerShellVersion: latest
+
           - task: PowerShell@2
             displayName: Deploy DBW Clusters
             inputs:

infrastructure-as-code/scripts/DatabricksSecrets.ps1

@@ -1,16 +1,23 @@
 param(
     [Parameter(Mandatory)] [string] [ValidateSet("dev", "qa", "prod", "sandbox")] $Environment,
-    [Parameter(Mandatory)] [string] $DeploymentOutputFile,
     [string] $SolutionParametersFile = "./infrastructure-as-code/infrastructure/parameters/parameters.$Environment.json"
 )
 
 $ErrorActionPreference = "Stop"
 
-Write-Host "Getting variables from $DeploymentOutputFile file..." -ForegroundColor Green
-$DeploymentOutput = Get-Content -Path $DeploymentOutputFile | ConvertFrom-Json -AsHashtable
-$DataLakeName = $DeploymentOutput["dataLakeName"]
-$DatabricksName = $DeploymentOutput["databricksName"]
-$keyVaultName = $DeploymentOutput["keyVaultName"]
+Write-Host "Getting variables from Library file..." -ForegroundColor Green
+#$DeploymentOutput = Get-Content -Path $DeploymentOutputFile | ConvertFrom-Json -AsHashtable
+# $DataLakeName = $DeploymentOutput["dataLakeName"]
+# $DatabricksName = $DeploymentOutput["databricksName"]
+# $keyVaultName = $DeploymentOutput["keyVaultName"]
+
+$DataLakeName = $dataLakeName
+$DatabricksName = $databricksName
+$keyVaultName = $keyVaultName
+
+Write-Host "DataLake: " $DataLakeName 
+Write-Host "DataBricks: " $DatabricksName 
+Write-Host "Key Valt: " $keyVaultName
 
 Write-Host "Getting variables from $SolutionParametersFile file..." -ForegroundColor Green
 $ParameterContent = Get-Content -Path $SolutionParametersFile | ConvertFrom-Json
@@ -32,7 +39,9 @@ if ($servicePrincipal) {
     $endDate = $startDate.AddMonths(6)
 
     $clientSecret = New-AzADSpCredential -ObjectId $servicePrincipal.Id -StartDate $startDate -EndDate $endDate
-    Write-Host "Secret generated " $clientSecret -ForegroundColor Yellow
+    $UnsecureSecret = ConvertFrom-SecureString -SecureString $clientSecret.Secret -AsPlainText
+
+    Write-Host "New Secret was generated for Service Principal " $UnsecureSecret -ForegroundColor Yellow
 
     Write-Host "Getting Azure resources..." -ForegroundColor Green
     $kv = Get-AzKeyVault -VaultName $KeyVaultName
@@ -48,20 +57,19 @@ if ($servicePrincipal) {
 
     Write-Host "Setting service principal secrets on Key Vault..." -ForegroundColor Green
     Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name "tenantId" -SecretValue $(ConvertTo-SecureString $context.Tenant.Id -AsPlainText -Force)
-    Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name "clientId" -SecretValue $(ConvertTo-SecureString $ClientID -AsPlainText -Force)
-    Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name "clientSecret" -SecretValue $clientSecret
+    Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name "clientId" -SecretValue $(ConvertTo-SecureString $servicePrincipal.Id -AsPlainText -Force)
+    Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name "clientSecret" -SecretValue $clientSecret.Secret
 
     Write-Host "Assigning roles to the service principal on the data lake..." -ForegroundColor Green
-    $assigment = Get-AzRoleAssignment -ObjectId $principal.Id -Scope $lake.Id | Where-Object { $_.RoleDefinitionName -eq "Storage Blob Data Contributor" }
+    $assigment = Get-AzRoleAssignment -ObjectId $servicePrincipal.Id -Scope $lake.Id | Where-Object { $_.RoleDefinitionName -eq "Storage Blob Data Contributor" }
     if(! $assigment){
-        New-AzRoleAssignment -ObjectId $principal.Id -Scope $lake.Id -RoleDefinitionName "Storage Blob Data Contributor" 
+        New-AzRoleAssignment -ObjectId $servicePrincipal.Id -Scope $lake.Id -RoleDefinitionName "Storage Blob Data Contributor" 
     }
 
     Write-Host "Creating the Key Vault secret scope on Databricks..." -ForegroundColor Green
-    $accessToken = Get-AzAccessToken -ResourceUrl 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d
-    $env:DATABRICKS_TOKEN = $accessToken.Token
     $env:DATABRICKS_HOST = "https://$($dbw.Url)"
     Write-Host "URL DBW https://$($dbw.Url)"
+    Write-Host "Databricks Access Token " $DATABRICKS_TOKEN
     $scopesList = databricks secrets list-scopes --output json | ConvertFrom-Json
     if (! $scopesList.scopes.name -contains "dataops") {
         databricks secrets create-scope --scope 'dataops' --scope-backend-type AZURE_KEYVAULT --resource-id $kv.ResourceId --dns-name $kv.VaultUri