From e65a72844dfc7b1f2d38bd7984c89a0309ef0d0e Mon Sep 17 00:00:00 2001 From: Shital Savekar Date: Mon, 16 Apr 2018 15:02:54 -0700 Subject: [PATCH] Uploading BUILD_URL and LOGFILE path to DB --- AzureTestSuite.ps1 | 26 ++++++++++--- Extras/UploadFilesToStorageAccount.ps1 | 53 ++++++++++++++------------ RunAzureTests.ps1 | 11 ++---- TestLibs/RDFELibs.psm1 | 14 +++++++ 4 files changed, 67 insertions(+), 37 deletions(-) diff --git a/AzureTestSuite.ps1 b/AzureTestSuite.ps1 index 462ddbf..ce5664f 100644 --- a/AzureTestSuite.ps1 +++ b/AzureTestSuite.ps1 @@ -357,16 +357,23 @@ Function RunTestsOnCycle ($cycleName , $xmlConfig, $Distro, $testIterations ) { try { + $xmlSecrets = [xml](Get-Content $secretsFile) + $testLogFolder = "TestLogs" + $testLogStorageAccount = $xmlSecrets.secrets.testLogsStorageAccount + $testLogStorageAccountKey = $xmlSecrets.secrets.testLogsStorageAccountKey + $ticks= (Get-Date).Ticks + $uploadFileName = ".\temp\$($currentTestData.testName)-$ticks.zip" + $out = ZipFiles -zipfilename $uploadFileName -sourcedir $LogDir + $uploadLink = .\Extras\UploadFilesToStorageAccount.ps1 -filePaths $uploadFileName -destinationStorageAccount $testLogStorageAccount -destinationContainer "logs" -destinationFolder "$testLogFolder" -destinationStorageKey $testLogStorageAccountKey $utctime = (Get-Date).ToUniversalTime() $dbDateTimeUTC = "$($utctime.Year)-$($utctime.Month)-$($utctime.Day) $($utctime.Hour):$($utctime.Minute):$($utctime.Second)" - $xmlSecrets = [xml](Get-Content $secretsFile) $dataSource = $xmlSecrets.secrets.DatabaseServer $dbuser = $xmlSecrets.secrets.DatabaseUser $dbpassword = $xmlSecrets.secrets.DatabasePassword $database = $xmlSecrets.secrets.DatabaseName $dataTableName = "AzureTestResultsMasterTable" - $SQLQuery = "INSERT INTO $dataTableName (DateTimeUTC,Environment,TestCycle,ExecutionID,TestName,TestResult,ARMImage,OsVHD,KernelVersion,LISVersion,GuestDistro,AzureHost,Location,OverrideVMSize,Networking) VALUES " - $SQLQuery += "('$dbDateTimeUTC','$dbEnvironment','$dbTestCycle','$dbExecutionID','$dbTestName','$dbTestResult','$dbARMImage','$BaseOsVHD','$finalKernelVersion','NA','$GuestDistro','$HostVersion','$dbLocation','$dbOverrideVMSize','$dbNetworking')" + $SQLQuery = "INSERT INTO $dataTableName (DateTimeUTC,Environment,TestCycle,ExecutionID,TestName,TestResult,ARMImage,OsVHD,KernelVersion,LISVersion,GuestDistro,AzureHost,Location,OverrideVMSize,Networking, LogFile, BuildURL) VALUES " + $SQLQuery += "('$dbDateTimeUTC','$dbEnvironment','$dbTestCycle','$dbExecutionID','$dbTestName','$dbTestResult','$dbARMImage','$BaseOsVHD','$finalKernelVersion','NA','$GuestDistro','$HostVersion','$dbLocation','$dbOverrideVMSize','$dbNetworking', '$uploadLink', '$env:BUILD_URL`consoleFull')" $SQLQuery = $SQLQuery.TrimEnd(',') $connectionString = "Server=$dataSource;uid=$dbuser; pwd=$dbpassword;Database=$database;Encrypt=yes;TrustServerCertificate=no;Connection Timeout=30;" $connection = New-Object System.Data.SqlClient.SqlConnection @@ -474,16 +481,23 @@ Function RunTestsOnCycle ($cycleName , $xmlConfig, $Distro, $testIterations ) { try { + $xmlSecrets = [xml](Get-Content $secretsFile) + $testLogFolder = "TestLogs" + $testLogStorageAccount = $xmlSecrets.secrets.testLogsStorageAccount + $testLogStorageAccountKey = $xmlSecrets.secrets.testLogsStorageAccountKey + $ticks= (Get-Date).Ticks + $uploadFileName = ".\temp\$($currentTestData.testName)-$ticks.zip" + $out = ZipFiles -zipfilename $uploadFileName -sourcedir $LogDir + $uploadLink = .\Extras\UploadFilesToStorageAccount.ps1 -filePaths $uploadFileName -destinationStorageAccount $testLogStorageAccount -destinationContainer "logs" -destinationFolder "$testLogFolder" -destinationStorageKey $testLogStorageAccountKey $utctime = (Get-Date).ToUniversalTime() $dbDateTimeUTC = "$($utctime.Year)-$($utctime.Month)-$($utctime.Day) $($utctime.Hour):$($utctime.Minute):$($utctime.Second)" - $xmlSecrets = [xml](Get-Content $secretsFile) $dataSource = $xmlSecrets.secrets.DatabaseServer $dbuser = $xmlSecrets.secrets.DatabaseUser $dbpassword = $xmlSecrets.secrets.DatabasePassword $database = $xmlSecrets.secrets.DatabaseName $dataTableName = "AzureTestResultsMasterTable" - $SQLQuery = "INSERT INTO $dataTableName (DateTimeUTC,Environment,TestCycle,ExecutionID,TestName,TestResult,ARMImage,OsVHD,KernelVersion,LISVersion,GuestDistro,AzureHost,Location,OverrideVMSize,Networking) VALUES " - $SQLQuery += "('$dbDateTimeUTC','$dbEnvironment','$dbTestCycle','$dbExecutionID','$dbTestName','$($testResult[0])','$dbARMImage','$dbOsVHD','$finalKernelVersion','NA','$GuestDistro','$HostVersion','$dbLocation','$dbOverrideVMSize','$dbNetworking')," + $SQLQuery = "INSERT INTO $dataTableName (DateTimeUTC,Environment,TestCycle,ExecutionID,TestName,TestResult,ARMImage,OsVHD,KernelVersion,LISVersion,GuestDistro,AzureHost,Location,OverrideVMSize,Networking,LogFile,BuildURL) VALUES " + $SQLQuery += "('$dbDateTimeUTC','$dbEnvironment','$dbTestCycle','$dbExecutionID','$dbTestName','$($testResult[0])','$dbARMImage','$dbOsVHD','$finalKernelVersion','NA','$GuestDistro','$HostVersion','$dbLocation','$dbOverrideVMSize','$dbNetworking','$uploadLink', '$env:BUILD_URL`consoleFull')," foreach ($tempResult in $summary.Split('>')) { if ($tempResult) diff --git a/Extras/UploadFilesToStorageAccount.ps1 b/Extras/UploadFilesToStorageAccount.ps1 index f989a95..d595d98 100644 --- a/Extras/UploadFilesToStorageAccount.ps1 +++ b/Extras/UploadFilesToStorageAccount.ps1 @@ -6,21 +6,21 @@ param $destinationFolder, $destinationStorageKey, [string]$customSecretsFilePath=$null - ) #---------------------------------------------------------[Initialisations]-------------------------------------------------------- + if ( $customSecretsFilePath ) { - $secretsFile = $customSecretsFilePath - Write-Host "Using provided secrets file: $($secretsFile | Split-Path -Leaf)" + $secretsFile = $customSecretsFilePath + Write-Host "Using provided secrets file: $($secretsFile | Split-Path -Leaf)" } -if ($env:Azure_Secrets_File) +elseif ($env:Azure_Secrets_File) { - $secretsFile = $env:Azure_Secrets_File - Write-Host "Using predefined secrets file: $($secretsFile | Split-Path -Leaf) in Jenkins Global Environments." + $secretsFile = $env:Azure_Secrets_File + Write-Host "Using predefined secrets file: $($secretsFile | Split-Path -Leaf) in Jenkins Global Environments." } -if ( $secretsFile -eq $null ) +elseif ( $secretsFile -eq $null ) { Write-Host "ERROR: Azure Secrets file not found in Jenkins / user not provided -customSecretsFilePath" -ForegroundColor Red -BackgroundColor Black exit 1 @@ -29,30 +29,37 @@ if ( $secretsFile -eq $null ) if ( Test-Path $secretsFile) { - Write-Host "$($secretsFile | Split-Path -Leaf) found." + Write-Host "$($secretsFile | Split-Path -Leaf) found." $xmlSecrets = [xml](Get-Content $secretsFile) - .\AddAzureRmAccountFromSecretsFile.ps1 -customSecretsFilePath $secretsFile - $subscriptionID = $xmlSecrets.secrets.SubscriptionID + if ($AuthenticatedSession) + { + Write-Host "Using pre-authenticated session" + } + else + { + .\AddAzureRmAccountFromSecretsFile.ps1 -customSecretsFilePath $secretsFile + } + $subscriptionID = $xmlSecrets.secrets.SubscriptionID } else { - Write-Host "$($secretsFile | Split-Path -Leaf) file is not added in Jenkins Global Environments OR it is not bound to 'Azure_Secrets_File' variable." -ForegroundColor Red -BackgroundColor Black - Write-Host "Aborting." -ForegroundColor Red -BackgroundColor Black - exit 1 + Write-Host "$($secretsFile | Split-Path -Leaf) file is not added in Jenkins Global Environments OR it is not bound to 'Azure_Secrets_File' variable." -ForegroundColor Red -BackgroundColor Black + Write-Host "Aborting." -ForegroundColor Red -BackgroundColor Black + exit 1 } if ( Test-Path $secretsFile ) { - Write-Host "$($secretsFile | Split-Path -Leaf) found." - Write-Host "---------------------------------" - $xmlSecrets = [xml](Get-Content $secretsFile) + Write-Host "$($secretsFile | Split-Path -Leaf) found." + Write-Host "---------------------------------" + $xmlSecrets = [xml](Get-Content $secretsFile) } else { - Write-Host "$($secretsFile | Spilt-Path -Leaf) file is not added in Jenkins Global Environments OR it is not bound to 'Azure_Secrets_File' variable." - Write-Host "If you are using local secret file, then make sure file path is correct." - Write-Host "Aborting." - exit 1 + Write-Host "$($secretsFile | Spilt-Path -Leaf) file is not added in Jenkins Global Environments OR it is not bound to 'Azure_Secrets_File' variable." + Write-Host "If you are using local secret file, then make sure file path is correct." + Write-Host "Aborting." + exit 1 } #---------------------------------------------------------[Script Start]-------------------------------------------------------- @@ -64,9 +71,9 @@ try { if ($destinationStorageKey) { - Write-Host "Using user provided storage account key." + Write-Host "Using user provided storage account key." } - else + else { Write-Host "Getting $destinationStorageAccount storage account key..." $allResources = Get-AzureRmResource @@ -83,9 +90,7 @@ try $ticks = (Get-Date).Ticks #$fileName = "$LogDir\$($vmData.RoleName)-waagent.log.txt" $blobName = "$destinationFolder/$($fileName | Split-Path -Leaf)" - Write-Host $out = Set-AzureStorageBlobContent -File $filename -Container $containerName -Blob $blobName -Context $blobContext -Force -ErrorAction Stop - Write-Host "$($blobContext.BlobEndPoint)$containerName/$blobName : Success" $uploadedFiles += "$($blobContext.BlobEndPoint)$containerName/$blobName" } return $uploadedFiles diff --git a/RunAzureTests.ps1 b/RunAzureTests.ps1 index efe9da0..6182422 100644 --- a/RunAzureTests.ps1 +++ b/RunAzureTests.ps1 @@ -68,6 +68,8 @@ Set-Content -Value "No tests ran yet." -Path ".\report\testSummary.html" -Force .\Extras\CheckForNewKernelPackages.ps1 +New-Item -Name temp -ItemType Directory -Force -ErrorAction SilentlyContinue | Out-Null + if ( $customSecretsFilePath ) { $secretsFile = $customSecretsFilePath Write-Host "Using user provided secrets file: $($secretsFile | Split-Path -Leaf)" @@ -88,6 +90,7 @@ if ( Test-Path $secretsFile) { .\AddAzureRmAccountFromSecretsFile.ps1 -customSecretsFilePath $secretsFile $xmlSecrets = [xml](Get-Content $secretsFile) Set-Variable -Name xmlSecrets -Value $xmlSecrets -Scope Global + Set-Variable -Name AuthenticatedSession -Value $true -Scope Global } else { Write-Host "AzureSecrets.xml file is not added in Jenkins Global Environments OR it is not bound to 'Azure_Secrets_File' variable." -ForegroundColor Red -BackgroundColor Black @@ -352,17 +355,11 @@ Invoke-Expression -Command $cmd $LogDir = Get-Content .\report\lastLogDirectory.txt -ErrorAction SilentlyContinue -$currentDir = $PWD +$currentDir = (Get-Location).Path $out = Remove-Item *.json -Force $out = Remove-Item *.xml -Force $zipFile = "$(($TestCycle).Trim())-$shortRandomNumber-azure-buildlogs.zip" -function ZipFiles( $zipfilename, $sourcedir ) -{ - $sourcedir = $sourcedir.Trim('\') - .\tools\7za.exe a -mx5 $zipFile $sourcedir -r -} - $out = ZipFiles -zipfilename $zipFile -sourcedir $LogDir diff --git a/TestLibs/RDFELibs.psm1 b/TestLibs/RDFELibs.psm1 index 859b534..1b49b32 100644 --- a/TestLibs/RDFELibs.psm1 +++ b/TestLibs/RDFELibs.psm1 @@ -7093,4 +7093,18 @@ Function GetFilePathsFromLinuxFolder ([string]$folderToSearch, $IpAddress, $SSHP } return $LogFilesPaths, $LogFiles } + +function ZipFiles( $zipfilename, $sourcedir ) +{ + $currentDir = (Get-Location).Path + $7z = (Get-ChildItem .\tools\7za.exe).FullName + $sourcedir = $sourcedir.Trim('\') + cd $sourcedir + $out = Invoke-Expression "$7z a -mx5 $currentDir\$zipfilename * -r" + cd $currentDir + if ($out -match "Everything is Ok") + { + Write-Host "$currentDir\$zipfilename created successfully." + } +} #endregion