Azure
/
Avere
зеркало из https://github.com/Azure/Avere.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Azure rendering solution deployment framework

This commit is contained in:
Rick Shahid 2022-12-14 05:04:37 -08:00
Родитель 8033526d63
Коммит 34d79ed487
4 изменённых файлов: 16 добавлений и 92 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
src/terraform/examples/e2e/2.storage/config.auto.tfvars
Просмотреть файл

@ -13,13 +13,9 @@ storageAccounts = [
enableHttpsOnly = true # https://learn.microsoft.com/azure/storage/common/storage-require-secure-transfer
enableBlobNfsV3 = true # https://learn.microsoft.com/azure/storage/blobs/network-file-system-protocol-support
enableLargeFileShare = false # https://learn.microsoft.com/azure/storage/files/storage-how-to-create-file-share#advanced
enableSampleDataLoad = false
blobContainers = [ # https://learn.microsoft.com/azure/storage/blobs/storage-blobs-introduction
{
name = "show"
sampleFiles = [
"blender"
]
}
]
fileShares = [ # https://learn.microsoft.com/azure/storage/files/storage-files-introduction
@ -33,7 +29,6 @@ storageAccounts = [
enableHttpsOnly = true # https://learn.microsoft.com/azure/storage/common/storage-require-secure-transfer
enableBlobNfsV3 = false # https://learn.microsoft.com/azure/storage/blobs/network-file-system-protocol-support
enableLargeFileShare = true # https://learn.microsoft.com/azure/storage/files/storage-how-to-create-file-share#advanced
enableSampleDataLoad = false
blobContainers = [ # https://learn.microsoft.com/azure/storage/blobs/storage-blobs-introduction
]
fileShares = [ # https://learn.microsoft.com/azure/storage/files/storage-files-introduction
@ -42,9 +37,6 @@ storageAccounts = [
tier = "Premium"
sizeGiB = 5120
protocol = "SMB"
sampleFiles = [
"blender"
]
}
]
}

88
src/terraform/examples/e2e/2.storage/main.tf
Просмотреть файл

@ -46,20 +46,17 @@ variable "storageAccounts" {
enableHttpsOnly = bool
enableBlobNfsV3 = bool
enableLargeFileShare = bool
enableSampleDataLoad = bool
blobContainers = list(object(
{
name = string
sampleFiles = list(string)
name = string
}
))
fileShares = list(object(
{
name = string
tier = string
sizeGiB = number
protocol = string
sampleFiles = list(string)
name = string
tier = string
sizeGiB = number
protocol = string
}
))
}
@ -270,31 +267,7 @@ locals {
name = blobContainer.name
storageAccountName = storageAccount.name
}
] if storageAccount.enableSampleDataLoad
])
blobRootFiles = flatten([
for storageAccount in var.storageAccounts : [
for blobContainer in storageAccount.blobContainers : [
for blob in fileset(blobContainer.name, "*") : {
name = blob
containerName = blobContainer.name
storageAccountName = storageAccount.name
}
]
] if storageAccount.enableSampleDataLoad
])
blobDirectoryFiles = flatten([
for storageAccount in var.storageAccounts : [
for blobContainer in storageAccount.blobContainers : [
for localPath in blobContainer.sampleFiles : [
for blob in fileset(blobContainer.name, "/${localPath}/**") : {
name = blob
containerName = blobContainer.name
storageAccountName = storageAccount.name
}
]
]
] if storageAccount.enableSampleDataLoad
]
])
fileShares = flatten([
for storageAccount in var.storageAccounts : [
@ -305,21 +278,7 @@ locals {
accessProtocol = fileShare.protocol
storageAccountName = storageAccount.name
}
] if storageAccount.enableSampleDataLoad
])
fileShareFiles = flatten([
for storageAccount in var.storageAccounts : [
for fileShare in storageAccount.fileShares : [
for localPath in fileShare.sampleFiles : [
for file in fileset(fileShare.name, "/${localPath}/**") : {
name = file
directoryPath = dirname(file)
shareName = fileShare.name
storageAccountName = storageAccount.name
}
]
]
] if storageAccount.enableSampleDataLoad
]
])
netAppVolumes = flatten([
for capacityPool in var.netAppAccount.capacityPools : [
@ -434,15 +393,15 @@ resource "azurerm_storage_account" "storage" {
for serviceEndpointSubnet in local.serviceEndpointSubnets :
"${data.azurerm_resource_group.network.id}/providers/Microsoft.Network/virtualNetworks/${serviceEndpointSubnet.virtualNetworkName}/subnets/${serviceEndpointSubnet.name}"
]
ip_rules = each.value.enableSampleDataLoad ? [
ip_rules = [
jsondecode(data.http.current.response_body).ip
] : []
]
}
}
resource "time_sleep" "storage_data" {
for_each = {
for storageAccount in var.storageAccounts : storageAccount.name => storageAccount if storageAccount.enableSampleDataLoad
for storageAccount in var.storageAccounts : storageAccount.name => storageAccount
}
create_duration = "30s"
depends_on = [
@ -461,20 +420,6 @@ resource "azurerm_storage_container" "containers" {
]
}
resource "azurerm_storage_blob" "blobs" {
for_each = {
for blob in setunion(local.blobRootFiles, local.blobDirectoryFiles) : "${blob.storageAccountName}.${blob.name}" => blob
}
name = each.value.name
storage_account_name = each.value.storageAccountName
storage_container_name = each.value.containerName
source = "${path.cwd}/${each.value.containerName}/${each.value.name}"
type = "Block"
depends_on = [
azurerm_storage_container.containers
]
}
resource "azurerm_storage_share" "shares" {
for_each = {
for fileShare in local.fileShares : "${fileShare.storageAccountName}.${fileShare.name}" => fileShare
@ -489,19 +434,6 @@ resource "azurerm_storage_share" "shares" {
]
}
resource "azurerm_storage_share_file" "files" {
for_each = {
for fileShareFile in local.fileShareFiles : "${fileShareFile.storageAccountName}.${fileShareFile.name}" => fileShareFile
}
name = basename(each.value.name)
path = each.value.directoryPath
source = "${path.cwd}/${each.value.shareName}/${each.value.name}"
storage_share_id = "${azurerm_resource_group.storage.id}/providers/Microsoft.Storage/storageAccounts/${each.value.storageAccountName}/fileServices/default/shares/${each.value.shareName}"
depends_on = [
azurerm_storage_share.shares
]
}
#######################################################################################################
# NetApp Files (https://learn.microsoft.com/azure/azure-netapp-files/azure-netapp-files-introduction) #
#######################################################################################################

3
src/terraform/examples/e2e/4.image.builder/customize.ps1
Просмотреть файл

@ -66,7 +66,8 @@ if ($gpuPlatform -contains "GRID") {
$installFile = "amd-gpu.exe"
$downloadUrl = "https://go.microsoft.com/fwlink/?linkid=2175154"
(New-Object System.Net.WebClient).DownloadFile($downloadUrl, (Join-Path -Path $pwd.Path -ChildPath $installFile))
# Start-Process -FilePath .\$installFile -ArgumentList "/quiet /norestart" -Wait -RedirectStandardOutput "amd-gpu.output.txt" -RedirectStandardError "amd-gpu.error.txt"
Start-Process -FilePath .\$installFile -ArgumentList "/S" -Wait
Start-Process -FilePath "C:\AMD\AMD*\Setup.exe" -ArgumentList "-install" -Wait -RedirectStandardOutput "amd-gpu.output.txt" -RedirectStandardError "amd-gpu.error.txt"
Write-Host "Customize (End): AMD GPU"
}

9
src/terraform/examples/e2e/README.md
Просмотреть файл

@ -13,7 +13,7 @@ The following *core principles* are implemented throughout the Azure Artist Anyw
| - | - | - | - |
| [0 Global](#0-global) | Defines global config (e.g., Azure region, render manager) and deploys core security services. | Yes | Yes |
| [1 Network](#1-network) | Deploys [Virtual Network](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) and [Bastion](https://learn.microsoft.com/azure/bastion/bastion-overview) with [VPN](https://learn.microsoft.com/azure/vpn-gateway/vpn-gateway-about-vpngateways) or [ExpressRoute](https://learn.microsoft.com/azure/expressroute/expressroute-about-virtual-network-gateways) hybrid networking services. | Yes, if [Virtual Network](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) not deployed. Otherwise, No | Yes, if [Virtual Network](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) not deployed. Otherwise, No |
| [2 Storage](#2-storage) | Deploys [Blob (NFS)](https://learn.microsoft.com/azure/storage/blobs/network-file-system-protocol-support) / [Files](https://learn.microsoft.com/azure/storage/files/storage-files-introduction) (with data option), [NetApp Files](https://learn.microsoft.com/azure/azure-netapp-files/azure-netapp-files-introduction) or [Hammerspace](https://azuremarketplace.microsoft.com/marketplace/apps/hammerspace.hammerspace_4_6_5) storage services. | No | Yes |
| [2 Storage](#2-storage) | Deploys [Blob (NFS)](https://learn.microsoft.com/azure/storage/blobs/network-file-system-protocol-support), [Files](https://learn.microsoft.com/azure/storage/files/storage-files-introduction), [NetApp Files](https://learn.microsoft.com/azure/azure-netapp-files/azure-netapp-files-introduction), [Qumulo](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/qumulo1584033880660.qumulo-saas) and/or [Hammerspace](https://azuremarketplace.microsoft.com/marketplace/apps/hammerspace.hammerspace_4_6_5) storage services. | No | Yes |
| [3 Storage Cache](#3-storage-cache) | Deploys [HPC Cache](https://learn.microsoft.com/azure/hpc-cache/hpc-cache-overview) or [Avere vFXT](https://learn.microsoft.com/azure/avere-vfxt/avere-vfxt-overview) for highly-available and scalable storage file caching. | Yes | Maybe, depends on your<br>render scale requirements |
| [4 Image Builder](#4-image-builder) | Deploys [Compute Gallery](https://learn.microsoft.com/azure/virtual-machines/shared-image-galleries) images that are custom built via the managed [Image Builder](https://learn.microsoft.com/azure/virtual-machines/image-builder-overview) service. | No, specify your custom *imageId* reference [here](https://github.com/Azure/Avere/blob/main/src/terraform/examples/e2e/6.render.farm/config.auto.tfvars#L10) | No, specify your custom *imageId* reference [here](https://github.com/Azure/Avere/blob/main/src/terraform/examples/e2e/6.render.farm/config.auto.tfvars#L10) |
| [5 Render Manager](#5-render-manager) | Deploys [Virtual Machines](https://learn.microsoft.com/azure/virtual-machines) for render farm job scheduling with your custom render image. | No, continue to use your current render manager | No, continue to use your current render manager |
@ -22,7 +22,7 @@ The following *core principles* are implemented throughout the Azure Artist Anyw
| [8 GitOps](#8-gitops) | Enables [Terraform Plan](https://www.terraform.io/cli/commands/plan) and [Apply](https://www.terraform.io/cli/commands/apply) workflows via [GitHub Actions](https://docs.github.com/actions) triggered by [Pull Requests](https://docs.github.com/pull-requests). | No | No |
| [9 Render](#9-render) | Sample render farm job submission from [Linux](https://learn.microsoft.com/azure/virtual-machines/linux/overview) and/or [Windows](https://learn.microsoft.com/azure/virtual-machines/windows/overview) remote artist workstations. | No | No |
For example, the following sample images were rendering in Azure via the Azure Artist Anywhere (AAA) solution deployment framework.
For example, the following sample images were [rendered on Azure](https://user-images.githubusercontent.com/22285652/202864874-e48070dc-deaa-45ee-a8ed-60ff401955f0.mp4) via the Azure Artist Anywhere (AAA) solution deployment framework.
<p align="center">
<img src=".github/images/blender-splash-3.4.png" alt="Blender Splash" width="1024" />
@ -76,7 +76,6 @@ For Azure role assignment instructions, refer to either the Azure [portal](https
1. Run `cd ~/e2e/2.storage` in a local shell (Bash or PowerShell)
1. Review and edit the config values in `config.auto.tfvars` for your deployment.
* Optionally set the `enableSampleDataLoad` config value to true to enable upload of sample files to be renderd into Azure Blob and/or File storage
1. Run `terraform init -backend-config ../0.global/module/backend.config` to initialize the current local directory (append `-upgrade` if older providers are detected)
1. Run `terraform apply` to generate the Terraform deployment [Plan](https://www.terraform.io/docs/cli/run/index.html#planning) (append `-destroy` to delete Azure resources)
1. Review and confirm the displayed Terraform deployment plan to add, change and/or destroy Azure resources
@ -238,7 +237,7 @@ TBD
*The following job submission command can be submitted from a **Linux** and/or **Windows** artist workstation.*
<p><code>
deadlinecommand -SubmitCommandLineJob -name blender-splash -executable blender -arguments "-b -y -noaudio /mnt/show/read/blender/3.4/splash.blend --render-output /mnt/show/write/blender/3.4/splash#### --render-frame 1"
deadlinecommand -SubmitCommandLineJob -name blender-splash -executable blender -arguments "-b -a -y -noaudio /mnt/show/read/blender/3.4/splash.blend --render-output /mnt/show/write/blender/3.4/splash####"
</code></p>
#### 9.1.3 [Royal Render](https://royalrender.de) Render Farm (*Windows*)
@ -252,7 +251,7 @@ TBD
*The following job submission command can be submitted from a **Linux** and/or **Windows** artist workstation.*
<p><code>
deadlinecommand -SubmitCommandLineJob -name blender-splash -executable blender -arguments "-b -y -noaudio R:\blender\3.4\splash.blend --render-output W:\blender\3.4\splash#### --render-frame 1"
deadlinecommand -SubmitCommandLineJob -name blender-splash -executable blender -arguments "-b -a -y -noaudio R:\blender\3.4\splash.blend --render-output W:\blender\3.4\splash####"
</code></p>
### 9.2 [Physically-Based Ray Tracer (PBRT)](https://pbrt.org) [Moana Island](https://www.disneyanimation.com/resources/moana-island-scene/)