2022-06-29 00:31:15 +03:00
|
|
|
{
|
|
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
|
|
"contentVersion": "1.0.0.0",
|
|
|
|
"metadata": {
|
|
|
|
"_generator": {
|
|
|
|
"name": "bicep",
|
|
|
|
"version": "0.7.4.23292",
|
2022-06-29 21:22:03 +03:00
|
|
|
"templateHash": "2850344384487342811"
|
2022-06-29 00:31:15 +03:00
|
|
|
}
|
|
|
|
},
|
|
|
|
"variables": {
|
|
|
|
"shareName": "open-test-data",
|
|
|
|
"storageAccountName": "[uniqueString(resourceGroup().id, deployment().name)]",
|
|
|
|
"roleAssignmentName": "[guid(format('{0}contributor', resourceGroup().name))]",
|
|
|
|
"acrName": "[uniqueString(resourceGroup().id, deployment().name)]",
|
|
|
|
"managedIdentityName": "[uniqueString(resourceGroup().id, deployment().name)]",
|
2022-06-29 19:42:20 +03:00
|
|
|
"contributorRoleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
|
2022-06-29 21:11:14 +03:00
|
|
|
"containerImageName": "osdu-data-load-tno:latest",
|
|
|
|
"templateSpecName": "OpenTestDataLoad",
|
|
|
|
"templateSpecVersionName": "1.0"
|
2022-06-29 00:31:15 +03:00
|
|
|
},
|
|
|
|
"resources": [
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Storage/storageAccounts/fileServices/shares",
|
|
|
|
"apiVersion": "2021-04-01",
|
|
|
|
"name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', variables('shareName'))]",
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]"
|
|
|
|
]
|
|
|
|
},
|
2022-06-29 19:42:20 +03:00
|
|
|
{
|
|
|
|
"type": "Microsoft.Storage/storageAccounts/fileServices/shares",
|
|
|
|
"apiVersion": "2021-04-01",
|
|
|
|
"name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'output')]",
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]"
|
|
|
|
]
|
|
|
|
},
|
2022-06-29 00:31:15 +03:00
|
|
|
{
|
|
|
|
"type": "Microsoft.Storage/storageAccounts/fileServices",
|
|
|
|
"apiVersion": "2021-04-01",
|
|
|
|
"name": "[format('{0}/{1}', variables('storageAccountName'), 'default')]",
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
|
|
|
|
]
|
|
|
|
},
|
2022-06-29 19:42:20 +03:00
|
|
|
{
|
|
|
|
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
|
|
|
|
"apiVersion": "2021-09-30-preview",
|
|
|
|
"name": "[variables('managedIdentityName')]",
|
|
|
|
"location": "[resourceGroup().location]"
|
|
|
|
},
|
2022-06-29 00:31:15 +03:00
|
|
|
{
|
|
|
|
"type": "Microsoft.Storage/storageAccounts",
|
|
|
|
"apiVersion": "2021-04-01",
|
|
|
|
"name": "[variables('storageAccountName')]",
|
2022-06-29 19:42:20 +03:00
|
|
|
"location": "[resourceGroup().location]",
|
2022-06-29 00:31:15 +03:00
|
|
|
"sku": {
|
|
|
|
"name": "Standard_LRS"
|
|
|
|
},
|
|
|
|
"kind": "StorageV2"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.ContainerRegistry/registries",
|
|
|
|
"apiVersion": "2021-06-01-preview",
|
|
|
|
"name": "[variables('acrName')]",
|
2022-06-29 19:42:20 +03:00
|
|
|
"location": "[resourceGroup().location]",
|
2022-06-29 00:31:15 +03:00
|
|
|
"sku": {
|
|
|
|
"name": "Standard"
|
|
|
|
},
|
|
|
|
"identity": {
|
|
|
|
"type": "UserAssigned",
|
|
|
|
"userAssignedIdentities": {
|
|
|
|
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"properties": {
|
|
|
|
"anonymousPullEnabled": true
|
|
|
|
},
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Authorization/roleAssignments",
|
|
|
|
"apiVersion": "2020-10-01-preview",
|
|
|
|
"name": "[variables('roleAssignmentName')]",
|
|
|
|
"properties": {
|
|
|
|
"description": "Managed identity access for the RG",
|
|
|
|
"roleDefinitionId": "[variables('contributorRoleDefinitionId')]",
|
2022-06-29 19:42:20 +03:00
|
|
|
"principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))).principalId]",
|
2022-06-29 00:31:15 +03:00
|
|
|
"principalType": "ServicePrincipal"
|
|
|
|
},
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
|
|
"apiVersion": "2020-10-01",
|
2022-06-29 19:42:20 +03:00
|
|
|
"name": "fileshare-load",
|
|
|
|
"location": "[resourceGroup().location]",
|
2022-06-29 00:31:15 +03:00
|
|
|
"kind": "AzureCLI",
|
2022-06-29 19:42:20 +03:00
|
|
|
"identity": {
|
|
|
|
"type": "UserAssigned",
|
|
|
|
"userAssignedIdentities": {
|
|
|
|
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
|
|
|
|
}
|
|
|
|
},
|
2022-06-29 00:31:15 +03:00
|
|
|
"properties": {
|
|
|
|
"azCliVersion": "2.37.0",
|
|
|
|
"timeout": "PT2H",
|
|
|
|
"retentionInterval": "PT1H",
|
|
|
|
"cleanupPreference": "OnSuccess",
|
|
|
|
"environmentVariables": [
|
|
|
|
{
|
|
|
|
"name": "AZURE_STORAGE_ACCOUNT",
|
|
|
|
"value": "[variables('storageAccountName')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "AZURE_STORAGE_KEY",
|
|
|
|
"secureValue": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "AZURE_STORAGE_SHARE",
|
|
|
|
"value": "open-test-data"
|
|
|
|
}
|
|
|
|
],
|
2022-06-29 21:11:14 +03:00
|
|
|
"scriptContent": " #!/bin/bash\n set -e\n LOG=script_log.txt\n FILE_NAME=open-test-data.gz\n SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n DATA_DIR=\"/tmp/${AZURE_STORAGE_SHARE}\"\n\n echo -e \"Retrieving data from OSDU...\" 2>&1 | tee -a $LOG\n wget -O $FILE_NAME https://community.opengroup.org/osdu/platform/data-flow/data-loading/open-test-data/-/archive/Azure/M8/open-test-data-Azure-M8.tar.gz 2>&1 | tee -a $LOG\n\n # Create Directory structure\n echo -e \"Creating Directory structure...\" 2>&1 | tee -a $LOG\n mkdir -p $DATA_DIR/datasets/documents\n mkdir -p $DATA_DIR/datasets/markers\n mkdir -p $DATA_DIR/datasets/trajectories\n mkdir -p $DATA_DIR/datasets/well-logs\n mkdir -p $DATA_DIR/schema\n mkdir -p $DATA_DIR/templates\n mkdir -p $DATA_DIR/TNO/contrib\n mkdir -p $DATA_DIR/TNO/provided\n ls -l $DATA_DIR | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/documents --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/USGS_docs\n echo -e \"Extracted Dataset Documents\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/markers --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/markers\n echo -e \"Extracted Dataset Markers\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/trajectories --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/trajectories\n echo -e \"Extracted Dataset Trajectories\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/well-logs --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/well-logs \n echo -e \"Extracted Dataset Well Logs\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/schema --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/3-schema\n echo -e \"Extracted Schemas\" 2>&1 | tee -a $LOG\n \n tar -xzvf $FILE_NAME -C $DATA_DIR/templates --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/5-templates\n echo -e \"Extracted Templates\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/contrib --strip-components=5 open-test-data-Azure-M8/rc--3.0.0/1-data/3-provided/TNO\n echo -e \"Extracted TNO Contrib\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/provided --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/4-instances/TNO\n echo -e \"Extracted TNO Provided\" 2>&1 | tee -a $LOG\n\n # Upload to Azure Storage\n echo -e \"Files Uploading...\" 2>&1 | tee -a $LOG\n az storage file upload-batch --destination $AZURE_STORAGE_SHARE --source $DATA_DIR | tee -a $LOG\n echo -e \"Upload Complete\" 2>&1 | tee -a $LOG\n\n echo '{\"status\": {\"download\": \"Success\", \"extract\": \"Success\", \"upload\": \"Success\"}}' | jq > $AZ_SCRIPTS_OUTPUT_PATH\n "
|
2022-06-29 00:31:15 +03:00
|
|
|
},
|
|
|
|
"dependsOn": [
|
2022-06-29 19:42:20 +03:00
|
|
|
"[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]",
|
|
|
|
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
|
|
"apiVersion": "2020-10-01",
|
|
|
|
"name": "build-and-push-image",
|
|
|
|
"location": "[resourceGroup().location]",
|
|
|
|
"kind": "AzureCLI",
|
|
|
|
"identity": {
|
|
|
|
"type": "UserAssigned",
|
|
|
|
"userAssignedIdentities": {
|
|
|
|
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"properties": {
|
|
|
|
"azCliVersion": "2.30.0",
|
|
|
|
"timeout": "PT15M",
|
|
|
|
"retentionInterval": "PT1H",
|
|
|
|
"cleanupPreference": "OnSuccess",
|
|
|
|
"environmentVariables": [
|
|
|
|
{
|
|
|
|
"name": "CONTAINER_IMAGE_NAME",
|
|
|
|
"value": "[variables('containerImageName')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "REGISTRY_NAME",
|
|
|
|
"value": "[variables('acrName')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "AZURE_TENANT",
|
|
|
|
"value": "[subscription().tenantId]"
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"scriptContent": " #!/bin/bash\n set -e\n LOG=script_log.txt\n FILE_NAME=main.zip\n SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n DATA_DIR=\"/tmp/osdu-data-load-tno-main\"\n\n echo -e \"Retrieving Source...\" 2>&1 | tee -a $LOG\n wget -O $FILE_NAME https://github.com/Azure/osdu-data-load-tno/archive/refs/heads/main.zip 2>&1 | tee -a $LOG\n unzip $FILE_NAME -d /tmp\n echo -e \"Extracted Source\" 2>&1 | tee -a $LOG\n\n echo -e \"Build and Import Image: ${CONTAINER_IMAGE_NAME} into ACR: ${REGISTRY_NAME}\" 2>&1 | tee -a $LOG\n az acr build --build-arg AZURE_TENANT=$AZURE_TENANT --image ${CONTAINER_IMAGE_NAME} --registry ${REGISTRY_NAME} --file ${DATA_DIR}/Dockerfile $DATA_DIR | tee -a $LOG\n\n result=$(az acr repository list -n ${REGISTRY_NAME} -ojson)\n echo $result | jq -c > $AZ_SCRIPTS_OUTPUT_PATH\n "
|
|
|
|
},
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
|
2022-06-29 00:31:15 +03:00
|
|
|
]
|
2022-06-29 21:11:14 +03:00
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Resources/templateSpecs",
|
|
|
|
"apiVersion": "2021-05-01",
|
|
|
|
"name": "[variables('templateSpecName')]",
|
|
|
|
"location": "[resourceGroup().location]",
|
|
|
|
"properties": {
|
|
|
|
"description": "Load OSDU with Open Test Data.",
|
|
|
|
"displayName": "Open Test Data Load"
|
|
|
|
}
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"type": "Microsoft.Resources/templateSpecs/versions",
|
|
|
|
"apiVersion": "2021-05-01",
|
|
|
|
"name": "[format('{0}/{1}', variables('templateSpecName'), variables('templateSpecVersionName'))]",
|
|
|
|
"location": "[resourceGroup().location]",
|
|
|
|
"properties": {
|
|
|
|
"mainTemplate": {
|
|
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
|
|
"contentVersion": "1.0.0.0",
|
|
|
|
"parameters": {
|
|
|
|
"endpoint": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "FQDN Endpoint ie: myosdu.mydomain.com"
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"dataPartition": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Data Partition name ie: mypartition"
|
2022-06-29 21:22:03 +03:00
|
|
|
}
|
2022-06-29 21:11:14 +03:00
|
|
|
},
|
|
|
|
"viewerGroup": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Reader Group ie: data.default.viewers@contoso.com"
|
2022-06-29 21:22:03 +03:00
|
|
|
}
|
2022-06-29 21:11:14 +03:00
|
|
|
},
|
|
|
|
"ownerGroup": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Owner Group ie: data.default.owners@contoso.com"
|
2022-06-29 21:22:03 +03:00
|
|
|
}
|
2022-06-29 21:11:14 +03:00
|
|
|
},
|
|
|
|
"legalTag": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Legal Tag Name ie: legal-tag-load"
|
2022-06-29 21:22:03 +03:00
|
|
|
}
|
2022-06-29 21:11:14 +03:00
|
|
|
},
|
|
|
|
"clientId": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Client Id."
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"clientSecret": {
|
|
|
|
"type": "string",
|
|
|
|
"metadata": {
|
|
|
|
"description": "Client Secret."
|
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"variables": {
|
|
|
|
"acrName": "[variables('acrName')]",
|
|
|
|
"imageName": "[variables('containerImageName')]"
|
|
|
|
},
|
|
|
|
"resources": [
|
|
|
|
{
|
|
|
|
"type": "Microsoft.ContainerInstance/containerGroups",
|
|
|
|
"apiVersion": "2021-09-01",
|
|
|
|
"name": "[[concat('data-load-', parameters('name'))]",
|
|
|
|
"location": "[[parameters('location')]",
|
|
|
|
"properties": {
|
|
|
|
"containers": [
|
|
|
|
{
|
|
|
|
"name": "deploy",
|
|
|
|
"properties": {
|
|
|
|
"image": "[[concat(variables('acrName'), '.azurecr.io/', variables('imageName'))]",
|
|
|
|
"command": [],
|
|
|
|
"environmentVariables": [
|
|
|
|
{
|
|
|
|
"name": "OSDU_ENDPOINT",
|
|
|
|
"value": "[[concat('https://'), parameters('endpoint')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "DATA_PARTITION",
|
|
|
|
"value": "[[parameters('dataPartition')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "VIEWER_GROUP",
|
|
|
|
"value": "[[parameters('viewerGroup')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "OWNER_GROUP",
|
|
|
|
"value": "[[parameters('ownerGroup')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "LEGAL_TAG",
|
|
|
|
"value": "[[parameters('legalTag')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "CLIENT_ID",
|
|
|
|
"value": "[[parameters('clientId')]"
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"name": "CLIENT_SECRET",
|
|
|
|
"value": "[[parameters('clientSecret')]"
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"ports": [],
|
|
|
|
"resources": {
|
|
|
|
"requests": {
|
|
|
|
"cpu": 4,
|
|
|
|
"memoryInGB": 16
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"osType": "Linux",
|
|
|
|
"restartPolicy": "Never"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"dependsOn": [
|
|
|
|
"[resourceId('Microsoft.Resources/templateSpecs', variables('templateSpecName'))]"
|
|
|
|
]
|
2022-06-29 00:31:15 +03:00
|
|
|
}
|
|
|
|
],
|
|
|
|
"outputs": {
|
|
|
|
"scriptLogs": {
|
|
|
|
"type": "string",
|
2022-06-29 19:42:20 +03:00
|
|
|
"value": "[reference(format('{0}/logs/default', resourceId('Microsoft.Resources/deploymentScripts', 'fileshare-load')), '2020-10-01', 'Full').properties.log]"
|
2022-06-29 00:31:15 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|