{ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "metadata": { "_generator": { "name": "bicep", "version": "0.7.4.23292", "templateHash": "647616438664145023" } }, "variables": { "shareName": "open-test-data", "storageAccountName": "[uniqueString(resourceGroup().id, deployment().name)]", "roleAssignmentName": "[guid(format('{0}contributor', resourceGroup().name))]", "acrName": "[uniqueString(resourceGroup().id, deployment().name)]", "managedIdentityName": "[uniqueString(resourceGroup().id, deployment().name)]", "contributorRoleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", "containerImageName": "osdu-data-load-tno:latest", "templateSpecName": "osdu-data-load-tno", "templateSpecVersionName": "1.0" }, "resources": [ { "type": "Microsoft.Storage/storageAccounts/fileServices/shares", "apiVersion": "2021-04-01", "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', variables('shareName'))]", "dependsOn": [ "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]" ] }, { "type": "Microsoft.Storage/storageAccounts/fileServices/shares", "apiVersion": "2021-04-01", "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'output')]", "dependsOn": [ "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]" ] }, { "type": "Microsoft.Storage/storageAccounts/fileServices", "apiVersion": "2021-04-01", "name": "[format('{0}/{1}', variables('storageAccountName'), 'default')]", "dependsOn": [ "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" ] }, { "type": "Microsoft.ManagedIdentity/userAssignedIdentities", "apiVersion": "2021-09-30-preview", "name": "[variables('managedIdentityName')]", "location": "[resourceGroup().location]" }, { "type": "Microsoft.Storage/storageAccounts", "apiVersion": "2021-04-01", "name": "[variables('storageAccountName')]", "location": "[resourceGroup().location]", "sku": { "name": "Standard_LRS" }, "kind": "StorageV2" }, { "type": "Microsoft.ContainerRegistry/registries", "apiVersion": "2021-06-01-preview", "name": "[variables('acrName')]", "location": "[resourceGroup().location]", "sku": { "name": "Standard" }, "identity": { "type": "UserAssigned", "userAssignedIdentities": { "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {} } }, "properties": { "anonymousPullEnabled": true }, "dependsOn": [ "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]" ] }, { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "2020-10-01-preview", "name": "[variables('roleAssignmentName')]", "properties": { "description": "Managed identity access for the RG", "roleDefinitionId": "[variables('contributorRoleDefinitionId')]", "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))).principalId]", "principalType": "ServicePrincipal" }, "dependsOn": [ "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]" ] }, { "type": "Microsoft.Resources/deploymentScripts", "apiVersion": "2020-10-01", "name": "fileshare-load", "location": "[resourceGroup().location]", "kind": "AzureCLI", "identity": { "type": "UserAssigned", "userAssignedIdentities": { "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {} } }, "properties": { "azCliVersion": "2.37.0", "timeout": "PT2H", "retentionInterval": "PT1H", "cleanupPreference": "OnSuccess", "environmentVariables": [ { "name": "AZURE_STORAGE_ACCOUNT", "value": "[variables('storageAccountName')]" }, { "name": "AZURE_STORAGE_KEY", "secureValue": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]" }, { "name": "AZURE_STORAGE_SHARE", "value": "open-test-data" } ], "scriptContent": " #!/bin/bash\n set -e\n LOG=script_log.txt\n FILE_NAME=open-test-data.gz\n SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n DATA_DIR=\"/tmp/${AZURE_STORAGE_SHARE}\"\n\n echo -e \"Retrieving data from OSDU...\" 2>&1 | tee -a $LOG\n wget -O $FILE_NAME https://community.opengroup.org/osdu/platform/data-flow/data-loading/open-test-data/-/archive/Azure/M8/open-test-data-Azure-M8.tar.gz 2>&1 | tee -a $LOG\n\n # Create Directory structure\n echo -e \"Creating Directory structure...\" 2>&1 | tee -a $LOG\n mkdir -p $DATA_DIR/datasets/documents\n mkdir -p $DATA_DIR/datasets/markers\n mkdir -p $DATA_DIR/datasets/trajectories\n mkdir -p $DATA_DIR/datasets/well-logs\n mkdir -p $DATA_DIR/schema\n mkdir -p $DATA_DIR/templates\n mkdir -p $DATA_DIR/TNO/contrib\n mkdir -p $DATA_DIR/TNO/provided\n ls -l $DATA_DIR | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/documents --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/USGS_docs\n echo -e \"Extracted Dataset Documents\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/markers --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/markers\n echo -e \"Extracted Dataset Markers\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/trajectories --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/trajectories\n echo -e \"Extracted Dataset Trajectories\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/well-logs --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/well-logs \n echo -e \"Extracted Dataset Well Logs\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/schema --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/3-schema\n echo -e \"Extracted Schemas\" 2>&1 | tee -a $LOG\n \n tar -xzvf $FILE_NAME -C $DATA_DIR/templates --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/5-templates\n echo -e \"Extracted Templates\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/contrib --strip-components=5 open-test-data-Azure-M8/rc--3.0.0/1-data/3-provided/TNO\n echo -e \"Extracted TNO Contrib\" 2>&1 | tee -a $LOG\n\n tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/provided --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/4-instances/TNO\n echo -e \"Extracted TNO Provided\" 2>&1 | tee -a $LOG\n\n # Upload to Azure Storage\n echo -e \"Files Uploading...\" 2>&1 | tee -a $LOG\n az storage file upload-batch --destination $AZURE_STORAGE_SHARE --source $DATA_DIR | tee -a $LOG\n echo -e \"Upload Complete\" 2>&1 | tee -a $LOG\n\n echo '{\"status\": {\"download\": \"Success\", \"extract\": \"Success\", \"upload\": \"Success\"}}' | jq > $AZ_SCRIPTS_OUTPUT_PATH\n " }, "dependsOn": [ "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]" ] }, { "type": "Microsoft.Resources/deploymentScripts", "apiVersion": "2020-10-01", "name": "build-and-push-image", "location": "[resourceGroup().location]", "kind": "AzureCLI", "identity": { "type": "UserAssigned", "userAssignedIdentities": { "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {} } }, "properties": { "azCliVersion": "2.37.0", "timeout": "PT2H", "retentionInterval": "PT1H", "cleanupPreference": "OnSuccess", "environmentVariables": [ { "name": "CONTAINER_IMAGE_NAME", "value": "[variables('containerImageName')]" }, { "name": "REGISTRY_NAME", "value": "[variables('acrName')]" }, { "name": "AZURE_TENANT", "value": "[subscription().tenantId]" } ], "scriptContent": " #!/bin/bash\n set -e\n LOG=script_log.txt\n FILE_NAME=main.zip\n SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n DATA_DIR=\"/tmp/osdu-data-load-tno-main\"\n\n echo -e \"Retrieving Source...\" 2>&1 | tee -a $LOG\n wget -O $FILE_NAME https://github.com/Azure/osdu-data-load-tno/archive/refs/heads/main.zip 2>&1 | tee -a $LOG\n unzip $FILE_NAME -d /tmp\n echo -e \"Extracted Source\" 2>&1 | tee -a $LOG\n\n echo -e \"Build and Import Image: ${CONTAINER_IMAGE_NAME} into ACR: ${REGISTRY_NAME}\" 2>&1 | tee -a $LOG\n az acr build --build-arg AZURE_TENANT=$AZURE_TENANT --image ${CONTAINER_IMAGE_NAME} --registry ${REGISTRY_NAME} --file ${DATA_DIR}/Dockerfile $DATA_DIR | tee -a $LOG\n\n echo '{\"status\": {\"download\": \"Success\", \"extract\": \"Success\", \"docker\": \"Success\"}}' | jq > $AZ_SCRIPTS_OUTPUT_PATH\n " }, "dependsOn": [ "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]" ] }, { "type": "Microsoft.Resources/templateSpecs", "apiVersion": "2021-05-01", "name": "[variables('templateSpecName')]", "location": "[resourceGroup().location]", "properties": { "description": "Load OSDU with Open Test Data.", "displayName": "Open Test Data Load" } }, { "type": "Microsoft.Resources/templateSpecs/versions", "apiVersion": "2021-05-01", "name": "[format('{0}/{1}', variables('templateSpecName'), variables('templateSpecVersionName'))]", "location": "[resourceGroup().location]", "properties": { "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "endpoint": { "type": "string", "metadata": { "description": "FQDN Endpoint ie: myosdu.mydomain.com" } }, "dataPartition": { "type": "string", "metadata": { "description": "Data Partition name ie: mypartition" } }, "viewerGroup": { "type": "string", "metadata": { "description": "Reader Group ie: data.default.viewers@contoso.com" } }, "ownerGroup": { "type": "string", "metadata": { "description": "Owner Group ie: data.default.owners@contoso.com" } }, "legalTag": { "type": "string", "metadata": { "description": "Legal Tag Name ie: legal-tag-load" } }, "clientId": { "type": "string", "metadata": { "description": "Client Id." } }, "clientSecret": { "type": "string", "metadata": { "description": "Client Secret." } } }, "variables": { "acrName": "[variables('acrName')]", "imageName": "[variables('containerImageName')]" }, "resources": [ { "type": "Microsoft.ContainerInstance/containerGroups", "apiVersion": "2021-09-01", "name": "[[concat('data-load-', parameters('name'))]", "location": "[[parameters('location')]", "properties": { "containers": [ { "name": "deploy", "properties": { "image": "[[concat(variables('acrName'), '.azurecr.io/', variables('imageName'))]", "command": [], "environmentVariables": [ { "name": "OSDU_ENDPOINT", "value": "[[concat('https://'), parameters('endpoint')]" }, { "name": "DATA_PARTITION", "value": "[[parameters('dataPartition')]" }, { "name": "VIEWER_GROUP", "value": "[[parameters('viewerGroup')]" }, { "name": "OWNER_GROUP", "value": "[[parameters('ownerGroup')]" }, { "name": "LEGAL_TAG", "value": "[[parameters('legalTag')]" }, { "name": "CLIENT_ID", "value": "[[parameters('clientId')]" }, { "name": "CLIENT_SECRET", "value": "[[parameters('clientSecret')]" } ], "ports": [], "volumeMounts": [ { "name": "opentestdata", "mountPath": "/app/open-test-data" }, { "name": "output", "mountPath": "/app/output" } ], "resources": { "requests": { "cpu": 4, "memoryInGB": 16 } } } } ], "osType": "Linux", "restartPolicy": "Never", "volumes": [ { "name": "opentestdata", "azureFile": { "shareName": "[variables('shareName')]", "storageAccountName": "[variables('storageAccountName')]", "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]" } }, { "name": "output", "azureFile": { "shareName": "output", "storageAccountName": "[variables('storageAccountName')]", "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]" } } ] } } ] } }, "dependsOn": [ "[resourceId('Microsoft.Resources/templateSpecs', variables('templateSpecName'))]", "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" ] } ], "outputs": { "scriptLogs": { "type": "string", "value": "[reference(format('{0}/logs/default', resourceId('Microsoft.Resources/deploymentScripts', 'fileshare-load')), '2020-10-01', 'Full').properties.log]" } } }