osdu-data-load-tno/azuredeploy.json

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "metadata": {
    "_generator": {
      "name": "bicep",
      "version": "0.7.4.23292",
      "templateHash": "5123147708032134211"
    }
  },
  "variables": {
    "shareName": "open-test-data",
    "storageAccountName": "[uniqueString(resourceGroup().id, deployment().name)]",
    "roleAssignmentName": "[guid(format('{0}contributor', resourceGroup().name))]",
    "acrName": "[uniqueString(resourceGroup().id, deployment().name)]",
    "managedIdentityName": "[uniqueString(resourceGroup().id, deployment().name)]",
    "contributorRoleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
    "containerImageName": "osdu-data-load-tno:latest",
    "templateSpecName": "osdu-data-load-tno",
    "templateSpecVersionName": "1.0"
  },
  "resources": [
    {
      "type": "Microsoft.Storage/storageAccounts/fileServices/shares",
      "apiVersion": "2021-04-01",
      "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', variables('shareName'))]",
      "dependsOn": [
        "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]"
      ]
    },
    {
      "type": "Microsoft.Storage/storageAccounts/fileServices/shares",
      "apiVersion": "2021-04-01",
      "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'output')]",
      "dependsOn": [
        "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('storageAccountName'), 'default')]"
      ]
    },
    {
      "type": "Microsoft.Storage/storageAccounts/fileServices",
      "apiVersion": "2021-04-01",
      "name": "[format('{0}/{1}', variables('storageAccountName'), 'default')]",
      "dependsOn": [
        "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
      ]
    },
    {
      "type": "Microsoft.ManagedIdentity/userAssignedIdentities",
      "apiVersion": "2021-09-30-preview",
      "name": "[variables('managedIdentityName')]",
      "location": "[resourceGroup().location]"
    },
    {
      "type": "Microsoft.Storage/storageAccounts",
      "apiVersion": "2021-04-01",
      "name": "[variables('storageAccountName')]",
      "location": "[resourceGroup().location]",
      "sku": {
        "name": "Standard_LRS"
      },
      "kind": "StorageV2"
    },
    {
      "type": "Microsoft.ContainerRegistry/registries",
      "apiVersion": "2021-06-01-preview",
      "name": "[variables('acrName')]",
      "location": "[resourceGroup().location]",
      "sku": {
        "name": "Standard"
      },
      "identity": {
        "type": "UserAssigned",
        "userAssignedIdentities": {
          "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
        }
      },
      "properties": {
        "anonymousPullEnabled": true
      },
      "dependsOn": [
        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
      ]
    },
    {
      "type": "Microsoft.Authorization/roleAssignments",
      "apiVersion": "2020-10-01-preview",
      "name": "[variables('roleAssignmentName')]",
      "properties": {
        "description": "Managed identity access for the RG",
        "roleDefinitionId": "[variables('contributorRoleDefinitionId')]",
        "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))).principalId]",
        "principalType": "ServicePrincipal"
      },
      "dependsOn": [
        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
      ]
    },
    {
      "type": "Microsoft.Resources/deploymentScripts",
      "apiVersion": "2020-10-01",
      "name": "fileshare-load",
      "location": "[resourceGroup().location]",
      "kind": "AzureCLI",
      "identity": {
        "type": "UserAssigned",
        "userAssignedIdentities": {
          "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
        }
      },
      "properties": {
        "azCliVersion": "2.37.0",
        "timeout": "PT2H",
        "retentionInterval": "PT1H",
        "cleanupPreference": "OnSuccess",
        "environmentVariables": [
          {
            "name": "AZURE_STORAGE_ACCOUNT",
            "value": "[variables('storageAccountName')]"
          },
          {
            "name": "AZURE_STORAGE_KEY",
            "secureValue": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]"
          },
          {
            "name": "AZURE_STORAGE_SHARE",
            "value": "open-test-data"
          }
        ],
        "scriptContent": "      #!/bin/bash\n      set -e\n      LOG=script_log.txt\n      FILE_NAME=open-test-data.gz\n      SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n      DATA_DIR=\"/tmp/${AZURE_STORAGE_SHARE}\"\n\n      echo -e \"Retrieving data from OSDU...\" 2>&1 | tee -a $LOG\n      wget -O $FILE_NAME https://community.opengroup.org/osdu/platform/data-flow/data-loading/open-test-data/-/archive/Azure/M8/open-test-data-Azure-M8.tar.gz 2>&1 | tee -a $LOG\n\n      # Create Directory structure\n      echo -e \"Creating Directory structure...\" 2>&1 | tee -a $LOG\n      mkdir -p $DATA_DIR/datasets/documents\n      mkdir -p $DATA_DIR/datasets/markers\n      mkdir -p $DATA_DIR/datasets/trajectories\n      mkdir -p $DATA_DIR/datasets/well-logs\n      mkdir -p $DATA_DIR/schema\n      mkdir -p $DATA_DIR/templates\n      mkdir -p $DATA_DIR/TNO/contrib\n      mkdir -p $DATA_DIR/TNO/provided\n      ls -l $DATA_DIR | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/documents --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/USGS_docs\n      echo -e \"Extracted Dataset Documents\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/markers --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/markers\n      echo -e \"Extracted Dataset Markers\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/trajectories --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/trajectories\n      echo -e \"Extracted Dataset Trajectories\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/datasets/well-logs --strip-components=5 open-test-data-Azure-M8/rc--1.0.0/1-data/3-provided/well-logs   \n      echo -e \"Extracted Dataset Well Logs\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/schema --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/3-schema\n      echo -e \"Extracted Schemas\" 2>&1 | tee -a $LOG\n      \n      tar -xzvf $FILE_NAME -C $DATA_DIR/templates --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/5-templates\n      echo -e \"Extracted Templates\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/contrib --strip-components=5 open-test-data-Azure-M8/rc--3.0.0/1-data/3-provided/TNO\n      echo -e \"Extracted TNO Contrib\" 2>&1 | tee -a $LOG\n\n      tar -xzvf $FILE_NAME -C $DATA_DIR/TNO/provided --strip-components=3 open-test-data-Azure-M8/rc--3.0.0/4-instances/TNO\n      echo -e \"Extracted TNO Provided\" 2>&1 | tee -a $LOG\n\n      # Upload to Azure Storage\n      echo -e \"Files Uploading...\" 2>&1 | tee -a $LOG\n      az storage file upload-batch --destination $AZURE_STORAGE_SHARE --source $DATA_DIR | tee -a $LOG\n      echo -e \"Upload Complete\" 2>&1 | tee -a $LOG\n\n      echo '{\"status\": {\"download\": \"Success\", \"extract\": \"Success\", \"upload\": \"Success\"}}' | jq > $AZ_SCRIPTS_OUTPUT_PATH\n    "
      },
      "dependsOn": [
        "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]",
        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
      ]
    },
    {
      "type": "Microsoft.Resources/deploymentScripts",
      "apiVersion": "2020-10-01",
      "name": "build-and-push-image",
      "location": "[resourceGroup().location]",
      "kind": "AzureCLI",
      "identity": {
        "type": "UserAssigned",
        "userAssignedIdentities": {
          "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName')))]": {}
        }
      },
      "properties": {
        "azCliVersion": "2.37.0",
        "timeout": "PT2H",
        "retentionInterval": "PT1H",
        "cleanupPreference": "OnSuccess",
        "environmentVariables": [
          {
            "name": "CONTAINER_IMAGE_NAME",
            "value": "[variables('containerImageName')]"
          },
          {
            "name": "REGISTRY_NAME",
            "value": "[variables('acrName')]"
          },
          {
            "name": "AZURE_TENANT",
            "value": "[subscription().tenantId]"
          }
        ],
        "scriptContent": "      #!/bin/bash\n      set -e\n      LOG=script_log.txt\n      FILE_NAME=main.zip\n      SCRIPT_DIR=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd )\n      DATA_DIR=\"/tmp/osdu-data-load-tno-main\"\n\n      echo -e \"Retrieving Source...\" 2>&1 | tee -a $LOG\n      wget -O $FILE_NAME https://github.com/Azure/osdu-data-load-tno/archive/refs/heads/main.zip 2>&1 | tee -a $LOG\n      unzip $FILE_NAME -d /tmp\n      echo -e \"Extracted Source\" 2>&1 | tee -a $LOG\n\n      echo -e \"Build and Import Image: ${CONTAINER_IMAGE_NAME} into ACR: ${REGISTRY_NAME}\" 2>&1 | tee -a $LOG\n      az acr build --build-arg AZURE_TENANT=$AZURE_TENANT --image ${CONTAINER_IMAGE_NAME} --registry ${REGISTRY_NAME} --file ${DATA_DIR}/Dockerfile $DATA_DIR | tee -a $LOG\n\n      echo '{\"status\": {\"download\": \"Success\", \"extract\": \"Success\", \"docker\": \"Success\"}}' | jq > $AZ_SCRIPTS_OUTPUT_PATH\n    "
      },
      "dependsOn": [
        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('managedIdentityName'))]"
      ]
    },
    {
      "type": "Microsoft.Resources/templateSpecs",
      "apiVersion": "2021-05-01",
      "name": "[variables('templateSpecName')]",
      "location": "[resourceGroup().location]",
      "properties": {
        "description": "Load OSDU with Open Test Data.",
        "displayName": "Open Test Data Load"
      }
    },
    {
      "type": "Microsoft.Resources/templateSpecs/versions",
      "apiVersion": "2021-05-01",
      "name": "[format('{0}/{1}', variables('templateSpecName'), variables('templateSpecVersionName'))]",
      "location": "[resourceGroup().location]",
      "properties": {
        "mainTemplate": {
          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "parameters": {
            "endpoint": {
              "type": "string",
              "metadata": {
                "description": "OSDU API Endpoint (FQDN)"
              }
            },
            "dataPartition": {
              "type": "string",
              "defaultValue": "opendes",
              "metadata": {
                "description": "OSDU Data Partition Name"
              }
            },
            "viewerGroup": {
              "type": "string",
              "defaultValue": "data.default.viewers@contoso.com",
              "metadata": {
                "description": "OSDU Reader Group ACL"
              }
            },
            "ownerGroup": {
              "type": "string",
              "defaultValue": "data.default.owners@contoso.com",
              "metadata": {
                "description": "OSDU Owner Group ACL"
              }
            },
            "legalTag": {
              "type": "string",
              "defaultValue": "open-test-data",
              "metadata": {
                "description": "OSDU Legal Tag Name"
              }
            },
            "clientId": {
              "type": "string",
              "metadata": {
                "description": "OSDU Client Id"
              }
            },
            "clientSecret": {
              "type": "string",
              "metadata": {
                "description": "OSDU Client Secret"
              }
            }
          },
          "variables": {
            "acrName": "[variables('acrName')]",
            "imageName": "[variables('containerImageName')]"
          },
          "resources": [
            {
              "type": "Microsoft.ContainerInstance/containerGroups",
              "apiVersion": "2021-09-01",
              "name": "[[concat('osdu-data-load-', parameters('endpoint'))]",
              "location": "[[parameters('location')]",
              "properties": {
                "containers": [
                  {
                    "name": "load",
                    "properties": {
                      "image": "[[concat(variables('acrName'), '.azurecr.io/', variables('imageName'))]",
                      "command": [],
                      "environmentVariables": [
                        {
                          "name": "OSDU_ENDPOINT",
                          "value": "[[concat('https://'), parameters('endpoint')]"
                        },
                        {
                          "name": "DATA_PARTITION",
                          "value": "[[parameters('dataPartition')]"
                        },
                        {
                          "name": "VIEWER_GROUP",
                          "value": "[[parameters('viewerGroup')]"
                        },
                        {
                          "name": "OWNER_GROUP",
                          "value": "[[parameters('ownerGroup')]"
                        },
                        {
                          "name": "LEGAL_TAG",
                          "value": "[[parameters('legalTag')]"
                        },
                        {
                          "name": "CLIENT_ID",
                          "value": "[[parameters('clientId')]"
                        },
                        {
                          "name": "CLIENT_SECRET",
                          "value": "[[parameters('clientSecret')]"
                        }
                      ],
                      "ports": [],
                      "volumeMounts": [
                        {
                          "name": "opentestdata",
                          "mountPath": "/app/open-test-data"
                        },
                        {
                          "name": "output",
                          "mountPath": "/app/output"
                        }
                      ],
                      "resources": {
                        "requests": {
                          "cpu": 4,
                          "memoryInGB": 16
                        }
                      }
                    }
                  }
                ],
                "osType": "Linux",
                "restartPolicy": "Never",
                "volumes": [
                  {
                    "name": "opentestdata",
                    "azureFile": {
                      "shareName": "[variables('shareName')]",
                      "storageAccountName": "[variables('storageAccountName')]",
                      "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]"
                    }
                  },
                  {
                    "name": "output",
                    "azureFile": {
                      "shareName": "output",
                      "storageAccountName": "[variables('storageAccountName')]",
                      "storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2021-04-01').keys[0].value]"
                    }
                  }
                ]
              }
            }
          ]
        }
      },
      "dependsOn": [
        "[resourceId('Microsoft.Resources/templateSpecs', variables('templateSpecName'))]",
        "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
      ]
    }
  ],
  "outputs": {
    "scriptLogs": {
      "type": "string",
      "value": "[reference(format('{0}/logs/default', resourceId('Microsoft.Resources/deploymentScripts', 'fileshare-load')), '2020-10-01', 'Full').properties.log]"
    }
  }
}