From 2ac4bcf617301a092c6ad54db09b7eb74f77d992 Mon Sep 17 00:00:00 2001
From: Matt Lowe <58700052+malowe101@users.noreply.github.com>
Date: Tue, 16 Jun 2020 09:02:57 -0400
Subject: [PATCH] Move content to new folder
Moving content for playbook into a folder instead of being out in the open
---
.../README.md.txt | 57 +-
.../azuredeploy.json | 870 +++++++++---------
2 files changed, 467 insertions(+), 460 deletions(-)
rename Playbooks/{ => Move-LogAnalytics-to-Storage}/README.md.txt (74%)
rename Playbooks/{ => Move-LogAnalytics-to-Storage}/azuredeploy.json (98%)
diff --git a/Playbooks/README.md.txt b/Playbooks/Move-LogAnalytics-to-Storage/README.md.txt
similarity index 74%
rename from Playbooks/README.md.txt
rename to Playbooks/Move-LogAnalytics-to-Storage/README.md.txt
index 37f9e42413..ddb65bd37a 100644
--- a/Playbooks/README.md.txt
+++ b/Playbooks/Move-LogAnalytics-to-Storage/README.md.txt
@@ -1,25 +1,32 @@
-Description:
-This Playbook runs on a daily schedule and moves 89 day old logs per data type to Blob storage in hourly incremements. The result of this Playbook is a structured file explorer within a data container in Azure that allows for easy file exploration and the ability to query the data from storage within a Log Analytics workspace.
-
-To deploy the template:
-- Go to the Azure Portal
-- In the top search bar, type deploy
-- Choose 'deploy a custom template'
-- Choose 'Build my own template in the editor'
-- Copy and paste the JSON from the GitHub template
-- Click save
-- Enter your resource group, workspace name, workspace subscription ID, workspace resource group, your email address, the name of the storage account that is going to be created, the SKU for the storage account, the storage account type, and a name for the container that is going to be built
-- Leave the name as is unless you would like to change it
-- Enter the names of the table that you do not want to back up to storage. We recommend any tables that you do not find useful or that are noisy. An example would be Heartbeat. The format should be 'Table1', 'Table2', etc
-- Click purchase
-
-You will need to authenticate a connection for Azure Monitor within the Playbook:
-
-- Click on the Azure Monitor actions
-- Chances are that the connection didn't establish, click the information icon next to the connection name to authorize the connection, it will bring up a login screen
-- Log in to your account
-- Confirm that the subscription, resource group, and workspace are all correct based on what you entered for the template
-- Make sure that the container that you named is listed under the Azure Blob option so that the logs are routed properly when the Playbook is run
-
-Note:
-- The Logic App will not save if there are any errors so make sure any issue is resolved before saving.
+Description:
+This Playbook runs on a daily schedule and moves 89 day old logs per data type to Blob storage in hourly incremements. The result of this Playbook is a structured file explorer within a data container in Azure that allows for easy file exploration and the ability to query the data from storage within a Log Analytics workspace.
+
+To deploy the template:
+- Go to the Azure Portal
+- In the top search bar, type deploy
+- Choose 'deploy a custom template'
+- Choose 'Build my own template in the editor'
+- Copy and paste the JSON from the GitHub template
+- Click save
+- Enter your resource group, workspace name, workspace subscription ID, workspace resource group, your email address, the name of the storage account that is going to be created, the SKU for the storage account, the storage account type, and a name for the container that is going to be built
+- Leave the name as is unless you would like to change it
+- Enter the names of the table that you do not want to back up to storage. We recommend any tables that you do not find useful or that are noisy. An example would be Heartbeat. The format should be 'Table1', 'Table2', etc
+- Click purchase
+
+You will need to authenticate a connection for Azure Monitor within the Playbook:
+
+- Click on the Azure Monitor actions
+- Chances are that the connection didn't establish, click the information icon next to the connection name to authorize the connection, it will bring up a login screen
+- Log in to your account
+- Confirm that the subscription, resource group, and workspace are all correct based on what you entered for the template
+- Make sure that the container that you named is listed under the Azure Blob option so that the logs are routed properly when the Playbook is run
+
+Note:
+- The Logic App will not save if there are any errors so make sure any issue is resolved before saving.
+
+
+ 
+
+
+
+
\ No newline at end of file
diff --git a/Playbooks/azuredeploy.json b/Playbooks/Move-LogAnalytics-to-Storage/azuredeploy.json
similarity index 98%
rename from Playbooks/azuredeploy.json
rename to Playbooks/Move-LogAnalytics-to-Storage/azuredeploy.json
index 32dbacf0ab..e3a22810ab 100644
--- a/Playbooks/azuredeploy.json
+++ b/Playbooks/Move-LogAnalytics-to-Storage/azuredeploy.json
@@ -1,435 +1,435 @@
-{
- "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
- "contentVersion": "1.0.0.0",
- "parameters": {
- "PlaybookName": {
- "defaultValue": "Move-LogAnalytics-to-Storage",
- "type": "String"
- },
- "ExclusionTable": {
- "defaultValue": "\"Heartbeat\", \"ConfigurationChange\", \"ConfigurationData\", \"ThreatIntelligenceIndicator\", \"IntuneDeviceComplianceOrg\", \"Perf\", \"Update\", \"UpdateSummary\", \"SecurityBaseline\", \"SecurityBaselineSummary\"",
- "type": "String"
- },
- "WorkspaceName": {
- "type": "String"
- },
- "WorkspaceSubscription": {
- "defaultValue": "Your subscription id",
- "type": "String"
- },
- "WorkspaceResourceGroup": {
- "type": "String"
- },
- "EmailAddress": {
- "defaultValue": "Your email address",
- "type": "string"
- },
- "StorageAccount": {
- "defaultValue": "",
- "type": "String"
- },
- "storageAccountSku": {
- "defaultValue": "Standard_LRS",
- "allowedValues": [
- "Standard_LRS",
- "Standard_GRS",
- "Standard_RAGRS",
- "Standard_ZRS",
- "Premium_LRS",
- "Premium_ZRS",
- "Standard_GZRS",
- "Standard_RAGZRS"
- ],
- "type": "String",
- "metadata": {
- "description": "Sku on which to run the Azure Storage account."
- }
- },
- "storageAccountKind": {
- "defaultValue": "StorageV2",
- "allowedValues": [
- "Storage",
- "StorageV2",
- "BlobStorage",
- "FileStorage",
- "BlockBlobStorage"
- ],
- "type": "String",
- "metadata": {
- "description": "Indicates the type of storage account."
- }
- },
- "storageAccountContainerName": {
- "defaultValue": "my-container",
- "type": "String",
- "metadata": {
- "description": "Set the name of the container to create in the Storage account."
- }
- }
- },
- "variables": {
- "azureblob": "[concat('azureblob-', parameters('PlaybookName'))]",
- "azuremonitorlogs": "[concat('azuremonitorlogs-', parameters('PlaybookName'))]",
- "storageaccount": "[concat('storageaccount-', parameters('StorageAccount'))]",
- "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccount'))]",
- "storagecontainer": "[concat('/', parameters('storageAccountContainerName'), '/')]"
- },
- "resources": [
- {
- "type": "Microsoft.Web/connections",
- "apiVersion": "2018-07-01-preview",
- "name": "[variables('storageaccount')]",
- "location": "[resourceGroup().location]",
- "dependsOn": [
- "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccount'))]"
- ],
- "properties": {
- "api": {
- "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureblob')]"
- },
- "parameterValues": {
- "accountName": "[parameters('StorageAccount')]",
- "accessKey": "[listKeys(variables('storageAccountId'), '2019-04-01').keys[0].value]"
- },
- "testLinks": [
- {
- "requestUri": "[uri('https://management.azure.com:443/', concat('subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Web/connections/', variables('storageaccount'), '/extensions/proxy/testconnection?api-version=2018-07-01-preview'))]",
- "method": "get"
- }
- ]
- }
- },
- {
- "type": "Microsoft.Storage/storageAccounts",
- "apiVersion": "2019-04-01",
- "name": "[parameters('StorageAccount')]",
- "location": "[resourceGroup().location]",
- "sku": {
- "name": "[parameters('storageAccountSku')]"
- },
- "kind": "[parameters('storageAccountKind')]",
- "properties": {
- "accessTier": "Cool",
- "supportsHttpsTrafficOnly": true
- },
- "resources": [
- {
- "type": "blobServices/containers",
- "apiVersion": "2019-06-01",
- "name": "[concat('default/', parameters('storageAccountContainerName'))]",
- "dependsOn": [
- "[parameters('StorageAccount')]"
- ],
- "properties": {
- "publicAccess": "Container"
- }
- }
- ]
- },
- {
- "type": "Microsoft.Web/connections",
- "apiVersion": "2016-06-01",
- "name": "[variables('azuremonitorlogs')]",
- "location": "[resourceGroup().location]",
- "properties": {
- "displayName": "[parameters('EmailAddress')]",
- "customParameterValues": {},
- "api": {
- "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azuremonitorlogs')]"
- }
- }
- },
- {
- "type": "Microsoft.Logic/workflows",
- "apiVersion": "2017-07-01",
- "name": "[parameters('PlaybookName')]",
- "location": "[resourceGroup().location]",
- "dependsOn": [
- "[resourceId('Microsoft.Web/connections', variables('storageaccount'))]",
- "[resourceId('Microsoft.Web/connections', variables('azuremonitorlogs'))]"
- ],
- "tags": {
- "LogicAppsCategory": "security"
- },
- "properties": {
- "state": "Enabled",
- "definition": {
- "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
- "contentVersion": "1.0.0.0",
- "parameters": {
- "$connections": {
- "defaultValue": {},
- "type": "Object"
- }
- },
- "triggers": {
- "Recurrence": {
- "recurrence": {
- "frequency": "Day",
- "interval": 1
- },
- "type": "Recurrence"
- }
- },
- "actions": {
- "Compose_Table_Names": {
- "runAfter": {
- "Run_query_and_list_results": [
- "Succeeded"
- ]
- },
- "type": "Compose",
- "inputs": "@body('Run_query_and_list_results')"
- },
- "For_each": {
- "foreach": "@body('Parse_JSON')?['value']",
- "actions": {
- "Set_variable": {
- "runAfter": {
- "Until": [
- "Succeeded"
- ]
- },
- "type": "SetVariable",
- "inputs": {
- "name": "HoursCount",
- "value": 0
- }
- },
- "Until": {
- "actions": {
- "Compose": {
- "runAfter": {
- "Run_query_and_list_results_2": [
- "Succeeded"
- ]
- },
- "type": "Compose",
- "inputs": "@body('Run_query_and_list_results_2')?['value']"
- },
- "Create_blob": {
- "runAfter": {
- "Compose": [
- "Succeeded"
- ]
- },
- "type": "ApiConnection",
- "inputs": {
- "body": "@outputs('Compose')",
- "host": {
- "connection": {
- "name": "@parameters('$connections')['azureblob']['connectionId']"
- }
- },
- "method": "post",
- "path": "/datasets/default/files",
- "queries": {
- "folderPath": "[concat(variables('storagecontainer'), '@{items(''For_each'')?[''DataType'']}')]",
- "name": "@{items('For_each')?['DataType']}-@{variables('StartDate')}-@{variables('HoursCount')}.json",
- "queryParametersSingleEncoded": true
- }
- },
- "runtimeConfiguration": {
- "contentTransfer": {
- "transferMode": "Chunked"
- }
- }
- },
- "Increment_variable": {
- "runAfter": {
- "Create_blob": [
- "Succeeded"
- ]
- },
- "type": "IncrementVariable",
- "inputs": {
- "name": "HoursCount",
- "value": 1
- }
- },
- "Run_query_and_list_results_2": {
- "runAfter": {},
- "type": "ApiConnection",
- "inputs": {
- "body": "@{items('For_each')?['DataType']}\n| where ingestion_time() between(datetime(@{formatDateTime(addHours(variables('StartDate'),variables('HoursCount')))}) .. datetime(@{formatDateTime(addHours(variables('StartDate'),add(int(variables('HoursCount')),1)))}))",
- "host": {
- "connection": {
- "name": "@parameters('$connections')['azuremonitorlogs']['connectionId']"
- }
- },
- "method": "post",
- "path": "/queryData",
- "queries": {
- "resourcegroups": "[parameters('WorkspaceResourceGroup')]",
- "resourcename": "[parameters('WorkspaceName')]",
- "resourcetype": "Log Analytics Workspace",
- "subscriptions": "[parameters('WorkspaceSubscription')]",
- "timerange": "between(datetime(@{formatDateTime(addHours(variables('StartDate'),variables('HoursCount')))}) .. datetime(@{formatDateTime(addHours(variables('StartDate'),add(int(variables('HoursCount')),1)))})"
- }
- }
- }
- },
- "runAfter": {},
- "expression": "@equals(variables('HoursCount'), 24)",
- "limit": {
- "count": 60,
- "timeout": "PT1H"
- },
- "type": "Until"
- }
- },
- "runAfter": {
- "Parse_JSON": [
- "Succeeded"
- ]
- },
- "type": "Foreach",
- "runtimeConfiguration": {
- "concurrency": {
- "repetitions": 1
- }
- }
- },
- "Initialize_EndDate_variable": {
- "runAfter": {
- "Initialize_StartDate_variable": [
- "Succeeded"
- ]
- },
- "type": "InitializeVariable",
- "inputs": {
- "variables": [
- {
- "name": "EndDate",
- "type": "string",
- "value": "@{formatDateTime(addDays(utcNow(), -28),'yyyy-MM-dd')}"
- }
- ]
- }
- },
- "Initialize_ExludedDataTypes_variable": {
- "runAfter": {
- "Initialize_EndDate_variable": [
- "Succeeded"
- ]
- },
- "type": "InitializeVariable",
- "inputs": {
- "variables": [
- {
- "name": "ExcludedDataTypes",
- "type": "string",
- "value": "[parameters('ExclusionTable')]"
- }
- ]
- }
- },
- "Initialize_StartDate_variable": {
- "runAfter": {
- "Initialize_variable_2": [
- "Succeeded"
- ]
- },
- "type": "InitializeVariable",
- "inputs": {
- "variables": [
- {
- "name": "StartDate",
- "type": "string",
- "value": "@{formatDateTime(addDays(utcNow(), -89),'yyyy-MM-dd')}"
- }
- ]
- }
- },
- "Initialize_variable_2": {
- "runAfter": {},
- "type": "InitializeVariable",
- "inputs": {
- "variables": [
- {
- "name": "HoursCount",
- "type": "integer",
- "value": 0
- }
- ]
- }
- },
- "Parse_JSON": {
- "runAfter": {
- "Compose_Table_Names": [
- "Succeeded"
- ]
- },
- "type": "ParseJson",
- "inputs": {
- "content": "@outputs('Compose_Table_Names')",
- "schema": {
- "properties": {
- "value": {
- "items": {
- "properties": {
- "DataType": {
- "type": "string"
- }
- },
- "required": [
- "DataType"
- ],
- "type": "object"
- },
- "type": "array"
- }
- },
- "type": "object"
- }
- }
- },
- "Run_query_and_list_results": {
- "runAfter": {
- "Initialize_ExludedDataTypes_variable": [
- "Succeeded"
- ]
- },
- "type": "ApiConnection",
- "inputs": {
- "body": "let excludedDataTypes = dynamic([@{variables('ExcludedDataTypes')}]);\nUsage \n| distinct DataType\n| where DataType !in (excludedDataTypes)",
- "host": {
- "connection": {
- "name": "@parameters('$connections')['azuremonitorlogs']['connectionId']"
- }
- },
- "method": "post",
- "path": "/queryData",
- "queries": {
- "resourcegroups": "[parameters('WorkspaceResourceGroup')]",
- "resourcename": "[parameters('WorkspaceName')]",
- "resourcetype": "Log Analytics Workspace",
- "subscriptions": "[parameters('WorkspaceSubscription')]",
- "timerange": "between(datetime(@{variables('StartDate')})..datetime(@{variables('EndDate')}))"
- }
- },
- "description": "Retrieves the distinct data table names from the Usage tables that are *not* in the ExcludedDataTypes list"
- }
- },
- "outputs": {}
- },
- "parameters": {
- "$connections": {
- "value": {
- "azureblob": {
- "connectionId": "[resourceId('Microsoft.Web/connections', variables('storageaccount'))]",
- "connectionName": "[variables('storageaccount')]",
- "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureblob')]"
- },
- "azuremonitorlogs": {
- "connectionId": "[resourceId('Microsoft.Web/connections', variables('azuremonitorlogs'))]",
- "connectionName": "[variables('azuremonitorlogs')]",
- "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azuremonitorlogs')]"
- }
- }
- }
- }
- }
- }
- ]
-}
+{
+ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+ "contentVersion": "1.0.0.0",
+ "parameters": {
+ "PlaybookName": {
+ "defaultValue": "Move-LogAnalytics-to-Storage",
+ "type": "String"
+ },
+ "ExclusionTable": {
+ "defaultValue": "\"Heartbeat\", \"ConfigurationChange\", \"ConfigurationData\", \"ThreatIntelligenceIndicator\", \"IntuneDeviceComplianceOrg\", \"Perf\", \"Update\", \"UpdateSummary\", \"SecurityBaseline\", \"SecurityBaselineSummary\"",
+ "type": "String"
+ },
+ "WorkspaceName": {
+ "type": "String"
+ },
+ "WorkspaceSubscription": {
+ "defaultValue": "Your subscription id",
+ "type": "String"
+ },
+ "WorkspaceResourceGroup": {
+ "type": "String"
+ },
+ "EmailAddress": {
+ "defaultValue": "Your email address",
+ "type": "string"
+ },
+ "StorageAccount": {
+ "defaultValue": "",
+ "type": "String"
+ },
+ "storageAccountSku": {
+ "defaultValue": "Standard_LRS",
+ "allowedValues": [
+ "Standard_LRS",
+ "Standard_GRS",
+ "Standard_RAGRS",
+ "Standard_ZRS",
+ "Premium_LRS",
+ "Premium_ZRS",
+ "Standard_GZRS",
+ "Standard_RAGZRS"
+ ],
+ "type": "String",
+ "metadata": {
+ "description": "Sku on which to run the Azure Storage account."
+ }
+ },
+ "storageAccountKind": {
+ "defaultValue": "StorageV2",
+ "allowedValues": [
+ "Storage",
+ "StorageV2",
+ "BlobStorage",
+ "FileStorage",
+ "BlockBlobStorage"
+ ],
+ "type": "String",
+ "metadata": {
+ "description": "Indicates the type of storage account."
+ }
+ },
+ "storageAccountContainerName": {
+ "defaultValue": "my-container",
+ "type": "String",
+ "metadata": {
+ "description": "Set the name of the container to create in the Storage account."
+ }
+ }
+ },
+ "variables": {
+ "azureblob": "[concat('azureblob-', parameters('PlaybookName'))]",
+ "azuremonitorlogs": "[concat('azuremonitorlogs-', parameters('PlaybookName'))]",
+ "storageaccount": "[concat('storageaccount-', parameters('StorageAccount'))]",
+ "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccount'))]",
+ "storagecontainer": "[concat('/', parameters('storageAccountContainerName'), '/')]"
+ },
+ "resources": [
+ {
+ "type": "Microsoft.Web/connections",
+ "apiVersion": "2018-07-01-preview",
+ "name": "[variables('storageaccount')]",
+ "location": "[resourceGroup().location]",
+ "dependsOn": [
+ "[resourceId('Microsoft.Storage/storageAccounts', parameters('StorageAccount'))]"
+ ],
+ "properties": {
+ "api": {
+ "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureblob')]"
+ },
+ "parameterValues": {
+ "accountName": "[parameters('StorageAccount')]",
+ "accessKey": "[listKeys(variables('storageAccountId'), '2019-04-01').keys[0].value]"
+ },
+ "testLinks": [
+ {
+ "requestUri": "[uri('https://management.azure.com:443/', concat('subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/Microsoft.Web/connections/', variables('storageaccount'), '/extensions/proxy/testconnection?api-version=2018-07-01-preview'))]",
+ "method": "get"
+ }
+ ]
+ }
+ },
+ {
+ "type": "Microsoft.Storage/storageAccounts",
+ "apiVersion": "2019-04-01",
+ "name": "[parameters('StorageAccount')]",
+ "location": "[resourceGroup().location]",
+ "sku": {
+ "name": "[parameters('storageAccountSku')]"
+ },
+ "kind": "[parameters('storageAccountKind')]",
+ "properties": {
+ "accessTier": "Cool",
+ "supportsHttpsTrafficOnly": true
+ },
+ "resources": [
+ {
+ "type": "blobServices/containers",
+ "apiVersion": "2019-06-01",
+ "name": "[concat('default/', parameters('storageAccountContainerName'))]",
+ "dependsOn": [
+ "[parameters('StorageAccount')]"
+ ],
+ "properties": {
+ "publicAccess": "Container"
+ }
+ }
+ ]
+ },
+ {
+ "type": "Microsoft.Web/connections",
+ "apiVersion": "2016-06-01",
+ "name": "[variables('azuremonitorlogs')]",
+ "location": "[resourceGroup().location]",
+ "properties": {
+ "displayName": "[parameters('EmailAddress')]",
+ "customParameterValues": {},
+ "api": {
+ "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azuremonitorlogs')]"
+ }
+ }
+ },
+ {
+ "type": "Microsoft.Logic/workflows",
+ "apiVersion": "2017-07-01",
+ "name": "[parameters('PlaybookName')]",
+ "location": "[resourceGroup().location]",
+ "dependsOn": [
+ "[resourceId('Microsoft.Web/connections', variables('storageaccount'))]",
+ "[resourceId('Microsoft.Web/connections', variables('azuremonitorlogs'))]"
+ ],
+ "tags": {
+ "LogicAppsCategory": "security"
+ },
+ "properties": {
+ "state": "Enabled",
+ "definition": {
+ "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
+ "contentVersion": "1.0.0.0",
+ "parameters": {
+ "$connections": {
+ "defaultValue": {},
+ "type": "Object"
+ }
+ },
+ "triggers": {
+ "Recurrence": {
+ "recurrence": {
+ "frequency": "Day",
+ "interval": 1
+ },
+ "type": "Recurrence"
+ }
+ },
+ "actions": {
+ "Compose_Table_Names": {
+ "runAfter": {
+ "Run_query_and_list_results": [
+ "Succeeded"
+ ]
+ },
+ "type": "Compose",
+ "inputs": "@body('Run_query_and_list_results')"
+ },
+ "For_each": {
+ "foreach": "@body('Parse_JSON')?['value']",
+ "actions": {
+ "Set_variable": {
+ "runAfter": {
+ "Until": [
+ "Succeeded"
+ ]
+ },
+ "type": "SetVariable",
+ "inputs": {
+ "name": "HoursCount",
+ "value": 0
+ }
+ },
+ "Until": {
+ "actions": {
+ "Compose": {
+ "runAfter": {
+ "Run_query_and_list_results_2": [
+ "Succeeded"
+ ]
+ },
+ "type": "Compose",
+ "inputs": "@body('Run_query_and_list_results_2')?['value']"
+ },
+ "Create_blob": {
+ "runAfter": {
+ "Compose": [
+ "Succeeded"
+ ]
+ },
+ "type": "ApiConnection",
+ "inputs": {
+ "body": "@outputs('Compose')",
+ "host": {
+ "connection": {
+ "name": "@parameters('$connections')['azureblob']['connectionId']"
+ }
+ },
+ "method": "post",
+ "path": "/datasets/default/files",
+ "queries": {
+ "folderPath": "[concat(variables('storagecontainer'), '@{items(''For_each'')?[''DataType'']}')]",
+ "name": "@{items('For_each')?['DataType']}-@{variables('StartDate')}-@{variables('HoursCount')}.json",
+ "queryParametersSingleEncoded": true
+ }
+ },
+ "runtimeConfiguration": {
+ "contentTransfer": {
+ "transferMode": "Chunked"
+ }
+ }
+ },
+ "Increment_variable": {
+ "runAfter": {
+ "Create_blob": [
+ "Succeeded"
+ ]
+ },
+ "type": "IncrementVariable",
+ "inputs": {
+ "name": "HoursCount",
+ "value": 1
+ }
+ },
+ "Run_query_and_list_results_2": {
+ "runAfter": {},
+ "type": "ApiConnection",
+ "inputs": {
+ "body": "@{items('For_each')?['DataType']}\n| where ingestion_time() between(datetime(@{formatDateTime(addHours(variables('StartDate'),variables('HoursCount')))}) .. datetime(@{formatDateTime(addHours(variables('StartDate'),add(int(variables('HoursCount')),1)))}))",
+ "host": {
+ "connection": {
+ "name": "@parameters('$connections')['azuremonitorlogs']['connectionId']"
+ }
+ },
+ "method": "post",
+ "path": "/queryData",
+ "queries": {
+ "resourcegroups": "[parameters('WorkspaceResourceGroup')]",
+ "resourcename": "[parameters('WorkspaceName')]",
+ "resourcetype": "Log Analytics Workspace",
+ "subscriptions": "[parameters('WorkspaceSubscription')]",
+ "timerange": "between(datetime(@{formatDateTime(addHours(variables('StartDate'),variables('HoursCount')))}) .. datetime(@{formatDateTime(addHours(variables('StartDate'),add(int(variables('HoursCount')),1)))})"
+ }
+ }
+ }
+ },
+ "runAfter": {},
+ "expression": "@equals(variables('HoursCount'), 24)",
+ "limit": {
+ "count": 60,
+ "timeout": "PT1H"
+ },
+ "type": "Until"
+ }
+ },
+ "runAfter": {
+ "Parse_JSON": [
+ "Succeeded"
+ ]
+ },
+ "type": "Foreach",
+ "runtimeConfiguration": {
+ "concurrency": {
+ "repetitions": 1
+ }
+ }
+ },
+ "Initialize_EndDate_variable": {
+ "runAfter": {
+ "Initialize_StartDate_variable": [
+ "Succeeded"
+ ]
+ },
+ "type": "InitializeVariable",
+ "inputs": {
+ "variables": [
+ {
+ "name": "EndDate",
+ "type": "string",
+ "value": "@{formatDateTime(addDays(utcNow(), -28),'yyyy-MM-dd')}"
+ }
+ ]
+ }
+ },
+ "Initialize_ExludedDataTypes_variable": {
+ "runAfter": {
+ "Initialize_EndDate_variable": [
+ "Succeeded"
+ ]
+ },
+ "type": "InitializeVariable",
+ "inputs": {
+ "variables": [
+ {
+ "name": "ExcludedDataTypes",
+ "type": "string",
+ "value": "[parameters('ExclusionTable')]"
+ }
+ ]
+ }
+ },
+ "Initialize_StartDate_variable": {
+ "runAfter": {
+ "Initialize_variable_2": [
+ "Succeeded"
+ ]
+ },
+ "type": "InitializeVariable",
+ "inputs": {
+ "variables": [
+ {
+ "name": "StartDate",
+ "type": "string",
+ "value": "@{formatDateTime(addDays(utcNow(), -89),'yyyy-MM-dd')}"
+ }
+ ]
+ }
+ },
+ "Initialize_variable_2": {
+ "runAfter": {},
+ "type": "InitializeVariable",
+ "inputs": {
+ "variables": [
+ {
+ "name": "HoursCount",
+ "type": "integer",
+ "value": 0
+ }
+ ]
+ }
+ },
+ "Parse_JSON": {
+ "runAfter": {
+ "Compose_Table_Names": [
+ "Succeeded"
+ ]
+ },
+ "type": "ParseJson",
+ "inputs": {
+ "content": "@outputs('Compose_Table_Names')",
+ "schema": {
+ "properties": {
+ "value": {
+ "items": {
+ "properties": {
+ "DataType": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "DataType"
+ ],
+ "type": "object"
+ },
+ "type": "array"
+ }
+ },
+ "type": "object"
+ }
+ }
+ },
+ "Run_query_and_list_results": {
+ "runAfter": {
+ "Initialize_ExludedDataTypes_variable": [
+ "Succeeded"
+ ]
+ },
+ "type": "ApiConnection",
+ "inputs": {
+ "body": "let excludedDataTypes = dynamic([@{variables('ExcludedDataTypes')}]);\nUsage \n| distinct DataType\n| where DataType !in (excludedDataTypes)",
+ "host": {
+ "connection": {
+ "name": "@parameters('$connections')['azuremonitorlogs']['connectionId']"
+ }
+ },
+ "method": "post",
+ "path": "/queryData",
+ "queries": {
+ "resourcegroups": "[parameters('WorkspaceResourceGroup')]",
+ "resourcename": "[parameters('WorkspaceName')]",
+ "resourcetype": "Log Analytics Workspace",
+ "subscriptions": "[parameters('WorkspaceSubscription')]",
+ "timerange": "between(datetime(@{variables('StartDate')})..datetime(@{variables('EndDate')}))"
+ }
+ },
+ "description": "Retrieves the distinct data table names from the Usage tables that are *not* in the ExcludedDataTypes list"
+ }
+ },
+ "outputs": {}
+ },
+ "parameters": {
+ "$connections": {
+ "value": {
+ "azureblob": {
+ "connectionId": "[resourceId('Microsoft.Web/connections', variables('storageaccount'))]",
+ "connectionName": "[variables('storageaccount')]",
+ "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureblob')]"
+ },
+ "azuremonitorlogs": {
+ "connectionId": "[resourceId('Microsoft.Web/connections', variables('azuremonitorlogs'))]",
+ "connectionName": "[variables('azuremonitorlogs')]",
+ "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azuremonitorlogs')]"
+ }
+ }
+ }
+ }
+ }
+ }
+ ]
+}