diff --git a/DataConnectors/GitHub/azuredeploy.json b/DataConnectors/GitHub/azuredeploy.json index 38f8ef6d00..00d6d99a75 100644 --- a/DataConnectors/GitHub/azuredeploy.json +++ b/DataConnectors/GitHub/azuredeploy.json @@ -6,8 +6,12 @@ "defaultValue": "Get-GitHubAuditEntry", "type": "String" }, - "TrafficPlaybookName": { - "defaultValue": "Get-GitHubTrafficLogs", + "RepoPlaybookName": { + "defaultValue": "Get-GitHubRepoLogs", + "type": "String" + }, + "VulnerabilityAlertPlaybookName": { + "defaultValue": "Get-GitHubVulnerabilityAlerts", "type": "String" }, "PersonalAccessToken": { @@ -56,12 +60,14 @@ "tenantId": "[subscription().tenantId]", "objectId": "[parameters('principalId')]", "permissions": { - "keys": [], + "keys": [ + ], "secrets": [ "Get", "List" ], - "certificates": [] + "certificates": [ + ] } } ], @@ -123,7 +129,7 @@ "accessTier": "Hot" } }, - { + { "type": "Microsoft.Storage/storageAccounts/blobServices", "apiVersion": "2019-06-01", "name": "[concat(variables('StorageAccountName'), '/default')]", @@ -162,7 +168,7 @@ "name": "[variables('AzureBlobConnectionName')]", "location": "[resourceGroup().location]", "dependsOn": [ -"[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName'))]" + "[resourceId('Microsoft.Storage/storageAccounts', variables('StorageAccountName'))]" ], "properties": { "displayName": "github", @@ -293,7 +299,7 @@ { "equals": [ "@body('Parse_JSON')?['lastContext']", - "@null" + "" ] } ] @@ -314,36 +320,12 @@ } }, "method": "get", - "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun.json'))}/content", + "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun-Audit.json'))}/content", "queries": { "inferContentType": true } } }, - "Get_secret": { - "runAfter": { - "Condition": [ - "Succeeded" - ] - }, - "type": "ApiConnection", - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['keyvault']['connectionId']" - } - }, - "method": "get", - "path": "[concat('/secrets/@{encodeURIComponent(''', variables('SecretName'), ''')}/value')]" - }, - "runtimeConfiguration": { - "secureData": { - "properties": [ - "outputs" - ] - } - } - }, "Parse_JSON": { "runAfter": { "Get_blob_content_3": [ @@ -391,12 +373,12 @@ } }, "method": "put", - "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun.json'))}" + "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun-Audit.json='))}" } } }, "runAfter": { - "Set_variable_6": [ + "Set_variable_5": [ "Succeeded" ] }, @@ -408,19 +390,7 @@ "type": "SetVariable", "inputs": { "name": "AuditQuery", - "value": "{\"query\": \"{organization(login:\\\"@{variables('OrgName')}\\\") {auditLog(first: 10 orderBy: {direction: ASC field: CREATED_AT} after: \\\"@{body('Parse_JSON_2')?['data']?['organization']?['auditLog']?['pageInfo']?['endCursor']}\\\"){edges{node{... on AuditEntry {action actorIp actorLocation {city country countryCode region regionCode} actorLogin actorResourcePath actorUrl createdAt operationType user {email} userLogin userResourcePath}}}pageInfo {endCursor hasNextPage hasPreviousPage startCursor}}}}\"}" - } - }, - "Set_variable_5": { - "runAfter": { - "Set_variable_4": [ - "Succeeded" - ] - }, - "type": "SetVariable", - "inputs": { - "name": "lastContext", - "value": "@{body('Parse_JSON_2')?['data']?['organization']?['auditLog']?['pageInfo']?['endCursor']}" + "value": "{\"query\": \"{organization(login:\\\"@{variables('OrgName')}\\\") {auditLog(first: 100 orderBy: {direction: ASC field: CREATED_AT} after: \\\"@{body('Parse_JSON_2')?['data']?['organization']?['auditLog']?['pageInfo']?['endCursor']}\\\"){edges{node{... on AuditEntry {action actorIp actorLocation {city country countryCode region regionCode} actorLogin actorResourcePath actorUrl createdAt operationType user {email} userLogin userResourcePath}}}pageInfo {endCursor hasNextPage hasPreviousPage startCursor}}}}\"}" } } } @@ -564,6 +534,18 @@ } } }, + "Set_variable_5": { + "runAfter": { + "Set_variable_6": [ + "Succeeded" + ] + }, + "type": "SetVariable", + "inputs": { + "name": "lastContext", + "value": "@{body('Parse_JSON_2')?['data']?['organization']?['auditLog']?['pageInfo']?['endCursor']}" + } + }, "Set_variable_6": { "runAfter": { "Condition_3": [ @@ -578,9 +560,6 @@ } }, "runAfter": { - "Get_secret": [ - "Succeeded" - ] }, "expression": "@equals(variables('hasNextPage'), false)", "limit": { @@ -595,11 +574,16 @@ "Succeeded" ] }, - "type": "Foreach" + "type": "Foreach", + "runtimeConfiguration": { + "concurrency": { + "repetitions": 1 + } + } }, "Get_blob_content": { "runAfter": { - "Initialize_variable_4": [ + "Get_secret": [ "Succeeded" ] }, @@ -617,6 +601,23 @@ } } }, + "Get_secret": { + "runAfter": { + "Initialize_variable_4": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['keyvault']['connectionId']" + } + }, + "method": "get", + "path": "[concat('/secrets/@{encodeURIComponent(''', variables('SecretName'), ''')}/value')]" + } + }, "Initialize_variable": { "runAfter": { "Initialize_variable_2": [ @@ -734,7 +735,7 @@ { "type": "Microsoft.Logic/workflows", "apiVersion": "2017-07-01", - "name": "[parameters('TrafficPlaybookName')]", + "name": "[parameters('RepoPlaybookName')]", "location": "[resourceGroup().location]", "dependsOn": [ "[resourceId('Microsoft.Web/connections', variables('AzureBlobConnectionName'))]", @@ -851,6 +852,66 @@ }, "type": "Foreach" }, + "For_each_7": { + "foreach": "@body('Parse_JSON_8')", + "actions": { + "Append_to_array_variable_6": { + "runAfter": { + }, + "type": "AppendToArrayVariable", + "inputs": { + "name": "Collaborators", + "value": "@addProperty(addProperty(addProperty(items('For_each_7'), 'Repository', items('For_each')['name']), 'LogType', 'Collaborators'), 'Organization', variables('OrgName'))" + } + } + }, + "runAfter": { + "Parse_JSON_8": [ + "Succeeded" + ] + }, + "type": "Foreach" + }, + "For_each_8": { + "foreach": "@body('Parse_JSON_7')", + "actions": { + "Append_to_array_variable_5": { + "runAfter": { + }, + "type": "AppendToArrayVariable", + "inputs": { + "name": "Commits", + "value": "@addProperty(addProperty(addProperty(items('For_each_8'), 'Repository', items('For_each')['name']), 'LogType', 'Commits'), 'Organization', variables('OrgName'))" + } + } + }, + "runAfter": { + "Parse_JSON_7": [ + "Succeeded" + ] + }, + "type": "Foreach" + }, + "For_each_9": { + "foreach": "@body('Parse_JSON_9')", + "actions": { + "Append_to_array_variable_7": { + "runAfter": { + }, + "type": "AppendToArrayVariable", + "inputs": { + "name": "Forks", + "value": "@addProperty(addProperty(addProperty(items('For_each_9'), 'Repository', items('For_each')['name']), 'LogType', 'Forks'), 'Organization', variables('OrgName'))" + } + } + }, + "runAfter": { + "Parse_JSON_9": [ + "Succeeded" + ] + }, + "type": "Foreach" + }, "HTTP_2": { "runAfter": { }, @@ -912,6 +973,54 @@ "uri": "https://api.github.com/repos/@{variables('OrgName')}/@{items('For_each')['name']}/traffic/clones" } }, + "HTTP_6": { + "runAfter": { + "Send_Data_4": [ + "Succeeded" + ] + }, + "type": "Http", + "inputs": { + "headers": { + "Authorization": "bearer @{body('Get_secret')?['value']}", + "Content-Type": "application/json" + }, + "method": "GET", + "uri": "https://api.github.com/repos/@{variables('OrgName')}/@{items('For_each')['name']}/commits" + } + }, + "HTTP_7": { + "runAfter": { + "Send_Data_5": [ + "Succeeded" + ] + }, + "type": "Http", + "inputs": { + "headers": { + "Authorization": "bearer @{body('Get_secret')?['value']}", + "Content-Type": "application/json" + }, + "method": "GET", + "uri": "https://api.github.com/repos/@{variables('OrgName')}/@{items('For_each')['name']}/collaborators" + } + }, + "HTTP_8": { + "runAfter": { + "Send_Data_6": [ + "Succeeded" + ] + }, + "type": "Http", + "inputs": { + "headers": { + "Authorization": "bearer @{body('Get_secret')?['value']}", + "Content-Type": "application/json" + }, + "method": "GET", + "uri": "https://api.github.com/repos/@{variables('OrgName')}/@{items('For_each')['name']}/forks" + } + }, "Parse_JSON_2": { "runAfter": { "HTTP_4": [ @@ -1012,6 +1121,57 @@ } } }, + "Parse_JSON_7": { + "runAfter": { + "HTTP_6": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@body('HTTP_6')", + "schema": { + "items": { + "type": "object" + }, + "type": "array" + } + } + }, + "Parse_JSON_8": { + "runAfter": { + "HTTP_7": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@body('HTTP_7')", + "schema": { + "items": { + "type": "object" + }, + "type": "array" + } + } + }, + "Parse_JSON_9": { + "runAfter": { + "HTTP_8": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@body('HTTP_8')", + "schema": { + "items": { + "type": "object" + }, + "type": "array" + } + } + }, "Send_Data": { "runAfter": { "For_each_2": [ @@ -1022,7 +1182,7 @@ "inputs": { "body": "@{variables('Referrers')}", "headers": { - "Log-Type": "GitHubTraffic" + "Log-Type": "GitHubRepoLogs" }, "host": { "connection": { @@ -1043,7 +1203,7 @@ "inputs": { "body": "@variables('Paths')", "headers": { - "Log-Type": "GitHubTraffic" + "Log-Type": "GitHubRepoLogs" }, "host": { "connection": { @@ -1064,7 +1224,7 @@ "inputs": { "body": "@variables('Views')", "headers": { - "Log-Type": "GitHubTraffic" + "Log-Type": "GitHubRepoLogs" }, "host": { "connection": { @@ -1085,7 +1245,70 @@ "inputs": { "body": "@variables('Clones')", "headers": { - "Log-Type": "GitHubTraffic" + "Log-Type": "GitHubRepoLogs" + }, + "host": { + "connection": { + "name": "@parameters('$connections')['azureloganalyticsdatacollector']['connectionId']" + } + }, + "method": "post", + "path": "/api/logs" + } + }, + "Send_Data_5": { + "runAfter": { + "For_each_8": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "body": "@variables('Commits')", + "headers": { + "Log-Type": "GitHubRepoLogs" + }, + "host": { + "connection": { + "name": "@parameters('$connections')['azureloganalyticsdatacollector']['connectionId']" + } + }, + "method": "post", + "path": "/api/logs" + } + }, + "Send_Data_6": { + "runAfter": { + "For_each_7": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "body": "@variables('Collaborators')", + "headers": { + "Log-Type": "GitHubRepoLogs" + }, + "host": { + "connection": { + "name": "@parameters('$connections')['azureloganalyticsdatacollector']['connectionId']" + } + }, + "method": "post", + "path": "/api/logs" + } + }, + "Send_Data_7": { + "runAfter": { + "For_each_9": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "body": "@variables('Forks')", + "headers": { + "Log-Type": "GitHubRepoLogs" }, "host": { "connection": { @@ -1104,33 +1327,9 @@ }, "type": "Foreach" }, - "Get_secret": { - "runAfter": { - "Set_variable": [ - "Succeeded" - ] - }, - "type": "ApiConnection", - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['keyvault']['connectionId']" - } - }, - "method": "get", - "path": "[concat('/secrets/@{encodeURIComponent(''', variables('SecretName'), ''')}/value')]" - }, - "runtimeConfiguration": { - "secureData": { - "properties": [ - "outputs" - ] - } - } - }, "HTTP": { "runAfter": { - "Get_secret": [ + "Set_variable": [ "Succeeded" ] }, @@ -1184,11 +1383,16 @@ "Succeeded" ] }, - "type": "Foreach" + "type": "Foreach", + "runtimeConfiguration": { + "concurrency": { + "repetitions": 1 + } + } }, "Get_blob_content": { "runAfter": { - "Initialize_variable_5": [ + "Get_secret": [ "Succeeded" ] }, @@ -1206,6 +1410,30 @@ } } }, + "Get_secret": { + "runAfter": { + "Initialize_variable_8": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['keyvault']['connectionId']" + } + }, + "method": "get", + "path": "[concat('/secrets/@{encodeURIComponent(''', variables('SecretName'), ''')}/value')]" + }, + "runtimeConfiguration": { + "secureData": { + "properties": [ + "outputs" + ] + } + } + }, "Initialize_variable": { "runAfter": { }, @@ -1283,6 +1511,54 @@ ] } }, + "Initialize_variable_6": { + "runAfter": { + "Initialize_variable_5": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "Commits", + "type": "array" + } + ] + } + }, + "Initialize_variable_7": { + "runAfter": { + "Initialize_variable_6": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "Collaborators", + "type": "array" + } + ] + } + }, + "Initialize_variable_8": { + "runAfter": { + "Initialize_variable_7": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "Forks", + "type": "array" + } + ] + } + }, "Parse_JSON_6": { "runAfter": { "Get_blob_content": [ @@ -1334,6 +1610,757 @@ } } } + }, + { + "type": "Microsoft.Logic/workflows", + "apiVersion": "2017-07-01", + "name": "[parameters('VulnerabilityAlertPlaybookName')]", + "location": "[resourceGroup().location]", + "dependsOn": [ + "[resourceId('Microsoft.Web/connections', variables('AzureBlobConnectionName'))]", + "[resourceId('Microsoft.Web/connections', variables('AzureLogAnalyticsDataCollectorConnectionName'))]", + "[resourceId('Microsoft.Web/connections', variables('KeyVaultConnectionName'))]", + "[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('StorageAccountName'), 'default', 'githublogicapp')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('KeyVaultName'), variables('SecretName'))]" + ], + "properties": { + "state": "Disabled", + "definition": { + "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "$connections": { + "defaultValue": { + }, + "type": "Object" + } + }, + "triggers": { + "Recurrence": { + "recurrence": { + "frequency": "Day", + "interval": 1 + }, + "type": "Recurrence" + } + }, + "actions": { + "For_each": { + "foreach": "@body('Parse_JSON')", + "actions": { + "For_each_2": { + "foreach": "@body('Parse_JSON_2')", + "actions": { + "Condition": { + "actions": { + "Create_blob": { + "runAfter": { + "Set_variable_3": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "body": "{\"lastcontext\": \"\", \"lastrun\":\"\"}", + "headers": { + "Content-Type": "application/json" + }, + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "post", + "path": "/datasets/default/files", + "queries": { + "folderPath": "/githublogicapp", + "name": "lastrun-@{variables('OrgName')}​-@{variables('RepoName')}​.json", + "queryParametersSingleEncoded": true + } + }, + "runtimeConfiguration": { + "contentTransfer": { + "transferMode": "Chunked" + } + } + }, + "Set_variable_3": { + "runAfter": { + }, + "type": "SetVariable", + "inputs": { + "name": "vulnerabilityAlertsQuery", + "value": "{\"query\": \"query {organization(login: \\\"@{variables('OrgName')}\\\") {repository(name: \\\"@{variables('RepoName')}\\\") { vulnerabilityAlerts(first: 100) { nodes { createdAt dismissReason dismissedAt id vulnerableManifestFilename vulnerableManifestPath vulnerableRequirements securityAdvisory { databaseId description ghsaId id origin permalink publishedAt severity summary withdrawnAt } } pageInfo { endCursor hasNextPage hasPreviousPage startCursor } } } } }\"}" + } + } + }, + "runAfter": { + "Filter_array": [ + "Succeeded" + ] + }, + "else": { + "actions": { + "Get_blob_content_2": { + "runAfter": { + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "get", + "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun-',variables('OrgName'),'-',variables('RepoName'),'.json'))}/content", + "queries": { + "inferContentType": true + } + } + }, + "Parse_JSON_3": { + "runAfter": { + "Get_blob_content_2": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@json(body('Get_blob_content_2'))", + "schema": { + "properties": { + "lastContext": { + "type": "string" + }, + "lastRun": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "Set_variable_4": { + "runAfter": { + "Parse_JSON_3": [ + "Succeeded" + ] + }, + "type": "SetVariable", + "inputs": { + "name": "lastContext", + "value": "@body('Parse_JSON_3')?['lastContext']" + } + }, + "Set_variable_5": { + "runAfter": { + "Set_variable_4": [ + "Succeeded" + ] + }, + "type": "SetVariable", + "inputs": { + "name": "vulnerabilityAlertsQuery", + "value": "{\"query\": \"query {organization(login: \\\"@{variables('OrgName')}\\\") {repository(name: \\\"@{variables('RepoName')}\\\") { vulnerabilityAlerts(first: 100, after: \\\"@{variables('lastContext')}\\\") { nodes { createdAt dismissReason dismissedAt id vulnerableManifestFilename vulnerableManifestPath vulnerableRequirements securityAdvisory { databaseId description ghsaId id origin permalink publishedAt severity summary withdrawnAt } } pageInfo { endCursor hasNextPage hasPreviousPage startCursor } } } } }\"}" + } + } + } + }, + "expression": { + "and": [ + { + "equals": [ + "@empty(body('Filter_array'))", + "@true" + ] + } + ] + }, + "type": "If" + }, + "Filter_array": { + "runAfter": { + "Set_variable_2": [ + "Succeeded" + ] + }, + "type": "Query", + "inputs": { + "from": "@body('List_blobs')?['value']", + "where": "@equals(item()?['Name'], concat('lastrun-', variables('OrgName'), '-', variables('RepoName'), '.json'))" + } + }, + "Set_variable_2": { + "runAfter": { + }, + "type": "SetVariable", + "inputs": { + "name": "RepoName", + "value": "@items('For_each_2')['name']" + } + }, + "Until": { + "actions": { + "Condition_2": { + "actions": { + "For_each_3": { + "foreach": "@body('Parse_JSON_4')?['data']?['organization']?['repository']?['vulnerabilityAlerts']?['nodes']", + "actions": { + "Append_to_array_variable": { + "runAfter": { + }, + "type": "AppendToArrayVariable", + "inputs": { + "name": "vulnerabilityAlertsArray", + "value": "@addProperty(addProperty(addProperty(items('For_each_3'), 'Repository', variables('RepoName')), 'LogType', 'vulnerabilityAlerts'), 'Organization', variables('OrgName'))" + } + } + }, + "runAfter": { + }, + "type": "Foreach" + }, + "Send_Data": { + "runAfter": { + "For_each_3": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "body": "@{variables('vulnerabilityAlertsArray')}", + "headers": { + "Log-Type": "GitHubRepoLogs" + }, + "host": { + "connection": { + "name": "@parameters('$connections')['azureloganalyticsdatacollector']['connectionId']" + } + }, + "method": "post", + "path": "/api/logs" + } + } + }, + "runAfter": { + "Parse_JSON_4": [ + "Succeeded" + ] + }, + "expression": { + "and": [ + { + "not": { + "equals": [ + "@length(body('Parse_JSON_4')?['data']?['organization']?['repository']?['vulnerabilityAlerts']?['nodes'])", + 0 + ] + } + } + ] + }, + "type": "If" + }, + "Condition_3": { + "actions": { + "Update_blob": { + "runAfter": { + }, + "type": "ApiConnection", + "inputs": { + "body": "{\n \"lastRun\": \"@{utcNow()}\",\n \"lastContext\": \"@{variables('lastContext')}\"\n}", + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "put", + "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/lastrun-',variables('OrgName'),'-',variables('RepoName'),'.json'))}" + } + } + }, + "runAfter": { + "Set_variable_7": [ + "Succeeded" + ] + }, + "else": { + "actions": { + "Set_variable_8": { + "runAfter": { + }, + "type": "SetVariable", + "inputs": { + "name": "vulnerabilityAlertsQuery", + "value": "{\"query\": \"query {organization(login: \\\"@{variables('OrgName')}\\\") {repository(name: \\\"@{variables('RepoName')}\\\") { vulnerabilityAlerts(first: 100, after: \\\"@{variables('lastContext')}\\\") { nodes { createdAt dismissReason dismissedAt id vulnerableManifestFilename vulnerableManifestPath vulnerableRequirements securityAdvisory { databaseId description ghsaId id origin permalink publishedAt severity summary withdrawnAt } } pageInfo { endCursor hasNextPage hasPreviousPage startCursor } } } } }\"}" + } + } + } + }, + "expression": { + "and": [ + { + "equals": [ + "@variables('hasNextPage')", + "" + ] + } + ] + }, + "type": "If" + }, + "HTTP_2": { + "runAfter": { + }, + "type": "Http", + "inputs": { + "body": "@variables('vulnerabilityAlertsQuery')", + "headers": { + "Authorization": "Bearer @{body('Get_secret')?['value']}", + "Content-Type": "application/json" + }, + "method": "POST", + "uri": "https://api.github.com/graphql" + } + }, + "Parse_JSON_4": { + "runAfter": { + "HTTP_2": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@body('HTTP_2')", + "schema": { + "properties": { + "data": { + "properties": { + "organization": { + "properties": { + "repository": { + "properties": { + "vulnerabilityAlerts": { + "properties": { + "nodes": { + "items": { + "properties": { + "createdAt": { + "type": "string" + }, + "dismissReason": { + }, + "dismissedAt": { + }, + "id": { + "type": "string" + }, + "securityAdvisory": { + "properties": { + "databaseId": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "ghsaId": { + "type": "string" + }, + "id": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "permalink": { + "type": "string" + }, + "publishedAt": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "withdrawnAt": { + } + }, + "type": "object" + }, + "vulnerableManifestFilename": { + "type": "string" + }, + "vulnerableManifestPath": { + "type": "string" + }, + "vulnerableRequirements": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "pageInfo": { + "properties": { + "endCursor": { + "type": [ + "string", + "null" + ] + }, + "hasNextPage": { + "type": "boolean" + }, + "hasPreviousPage": { + "type": "boolean" + }, + "startCursor": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + } + } + }, + "Set_variable_6": { + "runAfter": { + "Condition_2": [ + "Succeeded" + ] + }, + "type": "SetVariable", + "inputs": { + "name": "hasNextPage", + "value": "@body('Parse_JSON_4')?['data']?['organization']?['repository']?['vulnerabilityAlerts']?['pageInfo']?['hasNextPage']" + } + }, + "Set_variable_7": { + "runAfter": { + "Set_variable_6": [ + "Succeeded" + ] + }, + "type": "SetVariable", + "inputs": { + "name": "lastContext", + "value": "@body('Parse_JSON_4')?['data']?['organization']?['repository']?['vulnerabilityAlerts']?['pageInfo']?['endCursor']" + } + } + }, + "runAfter": { + "Condition": [ + "Succeeded" + ] + }, + "expression": "@equals(variables('hasNextPage'), false)", + "limit": { + "count": 60, + "timeout": "PT1H" + }, + "type": "Until" + } + }, + "runAfter": { + "Parse_JSON_2": [ + "Succeeded" + ] + }, + "type": "Foreach" + }, + "HTTP": { + "runAfter": { + "Set_variable": [ + "Succeeded" + ] + }, + "type": "Http", + "inputs": { + "headers": { + "Authorization": "Bearer @{body('Get_secret')?['value']}", + "Content-Type": "application/json" + }, + "method": "GET", + "uri": "https://api.github.com/orgs/@{variables('OrgName')}/repos" + } + }, + "Parse_JSON_2": { + "runAfter": { + "HTTP": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@body('HTTP')", + "schema": { + "items": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "type": "array" + } + } + }, + "Set_variable": { + "runAfter": { + }, + "type": "SetVariable", + "inputs": { + "name": "OrgName", + "value": "@items('For_each')['org']" + } + } + }, + "runAfter": { + "Parse_JSON": [ + "Succeeded" + ] + }, + "type": "Foreach", + "runtimeConfiguration": { + "concurrency": { + "repetitions": 1 + } + } + }, + "Get_blob_content": { + "runAfter": { + "Get_secret": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "get", + "path": "/datasets/default/files/@{encodeURIComponent(encodeURIComponent('/githublogicapp/ORGS.json'))}/content", + "queries": { + "inferContentType": true + } + } + }, + "Get_secret": { + "runAfter": { + "List_blobs": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['keyvault']['connectionId']" + } + }, + "method": "get", + "path": "[concat('/secrets/@{encodeURIComponent(''', variables('SecretName'), ''')}/value')]" + }, + "runtimeConfiguration": { + "secureData": { + "properties": [ + "outputs" + ] + } + } + }, + "Initialize_variable": { + "runAfter": { + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "OrgName", + "type": "string" + } + ] + } + }, + "Initialize_variable_2": { + "runAfter": { + "Initialize_variable": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "RepoName", + "type": "string" + } + ] + } + }, + "Initialize_variable_3": { + "runAfter": { + "Initialize_variable_2": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "vulnerabilityAlertsQuery", + "type": "string" + } + ] + } + }, + "Initialize_variable_4": { + "runAfter": { + "Initialize_variable_3": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "lastContext", + "type": "string" + } + ] + } + }, + "Initialize_variable_5": { + "runAfter": { + "Initialize_variable_4": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "hasNextPage", + "type": "boolean", + "value": true + } + ] + } + }, + "Initialize_variable_6": { + "runAfter": { + "Initialize_variable_5": [ + "Succeeded" + ] + }, + "type": "InitializeVariable", + "inputs": { + "variables": [ + { + "name": "vulnerabilityAlertsArray", + "type": "array" + } + ] + } + }, + "List_blobs": { + "runAfter": { + "Initialize_variable_6": [ + "Succeeded" + ] + }, + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['azureblob']['connectionId']" + } + }, + "method": "get", + "path": "/datasets/default/foldersV2/@{encodeURIComponent(encodeURIComponent('/githublogicapp'))}", + "queries": { + "nextPageMarker": "", + "useFlatListing": false + } + } + }, + "Parse_JSON": { + "runAfter": { + "Get_blob_content": [ + "Succeeded" + ] + }, + "type": "ParseJson", + "inputs": { + "content": "@json(body('Get_blob_content'))", + "schema": { + "items": { + "properties": { + "org": { + "type": "string" + } + }, + "required": [ + "org" + ], + "type": "object" + }, + "type": "array" + } + } + } + }, + "outputs": { + } + }, + "parameters": { + "$connections": { + "value": { + "azureblob": { + "connectionId": "[resourceId('Microsoft.Web/connections', variables('AzureBlobConnectionName'))]", + "connectionName": "[variables('AzureBlobConnectionName')]", + "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureblob')]" + }, + "azureloganalyticsdatacollector": { + "connectionId": "[resourceId('Microsoft.Web/connections', variables('AzureLogAnalyticsDataCollectorConnectionName'))]", + "connectionName": "[variables('AzureLogAnalyticsDataCollectorConnectionName')]", + "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/azureloganalyticsdatacollector')]" + }, + "keyvault": { + "connectionId": "[resourceId('Microsoft.Web/connections', variables('KeyVaultConnectionName'))]", + "connectionName": "[variables('KeyVaultConnectionName')]", + "id": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/keyvault')]" + } + } + } + } + } } ] } \ No newline at end of file diff --git a/DataConnectors/GitHub/lastrun.json b/DataConnectors/GitHub/lastrun-Audit.json similarity index 100% rename from DataConnectors/GitHub/lastrun.json rename to DataConnectors/GitHub/lastrun-Audit.json diff --git a/DataConnectors/GitHub/readme.md b/DataConnectors/GitHub/readme.md index 897cca5d2c..d720707400 100644 --- a/DataConnectors/GitHub/readme.md +++ b/DataConnectors/GitHub/readme.md @@ -1,7 +1,7 @@ # Ingest GitHub AuditLog and API Data Author: Nicholas DiCola -Get-GitHubAuditEntry playbook ingests GitHub AuditLog via (GraphQL)[https://developer.github.com/v4/interface/auditentry/] events and writes them to a custom log table called GitHub_CL. Get-GitHubTrafficLogs playbook ingests GitHub (Traffic Logs)[https://developer.github.com/v3/repos/traffic/] data and writes them to a custom log table called GitHubTraffic_CL. +Get-GitHubAuditEntry playbook ingests GitHub AuditLog via (GraphQL)[https://developer.github.com/v4/interface/auditentry/] events and writes them to a custom log table called GitHub_CL. Get-GitHubRepoLogs playbook ingests GitHub (Traffic Logs)[https://developer.github.com/v3/repos/traffic/] data and writes them to a custom log table called GitHubRepoLogs_CL. Get-GitHubVulnerabilityAlerts playbook ingests GitHub (Security Vulnerability)[https://developer.github.com/v4/object/securityvulnerability/] data and writes them to a custom log table called GitHubRepoLogs_CL There are a number of configuration steps required to deploy the Logic App playbooks. @@ -17,14 +17,16 @@ This user will be used to grant access to the Key Vault secret. "workspaceId": The Sentinel Workspace ID​ "workSpaceKey": The Sentinel Workspace Key ``` -4. There are two json files (ORGS.json and lastrun.json). +4. There are two json files (ORGS.json and lastrun-Audit.json). 5. Edit the ORGS.json file and update "org": "sampleorg" and replace sample org with your org name. If you have addtional orgs, add another line {"org": "sampleorg"} for each org. -6. Upload the ORGS.json and lastrun.json to the storage account githublogicapp container. +6. Upload the ORGS.json, and lastrun-Audit.json to the storage account githublogicapp container. 7. Go to the keyvault-GitHubPlaybooks connection resource. 8. Click Edit API Connection. 9. Click Authorize. Sign in as the user. Click Save. 10. The playbooks are deployed as disabled since the json files and connection has to be authorized. Go to each playbook and click Enable. +Note: there are two parsers (here)[https://github.com/Azure/Azure-Sentinel/tree/master/Parsers/GitHub] to make the logs useful + ## Deploy the Logic App template diff --git a/Detections/GitHub/ActivitiesFromANewCountry.yaml b/Detections/GitHub/ActivitiesFromANewCountry.yaml index 7ebd908038..77ec4db2a3 100644 --- a/Detections/GitHub/ActivitiesFromANewCountry.yaml +++ b/Detections/GitHub/ActivitiesFromANewCountry.yaml @@ -22,15 +22,15 @@ query: | let StartTime = 1h; let EndRunTime = StartTime - RunTime; let EndLearningTime = StartTime + LearningPeriod; - let GitHubCountryCodeLogs = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_actorLocation_countryCode_s != ""); + let GitHubCountryCodeLogs = (GitHubAudit + | where Country != ""); GitHubCountryCodeLogs | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) - | summarize makeset(node_actorLocation_countryCode_s) by node_actorLogin_s + | summarize makeset(Country) by Actor | join kind=innerunique ( GitHubCountryCodeLogs | where TimeGenerated between (ago(StartTime) .. ago(EndRunTime)) - | distinct node_actorLocation_countryCode_s, node_actorLogin_s - ) on node_actorLogin_s - | where set_node_actorLocation_countryCode_s !contains node_actorLocation_countryCode_s + | distinct Country, Actor + ) on Actor + | where set_Country !contains Country + | extend AccountCustomEntity = Actor, IPCustomEntity = IPAddress diff --git a/Detections/GitHub/BruteForce.yaml b/Detections/GitHub/BruteForce.yaml index 1b004bcf0b..81f65b27b0 100644 --- a/Detections/GitHub/BruteForce.yaml +++ b/Detections/GitHub/BruteForce.yaml @@ -39,4 +39,5 @@ query: | | summarize FailedLoginsCountInRunTime = count() by User = Identity ) on User | where FailedLoginsCountInRunTime > LearningThreshold + | extend AccountCustomEntity = UserPrincipalName, IPCustomEntity = IPAddress \ No newline at end of file diff --git a/Detections/GitHub/MultipleEntity_GitHub_CL.yaml b/Detections/GitHub/MultipleEntity_GitHub_CL.yaml index e5309595ce..660f847cc0 100644 --- a/Detections/GitHub/MultipleEntity_GitHub_CL.yaml +++ b/Detections/GitHub/MultipleEntity_GitHub_CL.yaml @@ -1,7 +1,7 @@ -id: -name: (Preview) TI map Domain entity to CommonSecurityLog +id: aac495a9-feb1-446d-b08e-a1164a539452 +name: TI map IP entity to GitHub_CL description: | - 'Identifies a match in CommonSecurityLog table from any Domain IOC from TI' + 'Identifies a match in GitHub_CL table from any IP IOC from TI' severity: Medium requiredDataConnectors: - connectorId: ThreatIntelligence @@ -15,7 +15,7 @@ queryPeriod: 14d triggerOperator: gt triggerThreshold: 0 tactics: - - ???? + - Impact query: | ThreatIntelligenceIndicator @@ -28,12 +28,11 @@ query: | | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) | join ( - GitHub_CL - // renaming time column so it is clear the log this came from - | extend GitHubActivity_TimeGenerated = TimeGenerated - | extend TimeGenerated = node_createdAt_t + GitHubAudit | where TimeGenerated >= ago(24h) + | extned GitHubAudit_TimeGenerated = TimeGenerated ) - on on $left.TI_ipEntity == $right.node_actorIp_s + on on $left.TI_ipEntity == $right.IPaddress | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, GitHubActivity_TimeGenerated, TI_ipEntity, actorIp_s, actorLogin_s, action_s, actorLocation_country_s, operationType_s, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress + | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, GitHubAudit_TimeGenerated, TI_ipEntity, IPaddress, Actor, Action, Country, OperationType, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress + | extend timestamp = GitHubAudit_TimeGenerated, IPCustomEntity = IPaddress, AccountCustomEntity = Actor diff --git a/Detections/GitHub/SigninBurstFromMultipleLocation.yaml b/Detections/GitHub/SigninBurstFromMultipleLocation.yaml index d72d795c02..0ba188bee8 100644 --- a/Detections/GitHub/SigninBurstFromMultipleLocation.yaml +++ b/Detections/GitHub/SigninBurstFromMultipleLocation.yaml @@ -24,5 +24,6 @@ query: | | where ResultType == 0 | summarize CountOfLocations = dcount(Location), Locations = make_set(Location) by User = Identity | where CountOfLocations > 1 + | extend AccountCustomEntity = UserPrincipalName, IPCustomEntity = IPAddress \ No newline at end of file diff --git a/Detections/GitHub/TwoFactorAuthDisable.yaml b/Detections/GitHub/TwoFactorAuthDisable.yaml index dfda1f7053..3a0cf3cc10 100644 --- a/Detections/GitHub/TwoFactorAuthDisable.yaml +++ b/Detections/GitHub/TwoFactorAuthDisable.yaml @@ -18,8 +18,8 @@ relevantTechniques: query: | let timeframe = 14d; - GitHub_CL - | extend TimeGenerated = node_createdAt_t + GitHubAudit | where TimeGenerated > ago(timeframe) - | where action_s == "org.disable_two_factor_requirement" - | project ["Action"]=node_action_s , ["TimeGenerated"]=TimeGenerated ,["User"]=node_actorLogin_s ,["Location"]=node_actorLocation_country_s + | where Action == "org.disable_two_factor_requirement" + | project TimeGenerated, Action, Actor, Country, IPaddress, Repository + | extend AccountCustomEntity = Actor, IPCustomEntity = IPaddress \ No newline at end of file diff --git a/Hunting Queries/GitHub/FirstTimeInviteUserAndAddMemberToRepo.yaml b/Hunting Queries/GitHub/FirstTimeInviteUserAndAddMemberToRepo.yaml index c58563ea68..0d13eeb61b 100644 --- a/Hunting Queries/GitHub/FirstTimeInviteUserAndAddMemberToRepo.yaml +++ b/Hunting Queries/GitHub/FirstTimeInviteUserAndAddMemberToRepo.yaml @@ -17,15 +17,14 @@ query: | let StartTime = 1h; let EndRunTime = StartTime - RunTime; let EndLearningTime = StartTime + LearningPeriod; - let GitHubOrgMemberLogs = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_action_s == "org.invite_member" or node_action_s == "org.update_member" or node_action_s == "org.add_member"); + let GitHubOrgMemberLogs = (GitHubAudit + where Action == "org.invite_member" or Action == "org.update_member" or Action == "org.add_member"); GitHubOrgMemberLogs | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) - | distinct node_actorLogin_s + | distinct Actor | join kind=rightanti ( GitHubOrgMemberLogs | where TimeGenerated between (ago(StartTime) .. ago(EndRunTime)) - | distinct node_actorLogin_s - ) on node_actorLogin_s + | distinct Actor + ) on Actor diff --git a/Hunting Queries/GitHub/FirstTimeRepoDelete.yaml b/Hunting Queries/GitHub/FirstTimeRepoDelete.yaml index e99436cf0d..d0330cc628 100644 --- a/Hunting Queries/GitHub/FirstTimeRepoDelete.yaml +++ b/Hunting Queries/GitHub/FirstTimeRepoDelete.yaml @@ -17,14 +17,13 @@ query: | let StartTime = 1h; let EndRunTime = StartTime - RunTime; let EndLearningTime = StartTime + LearningPeriod; - let GitHubRepositoryDestroyEvents = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_action_s == "repo.destroy"); + let GitHubRepositoryDestroyEvents = (GitHubAudit + | where Action == "repo.destroy"); GitHubRepositoryDestroyEvents | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) - | distinct node_actorLogin_s + | distinct Actor | join kind=rightanti ( GitHubRepositoryDestroyEvents | where TimeGenerated between (ago(StartTime) .. ago(EndRunTime)) - | distinct node_actorLogin_s - ) on node_actorLogin_s + | distinct Actor + ) on Actor diff --git a/Hunting Queries/GitHub/InactiveOrNewAccountAccessUsage.yaml b/Hunting Queries/GitHub/InactiveOrNewAccountAccessUsage.yaml index 02fb070ea2..2ea65f9216 100644 --- a/Hunting Queries/GitHub/InactiveOrNewAccountAccessUsage.yaml +++ b/Hunting Queries/GitHub/InactiveOrNewAccountAccessUsage.yaml @@ -17,35 +17,33 @@ query: | let StartTime = 1h; let EndRunTime = StartTime - RunTime; let EndLearningTime = StartTime + LearningPeriod; - let GitHubActorLogin = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_actorLogin_s != ""); - let GitHubUser = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_userLogin_s != ""); + let GitHubActorLogin = (GitHubAudit + | where Actor != ""); + let GitHubUser = (GitHubAudit + | where ImpactedUser != ""); let GitHubNewActorLogin = (GitHubActorLogin | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) - | summarize makeset(node_actorLogin_s) + | summarize makeset(Actor) | extend Dummy = 1 | join kind=innerunique ( GitHubActorLogin | where TimeGenerated between (ago(StartTime) .. ago(EndRunTime)) - | distinct node_actorLogin_s + | distinct Actor | extend Dummy = 1 ) on Dummy | project-away Dummy - | where set_node_actorLogin_s !contains node_actorLogin_s); + | where set_Actor !contains Actor); let GitHubNewUser = ( GitHubUser | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) - | summarize makeset(userLogin_s) + | summarize makeset(ImpactedUser) | extend Dummy = 1 | join kind=innerunique ( GitHubUser | where TimeGenerated between (ago(StartTime) .. ago(EndRunTime)) - | distinct node_userLogin_s + | distinct ImpactedUser | extend Dummy = 1 ) on Dummy | project-away Dummy - | where set_node_userLogin_s !contains node_userLogin_s); + | where set_ImpactedUser !contains ImpactedUser); union GitHubNewActorLogin, GitHubNewUser diff --git a/Hunting Queries/GitHub/MassDeletion.yaml b/Hunting Queries/GitHub/MassDeletion.yaml index 30107ab94f..3a05c182db 100644 --- a/Hunting Queries/GitHub/MassDeletion.yaml +++ b/Hunting Queries/GitHub/MassDeletion.yaml @@ -20,9 +20,8 @@ query: | let MinThreshold = 10.0; let EndRunTime = StartTime - RunTime; let EndLearningTime = StartTime + LearningPeriod; - let GitHubRepositoryDestroyEvents = (GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where action_s == "repo.destroy"); + let GitHubRepositoryDestroyEvents = (GitHubAudit + | where Action == "repo.destroy"); GitHubRepositoryDestroyEvents | where TimeGenerated between (ago(EndLearningTime) .. ago(StartTime)) | summarize count() by bin(TimeGenerated, BinTime) diff --git a/Hunting Queries/GitHub/OAuthAppRestrictionsDisabled.yaml b/Hunting Queries/GitHub/OAuthAppRestrictionsDisabled.yaml index 7516b7cf36..522f12a74c 100644 --- a/Hunting Queries/GitHub/OAuthAppRestrictionsDisabled.yaml +++ b/Hunting Queries/GitHub/OAuthAppRestrictionsDisabled.yaml @@ -13,8 +13,7 @@ relevantTechniques: query: | let timeframe = 14d; - GitHub_CL - | extend TimeGenerated = node_createdAt_t + GitHubAudit | where TimeGenerated > ago(timeframe) - | where node_action_s == "org.disable_oauth_app_restrictions" - | project ["Action"]=node_action_s , ["TimeGenerated"]=TimeGenerated ,["User"]=node_actorLogin_s ,["Location"]=node_actorLocation_country_s + | where Action == "org.disable_oauth_app_restrictions" + | project TimeGenerated, Action, Actor, Location diff --git a/Hunting Queries/GitHub/RepoClone-TimeSeriesAnomly.yaml b/Hunting Queries/GitHub/RepoClone-TimeSeriesAnomly.yaml index 89cf29ee55..808b94d6e1 100644 --- a/Hunting Queries/GitHub/RepoClone-TimeSeriesAnomly.yaml +++ b/Hunting Queries/GitHub/RepoClone-TimeSeriesAnomly.yaml @@ -5,7 +5,7 @@ description: | requiredDataConnectors: - connectorId: CustomConnector dataTypes: - - GithubTrafficLogs_CL + - GithubRepoLogs_CL tactics: - Collection relevantTechniques: @@ -17,7 +17,7 @@ query: | let max_t = toscalar(GithubTrafficLogs_CL | summarize max(timestamp_t)); GithubTrafficLogs_CL - | where LogType_s == "clones" - | make-series num=count(count_d) default=0 on timestamp_t in range(min_t, max_t, 1h) by Repository_s + | where Action == "clones" + | make-series num=count(count_d) default=0 on timestamp_t in range(min_t, max_t, 1h) by Repository | extend (anomalies, score, baseline) = series_decompose_anomalies(num, 1.5, -1, 'linefit') | render timechart diff --git a/Hunting Queries/GitHub/RepoSwitchedFromPrivateToPublic.yaml b/Hunting Queries/GitHub/RepoSwitchedFromPrivateToPublic.yaml index e9c59ec790..29cb76d2f1 100644 --- a/Hunting Queries/GitHub/RepoSwitchedFromPrivateToPublic.yaml +++ b/Hunting Queries/GitHub/RepoSwitchedFromPrivateToPublic.yaml @@ -13,10 +13,9 @@ relevantTechniques: query: | let timeframe = 14d; - GitHub_CL - | extend TimeGenerated = node_createdAt_t + GitHubAudit | where TimeGenerated > ago(timeframe) - | where node_action_s == "repo.access" - | where node_operationType_s == "MODIFY" + | where Action == "repo.access" + | where OperationType == "MODIFY" | where node_visibility_s == "PUBLIC" - | project ["Action"]=node_action_s , ["TimeGenerated"]=TimeGenerated, ["User"]=node_actorLogin_s ,["Location"]=node_actorLocation_country_s, ["Repo Name"]=node_repositoryName_s, ["Permission"]=node_visibility_s + | project TimeGenerated, Action, Actor, Country, Repository, Visability diff --git a/Hunting Queries/GitHub/UpdatePermissions.yaml b/Hunting Queries/GitHub/UpdatePermissions.yaml index fcf61b02f9..4199f4bf1d 100644 --- a/Hunting Queries/GitHub/UpdatePermissions.yaml +++ b/Hunting Queries/GitHub/UpdatePermissions.yaml @@ -12,7 +12,6 @@ relevantTechniques: - T1098 query: | - GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_action_s == "org.update_default_repository_permission" - | project ["Action"]=node_action_s, ["TimeGenerated"]=TimeGenerated, ["User"]=node_actorLogin_s, ["Location"]=node_actorLocation_country_s, ["Repo Name"]=node_repositoryName_s, ["Previous Permission"]=node_permissionWas_s, ["New Permission"]=node_permission_s + GitHubAudit + | where Action == "org.update_default_repository_permission" + | project TimeGenerated, Action, Actor, Country, Repository, PreviousPermission, CurrentPermission diff --git a/Hunting Queries/GitHub/UserGrantAccessAndGrantsAccessAnotherUser.yaml b/Hunting Queries/GitHub/UserGrantAccessAndGrantsAccessAnotherUser.yaml index a9d3337b26..b1961f11d8 100644 --- a/Hunting Queries/GitHub/UserGrantAccessAndGrantsAccessAnotherUser.yaml +++ b/Hunting Queries/GitHub/UserGrantAccessAndGrantsAccessAnotherUser.yaml @@ -12,18 +12,16 @@ relevantTechniques: - T1098 query: | - GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_userLogin_s != "" - | where node_action_s == "org.invite_member" or node_action_s == "org.add_member" or node_action_s == "team.add_member" or node_action_s == "repo.add_member" - | distinct node_userLogin_s, TimeGenerated, node_actorLogin_s - | project-rename firstUserAdded = node_userLogin_s, firstEventTime = TimeGenerated, firstAdderUser = node_actorLogin_s + GitHubAudit + | where ImpactedUser != "" + | where Action == "org.invite_member" or Action == "org.add_member" or Action == "team.add_member" or Action == "repo.add_member" + | distinct ImpactedUser, TimeGenerated, Actor + | project-rename firstUserAdded = ImpactedUser, firstEventTime = TimeGenerated, firstAdderUser = Actor | join kind= innerunique ( - GitHub_CL - | extend TimeGenerated = node_createdAt_t - | where node_userLogin_s != "" - | where node_action_s == "org.invite_member" or node_action_s == "org.add_member" or node_action_s == "team.add_member" or node_action_s == "repo.add_member" - | distinct node_userLogin_s, TimeGenerated, node_actorLogin_s - | project-rename secondUserAdded = node_userLogin_s, secondEventTime = TimeGenerated, secondAdderUser = node_actorLogin_s + GitHubAudit + | where ImpactedUser != "" + | where Action == "org.invite_member" or Action == "org.add_member" or Action == "team.add_member" or Action == "repo.add_member" + | distinct ImpactedUser, TimeGenerated, Actor + | project-rename secondUserAdded = ImpactedUser, secondEventTime = TimeGenerated, secondAdderUser = Actor ) on $right.secondAdderUser == $left.firstUserAdded | where secondEventTime between (firstEventTime .. (firstEventTime + 1h)) diff --git a/Parsers/GitHub/GitHubAuditLog.txt b/Parsers/GitHub/GitHubAuditLog.txt new file mode 100644 index 0000000000..2ea0e838ce --- /dev/null +++ b/Parsers/GitHub/GitHubAuditLog.txt @@ -0,0 +1,48 @@ +// GitHub Enterprise Audit Entry Data Parser +// Last Updated Date: Jun 7, 2020 +// +//This parser parses GitHub Enterprise Audit Entry extract the infromation from their various components. It is assumed that the playbook to ingest audit entry data into Sentinel is enabled +// +// Parser Notes: +// 1. This parser assumes logs are collected into a custom log table entitled GitHub_CL +// +// Usage Instruction : +// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias. +// To work with pre-built GitHub queries this Function should be given the alias of GitHubAudit. +// Functions usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. GitHubAudit | take 10). +// +// References : +// Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions +// Tech Community Blog on KQL Functions : https://techcommunity.microsoft.com/t5/Azure-Sentinel/Using-KQL-functions-to-speed-up-analysis-in-Azure-Sentinel/ba-p/712381 +// Tech Community Blog on GitHub data: <> +// +// + +GitHub_CL +| project TimeGenerated=node_createdAt_t, + Organization=columnifexists('node_organizationName_s', ""), + Action=node_action_s, + OperationType=node_operationType_s, + Repository=columnifexists('node_repositoryName_s',""), + Actor=node_actorLogin_s, + IPaddress=node_actorIp_s, + City=node_actorLocation_city_s, + Country=node_actorLocation_country_s, + ImpactedUser=columnifexists('node_userLogin_s', ""), + ImpactedUserEmail=columnifexists('node_user_email_s', ""), + InvitedUserPermission=node_permission_s, + Visability=columnifexists('node_visibility_s',""), + OauthApplication=columnifexists('node_oauthApplicationName_s',""), + OauthApplicationUrl=columnifexists('node_applicationUrl_s',""), + OauthApplicationState=columnifexists('node_state_s',""), + UserCanInviteCollaborators=columnifexists('node_canInviteOutsideCollaboratorsToRepositories_b',""), + MembershipType=columnifexists('node_membershipTypes_s',""), + CurrentPermission=columnifexists('node_permission_s',""), + PreviousPermission=columnifexists('node_permissionWas_s',""), + TeamName=columnifexists('node_teamName_s',""), + Reason=columnifexists('node_reason_s',""), + BlockedUser=columnifexists('node_blockedUserName_s',""), + CanCreateRepositories=columnifexists('canCreateRepositories_b',"") + + + diff --git a/Parsers/GitHub/GitHubRepoLog.txt b/Parsers/GitHub/GitHubRepoLog.txt new file mode 100644 index 0000000000..da725458dd --- /dev/null +++ b/Parsers/GitHub/GitHubRepoLog.txt @@ -0,0 +1,45 @@ +// GitHub Enterprise Repository Data Parser +// Last Updated Date: Jun 7, 2020 +// +//This parser parses GitHub Enterprise Repository Data extract the infromation from their various components. It is assumed that the playbook to ingest repository data into Sentinel is enabled +// +// Parser Notes: +// 1. This parser assumes logs are collected into a custom log table entitled GitHubRepoLogs_CL +// +// Usage Instruction : +// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias. +// To work with pre-built GitHub queries this Function should be given the alias of GitHubRepo +// Functions usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. GitHubRepo | take 10). +// +// References : +// Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions +// Tech Community Blog on KQL Functions : https://techcommunity.microsoft.com/t5/Azure-Sentinel/Using-KQL-functions-to-speed-up-analysis-in-Azure-Sentinel/ba-p/712381 +// Tech Community Blog on GitHub data: <> +// +// + +GitHubRepoLogs_CL +| project TimeGenerated = created_at_t, + Organization=columnifexists('Organization_s', ""), + Repository=columnifexists('Repository_s',""), + Action=columnifexists('LogType_s',""), + Actor=coalesce(login_s, owner_login_s), + ActorType=coalesce(owner_type_s, type_s), + IsPrivate=columnifexists('private_b',""), + ForksUrl=columnifexists('forks_url_s',""), + PushedAt=columnifexists('pushed_at_t',""), + IsDisabled=columnifexists('disabled_b',""), + AdminPermissions=columnifexists('permissions_admin_b',""), + PushPermissions=columnifexists('permissions_push_b',""), + PullPermissions=columnifexists('permissions_pull_b',""), + ForkCount=columnifexists('forks_count_d',""), + Count=columnifexists('count_d,',""), + UniqueUsersCount=columnifexists('uniques_d',""), + DismmisedAt=columnifexists('dismissedAt_t',""), + Reason=columnifexists('dismissReason_s',""), + vulnerableManifestFilename = columnifexists('vulnerableManifestFilename_s',""), + Description=columnifexists('securityAdvisory_description_s',""), + Link=columnifexists('securityAdvisory_permalink_s',""), + PublishedAt=columnifexists('securityAdvisory_publishedAt_t ',""), + Severity=columnifexists('securityAdvisory_severity_s',""), + Summary=columnifexists('securityAdvisory_summary_s',"") diff --git a/Workbooks/GitHubSecurityWorkbook.json b/Workbooks/GitHubSecurityWorkbook.json new file mode 100644 index 0000000000..77af76cdcc --- /dev/null +++ b/Workbooks/GitHubSecurityWorkbook.json @@ -0,0 +1,189 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "## GitHub - Security\n" + }, + "name": "text - 2" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "a9923eb9-9a02-4a48-bb72-e9be338eeb3b", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "type": 4, + "value": { + "durationMs": 1209600000 + }, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 300000 + }, + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 1209600000 + }, + { + "durationMs": 2419200000 + }, + { + "durationMs": 2592000000 + }, + { + "durationMs": 5184000000 + }, + { + "durationMs": 7776000000 + } + ] + }, + "resourceType": "microsoft.insights/components" + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "name": "parameters - 2" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHub_CL\n| extend TimeGenerated = node_createdAt_t\n| where node_action_s == \"org.add_member\" or node_action_s == \"org.remove_member\"\n| extend MemberName = node_userLogin_s\n| extend Action = iif(node_action_s==\"org.add_member\", \"Added\", \"Removed\")\n| extend Organization = node_organizationName_s\n| extend Permission = node_permission_s\n| sort by TimeGenerated desc\n| project MemberName, Action, Organization, Permission\n", + "size": 1, + "title": "Members Added or Removed", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "customWidth": "50", + "name": "membersaddedorremoved" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHub_CL\r\n| extend TimeGenerated = node_createdAt_t\r\n| where node_action_s == \"repo.create\"\r\n| extend RepoName = node_repositoryName_s\r\n| extend Actor = node_actorLogin_s\r\n| extend Private = node_visibility_s\r\n| sort by TimeGenerated desc\r\n| project RepoName, Actor, Private\r\n", + "size": 0, + "title": "Repositories Created", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "customWidth": "50", + "name": "repositoriescreated" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHub_CL\r\n| extend TimeGenerated = node_createdAt_t\r\n| where node_action_s == \"team.add_repository\" or node_action_s == \"team.remove_repository\"\r\n| extend Organization = node_organizationName_s\r\n| extend RepoName = node_repositoryName_s\r\n| extend TeamName = node_teamName_s\r\n| extend Action = iif(node_action_s==\"team.add_repository\", \"Added\", \"Removed\")\r\n| sort by TimeGenerated desc\r\n| project Organization, RepoName, TeamName, Action\r\n", + "size": 0, + "title": "Teams Added/Removed Repository", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "customWidth": "50", + "name": "teamsaddedremovedtorepository" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHub_CL\r\n| extend TimeGenerated = node_createdAt_t\r\n| where node_action_s == \"repo.access\" and node_operationType_s == \"MODIFY\" and node_visibility_s == \"PUBLIC\"\r\n| extend Organiation = node_organizationName_s\r\n| extend Repo = node_repositoryName_s\r\n| extend Actor = node_actorLogin_s\r\n| sort by TimeGenerated desc\r\n| project Organiation, Repo, Actor\r\n", + "size": 0, + "title": "Private Repos made Public", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces" + }, + "customWidth": "50", + "name": "privatereposmadepublic" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHubRepoLogs_CL\r\n| extend TimeGenerated = created_at_t\r\n| where LogType_s == \"Forks\"\r\n| summarize count() by bin(TimeGenerated, 1d), name_s", + "size": 0, + "title": "Fork Count by Repoistory over Time", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart" + }, + "customWidth": "50", + "name": "query - 6" + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "GitHubRepoLogs_CL\r\n| where LogType_s == \"Clones\"\r\n| extend TimeGenerated = timestamp_t\r\n| summarize count() by bin(TimeGenerated, 1d), Repository_s", + "size": 0, + "title": "Clone count by Repository Over Time", + "timeContext": { + "durationMs": 0 + }, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "visualization": "barchart" + }, + "customWidth": "50", + "name": "query - 7" + } + ], + "fromTemplateId": "sentinel-GitHubSecurity", + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" + } \ No newline at end of file diff --git a/Workbooks/Images/Logos/GitHub.svg b/Workbooks/Images/Logos/GitHub.svg new file mode 100644 index 0000000000..12de91c79b --- /dev/null +++ b/Workbooks/Images/Logos/GitHub.svg @@ -0,0 +1,34 @@ + + + + + + + + diff --git a/Workbooks/Images/Preview/GitHubSecurityBlack.png b/Workbooks/Images/Preview/GitHubSecurityBlack.png new file mode 100644 index 0000000000..59b2049c8d Binary files /dev/null and b/Workbooks/Images/Preview/GitHubSecurityBlack.png differ diff --git a/Workbooks/Images/Preview/GitHubSecurityWhite.png b/Workbooks/Images/Preview/GitHubSecurityWhite.png new file mode 100644 index 0000000000..294058378f Binary files /dev/null and b/Workbooks/Images/Preview/GitHubSecurityWhite.png differ diff --git a/Workbooks/WorkbooksMetadata.json b/Workbooks/WorkbooksMetadata.json index 8d15634b42..0193c77c86 100644 --- a/Workbooks/WorkbooksMetadata.json +++ b/Workbooks/WorkbooksMetadata.json @@ -674,5 +674,18 @@ "templateRelativePath": "Perimeter81OverviewWorkbook.json", "subtitle": "", "provider": "Perimeter 81" + }, + { + "workbookKey": "GitHubSecurityWorkbook", + "logoFileName": "github-logo.svg", + "description": "Gain insights to GitHub activities that may be interesting for security.", + "dataTypesDependencies": [ "Github_CL", "GitHubRepoLogs_CL" ], + "dataConnectorsDependencies": [ ], + "previewImagesFileNames": [ "GitHubSecurityWhite.png", "GitHubSecurityBlack.png"], + "version": "1.0", + "title": "GitHub Security", + "templateRelativePath": "GitHubSecurityWorkbook.json", + "subtitle": "", + "provider": "Azure Sentinel community" } ]