Create AzureOpenAIMonitoring.json
This commit is contained in:
Samik Roy
GitHub
Родитель
69c92a9b1c
Коммит
37fab55621
|
|
@ -0,0 +1,462 @@
|
|||
{
|
||||
"version": "Notebook/1.0",
|
||||
"items": [
|
||||
{
|
||||
"type": 1,
|
||||
"content": {
|
||||
"json": "# Azure OpenAI Monitoring Workbook \n---\n\n#### Welcome to this Azure OpenAI Monitoring Workbook\n#### This workbook will help to monitor your Azure Open AI Instances\n\n** Please enable diagnostics settings for the Open AI instance to view the workbook."
|
||||
},
|
||||
"name": "text - 2"
|
||||
},
|
||||
{
|
||||
"type": 9,
|
||||
"content": {
|
||||
"version": "KqlParameterItem/1.0",
|
||||
"parameters": [
|
||||
{
|
||||
"id": "49f60bf3-ce66-45a0-a13c-fdbc90d67e8d",
|
||||
"version": "KqlParameterItem/1.0",
|
||||
"name": "Timespan",
|
||||
"type": 4,
|
||||
"typeSettings": {
|
||||
"selectableValues": [
|
||||
{
|
||||
"durationMs": 300000
|
||||
},
|
||||
{
|
||||
"durationMs": 900000
|
||||
},
|
||||
{
|
||||
"durationMs": 1800000
|
||||
},
|
||||
{
|
||||
"durationMs": 3600000
|
||||
},
|
||||
{
|
||||
"durationMs": 14400000
|
||||
},
|
||||
{
|
||||
"durationMs": 43200000
|
||||
},
|
||||
{
|
||||
"durationMs": 86400000
|
||||
},
|
||||
{
|
||||
"durationMs": 172800000
|
||||
},
|
||||
{
|
||||
"durationMs": 259200000
|
||||
},
|
||||
{
|
||||
"durationMs": 604800000
|
||||
},
|
||||
{
|
||||
"durationMs": 1209600000
|
||||
},
|
||||
{
|
||||
"durationMs": 2419200000
|
||||
},
|
||||
{
|
||||
"durationMs": 2592000000
|
||||
},
|
||||
{
|
||||
"durationMs": 5184000000
|
||||
},
|
||||
{
|
||||
"durationMs": 7776000000
|
||||
}
|
||||
]
|
||||
},
|
||||
"timeContext": {
|
||||
"durationMs": 86400000
|
||||
},
|
||||
"value": {
|
||||
"durationMs": 7776000000
|
||||
}
|
||||
}
|
||||
],
|
||||
"style": "pills",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "parameters - 2"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\n| summarize count() by OperationName\n",
|
||||
"size": 1,
|
||||
"title": "Open AI Operations",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "tiles",
|
||||
"tileSettings": {
|
||||
"titleContent": {
|
||||
"columnMatch": "OperationName"
|
||||
},
|
||||
"secondaryContent": {
|
||||
"columnMatch": "count_",
|
||||
"formatter": 8,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
},
|
||||
"showBorder": false
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 2 - Copy"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "let data = (AzureDiagnostics\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\n| summarize count() by ResultSignature\n| extend Result = case (ResultSignature == 404, \"Not found\", ResultSignature == 200, \"Success\", \"Unknown\"));\nlet data1= (AzureDiagnostics\n| summarize make_list(DurationMs) by ResultSignature);\ndata\n| join kind=inner\n(data1)\n on ResultSignature",
|
||||
"size": 1,
|
||||
"title": "Open AI Operations",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "tiles",
|
||||
"tileSettings": {
|
||||
"titleContent": {
|
||||
"columnMatch": "Result",
|
||||
"formatter": 18,
|
||||
"formatOptions": {
|
||||
"thresholdsOptions": "icons",
|
||||
"thresholdsGrid": [
|
||||
{
|
||||
"operator": "==",
|
||||
"thresholdValue": "Success",
|
||||
"representation": "success",
|
||||
"text": "{0}{1}"
|
||||
},
|
||||
{
|
||||
"operator": "==",
|
||||
"thresholdValue": "Not found",
|
||||
"representation": "3",
|
||||
"text": "{0}{1}"
|
||||
},
|
||||
{
|
||||
"operator": "Default",
|
||||
"thresholdValue": null,
|
||||
"representation": "unknown",
|
||||
"text": "{0}{1}"
|
||||
}
|
||||
],
|
||||
"compositeBarSettings": {
|
||||
"labelText": "",
|
||||
"columnSettings": [
|
||||
{
|
||||
"columnName": "Result",
|
||||
"color": "lightBlue"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"subtitleContent": {
|
||||
"columnMatch": "list_DurationMs",
|
||||
"formatter": 10,
|
||||
"formatOptions": {
|
||||
"palette": "blueDarkDark"
|
||||
}
|
||||
},
|
||||
"secondaryContent": {
|
||||
"columnMatch": "count_",
|
||||
"formatter": 12,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
},
|
||||
"showBorder": false
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 2 - Copy - Copy"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\n| summarize count() by Resource",
|
||||
"size": 0,
|
||||
"title": "Open AI Instances Activity",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "tiles",
|
||||
"tileSettings": {
|
||||
"titleContent": {
|
||||
"columnMatch": "Resource"
|
||||
},
|
||||
"secondaryContent": {
|
||||
"columnMatch": "count_",
|
||||
"formatter": 8,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
},
|
||||
"showBorder": false
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 2"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| where Category == \"RequestResponse\"\r\n| summarize sum(DurationMs) by Resource",
|
||||
"size": 0,
|
||||
"title": "Open AI Total Duration ",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "tiles",
|
||||
"tileSettings": {
|
||||
"titleContent": {
|
||||
"columnMatch": "sum_DurationMs",
|
||||
"formatter": 12,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
},
|
||||
"numberFormat": {
|
||||
"unit": 23,
|
||||
"options": {
|
||||
"style": "decimal"
|
||||
}
|
||||
}
|
||||
},
|
||||
"secondaryContent": {
|
||||
"columnMatch": "Resource"
|
||||
},
|
||||
"showBorder": false
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 18"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\n| summarize count() by ResourceType\n",
|
||||
"size": 1,
|
||||
"title": "Open AI Resource Types",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "tiles",
|
||||
"tileSettings": {
|
||||
"titleContent": {
|
||||
"columnMatch": "ResourceType"
|
||||
},
|
||||
"secondaryContent": {
|
||||
"columnMatch": "count_",
|
||||
"formatter": 8,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
},
|
||||
"showBorder": false
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 2 - Copy - Copy"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| summarize ['Duration in Miliseconds'] = make_list(DurationMs) by Resource",
|
||||
"size": 0,
|
||||
"title": "Request Trend",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"gridSettings": {
|
||||
"formatters": [
|
||||
{
|
||||
"columnMatch": "Duration in Miliseconds",
|
||||
"formatter": 10,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"customWidth": "50",
|
||||
"name": "query - 5"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n|summarize count() by bin(TimeGenerated, {Timespan:grain} ), Resource",
|
||||
"size": 0,
|
||||
"title": "Open AI Activity Timeline",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "timechart",
|
||||
"chartSettings": {
|
||||
"showLegend": true,
|
||||
"customThresholdLine": "8",
|
||||
"customThresholdLineStyle": 5
|
||||
}
|
||||
},
|
||||
"name": "query - 6"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "let baseline = toscalar(AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n|summarize count() by bin(TimeGenerated, 15m)\r\n| summarize avg(count_));\r\nAzureDiagnostics\r\n|summarize count() by bin(TimeGenerated, 15m)\r\n| extend deviation = count_- baseline/ baseline \r\n| project-away count_",
|
||||
"size": 0,
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"visualization": "categoricalbar"
|
||||
},
|
||||
"name": "query - 7"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| where Category == \"RequestResponse\"\r\n| project OperationName, DurationMs, CallerIPAddress, ResourceGroup, ResourceId",
|
||||
"size": 0,
|
||||
"title": "Open AI Request Response",
|
||||
"noDataMessage": "No Open AI Request Response Found",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"gridSettings": {
|
||||
"formatters": [
|
||||
{
|
||||
"columnMatch": "OperationName",
|
||||
"formatter": 5
|
||||
},
|
||||
{
|
||||
"columnMatch": "DurationMs",
|
||||
"formatter": 8,
|
||||
"formatOptions": {
|
||||
"palette": "blue"
|
||||
}
|
||||
}
|
||||
],
|
||||
"filter": true,
|
||||
"hierarchySettings": {
|
||||
"treeType": 1,
|
||||
"groupBy": [
|
||||
"OperationName"
|
||||
],
|
||||
"expandTopLevel": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"name": "query - 9"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| where OperationName == \"ChatCompletions_Create\"\r\n//| where OperationName == \"Creates a completion for the chat message\"\r\n| project TimeGenerated,CallerIPAddress, ResourceId, DurationMs",
|
||||
"size": 0,
|
||||
"title": "Creating a completion for the chat message",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 10"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureActivity\r\n| where OperationNameValue == \"MICROSOFT.COGNITIVESERVICES/ACCOUNTS/DEPLOYMENTS/WRITE\"\r\n| where ActivitySubstatusValue == \"Created\"\r\n| extend Role = parse_json(tostring(parse_json(Authorization).evidence)).role\r\n| extend GivenName = parse_json(Claims).name\r\n| project CategoryValue, CallerIpAddress, Caller, GivenName, Role, ActivityStatusValue",
|
||||
"size": 0,
|
||||
"title": "New Open AI Deployment",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 11"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| join AzureActivity on ResourceGroup\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| where ActivityStatusValue == \"Failure\"\r\n| extend name_ = tostring(parse_json(Claims).name)\r\n| extend UPN = Caller\r\n| project ActivitySubstatus, ActivitySubstatusValue, HTTPRequest, Resource, OperationName, name_, UPN, CallerIpAddress",
|
||||
"size": 0,
|
||||
"title": "Azure Open AI service failures",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 12"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where ResourceProvider == \"MICROSOFT.COGNITIVESERVICES\"\r\n| where Category == \"RequestResponse\"\r\n| project OperationName, DurationMs, CallerIPAddress, ResourceGroup, Resource",
|
||||
"size": 0,
|
||||
"title": "Operations through Request Response",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces",
|
||||
"gridSettings": {
|
||||
"filter": true
|
||||
}
|
||||
},
|
||||
"name": "query - 13"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| join AzureActivity on ResourceGroup\r\n| where Resource contains \"GPT3\"\r\n| where ResourceProvider == \"MICROSOFT.LOGIC\"\r\n| where isnotempty(Caller)\r\n| where Caller hassuffix \"com\"\r\n| extend PlaybookName = resource_workflowName_s\r\n| extend Action = Resource\r\n| distinct Caller, PlaybookName, Action, CallerIpAddress",
|
||||
"size": 0,
|
||||
"title": "ChatGPT usage from Logic App API",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 14"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AzureDiagnostics\r\n| where parse_json(properties_s).apiName == \"Content Safety Service\"\r\n| where OperationName == \"Analyze Image\" or OperationName == \"Analyze Text\"\r\n| distinct CallerIPAddress",
|
||||
"size": 0,
|
||||
"title": "Azure OpenAI Content Safety Service: Detect analyzing text or images",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 16"
|
||||
},
|
||||
{
|
||||
"type": 3,
|
||||
"content": {
|
||||
"version": "KqlItem/1.0",
|
||||
"query": "AADNonInteractiveUserSignInLogs\r\n| where AppDisplayName == \"Azure OpenAI Studio\"\r\n| extend parse_json(LocationDetails).city\r\n| extend parse_json(LocationDetails).countryOrRegion\r\n| extend parse_json(LocationDetails).state\r\n| extend parse_json(tostring(parse_json(LocationDetails).geoCoordinates)).latitude\r\n| extend parse_json(tostring(parse_json(LocationDetails).geoCoordinates)).longitude\r\n| extend parse_json(DeviceDetail).browser\r\n| extend parse_json(DeviceDetail).displayName\r\n| extend parse_json(DeviceDetail).operatingSystem\r\n| extend parse_json(DeviceDetail).trustType\r\n| project UserDisplayName, UserPrincipalName, IPAddress, LocationDetails_city, LocationDetails_state, LocationDetails_countryOrRegion, LocationDetails_geoCoordinates_latitude, LocationDetails_geoCoordinates_longitude, DeviceDetail_browser, DeviceDetail_displayName, DeviceDetail_operatingSystem, DeviceDetail_trustType",
|
||||
"size": 0,
|
||||
"title": "User information using Azure OpenAI Studio",
|
||||
"timeContextFromParameter": "Timespan",
|
||||
"queryType": 0,
|
||||
"resourceType": "microsoft.operationalinsights/workspaces"
|
||||
},
|
||||
"name": "query - 17"
|
||||
}
|
||||
],
|
||||
"fallbackResourceIds": [],
|
||||
"fromTemplateId": "sentinel-Azure OpenAI Monitoring",
|
||||
"$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json"
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче