This commit is contained in:
yash
Родитель
4d3377a798
Коммит
e3e24981cc
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"workbench.colorCustomizations": {}
|
||||
}
|
||||
716
impact.ipynb
716
impact.ipynb
|
|
@ -1,363 +1,353 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Impact Investigation\n",
|
||||
"1. Run all cells.\n",
|
||||
"1. View report at the bottom."
|
||||
],
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"su = \"tfs-cus-1\"\n",
|
||||
"start = \"2019-07-20T16:00:00.0000000Z\"\n",
|
||||
"end = \"2019-07-20T16:33:36.0000000Z\"\n",
|
||||
"url = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK/delays.ipynb\"\n",
|
||||
"baseUrl = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK\"\n",
|
||||
"service = \"tfs\"\n",
|
||||
"hub = \"Build\"\n",
|
||||
"locationName = \"tfsprodcus1\""
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false,
|
||||
"tags": [
|
||||
"parameters"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"!pip install nimport azure-kusto-notebooks"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# Import the things we use\n",
|
||||
"\n",
|
||||
"# Note you can also use kql https://docs.microsoft.com/en-us/azure/data-explorer/kqlmagic\n",
|
||||
"# %kql is single line magic\n",
|
||||
"# %%kql is cell magic\n",
|
||||
"\n",
|
||||
"# https://nbviewer.jupyter.org/github/ipython/ipython/blob/4.0.x/examples/IPython%20Kernel/Rich%20Output.ipynb#HTML\n",
|
||||
"# https://ipython.readthedocs.io/en/stable/inte/magics.html\n",
|
||||
"from IPython.display import display, HTML, Markdown, Javascript, clear_output\n",
|
||||
"\n",
|
||||
"# http://pandas-docs.github.io/pandas-docs-travis/user_guide/reshaping.html\n",
|
||||
"import pandas as pd\n",
|
||||
"pd.options.display.html.table_schema = True\n",
|
||||
"from pandas import Series, DataFrame\n",
|
||||
"from datetime import datetime, timedelta, timezone\n",
|
||||
"from urllib.parse import urlencode, quote_plus\n",
|
||||
"from requests.utils import requote_uri\n",
|
||||
"import time\n",
|
||||
"import numpy as np\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"from nimport.utils import tokenize, open_nb\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import calendar as cal\n",
|
||||
"import concurrent.futures\n",
|
||||
"from azure.kusto.notebooks import utils as akn"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"params = {\n",
|
||||
" \"su\": su,\n",
|
||||
" \"start\": start,\n",
|
||||
" \"end\": end,\n",
|
||||
" \"url\": url,\n",
|
||||
" \"baseUrl\": baseUrl,\n",
|
||||
" \"service\": service\n",
|
||||
"}\n",
|
||||
"root = 'devops-pipelines' if os.path.basename(os.getcwd()) != 'devops-pipelines' else ''\n",
|
||||
"queryPath = os.path.join(root, 'queries')"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"client = akn.get_client('https://vso.kusto.windows.net', 'VSO')"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"icm_client = akn.get_client('https://icmcluster.kusto.windows.net', 'IcMDataWarehouse')"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"outputHidden": false,
|
||||
"inputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"q_loc = os.path.join(queryPath, \"LocationName.csl\")\n",
|
||||
"q_whatChanged = os.path.join(queryPath, \"WhatChanged.csl\")\n",
|
||||
"q_vipSwap = os.path.join(queryPath, \"VIPSwap.csl\")\n",
|
||||
"\n",
|
||||
"impactPath = os.path.join(queryPath, \"impact\")\n",
|
||||
"q_commands = os.path.join(impactPath, \"CommandsReason.csl\")\n",
|
||||
"q_commandsAT = os.path.join(impactPath, \"CommandsAT.csl\")\n",
|
||||
"q_commandsDb = os.path.join(impactPath, \"CommandsDb.csl\")\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" # materialize location name immediately as we need this for other queries\n",
|
||||
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_loc, params)\n",
|
||||
" locationNameResult = akn.to_dataframe_from_future(p1)\n",
|
||||
" locationName = locationNameResult[\"Tenant\"][0]\n",
|
||||
" params[\"locationName\"] = locationName\n",
|
||||
" p2 = executor.submit(akn.execute_file, client, 'VSO', q_whatChanged, params)\n",
|
||||
" p3 = executor.submit(akn.execute_file, client, 'VSO', q_vipSwap, params)\n",
|
||||
" \n",
|
||||
" p4 = executor.submit(akn.execute_file, client, 'VSO', q_commandsAT, params)\n",
|
||||
" p5 = executor.submit(akn.execute_file, client, 'VSO', q_commandsDb, params) \n",
|
||||
" p6 = executor.submit(akn.execute_file, client, 'VSO', q_commands, params)\n",
|
||||
" \n",
|
||||
" p7 = executor.submit(akn.execute_file, icm_client, 'IcmDataWarehouse', \n",
|
||||
" os.path.join(queryPath, 'ActiveIncidents.csl'), params)\n",
|
||||
"\n",
|
||||
"q_whatChanged_df = akn.to_dataframe_from_future(p2)\n",
|
||||
"\n",
|
||||
"vipSwapResultDf = akn.to_dataframe_from_future(p3)\n",
|
||||
"\n",
|
||||
"q_commandsAT_df = akn.to_dataframe_from_future(p4)\n",
|
||||
"\n",
|
||||
"q_commandsDb_df = akn.to_dataframe_from_future(p5)\n",
|
||||
"\n",
|
||||
"q_commands_df = akn.to_dataframe_from_future(p6)\n",
|
||||
"\n",
|
||||
"q_activeIncidentsResultDf = akn.to_dataframe_from_future(p7)"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"print('=' * 50)\n",
|
||||
"print('Report!')\n",
|
||||
"print('=' * 50, '\\n\\n')\n",
|
||||
"\n",
|
||||
"# jarvis params\n",
|
||||
"jarvisParams = {\n",
|
||||
" 'su': su, \n",
|
||||
" 'start': akn.get_time(start, -10), \n",
|
||||
" 'end': akn.get_time(end, 10), \n",
|
||||
" 'service': service \n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# jarvis\n",
|
||||
"jarvisLink = \"\"\"https://jarvis-west.dc.ad.msft.net/dashboard/VSO-ServiceInsights/DevOpsReports/TFS DevOpsReports\"\"\" \\\n",
|
||||
" \"\"\"?overrides=[{\"query\":\"//*[id='Service']\",\"key\":\"value\",\"replacement\":\"%(service)s\"},\"\"\" \\\n",
|
||||
" \"\"\"{\"query\":\"//*[id='RoleInstance']\",\"key\":\"value\",\"replacement\":\"\"},\"\"\" \\\n",
|
||||
" \"\"\"{\"query\":\"//*[id='ScaleUnit']\",\"key\":\"value\",\"replacement\":\"%(su)s\"}]\"\"\" \\\n",
|
||||
" \"\"\"&globalStartTime=%(start)s&globalEndTime=%(end)s&pinGlobalTimeRange=true\"\"\" % jarvisParams;\n",
|
||||
"print('Jarvis dashboard link:\\n', requote_uri(jarvisLink), '\\n')\n",
|
||||
"\n",
|
||||
"#\n",
|
||||
"# vip swap\n",
|
||||
"print()\n",
|
||||
"print('Vip Swap? =============================')\n",
|
||||
"if len(vipSwapResultDf.index) > 0:\n",
|
||||
" viptime = vipSwapResultDf[\"TIMESTAMP\"][0]\n",
|
||||
" starttime = akn.to_datetime(start)\n",
|
||||
" delta = starttime.replace(tzinfo=None) - viptime.replace(tzinfo=None)\n",
|
||||
" if delta.total_seconds() > 0:\n",
|
||||
" print(\"\"\"VIP SWAP happened: %s days %s hours %s minutes ago (%s) (issue start: %s)\"\"\" % (delta.days, delta.seconds//3600, (delta.seconds//60) % 60, viptime, start))\n",
|
||||
" else:\n",
|
||||
" print('...no swaps recorded in the given time range')\n",
|
||||
"else:\n",
|
||||
" print('...no swaps recorded in the given time range')\n",
|
||||
" \n",
|
||||
"# slow failed reason analysis\n",
|
||||
"print()\n",
|
||||
"print('Is it slow commands or failed commands? =============================')\n",
|
||||
"freq = q_commands_df[\"Frequency\"]\n",
|
||||
"coefficientOfVariance = freq.std()/freq.mean()\n",
|
||||
"failedCount = q_commands_df[q_commands_df[\"Reason\"] == \"failed\"][\"Frequency\"].values[0]\n",
|
||||
"slowCount = q_commands_df[q_commands_df[\"Reason\"] == \"slow\"][\"Frequency\"].values[0]\n",
|
||||
"reason = \"failed or slow\"\n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" if failedCount > slowCount:\n",
|
||||
" reason = \"failed\"\n",
|
||||
" else:\n",
|
||||
" reason = \"slow\"\n",
|
||||
"else:\n",
|
||||
" print(\"Slow and failed commands are too close, both might be contributing...\")\n",
|
||||
"if reason:\n",
|
||||
" print(\"Probably due to %s commands; Failed - %s, Slow - %s\" % (reason, failedCount, slowCount))\n",
|
||||
"\n",
|
||||
"# slow failed reason for AT?\n",
|
||||
"print()\n",
|
||||
"print('Is it %s because of AT? =============================' % (reason))\n",
|
||||
"failed = q_commandsAT_df[q_commandsAT_df[\"Reason\"] == \"failed\"]\n",
|
||||
"slow = q_commandsAT_df[q_commandsAT_df[\"Reason\"] == \"slow\"]\n",
|
||||
"data = q_commandsAT_df\n",
|
||||
"if reason == \"failed\":\n",
|
||||
" data = failed\n",
|
||||
"elif reason == \"slow\":\n",
|
||||
" data = slow\n",
|
||||
"\n",
|
||||
"coefficientOfVariance = data[\"Frequency\"].std()/data[\"Frequency\"].mean()\n",
|
||||
" \n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" print(\"Found variance in AT's for %s commands\" % (reason))\n",
|
||||
" print(data.head(30))\n",
|
||||
"else:\n",
|
||||
" print(\"Seems be same across AT's for %s commands\" % (reason))\n",
|
||||
" \n",
|
||||
"# slow failed reason for Db?\n",
|
||||
"print()\n",
|
||||
"print('Is it %s because of Db? =============================' % (reason))\n",
|
||||
"failed = q_commandsDb_df[q_commandsDb_df[\"Reason\"] == \"failed\"]\n",
|
||||
"slow = q_commandsDb_df[q_commandsDb_df[\"Reason\"] == \"slow\"]\n",
|
||||
"data = q_commandsDb_df\n",
|
||||
"if reason == \"failed\":\n",
|
||||
" data = failed\n",
|
||||
"elif reason == \"slow\":\n",
|
||||
" data = slow\n",
|
||||
"\n",
|
||||
"coefficientOfVariance = data[\"Frequency\"].std()/data[\"Frequency\"].mean()\n",
|
||||
" \n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" print(\"Found variance in Db's for %s commands\" % (reason))\n",
|
||||
" print(\"Suffix '%s' to database server name\" % (\".database.windows.net\"))\n",
|
||||
" print(\"Prefix '%s' to database name\" % (params[\"service\"] + \"_\" + params[\"locationName\"] + \"_\"))\n",
|
||||
" print(data.head(30))\n",
|
||||
"else:\n",
|
||||
" print(\"Seems be same across Db's for %s commands\" % (reason)) \n",
|
||||
" \n",
|
||||
"# what changed? analysis\n",
|
||||
"print()\n",
|
||||
"print('What changed? =============================')\n",
|
||||
"if(len(q_whatChanged_df.index) == 0):\n",
|
||||
" print(\"No relevant changes found...\")\n",
|
||||
"else:\n",
|
||||
" up_prefix = \"\";\n",
|
||||
" mit_prefix = \"\";\n",
|
||||
" text = \"\";\n",
|
||||
" for index, row in q_whatChanged_df.iterrows():\n",
|
||||
" if(row.title.lower().find('upgrade') != -1):\n",
|
||||
" if not up_prefix:\n",
|
||||
" up_prefix += \"Looks like, there's upgrade in progress...\\n\";\n",
|
||||
" text += \"\"\"%s %s %s \\n\"\"\" % (row.TIMESTAMP, row.title, row.buildNumber);\n",
|
||||
" if(row.title.lower().find('mitigation') != -1):\n",
|
||||
" if not mit_prefix:\n",
|
||||
" mit_prefix += \"Looks like, there are some mitigations by health agent...\\n\";\n",
|
||||
" state += states[\"healthagent\"];\n",
|
||||
" text += \"\"\"%s %s %s\"\"\" % (row.TIMESTAMP, row.title, row.buildNumber);\n",
|
||||
" \n",
|
||||
" if text:\n",
|
||||
" print(up_prefix + mit_prefix + text)\n",
|
||||
" else:\n",
|
||||
" print(q_whatChanged_df)\n",
|
||||
" \n",
|
||||
"# active incidents?\n",
|
||||
"print()\n",
|
||||
"print('Active incidents? =============================')\n",
|
||||
"otherIncidentsCount = 0;\n",
|
||||
"for index, row in q_activeIncidentsResultDf.iterrows():\n",
|
||||
" if(row.Title.find(\"TFS Customer Impact Monitor\") == -1):\n",
|
||||
" otherIncidentsCount+=1;\n",
|
||||
" \n",
|
||||
"if(otherIncidentsCount > 0):\n",
|
||||
" print(\"We found some incidents during the time period, check if they are related...\")\n",
|
||||
" # styling\n",
|
||||
" def make_clickable(url, text):\n",
|
||||
" return '{0}'.format(url)\n",
|
||||
"\n",
|
||||
" newDf = q_activeIncidentsResultDf.assign(URL=[*map(lambda x: make_clickable(\"\"\"https://icm.ad.msft.net/imp/v3/incidents/details/%s/home\"\"\" % (x), \"ICMLink\"), q_activeIncidentsResultDf.IncidentId)])\n",
|
||||
" print(\"ICM link to copy - \" + \"https://icm.ad.msft.net/imp/v3/incidents/details/INCIDENTID/home\")\n",
|
||||
" print(newDf[['IncidentId','Severity','Title']])\n",
|
||||
"else:\n",
|
||||
" print(\"No active incidents that could be related are found...\") "
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernel_info": {
|
||||
"name": "python3"
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"language": "python",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.7.4",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "ipython3",
|
||||
"nbconvert_exporter": "python",
|
||||
"file_extension": ".py"
|
||||
},
|
||||
"nteract": {
|
||||
"version": "0.14.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Impact Investigation\n",
|
||||
"1. Run all cells.\n",
|
||||
"1. View report at the bottom."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false,
|
||||
"tags": [
|
||||
"parameters"
|
||||
]
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# These are just defaults will be overwritten if you use nimport pip\n",
|
||||
"start = \"2019-08-05T13:10:00.0000000Z\"\n",
|
||||
"end = \"2019-08-05T14:10:00.0000000Z\"\n",
|
||||
"service = \"tfs\"\n",
|
||||
"su = \"tfs-sbr-1\"\n",
|
||||
"url = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK/impact.ipynb\"\n",
|
||||
"baseUrl = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"!pip install nimport azure-kusto-notebooks"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import the things we use\n",
|
||||
"\n",
|
||||
"# Note you can also use kql https://docs.microsoft.com/en-us/azure/data-explorer/kqlmagic\n",
|
||||
"# %kql is single line magic\n",
|
||||
"# %%kql is cell magic\n",
|
||||
"\n",
|
||||
"# https://nbviewer.jupyter.org/github/ipython/ipython/blob/4.0.x/examples/IPython%20Kernel/Rich%20Output.ipynb#HTML\n",
|
||||
"# https://ipython.readthedocs.io/en/stable/inte/magics.html\n",
|
||||
"from IPython.display import display, HTML, Markdown, Javascript, clear_output\n",
|
||||
"\n",
|
||||
"# http://pandas-docs.github.io/pandas-docs-travis/user_guide/reshaping.html\n",
|
||||
"import pandas as pd\n",
|
||||
"pd.options.display.html.table_schema = True\n",
|
||||
"from pandas import Series, DataFrame\n",
|
||||
"from datetime import datetime, timedelta, timezone\n",
|
||||
"from urllib.parse import urlencode, quote_plus\n",
|
||||
"from requests.utils import requote_uri\n",
|
||||
"import time\n",
|
||||
"import numpy as np\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"from nimport.utils import tokenize, open_nb\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import calendar as cal\n",
|
||||
"import concurrent.futures\n",
|
||||
"from azure.kusto.notebooks import utils as akn"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"params = {\n",
|
||||
" \"su\": su,\n",
|
||||
" \"start\": start,\n",
|
||||
" \"end\": end,\n",
|
||||
" \"url\": url,\n",
|
||||
" \"baseUrl\": baseUrl,\n",
|
||||
" \"service\": service\n",
|
||||
"}\n",
|
||||
"root = 'devops-pipelines' if os.path.basename(os.getcwd()) != 'devops-pipelines' else ''\n",
|
||||
"queryPath = os.path.join(root, 'queries')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"client = akn.get_client('https://vso.kusto.windows.net', 'VSO')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"icm_client = akn.get_client('https://icmcluster.kusto.windows.net', 'IcMDataWarehouse')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"q_loc = os.path.join(queryPath, \"LocationName.csl\")\n",
|
||||
"q_whatChanged = os.path.join(queryPath, \"WhatChanged.csl\")\n",
|
||||
"q_vipSwap = os.path.join(queryPath, \"VIPSwap.csl\")\n",
|
||||
"\n",
|
||||
"impactPath = os.path.join(queryPath, \"impact\")\n",
|
||||
"q_commands = os.path.join(impactPath, \"CommandsReason.csl\")\n",
|
||||
"q_commandsAT = os.path.join(impactPath, \"CommandsAT.csl\")\n",
|
||||
"q_commandsDb = os.path.join(impactPath, \"CommandsDb.csl\")\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" # materialize location name immediately as we need this for other queries\n",
|
||||
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_loc, params)\n",
|
||||
" locationNameResult = akn.to_dataframe_from_future(p1)\n",
|
||||
" locationName = locationNameResult[\"Tenant\"][0]\n",
|
||||
" params[\"locationName\"] = locationName\n",
|
||||
" p2 = executor.submit(akn.execute_file, client, 'VSO', q_whatChanged, params)\n",
|
||||
" p3 = executor.submit(akn.execute_file, client, 'VSO', q_vipSwap, params)\n",
|
||||
" \n",
|
||||
" p4 = executor.submit(akn.execute_file, client, 'VSO', q_commandsAT, params)\n",
|
||||
" p5 = executor.submit(akn.execute_file, client, 'VSO', q_commandsDb, params) \n",
|
||||
" p6 = executor.submit(akn.execute_file, client, 'VSO', q_commands, params)\n",
|
||||
" \n",
|
||||
" p7 = executor.submit(akn.execute_file, icm_client, 'IcmDataWarehouse', \n",
|
||||
" os.path.join(queryPath, 'ActiveIncidents.csl'), params)\n",
|
||||
"\n",
|
||||
"q_whatChanged_df = akn.to_dataframe_from_future(p2)\n",
|
||||
"\n",
|
||||
"vipSwapResultDf = akn.to_dataframe_from_future(p3)\n",
|
||||
"\n",
|
||||
"q_commandsAT_df = akn.to_dataframe_from_future(p4)\n",
|
||||
"\n",
|
||||
"q_commandsDb_df = akn.to_dataframe_from_future(p5)\n",
|
||||
"\n",
|
||||
"q_commands_df = akn.to_dataframe_from_future(p6)\n",
|
||||
"\n",
|
||||
"q_activeIncidentsResultDf = akn.to_dataframe_from_future(p7)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print('=' * 50)\n",
|
||||
"print('Report!')\n",
|
||||
"print('=' * 50, '\\n\\n')\n",
|
||||
"\n",
|
||||
"# jarvis params\n",
|
||||
"jarvisParams = {\n",
|
||||
" 'su': su, \n",
|
||||
" 'start': akn.get_time(start, -10), \n",
|
||||
" 'end': akn.get_time(end, 10), \n",
|
||||
" 'service': service \n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# jarvis\n",
|
||||
"jarvisLink = \"\"\"https://jarvis-west.dc.ad.msft.net/dashboard/VSO-ServiceInsights/DevOpsReports/TFS DevOpsReports\"\"\" \\\n",
|
||||
" \"\"\"?overrides=[{\"query\":\"//*[id='Service']\",\"key\":\"value\",\"replacement\":\"%(service)s\"},\"\"\" \\\n",
|
||||
" \"\"\"{\"query\":\"//*[id='RoleInstance']\",\"key\":\"value\",\"replacement\":\"\"},\"\"\" \\\n",
|
||||
" \"\"\"{\"query\":\"//*[id='ScaleUnit']\",\"key\":\"value\",\"replacement\":\"%(su)s\"}]\"\"\" \\\n",
|
||||
" \"\"\"&globalStartTime=%(start)s&globalEndTime=%(end)s&pinGlobalTimeRange=true\"\"\" % jarvisParams;\n",
|
||||
"print('Jarvis dashboard link:\\n', requote_uri(jarvisLink), '\\n')\n",
|
||||
"\n",
|
||||
"#\n",
|
||||
"# vip swap\n",
|
||||
"print()\n",
|
||||
"print('Vip Swap? =============================')\n",
|
||||
"if len(vipSwapResultDf.index) > 0:\n",
|
||||
" viptime = vipSwapResultDf[\"TIMESTAMP\"][0]\n",
|
||||
" starttime = akn.to_datetime(start)\n",
|
||||
" delta = starttime.replace(tzinfo=None) - viptime.replace(tzinfo=None)\n",
|
||||
" if delta.total_seconds() > 0:\n",
|
||||
" print(\"\"\"VIP SWAP happened: %s days %s hours %s minutes ago (%s) (issue start: %s)\"\"\" % (delta.days, delta.seconds//3600, (delta.seconds//60) % 60, viptime, start))\n",
|
||||
" else:\n",
|
||||
" print('...no swaps recorded in the given time range')\n",
|
||||
"else:\n",
|
||||
" print('...no swaps recorded in the given time range')\n",
|
||||
" \n",
|
||||
"# slow failed reason analysis\n",
|
||||
"print()\n",
|
||||
"print('Is it slow commands or failed commands? =============================')\n",
|
||||
"freq = q_commands_df[\"Frequency\"]\n",
|
||||
"coefficientOfVariance = freq.std()/freq.mean()\n",
|
||||
"failedCount = q_commands_df[q_commands_df[\"Reason\"] == \"failed\"][\"Frequency\"].values[0]\n",
|
||||
"slowCount = q_commands_df[q_commands_df[\"Reason\"] == \"slow\"][\"Frequency\"].values[0]\n",
|
||||
"reason = \"failed or slow\"\n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" if failedCount > slowCount:\n",
|
||||
" reason = \"failed\"\n",
|
||||
" else:\n",
|
||||
" reason = \"slow\"\n",
|
||||
"else:\n",
|
||||
" print(\"Slow and failed commands are too close, both might be contributing...\")\n",
|
||||
"if reason:\n",
|
||||
" print(\"Probably due to %s commands; Failed - %s, Slow - %s\" % (reason, failedCount, slowCount))\n",
|
||||
"\n",
|
||||
"# slow failed reason for AT?\n",
|
||||
"print()\n",
|
||||
"print('Is it %s because of AT? =============================' % (reason))\n",
|
||||
"failed = q_commandsAT_df[q_commandsAT_df[\"Reason\"] == \"failed\"]\n",
|
||||
"slow = q_commandsAT_df[q_commandsAT_df[\"Reason\"] == \"slow\"]\n",
|
||||
"data = q_commandsAT_df\n",
|
||||
"if reason == \"failed\":\n",
|
||||
" data = failed\n",
|
||||
"elif reason == \"slow\":\n",
|
||||
" data = slow\n",
|
||||
"\n",
|
||||
"coefficientOfVariance = data[\"Frequency\"].std()/data[\"Frequency\"].mean()\n",
|
||||
" \n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" print(\"Found variance in AT's for %s commands\" % (reason))\n",
|
||||
" print(data.head(30))\n",
|
||||
"else:\n",
|
||||
" print(\"Seems be same across AT's for %s commands\" % (reason))\n",
|
||||
" \n",
|
||||
"# slow failed reason for Db?\n",
|
||||
"print()\n",
|
||||
"print('Is it %s because of Db? =============================' % (reason))\n",
|
||||
"failed = q_commandsDb_df[q_commandsDb_df[\"Reason\"] == \"failed\"]\n",
|
||||
"slow = q_commandsDb_df[q_commandsDb_df[\"Reason\"] == \"slow\"]\n",
|
||||
"data = q_commandsDb_df\n",
|
||||
"if reason == \"failed\":\n",
|
||||
" data = failed\n",
|
||||
"elif reason == \"slow\":\n",
|
||||
" data = slow\n",
|
||||
"\n",
|
||||
"coefficientOfVariance = data[\"Frequency\"].std()/data[\"Frequency\"].mean()\n",
|
||||
" \n",
|
||||
"if coefficientOfVariance > 0.5:\n",
|
||||
" print(\"Found variance in Db's for %s commands\" % (reason))\n",
|
||||
" print(\"Suffix '%s' to database server name\" % (\".database.windows.net\"))\n",
|
||||
" print(\"Prefix '%s' to database name\" % (params[\"service\"] + \"_\" + params[\"locationName\"] + \"_\"))\n",
|
||||
" print(data.head(30))\n",
|
||||
"else:\n",
|
||||
" print(\"Seems be same across Db's for %s commands\" % (reason)) \n",
|
||||
" \n",
|
||||
"# what changed? analysis\n",
|
||||
"print()\n",
|
||||
"print('What changed? =============================')\n",
|
||||
"if(len(q_whatChanged_df.index) == 0):\n",
|
||||
" print(\"No relevant changes found...\")\n",
|
||||
"else:\n",
|
||||
" up_prefix = \"\";\n",
|
||||
" mit_prefix = \"\";\n",
|
||||
" text = \"\";\n",
|
||||
" for index, row in q_whatChanged_df.iterrows():\n",
|
||||
" if(row.title.lower().find('upgrade') != -1):\n",
|
||||
" if not up_prefix:\n",
|
||||
" up_prefix += \"Looks like, there's upgrade in progress...\\n\";\n",
|
||||
" text += \"\"\"%s %s %s \\n\"\"\" % (row.TIMESTAMP, row.title, row.buildNumber);\n",
|
||||
" if(row.title.lower().find('mitigation') != -1):\n",
|
||||
" if not mit_prefix:\n",
|
||||
" mit_prefix += \"Looks like, there are some mitigations by health agent...\\n\";\n",
|
||||
" text += \"\"\"%s %s %s\"\"\" % (row.TIMESTAMP, row.title, row.buildNumber);\n",
|
||||
" \n",
|
||||
" if text:\n",
|
||||
" print(up_prefix + mit_prefix + text)\n",
|
||||
" else:\n",
|
||||
" print(q_whatChanged_df)\n",
|
||||
" \n",
|
||||
"# active incidents?\n",
|
||||
"print()\n",
|
||||
"print('Active incidents? =============================')\n",
|
||||
"otherIncidentsCount = 0;\n",
|
||||
"for index, row in q_activeIncidentsResultDf.iterrows():\n",
|
||||
" if(row.Title.find(\"TFS Customer Impact Monitor\") == -1):\n",
|
||||
" otherIncidentsCount+=1;\n",
|
||||
" \n",
|
||||
"if(otherIncidentsCount > 0):\n",
|
||||
" print(\"We found some incidents during the time period, check if they are related...\")\n",
|
||||
" # styling\n",
|
||||
" def make_clickable(url, text):\n",
|
||||
" return '{0}'.format(url)\n",
|
||||
"\n",
|
||||
" newDf = q_activeIncidentsResultDf.assign(URL=[*map(lambda x: make_clickable(\"\"\"https://icm.ad.msft.net/imp/v3/incidents/details/%s/home\"\"\" % (x), \"ICMLink\"), q_activeIncidentsResultDf.IncidentId)])\n",
|
||||
" print(\"ICM link to copy - \" + \"https://icm.ad.msft.net/imp/v3/incidents/details/INCIDENTID/home\")\n",
|
||||
" print(newDf[['IncidentId','Severity','Title']])\n",
|
||||
"else:\n",
|
||||
" print(\"No active incidents that could be related are found...\") "
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernel_info": {
|
||||
"name": "python3"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.4"
|
||||
},
|
||||
"nteract": {
|
||||
"version": "0.14.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
|
|
|
|||
606
sla.ipynb
606
sla.ipynb
|
|
@ -1,308 +1,298 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# SLA Investigation\n",
|
||||
"1. Run all cells! (click on Menu > Cell > Run All Cells)\n",
|
||||
"1. View report at the bottom."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false,
|
||||
"tags": [
|
||||
"parameters"
|
||||
]
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triggerTime = \"2019-08-01T22:00:37.0000000Z\"\n",
|
||||
"lookback = '1h'\n",
|
||||
"scaleUnit = ''\n",
|
||||
"service = ''\n",
|
||||
"region = ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture \n",
|
||||
"\n",
|
||||
"# install packages, setup workspace root\n",
|
||||
"!pip install --upgrade pip azure-kusto-notebooks\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"import datetime\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"pd.options.display.html.table_schema = True\n",
|
||||
"import concurrent.futures\n",
|
||||
"from azure.kusto.notebooks import utils as akn\n",
|
||||
"\n",
|
||||
"# cwd should be workspace root\n",
|
||||
"if os.path.basename(os.getcwd()) == 'devops-pipelines':\n",
|
||||
" os.chdir(os.pardir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"client = akn.get_client('https://vso.kusto.windows.net', 'VSO')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# find orchestrations that violate SLA\n",
|
||||
"params = {\n",
|
||||
" 'TriggerTime': akn.to_kusto_datetime(triggerTime),\n",
|
||||
" 'Lookback': akn.to_kusto_timespan(lookback),\n",
|
||||
" 'Service': '\"' + service + '\"', \n",
|
||||
" 'Region': '\"' + region + '\"',\n",
|
||||
" 'ScaleUnit': '\"' + scaleUnit + '\"'\n",
|
||||
"}\n",
|
||||
"query = os.path.join('devops-pipelines', 'queries', 'sla', 'SLADurationAnalysis.csl')\n",
|
||||
"violations = akn.execute_file(client, database='VSO', path=query, params=params)\n",
|
||||
"# violations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# collect problematic orchestration ids\n",
|
||||
"result = violations.primary_results[0]\n",
|
||||
"oid_column_index = next((c.ordinal for c in result.columns if c.column_name == 'OrchestrationId'), None)\n",
|
||||
"su_column_index = next((c.ordinal for c in result.columns if c.column_name == 'ScaleUnit'), None)\n",
|
||||
"\n",
|
||||
"# group\n",
|
||||
"by_su = {}\n",
|
||||
"for r in result.rows:\n",
|
||||
" su = r[su_column_index]\n",
|
||||
" oid = r[oid_column_index]\n",
|
||||
" l = by_su.get(su, [])\n",
|
||||
" by_su[su] = l\n",
|
||||
" l.append(oid)\n",
|
||||
"\n",
|
||||
"max_scale_units = []\n",
|
||||
"max_problems = 0\n",
|
||||
"for k,v in by_su.items():\n",
|
||||
" c = len(v)\n",
|
||||
" if c > max_problems:\n",
|
||||
" max_problems = c\n",
|
||||
" max_scale_units = [k]\n",
|
||||
" elif c == max_problems:\n",
|
||||
" max_scale_units.append(k)\n",
|
||||
"max_scale_units.sort()\n",
|
||||
"\n",
|
||||
"# for su, oids in by_su.items():\n",
|
||||
"# print(su)\n",
|
||||
"# for oid in oids:\n",
|
||||
"# print(' ', oid)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# collect visualization data sets\n",
|
||||
"query = os.path.join('devops-pipelines', 'queries', 'sla', 'SLAVisualization.csl')\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" hfs = [executor.submit(akn.execute_file, client, 'VSO', query, \n",
|
||||
" {\n",
|
||||
" 'ScaleUnit': '\"' + r[su_column_index] + '\"', \n",
|
||||
" 'OrchestrationId': '\"' + r[oid_column_index] + '\"'\n",
|
||||
" }) for r in result.rows]\n",
|
||||
" histories = [h.result() for h in concurrent.futures.as_completed(hfs)]\n",
|
||||
"\n",
|
||||
"# convert to data frames\n",
|
||||
"primary_results = [h.primary_results[0] for h in histories]\n",
|
||||
"dataframes = None\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" dataframe_futures = [executor.submit(akn.to_dataframe, r) for r in primary_results]\n",
|
||||
" dataframes = [dff.result() for dff in concurrent.futures.as_completed(dataframe_futures)]\n",
|
||||
"histories = None\n",
|
||||
"\n",
|
||||
"# try to filter out false positives? at least a certain number of phases must have been recorded.\n",
|
||||
"required_phases = ('RunAgentJob.SendJob', 'RunAgentJob.JobCompleted')\n",
|
||||
"filtered_dataframes = [df for df in dataframes if all([p in df['PhaseName'].values for p in required_phases])]\n",
|
||||
"number_of_false_positives = len(dataframes) - len(filtered_dataframes)\n",
|
||||
"dataframes = filtered_dataframes\n",
|
||||
"number_of_violations = len(dataframes)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if dataframes:\n",
|
||||
" # what was the worst phase?\n",
|
||||
" combined = pd.concat(dataframes, ignore_index=True)\n",
|
||||
" df = combined.loc[combined['Level'] == 2].groupby(['PhaseName']).size().to_frame('Count').nlargest(1, 'Count')\n",
|
||||
" worst_phaseName = df.index[0]\n",
|
||||
" worst_count = df.iat[0, 0]\n",
|
||||
" worst_team = worst_phaseName.split('.')[0]\n",
|
||||
" \n",
|
||||
" # what was the worst plan?\n",
|
||||
" violations_df = akn.to_dataframe(violations.primary_results[0])\n",
|
||||
" df = violations_df.groupby(['PlanId']).size().to_frame('Count').nlargest(1, 'Count')\n",
|
||||
" worst_plan_name = df.index[0]\n",
|
||||
" worst_plan_count = df.iat[0, 0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if number_of_false_positives:\n",
|
||||
" print('INSIGHT:', number_of_false_positives, 'plans are likely missing kusto data and were ignored.')\n",
|
||||
"if number_of_violations <= 0:\n",
|
||||
" print('INSIGHT: no problems detected')\n",
|
||||
"else:\n",
|
||||
" print('INSIGHT:', number_of_violations, 'plans are out of SLA.')\n",
|
||||
" print('INSIGHT:', max_problems, 'of them are in', ', '.join(max_scale_units))\n",
|
||||
" print('INSIGHT:', worst_plan_count, 'of the', number_of_violations, 'SLA violations were caused by PlanId', worst_plan_name, '.')\n",
|
||||
" print('INSIGHT:', '\"' + worst_phaseName + '\"', 'was the slowest phase in', worst_count, 'of the', number_of_violations, 'SLA violations.')\n",
|
||||
" print()\n",
|
||||
" print('ACTION: open icm against scale units:', max_scale_units)\n",
|
||||
" print(' initially route it to: ', worst_team)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# find top 25 worst orchestrations\n",
|
||||
"vdf = violations_df.nlargest(25, 'TotalSLADuration')\n",
|
||||
"# vdf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"plt.rcdefaults()\n",
|
||||
"\n",
|
||||
"number_of_graphs = len(vdf.index)\n",
|
||||
"fig, axes = plt.subplots(nrows=number_of_graphs, ncols=1, figsize=(8, 6 * number_of_graphs), constrained_layout=True)\n",
|
||||
"for i in range(number_of_graphs):\n",
|
||||
" df = dataframes[vdf.index[i]]\n",
|
||||
" ax = axes[i] if number_of_graphs > 1 else axes\n",
|
||||
" ax.axhline(0, color='k')\n",
|
||||
" \n",
|
||||
" x = df['PhaseName']\n",
|
||||
" xpos = np.arange(len(x))\n",
|
||||
" y = df['PercentDifference']\n",
|
||||
" plan_id = df['PlanId'].iloc[0]\n",
|
||||
" violation_row = violations_df.loc[violations_df['PlanId'] == plan_id]\n",
|
||||
" title = '\\n'.join([\n",
|
||||
" 'plan id:' + plan_id,\n",
|
||||
" 'scale unit:' + str(violation_row['ScaleUnit'].iloc[0]),\n",
|
||||
" 'definition:' + str(df['DefinitionName'].iloc[0]),\n",
|
||||
" 'plan duration: ' + str(violation_row['PlanDuration'].iloc[0]),\n",
|
||||
" 'sla duration: ' + str(violation_row['TotalSLADuration'].iloc[0]),\n",
|
||||
" ])\n",
|
||||
" ax.title.set_text(title)\n",
|
||||
" \n",
|
||||
" ax.bar(x=xpos, height=y)\n",
|
||||
" ax.set_xticks(xpos)\n",
|
||||
" ax.set_xticklabels(x, rotation=45, ha=\"right\")\n",
|
||||
"\n",
|
||||
"# output_filename = 'analysis.svg'\n",
|
||||
"# plt.savefig(output_filename, format='svg')"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
},
|
||||
"nteract": {
|
||||
"version": "nteract-on-jupyter@2.1.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# SLA Investigation\n",
|
||||
"1. Run all cells! (click on Menu > Cell > Run All Cells)\n",
|
||||
"1. View report at the bottom."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false,
|
||||
"tags": [
|
||||
"parameters"
|
||||
]
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triggerTime = \"2019-08-01T22:00:37.0000000Z\"\n",
|
||||
"lookback = '1h'\n",
|
||||
"scaleUnit = ''\n",
|
||||
"service = ''\n",
|
||||
"region = ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture \n",
|
||||
"\n",
|
||||
"# install packages, setup workspace root\n",
|
||||
"!pip install --upgrade pip azure-kusto-notebooks\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"import datetime\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"pd.options.display.html.table_schema = True\n",
|
||||
"import concurrent.futures\n",
|
||||
"from azure.kusto.notebooks import utils as akn\n",
|
||||
"\n",
|
||||
"# cwd should be workspace root\n",
|
||||
"if os.path.basename(os.getcwd()) == 'devops-pipelines':\n",
|
||||
" os.chdir(os.pardir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# authenticate kusto client\n",
|
||||
"# you will need to copy the token into a browser window for AAD auth. \n",
|
||||
"client = akn.get_client('https://vso.kusto.windows.net', 'VSO')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# find orchestrations that violate SLA\n",
|
||||
"params = {\n",
|
||||
" 'TriggerTime': akn.to_kusto_datetime(triggerTime),\n",
|
||||
" 'Lookback': akn.to_kusto_timespan(lookback),\n",
|
||||
" 'Service': '\"' + service + '\"', \n",
|
||||
" 'Region': '\"' + region + '\"',\n",
|
||||
" 'ScaleUnit': '\"' + scaleUnit + '\"'\n",
|
||||
"}\n",
|
||||
"query = os.path.join('devops-pipelines', 'queries', 'sla', 'SLADurationAnalysis.csl')\n",
|
||||
"violations = akn.execute_file(client, database='VSO', path=query, params=params)\n",
|
||||
"# violations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# collect problematic orchestration ids\n",
|
||||
"result = violations.primary_results[0]\n",
|
||||
"oid_column_index = next((c.ordinal for c in result.columns if c.column_name == 'OrchestrationId'), None)\n",
|
||||
"su_column_index = next((c.ordinal for c in result.columns if c.column_name == 'ScaleUnit'), None)\n",
|
||||
"\n",
|
||||
"# group\n",
|
||||
"by_su = {}\n",
|
||||
"for r in result.rows:\n",
|
||||
" su = r[su_column_index]\n",
|
||||
" oid = r[oid_column_index]\n",
|
||||
" l = by_su.get(su, [])\n",
|
||||
" by_su[su] = l\n",
|
||||
" l.append(oid)\n",
|
||||
"\n",
|
||||
"max_scale_units = []\n",
|
||||
"max_problems = 0\n",
|
||||
"for k,v in by_su.items():\n",
|
||||
" c = len(v)\n",
|
||||
" if c > max_problems:\n",
|
||||
" max_problems = c\n",
|
||||
" max_scale_units = [k]\n",
|
||||
" elif c == max_problems:\n",
|
||||
" max_scale_units.append(k)\n",
|
||||
"max_scale_units.sort()\n",
|
||||
"\n",
|
||||
"# for su, oids in by_su.items():\n",
|
||||
"# print(su)\n",
|
||||
"# for oid in oids:\n",
|
||||
"# print(' ', oid)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# collect visualization data sets\n",
|
||||
"query = os.path.join('devops-pipelines', 'queries', 'sla', 'SLAVisualization.csl')\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" hfs = [executor.submit(akn.execute_file, client, 'VSO', query, \n",
|
||||
" {\n",
|
||||
" 'ScaleUnit': '\"' + r[su_column_index] + '\"', \n",
|
||||
" 'OrchestrationId': '\"' + r[oid_column_index] + '\"'\n",
|
||||
" }) for r in result.rows]\n",
|
||||
" histories = [h.result() for h in concurrent.futures.as_completed(hfs)]\n",
|
||||
"\n",
|
||||
"# convert to data frames\n",
|
||||
"primary_results = [h.primary_results[0] for h in histories]\n",
|
||||
"dataframes = None\n",
|
||||
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
||||
" dataframe_futures = [executor.submit(akn.to_dataframe, r) for r in primary_results]\n",
|
||||
" dataframes = [dff.result() for dff in concurrent.futures.as_completed(dataframe_futures)]\n",
|
||||
"histories = None\n",
|
||||
"\n",
|
||||
"# try to filter out false positives? at least a certain number of phases must have been recorded.\n",
|
||||
"required_phases = ('RunAgentJob.SendJob', 'RunAgentJob.JobCompleted')\n",
|
||||
"filtered_dataframes = [df for df in dataframes if all([p in df['PhaseName'].values for p in required_phases])]\n",
|
||||
"number_of_false_positives = len(dataframes) - len(filtered_dataframes)\n",
|
||||
"dataframes = filtered_dataframes\n",
|
||||
"number_of_violations = len(dataframes)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if dataframes:\n",
|
||||
" # what was the worst phase?\n",
|
||||
" combined = pd.concat(dataframes, ignore_index=True)\n",
|
||||
" df = combined.loc[combined['Level'] == 2].groupby(['PhaseName']).size().to_frame('Count').nlargest(1, 'Count')\n",
|
||||
" worst_phaseName = df.index[0]\n",
|
||||
" worst_count = df.iat[0, 0]\n",
|
||||
" worst_team = worst_phaseName.split('.')[0]\n",
|
||||
" \n",
|
||||
" # what was the worst plan?\n",
|
||||
" violations_df = akn.to_dataframe(violations.primary_results[0])\n",
|
||||
" df = violations_df.groupby(['PlanId']).size().to_frame('Count').nlargest(1, 'Count')\n",
|
||||
" worst_plan_name = df.index[0]\n",
|
||||
" worst_plan_count = df.iat[0, 0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if number_of_false_positives:\n",
|
||||
" print('INSIGHT:', number_of_false_positives, 'plans are likely missing kusto data and were ignored.')\n",
|
||||
"if number_of_violations <= 0:\n",
|
||||
" print('INSIGHT: no problems detected')\n",
|
||||
"else:\n",
|
||||
" print('INSIGHT:', number_of_violations, 'plans are out of SLA.')\n",
|
||||
" print('INSIGHT:', max_problems, 'of them are in', ', '.join(max_scale_units))\n",
|
||||
" print('INSIGHT:', worst_plan_count, 'of the', number_of_violations, 'SLA violations were caused by PlanId', worst_plan_name, '.')\n",
|
||||
" print('INSIGHT:', '\"' + worst_phaseName + '\"', 'was the slowest phase in', worst_count, 'of the', number_of_violations, 'SLA violations.')\n",
|
||||
" print()\n",
|
||||
" print('ACTION: open icm against scale units:', max_scale_units)\n",
|
||||
" print(' initially route it to: ', worst_team)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# find top 25 worst orchestrations\n",
|
||||
"vdf = violations_df.nlargest(25, 'TotalSLADuration')\n",
|
||||
"# vdf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"inputHidden": false,
|
||||
"outputHidden": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"plt.rcdefaults()\n",
|
||||
"\n",
|
||||
"number_of_graphs = len(vdf.index)\n",
|
||||
"fig, axes = plt.subplots(nrows=number_of_graphs, ncols=1, figsize=(8, 6 * number_of_graphs), constrained_layout=True)\n",
|
||||
"for i in range(number_of_graphs):\n",
|
||||
" df = dataframes[vdf.index[i]]\n",
|
||||
" ax = axes[i] if number_of_graphs > 1 else axes\n",
|
||||
" ax.axhline(0, color='k')\n",
|
||||
" \n",
|
||||
" x = df['PhaseName']\n",
|
||||
" xpos = np.arange(len(x))\n",
|
||||
" y = df['PercentDifference']\n",
|
||||
" plan_id = df['PlanId'].iloc[0]\n",
|
||||
" violation_row = violations_df.loc[violations_df['PlanId'] == plan_id]\n",
|
||||
" title = '\\n'.join([\n",
|
||||
" 'plan id:' + plan_id,\n",
|
||||
" 'scale unit:' + str(violation_row['ScaleUnit'].iloc[0]),\n",
|
||||
" 'definition:' + str(df['DefinitionName'].iloc[0]),\n",
|
||||
" 'plan duration: ' + str(violation_row['PlanDuration'].iloc[0]),\n",
|
||||
" 'sla duration: ' + str(violation_row['TotalSLADuration'].iloc[0]),\n",
|
||||
" ])\n",
|
||||
" ax.title.set_text(title)\n",
|
||||
" \n",
|
||||
" ax.bar(x=xpos, height=y)\n",
|
||||
" ax.set_xticks(xpos)\n",
|
||||
" ax.set_xticklabels(x, rotation=45, ha=\"right\")\n",
|
||||
"\n",
|
||||
"# output_filename = 'analysis.svg'\n",
|
||||
"# plt.savefig(output_filename, format='svg')"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
},
|
||||
"nteract": {
|
||||
"version": "nteract-on-jupyter@2.1.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче