318 строки
10 KiB
Plaintext
318 строки
10 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# SQL Investigation\n",
|
|
"1. Run all cells.\n",
|
|
"1. View report at the bottom."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false,
|
|
"tags": [
|
|
"parameters"
|
|
]
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# These are just defaults will be overwritten if you use nimport pip\n",
|
|
"db = \"Tfs_tfsprodcus2_37253a68-972a-4bf4-8c5f-a259ba4d42cd\"\n",
|
|
"start = \"2019-07-31T17:30:00.0000000Z\"\n",
|
|
"end = \"2019-07-31T18:30:36.0000000Z\"\n",
|
|
"url = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK/delays.ipynb\"\n",
|
|
"baseUrl = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%capture\n",
|
|
"!pip install nimport azure-kusto-notebooks"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Import the things we use\n",
|
|
"\n",
|
|
"# Note you can also use kql https://docs.microsoft.com/en-us/azure/data-explorer/kqlmagic\n",
|
|
"# %kql is single line magic\n",
|
|
"# %%kql is cell magic\n",
|
|
"\n",
|
|
"# https://nbviewer.jupyter.org/github/ipython/ipython/blob/4.0.x/examples/IPython%20Kernel/Rich%20Output.ipynb#HTML\n",
|
|
"# https://ipython.readthedocs.io/en/stable/inte/magics.html\n",
|
|
"from IPython.display import display, HTML, Markdown, Javascript, clear_output\n",
|
|
"\n",
|
|
"# http://pandas-docs.github.io/pandas-docs-travis/user_guide/reshaping.html\n",
|
|
"import pandas as pd\n",
|
|
"pd.options.display.html.table_schema = True\n",
|
|
"from pandas import Series, DataFrame\n",
|
|
"from datetime import datetime, timedelta, timezone\n",
|
|
"from urllib.parse import urlencode, quote_plus\n",
|
|
"from requests.utils import requote_uri\n",
|
|
"import time\n",
|
|
"import numpy as np\n",
|
|
"from matplotlib import pyplot as plt\n",
|
|
"from nimport.utils import tokenize, open_nb\n",
|
|
"import json\n",
|
|
"import os\n",
|
|
"import calendar as cal\n",
|
|
"import concurrent.futures\n",
|
|
"from azure.kusto.notebooks import utils as akn"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"params = {\n",
|
|
" \"db\": db,\n",
|
|
" \"start\": start,\n",
|
|
" \"end\": end,\n",
|
|
" \"url\": url,\n",
|
|
" \"baseUrl\": baseUrl\n",
|
|
"}\n",
|
|
"root = 'devops-pipelines' if os.path.basename(os.getcwd()) != 'devops-pipelines' else ''\n",
|
|
"queryPath = os.path.join(root, 'queries')\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# authenticate kusto client\n",
|
|
"# you will need to copy the token into a browser window for AAD auth. \n",
|
|
"client = akn.get_client('https://vso.kusto.windows.net')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"sqlPath = os.path.join(queryPath, 'sql')\n",
|
|
"q_data = os.path.join(sqlPath, \"GetData.csl\")\n",
|
|
"q_whatsSlow = os.path.join(sqlPath, \"WhatsSlow.csl\")\n",
|
|
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
|
" # materialize so that we have all information we might need\n",
|
|
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_data, params)\n",
|
|
" q_data_df = akn.to_dataframe_from_future(p1)\n",
|
|
" params[\"service\"] = q_data_df[\"Service\"][0]\n",
|
|
" params[\"su\"] =q_data_df[\"ScaleUnit\"][0]\n",
|
|
" \n",
|
|
" p2 = executor.submit(akn.execute_file, client, 'VSO', q_whatsSlow, params)\n",
|
|
"\n",
|
|
"q_whatsSlow_df = akn.to_dataframe_from_future(p2) \n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Initialize for further analysis later\n",
|
|
"q_cpuTop_df = None\n",
|
|
"q_cpuXEvent_df = None\n",
|
|
"q_cpuJob_df = None\n",
|
|
"q_cpuActivity_df = None"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def cpuAnalysis():\n",
|
|
" global q_cpuTop_df\n",
|
|
" global q_cpuXEvent_df\n",
|
|
" q_cpuTop = os.path.join(sqlPath, \"CpuTop.csl\")\n",
|
|
" q_cpuXEvent = os.path.join(sqlPath, \"CpuXevent.csl\")\n",
|
|
" with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
|
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_cpuTop, params)\n",
|
|
" p2 = executor.submit(akn.execute_file, client, 'VSO', q_cpuXEvent, params)\n",
|
|
"\n",
|
|
" q_cpuTop_df = akn.to_dataframe_from_future(p1)\n",
|
|
" \n",
|
|
" q_cpuXEvent_df = akn.to_dataframe_from_future(p2)\n",
|
|
" maxTime = q_cpuXEvent_df[\"sum_CpuTime\"].max()\n",
|
|
" q_cpuXEvent_df['CpuTimeDiff'] = q_cpuXEvent_df[\"sum_CpuTime\"].map(lambda x: x/maxTime)\n",
|
|
"\n",
|
|
"def cpuAnalysisJob():\n",
|
|
" global q_cpuJob_df\n",
|
|
" q_cpuJob = os.path.join(sqlPath, \"CpuJob.csl\")\n",
|
|
" with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
|
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_cpuJob, params)\n",
|
|
"\n",
|
|
" q_cpuJob_df = akn.to_dataframe_from_future(p1)\n",
|
|
"\n",
|
|
"def cpuAnalysisActivity():\n",
|
|
" global q_cpuActivity_df\n",
|
|
" q_cpuActivity = os.path.join(sqlPath, \"CpuActivity.csl\")\n",
|
|
" with concurrent.futures.ThreadPoolExecutor() as executor:\n",
|
|
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_cpuActivity, params)\n",
|
|
"\n",
|
|
" q_cpuActivity_df = akn.to_dataframe_from_future(p1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"inputHidden": false,
|
|
"outputHidden": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"print('=' * 50)\n",
|
|
"print('Report!')\n",
|
|
"print('=' * 50, '\\n\\n')\n",
|
|
"\n",
|
|
"jarvisParams = {'su': params[\"su\"], 'start': akn.get_time(start, -10), 'end': akn.get_time(end, 10), 'service': params[\"service\"], 'db': db }\n",
|
|
"\n",
|
|
"jaJarvisLink = \"\"\"https://jarvis-west.dc.ad.msft.net/dashboard/VSO-ServiceInsights/PlatformViews/SQLAzureDatabase\"\"\" \\\n",
|
|
" \"\"\"?overrides=[{\"query\":\"//*[id='Service']\",\"key\":\"value\",\"replacement\":\"%(service)s\"},\"\"\" \\\n",
|
|
" \"\"\"{\"query\":\"//*[id='ScaleUnit']\",\"key\":\"value\",\"replacement\":\"%(su)s\"},\"\"\" \\\n",
|
|
" \"\"\"{\"query\":\"//*[id='__DatabaseName']\",\"key\":\"value\",\"replacement\":\"%(db)s\"}]\"\"\" \\\n",
|
|
" \"\"\"&globalStartTime=%(start)s&globalEndTime=%(end)s&pinGlobalTimeRange=true\"\"\" % jarvisParams;\n",
|
|
"print('Jarvis dashboard link for sql:\\n', requote_uri(jaJarvisLink), '\\n')\n",
|
|
"\n",
|
|
"print()\n",
|
|
"print(\"Parameters used:\")\n",
|
|
"display(params)\n",
|
|
"\n",
|
|
"print()\n",
|
|
"\n",
|
|
"## Where is the database at?\n",
|
|
"print(\"Database is at: \")\n",
|
|
"so = q_whatsSlow_df[\"ServiceObjective\"].unique()\n",
|
|
"if so.size > 1:\n",
|
|
" print(\"We found different service objectives..looks like db was changed?\")\n",
|
|
"print(so) \n",
|
|
"\n",
|
|
"print()\n",
|
|
"\n",
|
|
"## What's slow?\n",
|
|
"cpu = q_whatsSlow_df[\"avg_AverageCpuPercentage\"]\n",
|
|
"memory = q_whatsSlow_df[\"avg_AverageMemoryUsagePercentage\"]\n",
|
|
"logWrite= q_whatsSlow_df[\"avg_AverageLogWriteUtilizationPercentage\"]\n",
|
|
"worker= q_whatsSlow_df[\"max_MaximumWorkerPercentage\"]\n",
|
|
"cpu_coefficientOfVariance = cpu.std()/cpu.mean()\n",
|
|
"memory_coefficientOfVariance = memory.std()/memory.mean()\n",
|
|
"logWrite_coefficientOfVariance = logWrite.std()/logWrite.mean()\n",
|
|
"worker_coefficientOfVariance = worker.std()/worker.mean()\n",
|
|
"maxVar = 0.5\n",
|
|
"\n",
|
|
"reasons = \"Possibly due to: \"\n",
|
|
"if cpu_coefficientOfVariance >= maxVar:\n",
|
|
" reasons+= \"cpu (max: %s), \" % (cpu.max())\n",
|
|
"if memory_coefficientOfVariance >= maxVar:\n",
|
|
" reasons+= \"memory (max: %s), \" % (memory.max())\n",
|
|
"if logWrite_coefficientOfVariance >= maxVar:\n",
|
|
" reasons+= \"logwrite (max: %s), \" % (logWrite.max())\n",
|
|
"if worker_coefficientOfVariance >= maxVar:\n",
|
|
" reasons+= \"worker (max: %s), \" % (worker.max())\n",
|
|
"print(reasons)\n",
|
|
"\n",
|
|
"if cpu.max() >= 80:\n",
|
|
" print(\"We found high CPU, let's start with CPU analysis...\")\n",
|
|
" \n",
|
|
" cpuAnalysis()\n",
|
|
" \n",
|
|
" #print()\n",
|
|
" #print(\"Top CPU commands:\")\n",
|
|
" #display(q_cpuTop_df)\n",
|
|
" \n",
|
|
" print()\n",
|
|
" print(\"Who's causing these commands?:\")\n",
|
|
" commandsToConsider = q_cpuXEvent_df[q_cpuXEvent_df[\"CpuTimeDiff\"] >= 0.5]\n",
|
|
" jobCommand = commandsToConsider[commandsToConsider[\"TypeName\"].str.contains('Job')]\n",
|
|
" if len(jobCommand) >= 1:\n",
|
|
" print(\"Possibly due to a job...\")\n",
|
|
" display(jobCommand)\n",
|
|
" cpuAnalysisJob()\n",
|
|
" \n",
|
|
" print()\n",
|
|
" display(q_cpuJob_df)\n",
|
|
" \n",
|
|
" activityCommand = commandsToConsider[commandsToConsider[\"TypeName\"].str.contains('Activity')]\n",
|
|
" if len(activityCommand) >= 1 and activityCommand[\"ObjectName\"][0]:\n",
|
|
" print(\"Possibly due to user activity...\")\n",
|
|
" display(activityCommand)\n",
|
|
" cpuAnalysisActivity()\n",
|
|
" \n",
|
|
" print()\n",
|
|
" display(q_cpuActivity_df)\n",
|
|
" "
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernel_info": {
|
|
"name": "python3"
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.4"
|
|
},
|
|
"nteract": {
|
|
"version": "0.14.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|