devops-pipelines/ja.ipynb

213 строки
6.9 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"source": [
"# Jobagent Investigation\n",
"1. Run all cells.\n",
"1. View report at the bottom."
],
"metadata": {}
},
{
"cell_type": "code",
"source": [
"# These are just defaults will be overwritten if you use nimport pip\n",
"su = \"tfs-cus-3\"\n",
"start = \"2019-07-30T15:05:00.0000000Z\"\n",
"end = \"2019-07-30T15:39:36.0000000Z\"\n",
"url = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK/delays.ipynb\"\n",
"baseUrl = \"https://notebooksv2.azure.com/yaananth/projects/06OasuNRs6rK\"\n",
"service = \"tfs\"\n",
"hub = \"Build\"\n",
"locationName = \"tfsprodcus3\""
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"inputHidden": false,
"outputHidden": false,
"tags": [
"parameters"
]
}
},
{
"cell_type": "code",
"source": [
"%%capture\n",
"!pip install nimport azure-kusto-notebooks"
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"inputHidden": false,
"outputHidden": false
}
},
{
"cell_type": "code",
"source": [
"# Import the things we use\n",
"\n",
"# Note you can also use kql https://docs.microsoft.com/en-us/azure/data-explorer/kqlmagic\n",
"# %kql is single line magic\n",
"# %%kql is cell magic\n",
"\n",
"# https://nbviewer.jupyter.org/github/ipython/ipython/blob/4.0.x/examples/IPython%20Kernel/Rich%20Output.ipynb#HTML\n",
"# https://ipython.readthedocs.io/en/stable/inte/magics.html\n",
"from IPython.display import display, HTML, Markdown, Javascript, clear_output\n",
"\n",
"# http://pandas-docs.github.io/pandas-docs-travis/user_guide/reshaping.html\n",
"import pandas as pd\n",
"pd.options.display.html.table_schema = True\n",
"from pandas import Series, DataFrame\n",
"from datetime import datetime, timedelta, timezone\n",
"from urllib.parse import urlencode, quote_plus\n",
"from requests.utils import requote_uri\n",
"import time\n",
"import numpy as np\n",
"from matplotlib import pyplot as plt\n",
"from nimport.utils import tokenize, open_nb\n",
"import json\n",
"import os\n",
"import calendar as cal\n",
"import concurrent.futures\n",
"from azure.kusto.notebooks import utils as akn"
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"inputHidden": false,
"outputHidden": false
}
},
{
"cell_type": "code",
"source": [
"params = {\n",
" \"su\": su,\n",
" \"start\": start,\n",
" \"end\": end,\n",
" \"url\": url,\n",
" \"baseUrl\": baseUrl,\n",
" \"service\": service\n",
"}\n",
"root = 'devops-pipelines' if os.path.basename(os.getcwd()) != 'devops-pipelines' else ''\n",
"queryPath = os.path.join(root, 'queries') "
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"inputHidden": false,
"outputHidden": false
}
},
{
"cell_type": "code",
"source": [
"# authenticate kusto client\n",
"# you will need to copy the token into a browser window for AAD auth. \n",
"client = akn.get_client('https://vso.kusto.windows.net', 'VSO')"
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"inputHidden": false,
"outputHidden": false
}
},
{
"cell_type": "code",
"source": [
"q_slow = os.path.join(queryPath, \"SlowActivities.csl\")\n",
"q_sqlSlow = os.path.join(queryPath, \"SlowSql.csl\")\n",
"\n",
"jaPath = os.path.join(queryPath, 'ja')\n",
"q_jobSql = os.path.join(jaPath, \"JASqlTime.csl\")\n",
"with concurrent.futures.ThreadPoolExecutor() as executor:\n",
" p1 = executor.submit(akn.execute_file, client, 'VSO', q_slow, params)\n",
" p2 = executor.submit(akn.execute_file, client, 'VSO', q_sqlSlow, params)\n",
" p3 = executor.submit(akn.execute_file, client, 'VSO', q_jobSql, params)\n",
"\n",
"q_slowResult_df = akn.to_dataframe_from_future(p1)\n",
"\n",
"q_sqlSlowResult_df = akn.to_dataframe_from_future(p2)\n",
"\n",
"q_jobSqlResult_df = akn.to_dataframe_from_future(p3) \n"
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"outputHidden": false,
"inputHidden": false
}
},
{
"cell_type": "code",
"source": [
"print('=' * 50)\n",
"print('Report!')\n",
"print('=' * 50, '\\n\\n')\n",
"\n",
"jarvisParams = {'su': su, 'start': akn.get_time(start, -10), 'end': akn.get_time(end, 10), 'service': service }\n",
"\n",
"jaJarvisLink = \"\"\"https://jarvis-west.dc.ad.msft.net/dashboard/VSO-ServiceInsights/PlatformViews/Compute-JA\"\"\" \\\n",
" \"\"\"?overrides=[{\"query\":\"//*[id='Service']\",\"key\":\"value\",\"replacement\":\"%(service)s\"},\"\"\" \\\n",
" \"\"\"{\"query\":\"//*[id='RoleInstance']\",\"key\":\"value\",\"replacement\":\"\"},\"\"\" \\\n",
" \"\"\"{\"query\":\"//*[id='ScaleUnit']\",\"key\":\"value\",\"replacement\":\"%(su)s\"}]\"\"\" \\\n",
" \"\"\"&globalStartTime=%(start)s&globalEndTime=%(end)s&pinGlobalTimeRange=true\"\"\" % jarvisParams;\n",
"print('Jarvis dashboard link for job agents:\\n', requote_uri(jaJarvisLink), '\\n')\n",
"\n",
"print('Top slow activities:')\n",
"display(q_slowResult_df)\n",
"\n",
"print('Top sql slow activities:')\n",
"display(q_sqlSlowResult_df)\n",
"\n",
"print('Top sql executime times from jobs:')\n",
"display(q_jobSqlResult_df)"
],
"outputs": [],
"execution_count": null,
"metadata": {
"collapsed": false,
"outputHidden": false,
"inputHidden": false
}
}
],
"metadata": {
"kernel_info": {
"name": "python3"
},
"kernelspec": {
"name": "python3",
"language": "python",
"display_name": "Python 3"
},
"language_info": {
"name": "python",
"version": "3.7.4",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py"
},
"nteract": {
"version": "0.14.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
}