Send notification emails
This commit is contained in:
Troy Dai
Родитель
d44060828d
Коммит
ee0abae9bd
147
app/monitor.py
147
app/monitor.py
|
|
@ -3,12 +3,15 @@ import os
|
|||
import time
|
||||
import itertools
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
import coloredlogs
|
||||
import requests
|
||||
from tabulate import tabulate
|
||||
|
||||
from kubernetes import client, config
|
||||
from kubernetes.client import CoreV1Api
|
||||
|
||||
|
||||
class InternalAuth(object): # pylint: disable=too-few-public-methods
|
||||
|
|
@ -17,6 +20,116 @@ class InternalAuth(object): # pylint: disable=too-few-public-methods
|
|||
return req
|
||||
|
||||
|
||||
def get_kube_api(local: False) -> CoreV1Api:
|
||||
if local:
|
||||
config.load_kube_config()
|
||||
else:
|
||||
config.load_incluster_config()
|
||||
return client.CoreV1Api()
|
||||
|
||||
|
||||
def get_store_uri(store_uri: str, local: bool = False) -> str:
|
||||
return f'https://{store_uri}' if local else f'http://{store_uri}'
|
||||
|
||||
|
||||
def get_namespace() -> str:
|
||||
try:
|
||||
with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file_handle:
|
||||
return file_handle.read().strip()
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
return os.environ.get('A01_REPORT_NAMESPACE', 'az')
|
||||
|
||||
|
||||
def send_report(tasks: list, run: dict) -> None:
|
||||
# Move this part to a standalone reporting service
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from smtplib import SMTP
|
||||
|
||||
smtp_server = os.environ.get('A01_REPORT_SMTP_SERVER', None)
|
||||
smtp_user = os.environ.get('A01_REPORT_SENDER_ADDRESS', None)
|
||||
smtp_pass = os.environ.get('A01_REPORT_SENDER_PASSWORD', None)
|
||||
receivers = os.environ.get('A01_REPORT_RECEIVER', '')
|
||||
|
||||
if not smtp_server or not smtp_user or not smtp_pass:
|
||||
logger.warning('Missing SMTP cred. Skip sending email.')
|
||||
sys.exit(1)
|
||||
|
||||
statuses = defaultdict(lambda: 0)
|
||||
results = defaultdict(lambda: 0)
|
||||
|
||||
failure = []
|
||||
|
||||
for task in tasks:
|
||||
status = task['status']
|
||||
result = task['result']
|
||||
|
||||
statuses[status] = statuses[status] + 1
|
||||
results[result] = results[result] + 1
|
||||
|
||||
if result != 'Passed':
|
||||
failure.append(
|
||||
(task['id'],
|
||||
task['name'].rsplit('.')[-1],
|
||||
task['status'],
|
||||
task['result'],
|
||||
(task.get('result_details') or dict()).get('duration')))
|
||||
|
||||
status_summary = ' | '.join([f'{status_name}: {count}' for status_name, count in statuses.items()])
|
||||
result_summary = ' | '.join([f'{result or "Not run"}: {count}' for result, count in results.items()])
|
||||
|
||||
creation = datetime.strptime(run['creation'], '%Y-%m-%dT%H:%M:%SZ') - timedelta(hours=8)
|
||||
|
||||
summaries = [('Id', run['id']),
|
||||
('Creation', str(creation) + ' PST'),
|
||||
('Creator', run['details']['creator']),
|
||||
('Remark', run['details']['remark']),
|
||||
('Live', run['details']['live']),
|
||||
('Task', status_summary),
|
||||
('Image', run['settings']['droid_image']),
|
||||
('Result', result_summary)]
|
||||
|
||||
content = f"""\
|
||||
<html>
|
||||
<body>
|
||||
<div>
|
||||
<h2>Summary</h2>
|
||||
{tabulate(summaries, tablefmt="html")}
|
||||
</div>
|
||||
<div>
|
||||
<h2>Failures</h2>
|
||||
{tabulate(failure, headers=("id", "name", "status", "result", "duration(ms)"), tablefmt="html")}
|
||||
</div>
|
||||
<div>
|
||||
<h2>More details</h2>
|
||||
<p>Install the latest release of A01 client to download log and recordings.</p>
|
||||
<code>
|
||||
$ virtualenv env --python=python3.6 <br>
|
||||
$ . env/bin/activate <br>
|
||||
$ pip install https://github.com/troydai/a01client/releases/download/0.4.0/a01ctl-0.4.0-py3-none-any.whl<br>
|
||||
$ a01 login<br>
|
||||
$ a01 get runs -l {run['id']}<br>
|
||||
</code>
|
||||
<p>Contact: trdai@microsoft.com</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
mail = MIMEMultipart()
|
||||
mail['Subject'] = f'Azure CLI Automation Run {str(creation)} - {result_summary}.'
|
||||
mail['From'] = smtp_user
|
||||
mail['To'] = receivers
|
||||
mail.attach(MIMEText(content, 'html'))
|
||||
|
||||
logger.info('Sending emails.')
|
||||
with SMTP(smtp_server) as server:
|
||||
server.starttls()
|
||||
server.login(smtp_user, smtp_pass)
|
||||
server.send_message(mail)
|
||||
|
||||
|
||||
def main(store: str, run: str, sleep: int = 5, local: bool = False) -> None:
|
||||
session = requests.Session()
|
||||
session.auth = InternalAuth()
|
||||
|
|
@ -56,52 +169,32 @@ def main(store: str, run: str, sleep: int = 5, local: bool = False) -> None:
|
|||
if 'initialized' not in status:
|
||||
if 'scheduled' not in status:
|
||||
logger.info(f'Run {run_id} is finished.')
|
||||
send_report(tasks, session.get(f'{store}/run/{run}').json())
|
||||
sys.exit(0)
|
||||
elif len(status['scheduled']) - len(lost_task) == 0:
|
||||
lost_tasks = ', '.join(lost_task)
|
||||
logger.warning(f'Despite tasks {lost_tasks} are lost. Run {run_id} is finished.')
|
||||
send_report(tasks, session.get(f'{store}/run/{run}').json())
|
||||
sys.exit(0)
|
||||
|
||||
time.sleep(sleep)
|
||||
|
||||
|
||||
def get_kube_api(local: False):
|
||||
if local:
|
||||
config.load_kube_config()
|
||||
else:
|
||||
config.load_incluster_config()
|
||||
return client.CoreV1Api()
|
||||
|
||||
|
||||
def get_store_uri(store_uri: str, local: bool = False) -> str:
|
||||
return f'https://{store_uri}' if local else f'http://{store_uri}'
|
||||
|
||||
|
||||
def get_namespace():
|
||||
try:
|
||||
with open('/var/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file_handle:
|
||||
return file_handle.read().strip()
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
return os.environ.get('A01_REPORT_NAMESPACE', 'az')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# pylint: disable=invalid-name
|
||||
coloredlogs.install(level=logging.INFO)
|
||||
logger = logging.getLogger('a01report')
|
||||
|
||||
interval = int(os.environ.get('A01_REPORT_INTERVAL', 5))
|
||||
interval = int(os.environ.get('A01_MONITOR_INTERVAL', 5))
|
||||
logger.info(f'Interval: {interval}s')
|
||||
|
||||
is_local = os.environ.get('A01_REPORT_LOCAL', False)
|
||||
is_local = os.environ.get('A01_MONITOR_LOCAL', False)
|
||||
logger.info(f'Is local: {is_local}')
|
||||
|
||||
run_id = os.environ['A01_REPORT_RUN_ID']
|
||||
run_id = os.environ['A01_MONITOR_RUN_ID']
|
||||
logger.info(f'Run id: {run_id}')
|
||||
|
||||
task_store = os.environ['A01_STORE_NAME']
|
||||
task_store = os.environ.get('A01_STORE_NAME', 'task-store-web-service-internal')
|
||||
logger.info(f'Store: {task_store}')
|
||||
|
||||
main(store=get_store_uri(task_store, is_local), run=run_id, sleep=interval, local=is_local)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
kubernetes==4.0.0
|
||||
requests==2.18.4
|
||||
coloredlogs==8.0
|
||||
tabulate==0.8.2
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче