github
/
incubator-airflow
зеркало из https://github.com/github/incubator-airflow.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Adding cache support for charts

This commit is contained in:
Maxime 2015-01-02 23:00:01 +00:00
Родитель 241896e03f
Коммит 30b2eecd1c
3 изменённых файлов: 140 добавлений и 42 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
airflow/models.py
Просмотреть файл

@ -1161,3 +1161,4 @@ class Chart(Base):
default_params = Column(String(5000), default="{}")
owner = relationship("User", cascade=False, cascade_backrefs=False)
db = relationship("DatabaseConnection")
x_is_date = Column(Boolean, default=True)

156
airflow/www/app.py
Просмотреть файл

@ -2,12 +2,14 @@ from datetime import datetime, timedelta
import dateutil.parser
import json
import logging
import sys
from flask import Flask, url_for, Markup, Blueprint, redirect, flash, Response
from flask.ext.admin import Admin, BaseView, expose, AdminIndexView
from flask.ext.admin.form import DateTimePickerWidget
from flask.ext.admin import base
from flask.ext.admin.contrib.sqla import ModelView
from flask.ext.cache import Cache
from flask import request
from wtforms import Form, DateTimeField, SelectField, TextAreaField
from cgi import escape
@ -46,6 +48,14 @@ app = Flask(__name__)
login_manager.init_app(app)
app.secret_key = 'airflowified'
cache = Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
def make_cache_key(*args, **kwargs):
path = request.path
args = str(hash(frozenset(request.args.items())))
return (path + args).encode('ascii', 'ignore')
# Init for chartkick, the python wrapper for highcharts
ck = Blueprint(
'ck_page', __name__,
@ -199,6 +209,7 @@ class Airflow(BaseView):
@expose('/chart_data')
@login_required
@cache.cached(timeout=3600, key_prefix=make_cache_key)
def chart_data(self):
session = settings.Session()
chart_id = request.args.get('chart_id')
@ -259,47 +270,111 @@ class Airflow(BaseView):
# Trying to convert time to something Highcharts likes
x_col = 1 if chart.sql_layout == 'series' else 0
x_is_dt = True
df[df.columns[x_col]] = pd.to_datetime(df[df.columns[x_col]])
try:
# From string to datetime
df[df.columns[x_col]] = pd.to_datetime(df[df.columns[x_col]])
except Exception as e:
x_is_dt = False
raise Exception(str(e))
if x_is_dt:
if chart.x_is_date:
try:
# From string to datetime
df[df.columns[x_col]] = pd.to_datetime(
df[df.columns[x_col]])
except Exception as e:
raise Exception(str(e))
df[df.columns[x_col]] = df[df.columns[x_col]].apply(
lambda x: int(x.strftime("%s")) * 1000)
if chart.sql_layout == 'series':
# User provides columns (series, x, y)
series = []
colorAxis = None
if chart.chart_type == 'heatmap':
color_perc_lbound = float(request.args.get('color_perc_lbound', 0))
color_perc_rbound = float(request.args.get('color_perc_rbound', 1))
color_scheme = request.args.get('color_scheme', 'blue_red')
if color_scheme == 'blue_red':
stops = [
[color_perc_lbound, '#3060CF'],
[color_perc_lbound + ((color_perc_rbound - color_perc_lbound)/2), '#FFFBBC'],
[color_perc_rbound, '#C4463A']
]
elif color_scheme == 'blue_scale':
stops = [
[color_perc_lbound, '#FFFFFF'],
[color_perc_rbound, '#2222FF']
]
elif color_scheme == 'fire':
diff = float(color_perc_rbound - color_perc_lbound)
stops = [
[color_perc_lbound, '#FFFFFF'],
[color_perc_lbound + 0.33*diff, '#FFFF00'],
[color_perc_lbound + 0.66*diff, '#FF0000'],
[color_perc_rbound, '#000000']
]
else:
stops = [
[color_perc_lbound, '#FFFFFF'],
[color_perc_lbound + ((color_perc_rbound - color_perc_lbound)/2), '#888888'],
[color_perc_rbound, '#000000'],
]
xaxis_label = df.columns[1]
yaxis_label = df.columns[2]
df[df.columns[2]] = df[df.columns[2]].astype(np.float)
df = df.pivot_table(
index=df.columns[1],
columns=df.columns[0],
values=df.columns[2], aggfunc=np.sum)
else:
# User provides columns (x, y, metric1, metric2, ...)
xaxis_label = df.columns[0]
yaxis_label = 'y'
df.index = df[df.columns[0]]
df = df.sort('ds')
del df[df.columns[0]]
for col in df.columns:
df[col] = df[col].astype(np.float)
series = []
for col in df.columns:
data = []
for row in df.itertuples():
data.append({
'x': row[2],
'y': row[3],
'value': row[4],
})
x_format = '{point.x:%Y-%m-%d}' if chart.x_is_date else '{point.x}'
series.append({
'name': col,
'data': [
(i, v)
for i, v in df[col].iteritems() if not np.isnan(v)]
'data': data,
'borderWidth': 0,
'colsize': 24 * 36e5,
'turboThreshold': sys.float_info.max,
'tooltip': {
'headerFormat': '',
'pointFormat': (
df.columns[1] + ': ' + x_format + '<br/>' +
df.columns[2] + ': {point.y}<br/>' +
df.columns[3] + ': <b>{point.value}</b>'
),
},
})
series = [serie for serie in sorted(
series, key=lambda s: s['data'][0][1], reverse=True)]
colorAxis = {
'stops': stops,
'minColor': '#FFFFFF',
'maxColor': '#000000',
'min': 50,
'max': 2200,
}
else:
if chart.sql_layout == 'series':
# User provides columns (series, x, y)
xaxis_label = df.columns[1]
yaxis_label = df.columns[2]
df[df.columns[2]] = df[df.columns[2]].astype(np.float)
df = df.pivot_table(
index=df.columns[1],
columns=df.columns[0],
values=df.columns[2], aggfunc=np.sum)
else:
# User provides columns (x, y, metric1, metric2, ...)
xaxis_label = df.columns[0]
yaxis_label = 'y'
df.index = df[df.columns[0]]
df = df.sort('ds')
del df[df.columns[0]]
for col in df.columns:
df[col] = df[col].astype(np.float)
for col in df.columns:
series.append({
'name': col,
'data': [
(i, v)
for i, v in df[col].iteritems() if not np.isnan(v)]
})
series = [serie for serie in sorted(
series, key=lambda s: s['data'][0][1], reverse=True)]
chart_type = chart.chart_type
if chart.chart_type == "stacked_area":
@ -325,12 +400,18 @@ class Airflow(BaseView):
'title': {'text': ''},
'xAxis': {
'title': {'text': xaxis_label},
'type': 'datetime' if x_is_dt else None,
'type': 'datetime' if chart.x_is_date else None,
},
'yAxis': {
'min': 0,
'title': {'text': yaxis_label},
},
'colorAxis': colorAxis,
'tooltip': {
'useHTML': True,
'backgroundColor': None,
'borderWidth': 0,
},
'series': series,
}
@ -345,6 +426,7 @@ class Airflow(BaseView):
def date_handler(obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
response = Response(
response=json.dumps(payload, indent=4, default=date_handler),
status=200,
@ -970,6 +1052,7 @@ class ChartModelView(LoginMixin, ModelView):
'db',
'chart_type',
'show_datatable',
'x_is_date',
'y_log_scale',
'show_sql',
'height',
@ -994,6 +1077,7 @@ class ChartModelView(LoginMixin, ModelView):
('stacked_area', 'Stacked Area Chart'),
('percent_area', 'Percent Area Chart'),
('datatable', 'No chart, data table only'),
('heatmap', 'Heatmap'),
],
'sql_layout': [
('series', 'SELECT series, x, y FROM ...'),
@ -1002,7 +1086,7 @@ class ChartModelView(LoginMixin, ModelView):
}
def on_model_change(self, form, model, is_created):
if not model.user_id and flask_login.current_user:
if AUTHENTICATE and not model.user_id and flask_login.current_user:
model.user_id = flask_login.current_user.id
mv = ChartModelView(

25
airflow/www/templates/airflow/highchart.html
Просмотреть файл

@ -13,7 +13,15 @@
border: none;
background: none;
background-color: #FFF;
};
}
.highcharts-tooltip>span {
background: rgba(255,255,255,0.85);
border: 1px solid silver;
border-radius: 3px;
box-shadow: 1px 1px 2px #888;
padding: 8px;
z-index: 2;
}
</style>
{% endblock %}
{% block title %}
@ -25,8 +33,8 @@
<div id="container">
<h2>
<span id="label">{{ label }}</span>
<a href="/admin/chart/edit/?id={{ chart.id }}">
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
<a href="/admin/chart/edit/?id={{ chart.id }}" >
<span class="glyphicon glyphicon-edit" aria-hidden="true" ></span>
</a>
</h2>
<div id="error" style="display: none;" class="alert alert-danger" role="alert">
@ -64,7 +72,7 @@
</div>
<div id="chart_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo">
<div class="panel-body">
<div id="hc" style="height:{{ chart.height }}px;">
<div id="chart_body">
<img src="{{ url_for('static', filename='loading.gif') }}" width="50px">
</div>
</div>
@ -100,6 +108,8 @@
<script src="{{ url_for('static', filename='highcharts.js') }}"></script>
<script src="{{ url_for('static', filename='highcharts-more.js') }}">
</script>
<script src="{{ url_for('static', filename='heatmap.js') }}"></script>
<script src="{{ url_for('static', filename='heatmap-canvas.js') }}"></script>
<script>
function error(msg){
$('#error_msg').html(msg);
@ -115,14 +125,17 @@
"#FFAA91", "#B4A76C", "#9CA299", "#565A5C"
],
});
url = "{{ url_for('airflow.chart_data', chart_id=chart.id) }}";
url = "{{ url_for('airflow.chart_data', chart_id=chart.id) }}" + location.search;
$.getJSON(url, function(payload) {
console.log(payload);
$('#loading').hide();
$("#sql_panel_body").html(payload.sql_html);
$("#label").html(payload.label);
if (payload.state == "SUCCESS") {
{% if chart.chart_type != "datatable" %}
$('#hc').highcharts(payload.hc);
$('#chart_body').css('width', '100%');
$('#chart_body').css('height', '{{ chart.height }}');
$('#chart_body').highcharts(payload.hc);
{% endif %}
{% if chart.show_datatable or chart.chart_type == "datatable" %}
$('#datatable').dataTable( {