Merge pull request #68 from mistercrunch/chart

Chart - more cosmetic polish
This commit is contained in:
Maxime Beauchemin 2015-01-04 08:18:17 -08:00
Родитель 241896e03f 462f9b999a
Коммит 9fda01c5d2
14 изменённых файлов: 548 добавлений и 114 удалений

1
.gitignore поставляемый
Просмотреть файл

@ -11,3 +11,4 @@ logs
*.cfg
MANIFEST
secrets.py
*.egg-info

Просмотреть файл

@ -2,6 +2,7 @@
from airflow.configuration import conf
from airflow import settings
from airflow import utils
from airflow import jobs
from airflow.models import DagBag, TaskInstance, DagPickle
@ -42,7 +43,7 @@ def backfill(args):
def run(args):
settings.pessimistic_connection_handling()
utils.pessimistic_connection_handling()
# Setting up logging
directory = conf.get('core', 'BASE_LOG_FOLDER') + \

Просмотреть файл

@ -1160,4 +1160,6 @@ class Chart(Base):
height = Column(Integer, default=600)
default_params = Column(String(5000), default="{}")
owner = relationship("User", cascade=False, cascade_backrefs=False)
x_is_date = Column(Boolean, default=True)
db = relationship("DatabaseConnection")
iteration_no = Column(Integer, default=0)

Просмотреть файл

@ -2,9 +2,7 @@ import sys
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy import create_engine
from sqlalchemy import event
from sqlalchemy import exc
from sqlalchemy.pool import Pool
from airflow.configuration import conf
@ -16,20 +14,6 @@ _____ |__|_______/ ____\ | ______ _ __
(____ /__||__| |__| |____/\____/ \/\_/
\/"""
def pessimistic_connection_handling():
@event.listens_for(Pool, "checkout")
def ping_connection(dbapi_connection, connection_record, connection_proxy):
'''
Disconnect Handling - Pessimistic, taken from:
http://docs.sqlalchemy.org/en/rel_0_9/core/pooling.html
'''
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT 1")
except:
raise exc.DisconnectionError()
cursor.close()
BASE_FOLDER = conf.get('core', 'BASE_FOLDER')
BASE_LOG_URL = "/admin/airflow/log"

Просмотреть файл

@ -3,6 +3,10 @@ from functools import wraps
import inspect
import logging
import re
from sqlalchemy import event
from sqlalchemy.pool import Pool
from airflow.configuration import conf
@ -32,6 +36,22 @@ class State(object):
return [None, cls.FAILED, cls.UP_FOR_RETRY]
def pessimistic_connection_handling():
@event.listens_for(Pool, "checkout")
def ping_connection(dbapi_connection, connection_record, connection_proxy):
'''
Disconnect Handling - Pessimistic, taken from:
http://docs.sqlalchemy.org/en/rel_0_9/core/pooling.html
'''
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT 1")
except:
raise exc.DisconnectionError()
cursor.close()
def validate_key(k, max_length=250):
if type(k) is not str:
raise TypeError("The key has to be a string")

Просмотреть файл

@ -2,16 +2,16 @@ from datetime import datetime, timedelta
import dateutil.parser
import json
import logging
import sys
from flask import Flask, url_for, Markup, Blueprint, redirect, flash, Response
from flask.ext.admin import Admin, BaseView, expose, AdminIndexView
from flask.ext.admin.form import DateTimePickerWidget
from flask.ext.admin import base
from flask.ext.admin.contrib.sqla import ModelView
from flask.ext.cache import Cache
from flask import request
from wtforms import Form, DateTimeField, SelectField, TextAreaField
from cgi import escape
from wtforms.compat import text_type
import wtforms
from pygments import highlight
@ -19,7 +19,6 @@ from pygments.lexers import PythonLexer, SqlLexer, BashLexer
from pygments.formatters import HtmlFormatter
import jinja2
import markdown
import chartkick
@ -30,22 +29,30 @@ from airflow.models import State
from airflow import settings
from airflow.configuration import conf
from airflow import utils
from airflow.www import utils as wwwutils
from airflow.www.login import login_manager
import flask_login
from flask_login import login_required
AUTHENTICATE = conf.getboolean('core', 'AUTHENTICATE')
if AUTHENTICATE is False:
login_required = lambda x: x
dagbag = models.DagBag(conf.get('core', 'DAGS_FOLDER'))
session = Session()
utils.pessimistic_connection_handling()
app = Flask(__name__)
app.config['SQLALCHEMY_POOL_RECYCLE'] = 3600
login_manager.init_app(app)
app.secret_key = 'airflowified'
cache = Cache(
app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
# Init for chartkick, the python wrapper for highcharts
ck = Blueprint(
'ck_page', __name__,
@ -54,26 +61,6 @@ app.register_blueprint(ck, url_prefix='/ck')
app.jinja_env.add_extension("chartkick.ext.charts")
class AceEditorWidget(wtforms.widgets.TextArea):
"""
Renders an ACE code editor.
"""
def __call__(self, field, **kwargs):
kwargs.setdefault('id', field.id)
html = '''
<div id="{el_id}" style="height:100px;">{contents}</div>
<textarea
id="{el_id}_ace" name="{form_name}"
style="display:none;visibility:hidden;">
</textarea>
'''.format(
el_id=kwargs.get('id', field.id),
contents=escape(text_type(field._value())),
form_name=field.id,
)
return wtforms.widgets.core.HTMLString(html)
class DateTimeForm(Form):
# Date filter form needed for gantt and graph view
execution_date = DateTimeField(
@ -126,18 +113,8 @@ admin = Admin(
admin.add_link(
base.MenuLink(
category='Tools',
name='Query',
name='Ad Hoc Query',
url='/admin/airflow/query'))
admin.add_link(
base.MenuLink(
category='Docs',
name='@readthedocs.org',
url='http://airflow.readthedocs.org/en/latest/'))
admin.add_link(
base.MenuLink(
category='Docs',
name='Github',
url='https://github.com/mistercrunch/Airflow'))
class Airflow(BaseView):
@ -161,7 +138,7 @@ class Airflow(BaseView):
class QueryForm(Form):
db_id = SelectField("Layout", choices=db_choices)
sql = TextAreaField("SQL", widget=AceEditorWidget())
sql = TextAreaField("SQL", widget=wwwutils.AceEditorWidget())
data = {
'db_id': db_id_str,
'sql': sql,
@ -199,6 +176,8 @@ class Airflow(BaseView):
@expose('/chart_data')
@login_required
@wwwutils.gzipped
@cache.cached(timeout=3600, key_prefix=wwwutils.make_cache_key)
def chart_data(self):
session = settings.Session()
chart_id = request.args.get('chart_id')
@ -259,47 +238,120 @@ class Airflow(BaseView):
# Trying to convert time to something Highcharts likes
x_col = 1 if chart.sql_layout == 'series' else 0
x_is_dt = True
df[df.columns[x_col]] = pd.to_datetime(df[df.columns[x_col]])
try:
# From string to datetime
df[df.columns[x_col]] = pd.to_datetime(df[df.columns[x_col]])
except Exception as e:
x_is_dt = False
raise Exception(str(e))
if x_is_dt:
if chart.x_is_date:
try:
# From string to datetime
df[df.columns[x_col]] = pd.to_datetime(
df[df.columns[x_col]])
except Exception as e:
raise Exception(str(e))
df[df.columns[x_col]] = df[df.columns[x_col]].apply(
lambda x: int(x.strftime("%s")) * 1000)
if chart.sql_layout == 'series':
# User provides columns (series, x, y)
series = []
colorAxis = None
if chart.chart_type == 'heatmap':
color_perc_lbound = float(
request.args.get('color_perc_lbound', 0))
color_perc_rbound = float(
request.args.get('color_perc_rbound', 1))
color_scheme = request.args.get('color_scheme', 'blue_red')
if color_scheme == 'blue_red':
stops = [
[color_perc_lbound, '#00D1C1'],
[
color_perc_lbound +
((color_perc_rbound - color_perc_lbound)/2),
'#FFFFCC'
],
[color_perc_rbound, '#FF5A5F']
]
elif color_scheme == 'blue_scale':
stops = [
[color_perc_lbound, '#FFFFFF'],
[color_perc_rbound, '#2222FF']
]
elif color_scheme == 'fire':
diff = float(color_perc_rbound - color_perc_lbound)
stops = [
[color_perc_lbound, '#FFFFFF'],
[color_perc_lbound + 0.33*diff, '#FFFF00'],
[color_perc_lbound + 0.66*diff, '#FF0000'],
[color_perc_rbound, '#000000']
]
else:
stops = [
[color_perc_lbound, '#FFFFFF'],
[
color_perc_lbound +
((color_perc_rbound - color_perc_lbound)/2),
'#888888'
],
[color_perc_rbound, '#000000'],
]
xaxis_label = df.columns[1]
yaxis_label = df.columns[2]
df[df.columns[2]] = df[df.columns[2]].astype(np.float)
df = df.pivot_table(
index=df.columns[1],
columns=df.columns[0],
values=df.columns[2], aggfunc=np.sum)
else:
# User provides columns (x, y, metric1, metric2, ...)
xaxis_label = df.columns[0]
yaxis_label = 'y'
df.index = df[df.columns[0]]
df = df.sort('ds')
del df[df.columns[0]]
for col in df.columns:
df[col] = df[col].astype(np.float)
series = []
for col in df.columns:
data = []
for row in df.itertuples():
data.append({
'x': row[2],
'y': row[3],
'value': row[4],
})
x_format = '{point.x:%Y-%m-%d}' \
if chart.x_is_date else '{point.x}'
series.append({
'name': col,
'data': [
(i, v)
for i, v in df[col].iteritems() if not np.isnan(v)]
'data': data,
'borderWidth': 0,
'colsize': 24 * 36e5,
'turboThreshold': sys.float_info.max,
'tooltip': {
'headerFormat': '',
'pointFormat': (
df.columns[1] + ': ' + x_format + '<br/>' +
df.columns[2] + ': {point.y}<br/>' +
df.columns[3] + ': <b>{point.value}</b>'
),
},
})
series = [serie for serie in sorted(
series, key=lambda s: s['data'][0][1], reverse=True)]
colorAxis = {
'stops': stops,
'minColor': '#FFFFFF',
'maxColor': '#000000',
'min': 50,
'max': 2200,
}
else:
if chart.sql_layout == 'series':
# User provides columns (series, x, y)
xaxis_label = df.columns[1]
yaxis_label = df.columns[2]
df[df.columns[2]] = df[df.columns[2]].astype(np.float)
df = df.pivot_table(
index=df.columns[1],
columns=df.columns[0],
values=df.columns[2], aggfunc=np.sum)
else:
# User provides columns (x, y, metric1, metric2, ...)
xaxis_label = df.columns[0]
yaxis_label = 'y'
df.index = df[df.columns[0]]
df = df.sort('ds')
del df[df.columns[0]]
for col in df.columns:
df[col] = df[col].astype(np.float)
for col in df.columns:
series.append({
'name': col,
'data': [
(i, v)
for i, v in df[col].iteritems() if not np.isnan(v)]
})
series = [serie for serie in sorted(
series, key=lambda s: s['data'][0][1], reverse=True)]
chart_type = chart.chart_type
if chart.chart_type == "stacked_area":
@ -325,12 +377,18 @@ class Airflow(BaseView):
'title': {'text': ''},
'xAxis': {
'title': {'text': xaxis_label},
'type': 'datetime' if x_is_dt else None,
'type': 'datetime' if chart.x_is_date else None,
},
'yAxis': {
'min': 0,
'title': {'text': yaxis_label},
},
'colorAxis': colorAxis,
'tooltip': {
'useHTML': True,
'backgroundColor': None,
'borderWidth': 0,
},
'series': series,
}
@ -345,6 +403,7 @@ class Airflow(BaseView):
def date_handler(obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else obj
response = Response(
response=json.dumps(payload, indent=4, default=date_handler),
status=200,
@ -394,6 +453,13 @@ class Airflow(BaseView):
def noaccess(self):
return self.render('airflow/noaccess.html')
@expose('/headers')
def headers(self):
d = {k: v for k, v in request.headers}
return Response(
response=json.dumps(d, indent=4),
status=200, mimetype="application/json")
@expose('/login')
def login(u):
session = settings.Session()
@ -404,11 +470,14 @@ class Airflow(BaseView):
has_access = role in request.headers.get('X-Internalauth-Groups')
d = {k: v for k, v in request.headers}
import urllib2
cookie = urllib2.unquote(d.get('Cookie'))
cookie = ''.join(cookie.split('j:')[1:]).split('; _ga=')[0]
cookie = json.loads(cookie)
email = str(cookie['data']['userData']['mail'][0])
try:
import urllib2
cookie = urllib2.unquote(d.get('Cookie'))
cookie = ''.join(cookie.split('j:')[1:]).split('; _ga=')[0]
cookie = json.loads(cookie)
email = str(cookie['data']['userData']['mail'][0])
except:
email = ""
if has_access:
user = session.query(models.User).filter(
models.User.username == username).first()
@ -416,6 +485,7 @@ class Airflow(BaseView):
user = models.User(username=username)
user.email = email
session.merge(user)
session.commit()
flask_login.login_user(user)
session.commit()
session.close()
@ -959,7 +1029,9 @@ def label_link(v, c, m, p):
default_params = eval(m.default_params)
except:
default_params = {}
url = url_for('airflow.chart', chart_id=m.id, **default_params)
url = url_for(
'airflow.chart', chart_id=m.id, iteration_no=m.iteration_no,
**default_params)
return Markup("<a href='{url}'>{m.label}</a>".format(**locals()))
@ -970,6 +1042,7 @@ class ChartModelView(LoginMixin, ModelView):
'db',
'chart_type',
'show_datatable',
'x_is_date',
'y_log_scale',
'show_sql',
'height',
@ -984,6 +1057,45 @@ class ChartModelView(LoginMixin, ModelView):
edit_template = 'airflow/chart/edit.html'
column_filters = ('owner.username', 'db_id',)
column_searchable_list = ('owner.username', 'label', 'sql')
column_descriptions = {
'label': "Can include {{ templated_fields }} and {{ macros }}",
'chart_type': "The type of chart to be displayed",
'sql': "Can include {{ templated_fields }} and {{ macros }}.",
'height': "Height of the chart, in pixels.",
'x_is_date': (
"Whether the X axis should be casted as a date field. Expect most "
"intelligible date formats to get casted properly."
),
'owner': (
"The chart's owner, mostly used for reference and filtering in "
"the list view."
),
'show_datatable':
"Whether to display an interactive data table under the chart.",
'default_params': (
'A dictionary of {"key": "values",} that define what the '
'templated fields (parameters) values should be by default. '
'To be valid, it needs to "eval" as a Python dict. '
'The key values will show up in the url\'s querystring '
'and can be altered there.'
),
'show_sql': "Whether to display the SQL statement as a collapsible "
"section in the chart page.",
'y_log_scale': "Whether to use a log scale for the Y axis.",
'sql_layout': (
"Defines the layout of the SQL that the application should "
"expect. Depending on the tables you are sourcing from, it may "
"make more sense to pivot / unpivot the metrics."
),
}
column_labels = {
'db': "Source Database",
'sql': "SQL",
'height': "Chart Height",
'sql_layout': "SQL Layout",
'show_sql': "Display the SQL Statement",
'default_params': "Default Parameters",
}
form_choices = {
'chart_type': [
('line', 'Line Chart'),
@ -993,6 +1105,7 @@ class ChartModelView(LoginMixin, ModelView):
('area', 'Overlapping Area Chart'),
('stacked_area', 'Stacked Area Chart'),
('percent_area', 'Percent Area Chart'),
('heatmap', 'Heatmap'),
('datatable', 'No chart, data table only'),
],
'sql_layout': [
@ -1002,10 +1115,22 @@ class ChartModelView(LoginMixin, ModelView):
}
def on_model_change(self, form, model, is_created):
if not model.user_id and flask_login.current_user:
model.iteration_no += 1
if AUTHENTICATE and not model.user_id and flask_login.current_user:
model.user_id = flask_login.current_user.id
mv = ChartModelView(
models.Chart, session,
name="Charts", category="Tools")
admin.add_view(mv)
admin.add_link(
base.MenuLink(
category='Docs',
name='@readthedocs.org',
url='http://airflow.readthedocs.org/en/latest/'))
admin.add_link(
base.MenuLink(
category='Docs',
name='Github',
url='https://github.com/mistercrunch/Airflow'))

Просмотреть файл

@ -15,7 +15,6 @@ def load_user(userid):
session = settings.Session()
user = session.query(User).filter(User.id == userid).first()
if not user:
#user = User(username=username)
raise Exception(userid)
session.expunge_all()
session.commit()

Просмотреть файл

@ -0,0 +1,175 @@
/**
* This plugin extends Highcharts in two ways:
* - Use HTML5 canvas instead of SVG for rendering of the heatmap squares. Canvas
* outperforms SVG when it comes to thousands of single shapes.
* - Add a K-D-tree to find the nearest point on mouse move. Since we no longer have SVG shapes
* to capture mouseovers, we need another way of detecting hover points for the tooltip.
*/
(function (H) {
var wrap = H.wrap,
seriesTypes = H.seriesTypes;
/**
* Recursively builds a K-D-tree
*/
function KDTree(points, depth) {
var axis, median, length = points && points.length;
if (length) {
// alternate between the axis
axis = ['plotX', 'plotY'][depth % 2];
// sort point array
points.sort(function (a, b) {
return a[axis] - b[axis];
});
median = Math.floor(length / 2);
// build and return node
return {
point: points[median],
left: KDTree(points.slice(0, median), depth + 1),
right: KDTree(points.slice(median + 1), depth + 1)
};
}
}
/**
* Recursively searches for the nearest neighbour using the given K-D-tree
*/
function nearest(search, tree, depth) {
var point = tree.point,
axis = ['plotX', 'plotY'][depth % 2],
tdist,
sideA,
sideB,
ret = point,
nPoint1,
nPoint2;
// Get distance
point.dist = Math.pow(search.plotX - point.plotX, 2) +
Math.pow(search.plotY - point.plotY, 2);
// Pick side based on distance to splitting point
tdist = search[axis] - point[axis];
sideA = tdist < 0 ? 'left' : 'right';
// End of tree
if (tree[sideA]) {
nPoint1 = nearest(search, tree[sideA], depth + 1);
ret = (nPoint1.dist < ret.dist ? nPoint1 : point);
sideB = tdist < 0 ? 'right' : 'left';
if (tree[sideB]) {
// compare distance to current best to splitting point to decide wether to check side B or not
if (Math.abs(tdist) < ret.dist) {
nPoint2 = nearest(search, tree[sideB], depth + 1);
ret = (nPoint2.dist < ret.dist ? nPoint2 : ret);
}
}
}
return ret;
}
// Extend the heatmap to use the K-D-tree to search for nearest points
H.seriesTypes.heatmap.prototype.setTooltipPoints = function () {
var series = this;
this.tree = null;
setTimeout(function () {
series.tree = KDTree(series.points, 0);
});
};
H.seriesTypes.heatmap.prototype.getNearest = function (search) {
if (this.tree) {
return nearest(search, this.tree, 0);
}
};
H.wrap(H.Pointer.prototype, 'runPointActions', function (proceed, e) {
var chart = this.chart;
proceed.call(this, e);
// Draw independent tooltips
H.each(chart.series, function (series) {
var point;
if (series.getNearest) {
point = series.getNearest({
plotX: e.chartX - chart.plotLeft,
plotY: e.chartY - chart.plotTop
});
if (point) {
point.onMouseOver(e);
}
}
})
});
/**
* Get the canvas context for a series
*/
H.Series.prototype.getContext = function () {
var canvas;
if (!this.ctx) {
canvas = document.createElement('canvas');
canvas.setAttribute('width', this.chart.plotWidth);
canvas.setAttribute('height', this.chart.plotHeight);
canvas.style.position = 'absolute';
canvas.style.left = this.group.translateX + 'px';
canvas.style.top = this.group.translateY + 'px';
canvas.style.zIndex = 0;
canvas.style.cursor = 'crosshair';
this.chart.container.appendChild(canvas);
if (canvas.getContext) {
this.ctx = canvas.getContext('2d');
}
}
return this.ctx;
}
/**
* Wrap the drawPoints method to draw the points in canvas instead of the slower SVG,
* that requires one shape each point.
*/
H.wrap(H.seriesTypes.heatmap.prototype, 'drawPoints', function (proceed) {
var ctx;
if (this.chart.renderer.forExport) {
// Run SVG shapes
proceed.call(this);
} else {
if (ctx = this.getContext()) {
// draw the columns
H.each(this.points, function (point) {
var plotY = point.plotY,
shapeArgs;
if (plotY !== undefined && !isNaN(plotY) && point.y !== null) {
shapeArgs = point.shapeArgs;
ctx.fillStyle = point.pointAttr[''].fill;
ctx.fillRect(shapeArgs.x, shapeArgs.y, shapeArgs.width, shapeArgs.height);
}
});
} else {
this.chart.showLoading("Your browser doesn't support HTML5 canvas, <br>please use a modern browser");
// Uncomment this to provide low-level (slow) support in oldIE. It will cause script errors on
// charts with more than a few thousand points.
//proceed.call(this);
}
}
});
}(Highcharts));

Просмотреть файл

@ -0,0 +1,23 @@
/*
Highcharts JS v4.0.4 (2014-09-02)
(c) 2011-2014 Torstein Honsi
License: www.highcharts.com/license
*/
(function(h){var k=h.Axis,y=h.Chart,l=h.Color,z=h.Legend,t=h.LegendSymbolMixin,u=h.Series,v=h.SVGRenderer,w=h.getOptions(),i=h.each,r=h.extend,A=h.extendClass,m=h.merge,o=h.pick,x=h.numberFormat,p=h.seriesTypes,s=h.wrap,n=function(){},q=h.ColorAxis=function(){this.isColorAxis=!0;this.init.apply(this,arguments)};r(q.prototype,k.prototype);r(q.prototype,{defaultColorAxisOptions:{lineWidth:0,gridLineWidth:1,tickPixelInterval:72,startOnTick:!0,endOnTick:!0,offset:0,marker:{animation:{duration:50},color:"gray",
width:0.01},labels:{overflow:"justify"},minColor:"#EFEFFF",maxColor:"#003875",tickLength:5},init:function(b,a){var c=b.options.legend.layout!=="vertical",d;d=m(this.defaultColorAxisOptions,{side:c?2:1,reversed:!c},a,{isX:c,opposite:!c,showEmpty:!1,title:null,isColor:!0});k.prototype.init.call(this,b,d);a.dataClasses&&this.initDataClasses(a);this.initStops(a);this.isXAxis=!0;this.horiz=c;this.zoomEnabled=!1},tweenColors:function(b,a,c){var d=a.rgba[3]!==1||b.rgba[3]!==1;return(d?"rgba(":"rgb(")+Math.round(a.rgba[0]+
(b.rgba[0]-a.rgba[0])*(1-c))+","+Math.round(a.rgba[1]+(b.rgba[1]-a.rgba[1])*(1-c))+","+Math.round(a.rgba[2]+(b.rgba[2]-a.rgba[2])*(1-c))+(d?","+(a.rgba[3]+(b.rgba[3]-a.rgba[3])*(1-c)):"")+")"},initDataClasses:function(b){var a=this,c=this.chart,d,e=0,f=this.options,g=b.dataClasses.length;this.dataClasses=d=[];this.legendItems=[];i(b.dataClasses,function(b,h){var i,b=m(b);d.push(b);if(!b.color)f.dataClassColor==="category"?(i=c.options.colors,b.color=i[e++],e===i.length&&(e=0)):b.color=a.tweenColors(l(f.minColor),
l(f.maxColor),g<2?0.5:h/(g-1))})},initStops:function(b){this.stops=b.stops||[[0,this.options.minColor],[1,this.options.maxColor]];i(this.stops,function(a){a.color=l(a[1])})},setOptions:function(b){k.prototype.setOptions.call(this,b);this.options.crosshair=this.options.marker;this.coll="colorAxis"},setAxisSize:function(){var b=this.legendSymbol,a=this.chart,c,d,e;if(b)this.left=c=b.attr("x"),this.top=d=b.attr("y"),this.width=e=b.attr("width"),this.height=b=b.attr("height"),this.right=a.chartWidth-
c-e,this.bottom=a.chartHeight-d-b,this.len=this.horiz?e:b,this.pos=this.horiz?c:d},toColor:function(b,a){var c,d=this.stops,e,f=this.dataClasses,g,j;if(f)for(j=f.length;j--;){if(g=f[j],e=g.from,d=g.to,(e===void 0||b>=e)&&(d===void 0||b<=d)){c=g.color;if(a)a.dataClass=j;break}}else{this.isLog&&(b=this.val2lin(b));c=1-(this.max-b)/(this.max-this.min||1);for(j=d.length;j--;)if(c>d[j][0])break;e=d[j]||d[j+1];d=d[j+1]||e;c=1-(d[0]-c)/(d[0]-e[0]||1);c=this.tweenColors(e.color,d.color,c)}return c},getOffset:function(){var b=
this.legendGroup,a=this.chart.axisOffset[this.side];if(b){k.prototype.getOffset.call(this);if(!this.axisGroup.parentGroup)this.axisGroup.add(b),this.gridGroup.add(b),this.labelGroup.add(b),this.added=!0;this.chart.axisOffset[this.side]=a}},setLegendColor:function(){var b,a=this.options;b=this.horiz?[0,0,1,0]:[0,0,0,1];this.legendColor={linearGradient:{x1:b[0],y1:b[1],x2:b[2],y2:b[3]},stops:a.stops||[[0,a.minColor],[1,a.maxColor]]}},drawLegendSymbol:function(b,a){var c=b.padding,d=b.options,e=this.horiz,
f=o(d.symbolWidth,e?200:12),g=o(d.symbolHeight,e?12:200),j=o(d.labelPadding,e?16:30),d=o(d.itemDistance,10);this.setLegendColor();a.legendSymbol=this.chart.renderer.rect(0,b.baseline-11,f,g).attr({zIndex:1}).add(a.legendGroup);a.legendSymbol.getBBox();this.legendItemWidth=f+c+(e?d:j);this.legendItemHeight=g+c+(e?j:0)},setState:n,visible:!0,setVisible:n,getSeriesExtremes:function(){var b;if(this.series.length)b=this.series[0],this.dataMin=b.valueMin,this.dataMax=b.valueMax},drawCrosshair:function(b,
a){var c=!this.cross,d=a&&a.plotX,e=a&&a.plotY,f,g=this.pos,j=this.len;if(a)f=this.toPixels(a.value),f<g?f=g-2:f>g+j&&(f=g+j+2),a.plotX=f,a.plotY=this.len-f,k.prototype.drawCrosshair.call(this,b,a),a.plotX=d,a.plotY=e,!c&&this.cross&&this.cross.attr({fill:this.crosshair.color}).add(this.labelGroup)},getPlotLinePath:function(b,a,c,d,e){return e?this.horiz?["M",e-4,this.top-6,"L",e+4,this.top-6,e,this.top,"Z"]:["M",this.left,e,"L",this.left-6,e+6,this.left-6,e-6,"Z"]:k.prototype.getPlotLinePath.call(this,
b,a,c,d)},update:function(b,a){i(this.series,function(a){a.isDirtyData=!0});k.prototype.update.call(this,b,a);this.legendItem&&(this.setLegendColor(),this.chart.legend.colorizeItem(this,!0))},getDataClassLegendSymbols:function(){var b=this,a=this.chart,c=this.legendItems,d=a.options.legend,e=d.valueDecimals,f=d.valueSuffix||"",g;c.length||i(this.dataClasses,function(d,h){var k=!0,l=d.from,m=d.to;g="";l===void 0?g="< ":m===void 0&&(g="> ");l!==void 0&&(g+=x(l,e)+f);l!==void 0&&m!==void 0&&(g+=" - ");
m!==void 0&&(g+=x(m,e)+f);c.push(r({chart:a,name:g,options:{},drawLegendSymbol:t.drawRectangle,visible:!0,setState:n,setVisible:function(){k=this.visible=!k;i(b.series,function(a){i(a.points,function(a){a.dataClass===h&&a.setVisible(k)})});a.legend.colorizeItem(this,k)}},d))});return c},name:""});i(["fill","stroke"],function(b){HighchartsAdapter.addAnimSetter(b,function(a){a.elem.attr(b,q.prototype.tweenColors(l(a.start),l(a.end),a.pos))})});s(y.prototype,"getAxes",function(b){var a=this.options.colorAxis;
b.call(this);this.colorAxis=[];a&&new q(this,a)});s(z.prototype,"getAllItems",function(b){var a=[],c=this.chart.colorAxis[0];c&&(c.options.dataClasses?a=a.concat(c.getDataClassLegendSymbols()):a.push(c),i(c.series,function(a){a.options.showInLegend=!1}));return a.concat(b.call(this))});h={pointAttrToOptions:{stroke:"borderColor","stroke-width":"borderWidth",fill:"color",dashstyle:"dashStyle"},pointArrayMap:["value"],axisTypes:["xAxis","yAxis","colorAxis"],optionalAxis:"colorAxis",trackerGroups:["group",
"markerGroup","dataLabelsGroup"],getSymbol:n,parallelArrays:["x","y","value"],colorKey:"value",translateColors:function(){var b=this,a=this.options.nullColor,c=this.colorAxis,d=this.colorKey;i(this.data,function(e){var f=e[d];if(f=f===null?a:c&&f!==void 0?c.toColor(f,e):e.color||b.color)e.color=f})}};s(v.prototype,"buildText",function(b,a){var c=a.styles&&a.styles.HcTextStroke;b.call(this,a);c&&a.applyTextStroke&&a.applyTextStroke(c)});v.prototype.Element.prototype.applyTextStroke=function(b){var a=
this.element,c,d,b=b.split(" ");c=a.getElementsByTagName("tspan");d=a.firstChild;this.ySetter=this.xSetter;i([].slice.call(c),function(c,f){var g;f===0&&(c.setAttribute("x",a.getAttribute("x")),(f=a.getAttribute("y"))!==null&&c.setAttribute("y",f));g=c.cloneNode(1);g.setAttribute("stroke",b[1]);g.setAttribute("stroke-width",b[0]);g.setAttribute("stroke-linejoin","round");a.insertBefore(g,d)})};w.plotOptions.heatmap=m(w.plotOptions.scatter,{animation:!1,borderWidth:0,nullColor:"#F8F8F8",dataLabels:{formatter:function(){return this.point.value},
verticalAlign:"middle",crop:!1,overflow:!1,style:{color:"white",fontWeight:"bold",HcTextStroke:"1px rgba(0,0,0,0.5)"}},marker:null,tooltip:{pointFormat:"{point.x}, {point.y}: {point.value}<br/>"},states:{normal:{animation:!0},hover:{brightness:0.2}}});p.heatmap=A(p.scatter,m(h,{type:"heatmap",pointArrayMap:["y","value"],hasPointSpecificOptions:!0,supportsDrilldown:!0,getExtremesFromAll:!0,init:function(){p.scatter.prototype.init.apply(this,arguments);this.pointRange=this.options.colsize||1;this.yAxis.axisPointRange=
this.options.rowsize||1},translate:function(){var b=this.options,a=this.xAxis,c=this.yAxis;this.generatePoints();i(this.points,function(d){var e=(b.colsize||1)/2,f=(b.rowsize||1)/2,g=Math.round(a.len-a.translate(d.x-e,0,1,0,1)),e=Math.round(a.len-a.translate(d.x+e,0,1,0,1)),h=Math.round(c.translate(d.y-f,0,1,0,1)),f=Math.round(c.translate(d.y+f,0,1,0,1));d.plotX=(g+e)/2;d.plotY=(h+f)/2;d.shapeType="rect";d.shapeArgs={x:Math.min(g,e),y:Math.min(h,f),width:Math.abs(e-g),height:Math.abs(f-h)}});this.translateColors();
this.chart.hasRendered&&i(this.points,function(a){a.shapeArgs.fill=a.options.color||a.color})},drawPoints:p.column.prototype.drawPoints,animate:n,getBox:n,drawLegendSymbol:t.drawRectangle,getExtremes:function(){u.prototype.getExtremes.call(this,this.valueData);this.valueMin=this.dataMin;this.valueMax=this.dataMax;u.prototype.getExtremes.call(this)}}))})(Highcharts);

Просмотреть файл

@ -19,7 +19,16 @@
textarea.val(editor.getSession().getValue());
});
editor.focus();
// Getting column_descriptions in tooltips
$(":checkbox").removeClass("form-control");
$("span.help-block").each(function(){
$(this).parent().attr("title", $(this).text());
$(this).parent().attr("data-toggle", "tooltip");
$(this).remove();
});
$('[data-toggle="tooltip"]').tooltip();
});
</script>
{% endblock %}

Просмотреть файл

@ -13,7 +13,26 @@
border: none;
background: none;
background-color: #FFF;
};
}
.highcharts-tooltip>span {
background: rgba(255,255,255,0.85);
border: 1px solid silver;
border-radius: 3px;
box-shadow: 1px 1px 2px #888;
padding: 8px;
z-index: 2;
}
.panel-heading .accordion-toggle:after {
/* symbol for "opening" panels */
font-family: 'Glyphicons Halflings'; /* essential for enabling glyphicon */
content: "\e114"; /* adjust as needed, taken from bootstrap.css */
float: right; /* adjust as needed */
color: grey; /* adjust as needed */
}
.panel-heading .accordion-toggle.collapsed:after {
/* symbol for "collapsed" panels */
content: "\e080"; /* adjust as needed, taken from bootstrap.css */
}
</style>
{% endblock %}
{% block title %}
@ -25,8 +44,8 @@
<div id="container">
<h2>
<span id="label">{{ label }}</span>
<a href="/admin/chart/edit/?id={{ chart.id }}">
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
<a href="/admin/chart/edit/?id={{ chart.id }}" >
<span class="glyphicon glyphicon-edit" aria-hidden="true" ></span>
</a>
</h2>
<div id="error" style="display: none;" class="alert alert-danger" role="alert">
@ -37,10 +56,8 @@
<div class="panel panel-default">
<div class="panel-heading" role="tab" id="headingTwo">
<h4 class="panel-title">
<a class="collapsed" data-toggle="collapse" data-parent="#accordion" href="#sql_panel" aria-expanded="true" aria-controls="sql_panel">
<a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#sql_panel" aria-expanded="true" aria-controls="sql_panel">
SQL
<span class="glyphicon glyphicon-chevron-up" aria-hidden="true"></span>
<span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span>
</a>
</h4>
</div>
@ -55,16 +72,14 @@
<div id="chart_section" class="panel panel-default">
<div class="panel-heading" role="tab" id="headingTwo">
<h4 class="panel-title">
<a class="collapsed" data-toggle="collapse" data-parent="#accordion" href="#chart_panel" aria-expanded="true" aria-controls="chart_panel">
<a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#chart_panel" aria-expanded="true" aria-controls="chart_panel">
Chart
<span class="glyphicon glyphicon-chevron-up" aria-hidden="true"></span>
<span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span>
</a>
</h4>
</div>
<div id="chart_panel" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingTwo">
<div class="panel-body">
<div id="hc" style="height:{{ chart.height }}px;">
<div id="chart_body">
<img src="{{ url_for('static', filename='loading.gif') }}" width="50px">
</div>
</div>
@ -75,10 +90,8 @@
<div id="datatable_section" class="panel panel-default">
<div class="panel-heading" role="tab" id="headingTwo">
<h4 class="panel-title">
<a class="collapsed" data-toggle="collapse" data-parent="#accordion" href="#datatable_panel" aria-expanded="true" aria-controls="datatable_panel">
<a class="accordion-toggle" data-toggle="collapse" data-parent="#accordion" href="#datatable_panel" aria-expanded="true" aria-controls="datatable_panel">
Data
<span class="glyphicon glyphicon-chevron-up" aria-hidden="true"></span>
<span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span>
</a>
</h4>
</div>
@ -100,6 +113,8 @@
<script src="{{ url_for('static', filename='highcharts.js') }}"></script>
<script src="{{ url_for('static', filename='highcharts-more.js') }}">
</script>
<script src="{{ url_for('static', filename='heatmap.js') }}"></script>
<script src="{{ url_for('static', filename='heatmap-canvas.js') }}"></script>
<script>
function error(msg){
$('#error_msg').html(msg);
@ -115,14 +130,17 @@
"#FFAA91", "#B4A76C", "#9CA299", "#565A5C"
],
});
url = "{{ url_for('airflow.chart_data', chart_id=chart.id) }}";
url = "{{ url_for('airflow.chart_data', chart_id=chart.id) }}" + location.search;
$.getJSON(url, function(payload) {
console.log(payload);
$('#loading').hide();
$("#sql_panel_body").html(payload.sql_html);
$("#label").html(payload.label);
if (payload.state == "SUCCESS") {
{% if chart.chart_type != "datatable" %}
$('#hc').highcharts(payload.hc);
$('#chart_body').css('width', '100%');
$('#chart_body').css('height', '{{ chart.height }}');
$('#chart_body').highcharts(payload.hc);
{% endif %}
{% if chart.show_datatable or chart.chart_type == "datatable" %}
$('#datatable').dataTable( {

Просмотреть файл

@ -11,7 +11,7 @@
{% endblock %}
{% block body %}
<h2>Query</h2>
<h2>Ad Hoc Query</h2>
<form method="get">
<div class="form-inline">
{{ form.db_id }}
@ -34,21 +34,23 @@
$( document ).ready(function() {
var editor = ace.edit("sql");
var textarea = $('textarea[name="sql"]').hide();
function sync() {
textarea.val(editor.getSession().getValue());
}
editor.setTheme("ace/theme/crimson_editor");
editor.setOptions({
minLines: 3,
maxLines: Infinity,
});
editor.getSession().setMode("ace/mode/sql");
editor.getSession().on('change', function(){
textarea.val(editor.getSession().getValue());
});
editor.getSession().on('change', sync);
editor.focus();
$('table.dataframe').dataTable({
"scrollX": true,
"iDisplayLength": 25,
});
$('select').addClass("form-control");
sync();
});
</script>
{% endblock %}

74
airflow/www/utils.py Normal file
Просмотреть файл

@ -0,0 +1,74 @@
from cgi import escape
from cStringIO import StringIO as IO
import gzip
import functools
from flask import after_this_request, request
import wtforms
from wtforms.compat import text_type
def gzipped(f):
'''
Decorator to make a view compressed
'''
@functools.wraps(f)
def view_func(*args, **kwargs):
@after_this_request
def zipper(response):
accept_encoding = request.headers.get('Accept-Encoding', '')
if 'gzip' not in accept_encoding.lower():
return response
response.direct_passthrough = False
if (response.status_code < 200 or
response.status_code >= 300 or
'Content-Encoding' in response.headers):
return response
gzip_buffer = IO()
gzip_file = gzip.GzipFile(mode='wb',
fileobj=gzip_buffer)
gzip_file.write(response.data)
gzip_file.close()
response.data = gzip_buffer.getvalue()
response.headers['Content-Encoding'] = 'gzip'
response.headers['Vary'] = 'Accept-Encoding'
response.headers['Content-Length'] = len(response.data)
return response
return f(*args, **kwargs)
return view_func
def make_cache_key(*args, **kwargs):
'''
Used by cache to get a unique key per URL
'''
path = request.path
args = str(hash(frozenset(request.args.items())))
return (path + args).encode('ascii', 'ignore')
class AceEditorWidget(wtforms.widgets.TextArea):
"""
Renders an ACE code editor.
"""
def __call__(self, field, **kwargs):
kwargs.setdefault('id', field.id)
html = '''
<div id="{el_id}" style="height:100px;">{contents}</div>
<textarea
id="{el_id}_ace" name="{form_name}"
style="display:none;visibility:hidden;">
</textarea>
'''.format(
el_id=kwargs.get('id', field.id),
contents=escape(text_type(field._value())),
form_name=field.id,
)
return wtforms.widgets.core.HTMLString(html)

Просмотреть файл

@ -4,6 +4,7 @@ flake8
flask
flask-admin
flask-bootstrap
flask-cache
hive-thrift-py
ipython[all]
jinja2