Improvements to the query tool

TODO.md

@@ -4,7 +4,6 @@ TODO
 * User login / security
 * Tree view: remove dummy root node
 * Backfill wizard
-* Fix datepicker
 
 #### Write unittests
 * For each existing operator

airflow/models.py

@@ -1139,4 +1139,4 @@ class Chart(Base):
     show_sql = Column(Boolean, default=True)
     height = Column(Integer, default=600)
     default_params = Column(String(5000), default="{}")
-    db = relationship("DatabaseConnection")
+    db = relationship("DatabaseConnection", order_by="DatabaseConnection.db_id")

airflow/www/app.py

@@ -9,7 +9,10 @@ from flask.ext.admin.form import DateTimePickerWidget
 from flask.ext.admin import base
 from flask.ext.admin.contrib.sqla import ModelView
 from flask import request
-from wtforms import Form, DateTimeField, SelectField, TextField, TextAreaField
+from wtforms import Form, DateTimeField, SelectField, TextAreaField
+from cgi import escape
+from wtforms.compat import text_type
+import wtforms
 
 from pygments import highlight
 from pygments.lexers import PythonLexer, SqlLexer, BashLexer
@@ -43,6 +46,23 @@ app.register_blueprint(ck, url_prefix='/ck')
 app.jinja_env.add_extension("chartkick.ext.charts")
 
 
+class AceEditorWidget(wtforms.widgets.TextArea):
+    """
+    Renders an ACE code editor.
+    """
+    def __call__(self, field, **kwargs):
+        kwargs.setdefault('id', field.id)
+        html = '''
+        <div id="{el_id}" style="height:100px;">{contents}</div>
+        <textarea id="{el_id}_ace" name="{form_name}" style="display:none;visibility:hidden;">
+        </textarea>
+        '''.format(
+            el_id=kwargs.get('id', field.id),
+            contents=escape(text_type(field._value())),
+            form_name=field.id,
+        )
+        return wtforms.widgets.core.HTMLString(html)
+
 # Date filter form needed for gantt and graph view
 class DateTimeForm(Form):
     execution_date = DateTimeField("Execution date", widget=DateTimePickerWidget())
@@ -81,6 +101,21 @@ class HomeView(AdminIndexView):
         dags = sorted(dagbag.dags.values(), key=lambda dag: dag.dag_id)
         return self.render('airflow/dags.html', dags=dags)
 admin = Admin(app, name="Airflow", index_view=HomeView(name='DAGs'))
+admin.add_link(
+    base.MenuLink(
+        category='Tools',
+        name='Query',
+        url='/admin/airflow/query'))
+admin.add_link(
+    base.MenuLink(
+        category='Docs',
+        name='@readthedocs.org',
+        url='http://airflow.readthedocs.org/en/latest/'))
+admin.add_link(
+    base.MenuLink(
+        category='Docs',
+        name='Github',
+        url='https://github.com/mistercrunch/Airflow'))
 
 
 class Airflow(BaseView):
@@ -95,13 +130,14 @@ class Airflow(BaseView):
     @expose('/query')
     def query(self):
         session = settings.Session()
-        dbs = session.query(models.DatabaseConnection)
+        dbs = session.query(models.DatabaseConnection).order_by(
+            models.DatabaseConnection.db_id)
         db_choices = [(db.db_id, db.db_id) for db in dbs]
         db_id_str = request.args.get('db_id')
         sql = request.args.get('sql')
         class QueryForm(Form):
             db_id = SelectField("Layout", choices=db_choices)
-            sql = TextAreaField("Execution date")
+            sql = TextAreaField("SQL", widget=AceEditorWidget())
         data = {
             'db_id': db_id_str,
             'sql': sql,
@@ -115,8 +151,15 @@ class Airflow(BaseView):
             try:
                 df = hook.get_pandas_df(sql)
                 has_data = len(df) > 0
+                df = df.fillna('')
                 results = df.to_html(
-                    classes="table table-striped table-bordered model-list")
+                    classes=(
+                        "table initialism table-striped "
+                        "table-bordered table-condensed"),
+                    index=False,
+                    na_rep='',
+
+                ) if has_data else ''
             except Exception as e:
                 flash(str(e), 'error')
                 error = True
@@ -130,7 +173,8 @@ class Airflow(BaseView):
         return self.render(
             'airflow/query.html', form=form,
             title="Query",
-            results=results, has_data=has_data)
+            results=results or '',
+            has_data=has_data)
 
     @expose('/chart')
     def chart(self):
@@ -782,20 +826,6 @@ class ReloadTaskView(BaseView):
         return redirect(url_for('index'))
 admin.add_view(ReloadTaskView(name='Reload DAGs', category="Admin"))
 
-if __name__ == "__main__":
-    logging.info("Starting the web server.")
-    app.run(debug=True)
-
-admin.add_link(
-    base.MenuLink(
-        category='Docs',
-        name='@readthedocs.org',
-        url='http://airflow.readthedocs.org/en/latest/'))
-admin.add_link(
-    base.MenuLink(
-        category='Docs',
-        name='Github',
-        url='https://github.com/mistercrunch/Airflow'))
 
 
 def label_link(v, c, m, p):
@@ -823,5 +853,5 @@ class ChartModelView(ModelView):
     }
 mv = ChartModelView(
     models.Chart, session,
-    name="Charts", category="Admin")
+    name="Charts", category="Tools")
 admin.add_view(mv)

airflow/www/static/main.css

@@ -39,13 +39,23 @@ pre code {
     overflow-wrap: normal;
     white-space: pre;
 }
-textarea {
-    width: 100%;
-    margin-top:2px;
-}
 input, select {
     margin: 0px;
 }
 .code {
     font-family: monospace;
 }
+
+#sql {
+    border: 1px solid #CCC;
+    border-radius: 5px;
+}
+.ace_editor div {
+    font: inherit!important
+}
+#ace_container {
+    margin: 10px 0px;
+}
+#sql_ace {
+    visibility: hidden;
+}

airflow/www/static/mode-sql.js

@@ -0,0 +1,92 @@
+define("ace/mode/sql_highlight_rules",["require","exports","module","ace/lib/oop","ace/mode/text_highlight_rules"], function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
+
+var SqlHighlightRules = function() {
+
+    var keywords = (
+        "select|insert|update|delete|from|where|and|or|group|by|order|limit|offset|having|as|case|" +
+        "when|else|end|type|left|right|join|on|outer|desc|asc|union"
+    );
+
+    var builtinConstants = (
+        "true|false|null"
+    );
+
+    var builtinFunctions = (
+        "count|min|max|avg|sum|rank|now|coalesce"
+    );
+
+    var keywordMapper = this.createKeywordMapper({
+        "support.function": builtinFunctions,
+        "keyword": keywords,
+        "constant.language": builtinConstants
+    }, "identifier", true);
+
+    this.$rules = {
+        "start" : [ {
+            token : "comment",
+            regex : "--.*$"
+        },  {
+            token : "comment",
+            start : "/\\*",
+            end : "\\*/"
+        }, {
+            token : "string",           // " string
+            regex : '".*?"'
+        }, {
+            token : "string",           // ' string
+            regex : "'.*?'"
+        }, {
+            token : "constant.numeric", // float
+            regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
+        }, {
+            token : keywordMapper,
+            regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+        }, {
+            token : "keyword.operator",
+            regex : "\\+|\\-|\\/|\\/\\/|%|<@>|@>|<@|&|\\^|~|<|>|<=|=>|==|!=|<>|="
+        }, {
+            token : "paren.lparen",
+            regex : "[\\(]"
+        }, {
+            token : "paren.rparen",
+            regex : "[\\)]"
+        }, {
+            token : "text",
+            regex : "\\s+"
+        } ]
+    };
+    this.normalizeRules();
+};
+
+oop.inherits(SqlHighlightRules, TextHighlightRules);
+
+exports.SqlHighlightRules = SqlHighlightRules;
+});
+
+define("ace/mode/sql",["require","exports","module","ace/lib/oop","ace/mode/text","ace/mode/sql_highlight_rules","ace/range"], function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var TextMode = require("./text").Mode;
+var SqlHighlightRules = require("./sql_highlight_rules").SqlHighlightRules;
+var Range = require("../range").Range;
+
+var Mode = function() {
+    this.HighlightRules = SqlHighlightRules;
+};
+oop.inherits(Mode, TextMode);
+
+(function() {
+
+    this.lineCommentStart = "--";
+
+    this.$id = "ace/mode/sql";
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+
+});

airflow/www/static/theme-crimson_editor.js

@@ -0,0 +1,118 @@
+define("ace/theme/crimson_editor",["require","exports","module","ace/lib/dom"], function(require, exports, module) {
+exports.isDark = false;
+exports.cssText = ".ace-crimson-editor .ace_gutter {\
+background: #ebebeb;\
+color: #333;\
+overflow : hidden;\
+}\
+.ace-crimson-editor .ace_gutter-layer {\
+width: 100%;\
+text-align: right;\
+}\
+.ace-crimson-editor .ace_print-margin {\
+width: 1px;\
+background: #e8e8e8;\
+}\
+.ace-crimson-editor {\
+background-color: #FFFFFF;\
+color: rgb(64, 64, 64);\
+}\
+.ace-crimson-editor .ace_cursor {\
+color: black;\
+}\
+.ace-crimson-editor .ace_invisible {\
+color: rgb(191, 191, 191);\
+}\
+.ace-crimson-editor .ace_identifier {\
+color: black;\
+}\
+.ace-crimson-editor .ace_keyword {\
+color: blue;\
+}\
+.ace-crimson-editor .ace_constant.ace_buildin {\
+color: rgb(88, 72, 246);\
+}\
+.ace-crimson-editor .ace_constant.ace_language {\
+color: rgb(255, 156, 0);\
+}\
+.ace-crimson-editor .ace_constant.ace_library {\
+color: rgb(6, 150, 14);\
+}\
+.ace-crimson-editor .ace_invalid {\
+text-decoration: line-through;\
+color: rgb(224, 0, 0);\
+}\
+.ace-crimson-editor .ace_fold {\
+}\
+.ace-crimson-editor .ace_support.ace_function {\
+color: rgb(192, 0, 0);\
+}\
+.ace-crimson-editor .ace_support.ace_constant {\
+color: rgb(6, 150, 14);\
+}\
+.ace-crimson-editor .ace_support.ace_type,\
+.ace-crimson-editor .ace_support.ace_class {\
+color: rgb(109, 121, 222);\
+}\
+.ace-crimson-editor .ace_keyword.ace_operator {\
+color: rgb(49, 132, 149);\
+}\
+.ace-crimson-editor .ace_string {\
+color: rgb(128, 0, 128);\
+}\
+.ace-crimson-editor .ace_comment {\
+color: rgb(76, 136, 107);\
+}\
+.ace-crimson-editor .ace_comment.ace_doc {\
+color: rgb(0, 102, 255);\
+}\
+.ace-crimson-editor .ace_comment.ace_doc.ace_tag {\
+color: rgb(128, 159, 191);\
+}\
+.ace-crimson-editor .ace_constant.ace_numeric {\
+color: rgb(0, 0, 64);\
+}\
+.ace-crimson-editor .ace_variable {\
+color: rgb(0, 64, 128);\
+}\
+.ace-crimson-editor .ace_xml-pe {\
+color: rgb(104, 104, 91);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_selection {\
+background: rgb(181, 213, 255);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_step {\
+background: rgb(252, 255, 0);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_stack {\
+background: rgb(164, 229, 101);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_bracket {\
+margin: -1px 0 0 -1px;\
+border: 1px solid rgb(192, 192, 192);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_active-line {\
+background: rgb(232, 242, 254);\
+}\
+.ace-crimson-editor .ace_gutter-active-line {\
+background-color : #dcdcdc;\
+}\
+.ace-crimson-editor .ace_meta.ace_tag {\
+color:rgb(28, 2, 255);\
+}\
+.ace-crimson-editor .ace_marker-layer .ace_selected-word {\
+background: rgb(250, 250, 255);\
+border: 1px solid rgb(200, 200, 250);\
+}\
+.ace-crimson-editor .ace_string.ace_regex {\
+color: rgb(192, 0, 192);\
+}\
+.ace-crimson-editor .ace_indent-guide {\
+background: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAACCAYAAACZgbYnAAAAE0lEQVQImWP4////f4bLly//BwAmVgd1/w11/gAAAABJRU5ErkJggg==\") right repeat-y;\
+}";
+
+exports.cssClass = "ace-crimson-editor";
+
+var dom = require("../lib/dom");
+dom.importCssString(exports.cssText, exports.cssClass);
+});

airflow/www/templates/airflow/dag.html

@@ -12,7 +12,8 @@
   <h2>DAG: {{ dag.dag_id }}</h2>
   <ul class="nav nav-pills">
     <li><a href="{{ url_for("airflow.tree", dag_id=dag.dag_id, num_runs=25) }}">
-      <i class="icon-plus-sign"></i>Tree View
+        <i class="icon-plus-sign"></i>
+      Tree View
     </a></li>
     <li><a href="{{ url_for("airflow.graph", dag_id=dag.dag_id) }}">
       <i class="icon-random"></i>

airflow/www/templates/airflow/query.html

@@ -13,12 +13,31 @@
   <form method="get">
   {{ form.db_id }}
   <input type="submit" class="btn btn-default" value="Run!"><br>
+  <div id='ace_container'>
   {{ form.sql(rows=10) }}
+  </div>
   </form>
   {{ results|safe }}
 {% endblock %}
 {% block tail %}
   {{ super() }}
+  <script src="{{ url_for('static', filename='ace.js') }}"></script>
+  <script src="{{ url_for('static', filename='mode-sql.js') }}"></script>
+  <script src="{{ url_for('static', filename='theme-crimson_editor.js') }}"></script>
   <script>
+    $( document ).ready(function() {
+        var editor = ace.edit("sql");
+        var textarea = $('textarea[name="sql"]').hide();
+        editor.setTheme("ace/theme/crimson_editor");
+        editor.setOptions({
+            minLines: 3,
+            maxLines: Infinity,
+        });
+        editor.getSession().setMode("ace/mode/sql");
+        editor.getSession().on('change', function(){
+            textarea.val(editor.getSession().getValue());
+        });
+        editor.focus(); 
+    });
   </script>
 {% endblock %}