fixes for last lib update

bzETL/extract_bugzilla.py

@@ -11,12 +11,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import unicode_literals
 
+from bzETL.transform_bugzilla import esfilter2sqlwhere
 from jx_python import jx
 from mo_dots.datas import Data
 from mo_logs import Log
 from mo_times.timer import Timer
 from pyLibrary import convert
-from pyLibrary.queries.jx_usingMySQL import esfilter2sqlwhere
 from pyLibrary.sql import SQL
 from pyLibrary.sql.mysql import int_list_packer
 

bzETL/transform_bugzilla.py

@@ -15,13 +15,15 @@ import re
 from datetime import date
 
 from jx_python import jx
-from mo_dots import listwrap
+from mo_dots import listwrap, wrap, coalesce
 from mo_future import text_type, long
 from mo_json import json2value, value2json
-from mo_logs import Log
+from mo_logs import Log, suppress_exception
 from mo_times import Date
 from pyLibrary import convert
 from pyLibrary.env import elasticsearch
+from pyLibrary.sql import SQL_TRUE, sql_iso, sql_list, SQL_AND, SQL_OR, SQL_NOT, SQL, SQL_IS_NULL, SQL_IS_NOT_NULL
+from pyLibrary.sql.mysql import int_list_packer
 
 USE_ATTACHMENTS_DOT = True  # REMOVE THIS, ASSUME False
 
@@ -151,3 +153,95 @@ def normalize(bug, old_school=False):
 
     return elasticsearch.scrub(bug)
 
+
+
+def esfilter2sqlwhere(db, esfilter):
+    return _esfilter2sqlwhere(db, esfilter)
+
+
+def _esfilter2sqlwhere(db, esfilter):
+    """
+    CONVERT ElassticSearch FILTER TO SQL FILTER
+    db - REQUIRED TO PROPERLY QUOTE VALUES AND COLUMN NAMES
+    """
+    esfilter = wrap(esfilter)
+
+    if esfilter is True:
+        return SQL_TRUE
+    elif esfilter["and"]:
+        return sql_iso(SQL_AND.join([esfilter2sqlwhere(db, a) for a in esfilter["and"]]))
+    elif esfilter["or"]:
+        return sql_iso(SQL_OR.join([esfilter2sqlwhere(db, a) for a in esfilter["or"]]))
+    elif esfilter["not"]:
+        return SQL_NOT + sql_iso(esfilter2sqlwhere(db, esfilter["not"]))
+    elif esfilter.term:
+        return sql_iso(SQL_AND.join([
+            db.quote_column(col) + SQL("=") + db.quote_value(val)
+            for col, val in esfilter.term.items()
+        ]))
+    elif esfilter.terms:
+        for col, v in esfilter.terms.items():
+            if len(v) == 0:
+                return "FALSE"
+
+            with suppress_exception:
+                int_list = convert.value2intlist(v)
+                has_null = False
+                for vv in v:
+                    if vv == None:
+                        has_null = True
+                        break
+                if int_list:
+                    filter = int_list_packer(col, int_list)
+                    if has_null:
+                        return esfilter2sqlwhere(db, {"or": [{"missing": col}, filter]})
+                    else:
+                        return esfilter2sqlwhere(db, filter)
+                else:
+                    if has_null:
+                        return esfilter2sqlwhere(db, {"missing": col})
+                    else:
+                        return "false"
+            return db.quote_column(col) + " in " + sql_iso(sql_list([db.quote_value(val) for val in v]))
+    elif esfilter.script:
+        return sql_iso(esfilter.script)
+    elif esfilter.range:
+        name2sign = {
+            "gt": SQL(">"),
+            "gte": SQL(">="),
+            "lte": SQL("<="),
+            "lt": SQL("<")
+        }
+
+        def single(col, r):
+            min = coalesce(r["gte"], r[">="])
+            max = coalesce(r["lte"], r["<="])
+            if min != None and max != None:
+                # SPECIAL CASE (BETWEEN)
+                sql = db.quote_column(col) + SQL(" BETWEEN ") + db.quote_value(min) + SQL_AND + db.quote_value(max)
+            else:
+                sql = SQL_AND.join(
+                    db.quote_column(col) + name2sign[sign] + db.quote_value(value)
+                    for sign, value in r.items()
+                )
+            return sql
+
+        output = sql_iso(SQL_AND.join([single(col, ranges) for col, ranges in esfilter.range.items()]))
+        return output
+    elif esfilter.missing:
+        if isinstance(esfilter.missing, text_type):
+            return sql_iso(db.quote_column(esfilter.missing) + SQL_IS_NULL)
+        else:
+            return sql_iso(db.quote_column(esfilter.missing.field) + SQL_IS_NULL)
+    elif esfilter.exists:
+        if isinstance(esfilter.exists, text_type):
+            return sql_iso(db.quote_column(esfilter.exists) + SQL_IS_NOT_NULL)
+        else:
+            return sql_iso(db.quote_column(esfilter.exists.field) + SQL_IS_NOT_NULL)
+    elif esfilter.match_all:
+        return SQL_TRUE
+    elif esfilter.instr:
+        return sql_iso(SQL_AND.join(["instr" + sql_iso(db.quote_column(col) + ", " + db.quote_value(val)) + ">0" for col, val in esfilter.instr.items()]))
+    else:
+        Log.error("Can not convert esfilter to SQL: {{esfilter}}", esfilter=esfilter)
+

tests/test_etl.py

@@ -29,7 +29,7 @@ from mo_math.randoms import Random
 from mo_threads import ThreadedQueue, Till
 from mo_times import Timer
 from pyLibrary import convert
-from pyLibrary.queries.jx_usingMySQL import esfilter2sqlwhere
+from bzETL.transform_bugzilla import esfilter2sqlwhere
 from pyLibrary.sql.mysql import all_db, MySQL
 from pyLibrary.testing import elasticsearch
 from util import database, compare_es

tests/util/compare_es.py

@@ -14,15 +14,14 @@ from __future__ import unicode_literals
 
 from datetime import datetime
 
-from mo_future import long
-from mo_logs import Log
-
 import jx_elasticsearch
-import jx_python
 from bzETL import transform_bugzilla, parse_bug_history
 from jx_python import jx
+from jx_python.containers.list_usingPythonList import ListContainer
 from mo_dots import coalesce, unwrap
+from mo_future import long
 from mo_json import json2value, value2json
+from mo_logs import Log
 from mo_math import Math
 from mo_times.timer import Timer
 from pyLibrary import convert
@@ -36,7 +35,7 @@ def get_all_bug_versions(es, bug_id, max_time=None):
     if isinstance(es, elasticsearch.Index):
         esq = jx_elasticsearch.new_instance(es.settings)
     elif isinstance(es, FakeES):
-        esq = jx_python.wrap_from(es.data.values())
+        esq = ListContainer(name="bugs", data=es.data.values())
     else:
         raise Log.error("unknown container")
 

tests/util/database.py

@@ -16,7 +16,7 @@ from mo_dots.datas import Data
 from mo_logs import Log
 from mo_times.timer import Timer
 from pyLibrary import convert
-from pyLibrary.queries.jx_usingMySQL import esfilter2sqlwhere
+from bzETL.transform_bugzilla import esfilter2sqlwhere
 from pyLibrary.sql.mysql import MySQL
 
 

vendor/jx_python/meta.py

@@ -21,7 +21,7 @@ from jx_base import STRUCT, Column, Table
 from jx_base.schema import Schema
 from jx_python import jx
 from mo_collections import UniqueIndex
-from mo_dots import Data, concat_field, get_attr, listwrap, unwraplist, NullType, FlatList, set_default, split_field, join_field, ROOT_PATH, wrap
+from mo_dots import Data, concat_field, get_attr, listwrap, unwraplist, NullType, FlatList, set_default, split_field, join_field, ROOT_PATH, wrap, coalesce
 from mo_future import none_type, text_type, long, PY2
 from mo_json.typed_encoder import untype_path, unnest_path
 from mo_logs import Log
@@ -249,7 +249,7 @@ def _get_schema_from_list(frum, table_name, parent, nested_path, columns):
                 )
                 columns.add(column)
             column.es_type = _merge_type[column.es_type][row_type]
-            column.jx_type = _merge_type[column.jx_type][row_type]
+            column.jx_type = _merge_type[coalesce(column.jx_type, "undefined")][row_type]
         else:
             for name, value in d.items():
                 full_name = concat_field(parent, name)

vendor/pyLibrary/sql/mysql.py

@@ -30,7 +30,7 @@ from mo_math import Math
 from mo_times import Date
 from pymysql import connect, InterfaceError, cursors
 
-from mo_future import text_type, utf8_json_encoder
+from mo_future import text_type, utf8_json_encoder, binary_type
 from pyLibrary.sql import SQL, SQL_NULL, SQL_SELECT, SQL_LIMIT, SQL_WHERE, SQL_LEFT_JOIN, SQL_FROM, SQL_AND, sql_list, sql_iso, SQL_ASC, SQL_TRUE, SQL_ONE, SQL_DESC, SQL_IS_NULL, sql_alias
 from pyLibrary.sql.sqlite import join_column
 

vendor/pyLibrary/sql/util.py

@@ -13,7 +13,7 @@ from __future__ import division
 from __future__ import unicode_literals
 
 from mo_dots import wrap
-from pyLibrary.queries.jx_usingMySQL import esfilter2sqlwhere
+from bzETL.transform_bugzilla import esfilter2sqlwhere
 
 
 def find_holes(db, table_name, column_name, _range, filter=None):