This commit is contained in:
Kyle Lahnakoski 2018-08-28 06:59:42 -04:00
Родитель e6ab8953ab
Коммит b70217c4fa
17 изменённых файлов: 243 добавлений и 89 удалений

4
vendor/jx_base/expressions.py поставляемый
Просмотреть файл

@ -2563,7 +2563,9 @@ class InOp(Expression):
def partial_eval(self):
value = self.value.partial_eval()
superset = self.superset.partial_eval()
if isinstance(value, Literal) and isinstance(superset, Literal):
if superset is NULL:
return FALSE
elif isinstance(value, Literal) and isinstance(superset, Literal):
return Literal(None, self())
else:
return self

11
vendor/jx_elasticsearch/es52/__init__.py поставляемый
Просмотреть файл

@ -11,15 +11,10 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from collections import Mapping
from copy import copy
import jx_base
from jx_base import container
from jx_base.container import Container
from jx_base.dimensions import Dimension
from jx_base.expressions import jx_expression
from jx_base.queries import is_variable_name
from jx_base.query import QueryOp
from jx_elasticsearch.es52.aggs import es_aggsop, is_aggsop
from jx_elasticsearch.es52.deep import is_deepop, es_deepop
@ -27,9 +22,9 @@ from jx_elasticsearch.es52.setop import is_setop, es_setop
from jx_elasticsearch.es52.util import aggregates
from jx_elasticsearch.meta import ElasticsearchMetadata, Table
from jx_python import jx
from mo_dots import Data, Null, unwrap, coalesce, split_field, literal_field, unwraplist, join_field, wrap, listwrap, FlatList, concat_field, set_default
from mo_json import scrub, value2json
from mo_json.typed_encoder import TYPE_PREFIX, EXISTS_TYPE
from mo_dots import Data, unwrap, coalesce, split_field, join_field, wrap, listwrap
from mo_json import value2json
from mo_json.typed_encoder import EXISTS_TYPE
from mo_kwargs import override
from mo_logs import Log, Except
from pyLibrary.env import elasticsearch, http

14
vendor/jx_elasticsearch/meta.py поставляемый
Просмотреть файл

@ -311,7 +311,7 @@ class ElasticsearchMetadata(Namespace):
"size": 0
})
count = result.hits.total
cardinality = 1001
cardinality = max(1001, count)
multi = 1001
elif column.es_column == "_id":
result = self.es_cluster.post("/" + es_index + "/_search", data={
@ -419,7 +419,7 @@ class ElasticsearchMetadata(Namespace):
e = Except.wrap(e)
TEST_TABLE = "testdata"
is_missing_index = any(w in e for w in ["IndexMissingException", "index_not_found_exception"])
is_test_table = any(column.es_index.startswith(t) for t in [TEST_TABLE_PREFIX, TEST_TABLE])
is_test_table = column.es_index.startswith((TEST_TABLE_PREFIX, TEST_TABLE))
if is_missing_index and is_test_table:
# WE EXPECT TEST TABLES TO DISAPPEAR
self.meta.columns.update({
@ -438,7 +438,7 @@ class ElasticsearchMetadata(Namespace):
"multi",
"partitions",
],
"where": {"eq": {"names.\\.": ".", "es_index": column.es_index, "es_column": column.es_column}}
"where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
})
Log.warning("Could not get {{col.es_index}}.{{col.es_column}} info", col=column, cause=e)
@ -487,7 +487,13 @@ class ElasticsearchMetadata(Namespace):
self._update_cardinality(column)
(DEBUG and not column.es_index.startswith(TEST_TABLE_PREFIX)) and Log.note("updated {{column.name}}", column=column)
except Exception as e:
Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
if '"status":404' in e:
self.meta.columns.update({
"clear": ".",
"where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
})
else:
Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
except Exception as e:
Log.warning("problem in cardinality monitor", cause=e)

26
vendor/jx_python/meta.py поставляемый
Просмотреть файл

@ -135,9 +135,18 @@ class ColumnList(Table, jx_base.Container):
command = wrap(command)
eq = command.where.eq
if eq.es_index:
if eq.es_column and len(eq)==2:
all_columns = self.data.get(eq.es_index, {}).values()
if len(eq) == 1:
# FASTEST
with self.locker:
columns = [
c
for cs in all_columns
for c in cs
]
elif eq.es_column and len(eq) == 2:
# FASTER
with self.locker:
all_columns = self.data.get(eq.es_index, {}).values()
columns = [
c
for cs in all_columns
@ -150,20 +159,25 @@ class ColumnList(Table, jx_base.Container):
with self.locker:
columns = [
c
for cs in self.data.get(eq.es_index, {}).values()
for cs in all_columns
for c in cs
if all(c[k] == v for k, v in eq.items())
if all(c[k] == v for k, v in eq.items()) # THIS LINE IS VERY SLOW
]
else:
columns = list(self)
columns = jx.filter(columns, command.where)
with self.locker:
self.dirty = True
for col in columns:
for k in command["clear"]:
if k == ".":
columns.remove(col)
lst = self.data[col.es_index]
cols = lst[col.names['.']]
cols.remove(col)
if len(cols) == 0:
del lst[col.names['.']]
if len(lst) == 0:
del self.data[col.es_index]
else:
col[k] = None

14
vendor/mo_collections/queue.py поставляемый
Просмотреть файл

@ -34,6 +34,9 @@ class Queue(object):
def __nonzero__(self):
return len(self.list) > 0
def __contains__(self, value):
return value in self.set
def __len__(self):
return self.list.__len__()
@ -44,7 +47,8 @@ class Queue(object):
self.list.append(value)
def push(self, value):
self.add(value)
self.set.add(value)
self.list.insert(0, value)
def extend(self, values):
for v in values:
@ -58,3 +62,11 @@ class Queue(object):
self.set.remove(output)
return output
def next(self):
if len(self.list) == 0:
return None
output = self.list.pop()
self.set.remove(output)
return output

8
vendor/mo_dots/__init__.py поставляемый
Просмотреть файл

@ -208,7 +208,7 @@ def _all_default(d, default, seen=None):
if default is None:
return
if isinstance(default, Data):
default = object.__getattribute__(default, b"_dict") # REACH IN AND GET THE dict
default = object.__getattribute__(default, SLOT) # REACH IN AND GET THE dict
# Log = _late_import()
# Log.error("strictly dict (or object) allowed: got {{type}}", type=default.__class__.__name__)
@ -421,7 +421,7 @@ def wrap(v):
if type_ is dict:
m = object.__new__(Data)
_set(m, "_dict", v)
_set(m, SLOT, v)
return m
elif type_ is none_type:
return Null
@ -489,7 +489,7 @@ def _wrap_leaves(value):
def unwrap(v):
_type = _get(v, "__class__")
if _type is Data:
d = _get(v, "_dict")
d = _get(v, SLOT)
return d
elif _type is FlatList:
return v.list
@ -569,6 +569,6 @@ def tuplewrap(value):
from mo_dots.nones import Null, NullType
from mo_dots.datas import Data
from mo_dots.datas import Data, SLOT
from mo_dots.lists import FlatList
from mo_dots.objects import DataObject

69
vendor/mo_dots/datas.py поставляемый
Просмотреть файл

@ -20,7 +20,7 @@ from mo_future import text_type, PY2, iteritems
_get = object.__getattribute__
_set = object.__setattr__
SLOT = str("_internal_dict")
DEBUG = False
@ -29,7 +29,7 @@ class Data(MutableMapping):
Please see README.md
"""
__slots__ = ["_dict"]
__slots__ = [SLOT]
def __init__(self, *args, **kwargs):
"""
@ -37,32 +37,32 @@ class Data(MutableMapping):
IS UNLIKELY TO BE USEFUL. USE wrap() INSTEAD
"""
if DEBUG:
d = _get(self, "_dict")
d = _get(self, SLOT)
for k, v in kwargs.items():
d[literal_field(k)] = unwrap(v)
else:
if args:
args0 = args[0]
if isinstance(args0, Data):
_set(self, "_dict", _get(args0, "_dict"))
_set(self, SLOT, _get(args0, SLOT))
elif isinstance(args0, dict):
_set(self, "_dict", args0)
_set(self, SLOT, args0)
else:
_set(self, "_dict", dict(args0))
_set(self, SLOT, dict(args0))
elif kwargs:
_set(self, "_dict", unwrap(kwargs))
_set(self, SLOT, unwrap(kwargs))
else:
_set(self, "_dict", {})
_set(self, SLOT, {})
def __bool__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
if isinstance(d, dict):
return bool(d)
else:
return d != None
def __nonzero__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
if isinstance(d, dict):
return True if d else False
else:
@ -75,22 +75,21 @@ class Data(MutableMapping):
return False
def __iter__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return d.__iter__()
def __getitem__(self, key):
if key == None:
return Null
if key == ".":
output = _get(self, "_dict")
output = _get(self, SLOT)
if isinstance(output, Mapping):
return self
else:
return output
key = text_type(key)
d = _get(self, "_dict")
d = _get(self, SLOT)
if key.find(".") >= 0:
seq = _split_field(key)
@ -119,11 +118,11 @@ class Data(MutableMapping):
# SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping;
# HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS unwrap()
v = unwrap(value)
_set(self, "_dict", v)
_set(self, SLOT, v)
return v
try:
d = _get(self, "_dict")
d = _get(self, SLOT)
value = unwrap(value)
if key.find(".") == -1:
if value is None:
@ -149,31 +148,31 @@ class Data(MutableMapping):
raise e
def __getattr__(self, key):
d = _get(self, "_dict")
d = _get(self, SLOT)
o = d.get(key)
if o == None:
return NullType(d, key)
return wrap(o)
def __setattr__(self, key, value):
d = _get(self, "_dict")
d = _get(self, SLOT)
value = unwrap(value)
if value is None:
d = _get(self, "_dict")
d = _get(self, SLOT)
d.pop(key, None)
else:
d[key] = value
return self
def __hash__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return hash_value(d)
def __eq__(self, other):
if self is other:
return True
d = _get(self, "_dict")
d = _get(self, SLOT)
if not isinstance(d, dict):
return d == other
@ -195,11 +194,11 @@ class Data(MutableMapping):
return not self.__eq__(other)
def get(self, key, default=None):
d = _get(self, "_dict")
d = _get(self, SLOT)
return d.get(key, default)
def items(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return [(k, wrap(v)) for k, v in d.items() if v != None or isinstance(v, Mapping)]
def leaves(self, prefix=None):
@ -210,42 +209,42 @@ class Data(MutableMapping):
def iteritems(self):
# LOW LEVEL ITERATION, NO WRAPPING
d = _get(self, "_dict")
d = _get(self, SLOT)
return ((k, wrap(v)) for k, v in iteritems(d))
def keys(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return set(d.keys())
def values(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return listwrap(list(d.values()))
def clear(self):
get_logger().error("clear() not supported")
def __len__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return dict.__len__(d)
def copy(self):
return Data(**self)
def __copy__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return Data(**d)
def __deepcopy__(self, memo):
d = _get(self, "_dict")
d = _get(self, SLOT)
return wrap(deepcopy(d, memo))
def __delitem__(self, key):
if key.find(".") == -1:
d = _get(self, "_dict")
d = _get(self, SLOT)
d.pop(key, None)
return
d = _get(self, "_dict")
d = _get(self, SLOT)
seq = _split_field(key)
for k in seq[:-1]:
d = d[k]
@ -253,7 +252,7 @@ class Data(MutableMapping):
def __delattr__(self, key):
key = text_type(key)
d = _get(self, "_dict")
d = _get(self, SLOT)
d.pop(key, None)
def setdefault(self, k, d=None):
@ -263,13 +262,13 @@ class Data(MutableMapping):
def __str__(self):
try:
return dict.__str__(_get(self, "_dict"))
return dict.__str__(_get(self, SLOT))
except Exception:
return "{}"
def __repr__(self):
try:
return "Data("+dict.__repr__(_get(self, "_dict"))+")"
return "Data("+dict.__repr__(_get(self, SLOT))+")"
except Exception as e:
return "Data()"
@ -450,7 +449,7 @@ class _DictUsingSelf(dict):
get_logger().error("clear() not supported")
def __len__(self):
d = _get(self, "_dict")
d = _get(self, SLOT)
return d.__len__()
def copy(self):

4
vendor/mo_dots/objects.py поставляемый
Просмотреть файл

@ -15,7 +15,7 @@ from collections import Mapping
from datetime import date, datetime
from decimal import Decimal
from mo_dots import wrap, unwrap, Data, FlatList, NullType, get_attr, set_attr
from mo_dots import wrap, unwrap, Data, FlatList, NullType, get_attr, set_attr, SLOT
from mo_future import text_type, binary_type, get_function_defaults, get_function_arguments, none_type, generator_types
_get = object.__getattribute__
@ -103,7 +103,7 @@ def datawrap(v):
if type_ is dict:
m = Data()
_set(m, "_dict", v) # INJECT m.__dict__=v SO THERE IS NO COPY
_set(m, SLOT, v) # INJECT m.__dict__=v SO THERE IS NO COPY
return m
elif type_ is Data:
return v

1
vendor/mo_files/url.py поставляемый
Просмотреть файл

@ -11,6 +11,7 @@ from collections import Mapping
from mo_dots import wrap, Data, coalesce, Null
from mo_future import urlparse, text_type, PY2, unichr
from mo_json import value2json
from mo_logs import Log

4
vendor/mo_json/__init__.py поставляемый
Просмотреть файл

@ -17,7 +17,7 @@ from collections import Mapping
from datetime import date, timedelta, datetime
from decimal import Decimal
from mo_dots import FlatList, NullType, Data, wrap_leaves, wrap, Null
from mo_dots import FlatList, NullType, Data, wrap_leaves, wrap, Null, SLOT
from mo_dots.objects import DataObject
from mo_future import text_type, none_type, long, binary_type, PY2
from mo_logs import Except, strings, Log
@ -158,7 +158,7 @@ def _scrub(value, is_done, stack, scrub_text, scrub_number):
elif type_ is Decimal:
return scrub_number(value)
elif type_ is Data:
return _scrub(_get(value, '_dict'), is_done, stack, scrub_text, scrub_number)
return _scrub(_get(value, SLOT), is_done, stack, scrub_text, scrub_number)
elif isinstance(value, Mapping):
_id = id(value)
if _id in is_done:

2
vendor/mo_json/encoder.py поставляемый
Просмотреть файл

@ -171,7 +171,7 @@ def _value2json(value, _buffer):
_dict2json(value, _buffer)
return
elif type is Data:
d = _get(value, "_dict") # MIGHT BE A VALUE NOT A DICT
d = _get(value, SLOT) # MIGHT BE A VALUE NOT A DICT
_value2json(d, _buffer)
return
elif type in (int, long, Decimal):

20
vendor/mo_logs/strings.py поставляемый
Просмотреть файл

@ -109,13 +109,19 @@ def unix(value):
return str(datetime2unix(value))
#
# @formatter
# def url(value):
# """
# convert FROM dict OR string TO URL PARAMETERS
# """
# return value2url_param(value)
value2url_param = None
@formatter
def url(value):
"""
convert FROM dict OR string TO URL PARAMETERS
"""
global value2url_param
if not value2url_param:
from mo_files.url import value2url_param
return value2url_param(value)
@formatter

2
vendor/mo_testing/fuzzytestcase.py поставляемый
Просмотреть файл

@ -159,7 +159,7 @@ def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, d
if not Math.is_number(expected):
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
if isinstance(expected, list) and len(expected)==0 and test == None:
if isinstance(expected, list) and len(expected) == 0 and test == None:
return
if isinstance(expected, Mapping) and not expected.keys() and test == None:
return

8
vendor/pyLibrary/env/elasticsearch.py поставляемый
Просмотреть файл

@ -986,8 +986,8 @@ class Cluster(object):
try:
response = http.put(url, **kwargs)
if response.status_code not in [200]:
Log.error(response.reason + ": " + utf82unicode(response.all_content))
self.debug and Log.note("response: {{response}}", response=utf82unicode(response.all_content)[0:300:])
Log.error(response.reason + ": " + utf82unicode(response.content))
self.debug and Log.note("response: {{response}}", response=utf82unicode(response.content)[0:300:])
details = json2value(utf82unicode(response.content))
if details.error:
@ -1035,7 +1035,7 @@ def _scrub(r):
return convert.value2number(r)
elif isinstance(r, Mapping):
if isinstance(r, Data):
r = object.__getattribute__(r, "_dict")
r = object.__getattribute__(r, SLOT)
output = {}
for k, v in r.items():
v = _scrub(v)
@ -1487,6 +1487,8 @@ def diff_schema(A, B):
output =[]
def _diff_schema(path, A, B):
for k, av in A.items():
if k == "_id" and path == ".":
continue # DO NOT ADD _id TO ANY SCHEMA DIFF
bv = B[k]
if bv == None:
output.append((concat_field(path, k), av))

31
vendor/pyLibrary/env/git.py поставляемый
Просмотреть файл

@ -8,18 +8,17 @@
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import mo_threads
from mo_logs.exceptions import suppress_exception
from pyLibrary.meta import cache
from mo_threads import Process
from pyLibrary.meta import cache
@cache
def get_git_revision():
def get_revision():
"""
GET THE CURRENT GIT REVISION
"""
@ -36,13 +35,12 @@ def get_git_revision():
with suppress_exception:
proc.join()
@cache
def get_remote_revision(url, branch):
"""
GET REVISION OF A REMOTE BRANCH
"""
mo_threads.DEBUG = True
proc = Process("git remote revision", ["git", "ls-remote", url, "refs/heads/" + branch])
try:
@ -58,5 +56,22 @@ def get_remote_revision(url, branch):
except Exception:
pass
return None
@cache
def get_branch():
"""
GET THE CURRENT GIT BRANCH
"""
proc = Process("git status", ["git", "status"])
try:
while True:
raw_line = proc.stdout.pop()
line = raw_line.decode('utf8').strip()
if line.startswith("On branch "):
return line[10:]
finally:
try:
proc.join()
except Exception:
pass

41
vendor/pyLibrary/graphs/__init__.py поставляемый
Просмотреть файл

@ -11,22 +11,29 @@ from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from collections import namedtuple
from mo_logs import Log
class Graph(object):
def __init__(self, node_type=None):
self.nodes = []
self.edges = []
self.nodes = set()
self.edges = set()
self.node_type = node_type
def verticies(self):
return self.nodes
def add_edge(self, edge):
self.edges.append(edge)
self.nodes |= {edge.parent, edge.child}
self.edges.add(edge)
def remove_children(self, node):
self.edges = [e for e in self.edges if e[0] != node]
self.edges = [e for e in self.edges if e.parent != node]
def get_children(self, node):
#FIND THE REVISION
#
# FIND THE REVISION
return [c for p, c in self.edges if p == node]
def get_parents(self, node):
@ -41,3 +48,25 @@ class Graph(object):
"""
return set([p if c == node else c for p, c in self.edges])
Edge = namedtuple("Edge", ["parent", "child"])
class Tree(Graph):
def get_parent(self, node):
output = [p for p, c in self.edges if c == node]
num = len(output)
if num == 0:
return None
elif num == 1:
return output[0]
else:
Log.error("not expected")
def get_path_to_root(self, node):
output = []
while node:
output.append(node)
node = self.get_parent(node)
return output

73
vendor/pyLibrary/graphs/algorithms.py поставляемый
Просмотреть файл

@ -11,7 +11,11 @@ from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from collections import deque
from mo_collections.queue import Queue
from mo_math import INTERSECT
from pyLibrary.graphs import Graph, Edge, Tree
from pyLibrary.graphs.paths import Step, Path
from mo_dots import Data
@ -99,3 +103,72 @@ def dominator(graph, head):
bfs(graph, find_dominator, head)
return dom.output
def dominator_tree(graph):
"""
RETURN DOMINATOR TREE
ALL NODES WITH None AS DOMINATOR ARE ROOT NODES
roots = dominator_tree(graph).get_children(None)
"""
todo = Queue()
done = set()
dominator = Tree(None)
def next_node():
nodes = list(graph.nodes)
while True:
if todo:
output = todo.pop()
elif nodes:
output = nodes.pop()
else:
return
if output not in done:
yield output
for node in next_node():
parents = graph.get_parents(node)
if not parents:
# node WITHOUT parents IS A ROOT
done.add(node)
dominator.add_edge(Edge(None, node))
continue
not_done = [p for p in parents if p not in done]
if not_done:
# THERE ARE MORE parents TO DO FIRST
more_todo = [p for p in not_done if p not in todo]
if not more_todo:
# ALL PARENTS ARE PART OF A CYCLE, MAKE node A ROOT
done.add(node)
dominator.add_edge(Edge(None, node))
else:
# DO THE PARENTS BEFORE node
todo.push(node)
for p in more_todo:
todo.push(p)
continue
# WE CAN GET THE DOMINATORS FOR ALL parents
if len(parents) == 1:
# SHORTCUT
dominator.add_edge(Edge(parents[0], node))
done.add(node)
continue
common_path = list(reversed(dominator.get_path_to_root(parents[0])))
for p in parents[1:]:
pfr = list(reversed(dominator.get_path_to_root(p)))
# FIND COMMON PATH FROM root
for i, (a, b) in enumerate(zip(common_path, pfr)):
if a != b:
common_path = common_path[:i]
break
dom = common_path[-1]
dominator.add_edge(Edge(dom, node))
done.add(node)
return dominator