lib updates
This commit is contained in:
Родитель
e6ab8953ab
Коммит
b70217c4fa
|
@ -2563,7 +2563,9 @@ class InOp(Expression):
|
|||
def partial_eval(self):
|
||||
value = self.value.partial_eval()
|
||||
superset = self.superset.partial_eval()
|
||||
if isinstance(value, Literal) and isinstance(superset, Literal):
|
||||
if superset is NULL:
|
||||
return FALSE
|
||||
elif isinstance(value, Literal) and isinstance(superset, Literal):
|
||||
return Literal(None, self())
|
||||
else:
|
||||
return self
|
||||
|
|
|
@ -11,15 +11,10 @@ from __future__ import absolute_import
|
|||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from collections import Mapping
|
||||
from copy import copy
|
||||
|
||||
import jx_base
|
||||
from jx_base import container
|
||||
from jx_base.container import Container
|
||||
from jx_base.dimensions import Dimension
|
||||
from jx_base.expressions import jx_expression
|
||||
from jx_base.queries import is_variable_name
|
||||
from jx_base.query import QueryOp
|
||||
from jx_elasticsearch.es52.aggs import es_aggsop, is_aggsop
|
||||
from jx_elasticsearch.es52.deep import is_deepop, es_deepop
|
||||
|
@ -27,9 +22,9 @@ from jx_elasticsearch.es52.setop import is_setop, es_setop
|
|||
from jx_elasticsearch.es52.util import aggregates
|
||||
from jx_elasticsearch.meta import ElasticsearchMetadata, Table
|
||||
from jx_python import jx
|
||||
from mo_dots import Data, Null, unwrap, coalesce, split_field, literal_field, unwraplist, join_field, wrap, listwrap, FlatList, concat_field, set_default
|
||||
from mo_json import scrub, value2json
|
||||
from mo_json.typed_encoder import TYPE_PREFIX, EXISTS_TYPE
|
||||
from mo_dots import Data, unwrap, coalesce, split_field, join_field, wrap, listwrap
|
||||
from mo_json import value2json
|
||||
from mo_json.typed_encoder import EXISTS_TYPE
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log, Except
|
||||
from pyLibrary.env import elasticsearch, http
|
||||
|
|
|
@ -311,7 +311,7 @@ class ElasticsearchMetadata(Namespace):
|
|||
"size": 0
|
||||
})
|
||||
count = result.hits.total
|
||||
cardinality = 1001
|
||||
cardinality = max(1001, count)
|
||||
multi = 1001
|
||||
elif column.es_column == "_id":
|
||||
result = self.es_cluster.post("/" + es_index + "/_search", data={
|
||||
|
@ -419,7 +419,7 @@ class ElasticsearchMetadata(Namespace):
|
|||
e = Except.wrap(e)
|
||||
TEST_TABLE = "testdata"
|
||||
is_missing_index = any(w in e for w in ["IndexMissingException", "index_not_found_exception"])
|
||||
is_test_table = any(column.es_index.startswith(t) for t in [TEST_TABLE_PREFIX, TEST_TABLE])
|
||||
is_test_table = column.es_index.startswith((TEST_TABLE_PREFIX, TEST_TABLE))
|
||||
if is_missing_index and is_test_table:
|
||||
# WE EXPECT TEST TABLES TO DISAPPEAR
|
||||
self.meta.columns.update({
|
||||
|
@ -438,7 +438,7 @@ class ElasticsearchMetadata(Namespace):
|
|||
"multi",
|
||||
"partitions",
|
||||
],
|
||||
"where": {"eq": {"names.\\.": ".", "es_index": column.es_index, "es_column": column.es_column}}
|
||||
"where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
|
||||
})
|
||||
Log.warning("Could not get {{col.es_index}}.{{col.es_column}} info", col=column, cause=e)
|
||||
|
||||
|
@ -487,7 +487,13 @@ class ElasticsearchMetadata(Namespace):
|
|||
self._update_cardinality(column)
|
||||
(DEBUG and not column.es_index.startswith(TEST_TABLE_PREFIX)) and Log.note("updated {{column.name}}", column=column)
|
||||
except Exception as e:
|
||||
Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
|
||||
if '"status":404' in e:
|
||||
self.meta.columns.update({
|
||||
"clear": ".",
|
||||
"where": {"eq": {"es_index": column.es_index, "es_column": column.es_column}}
|
||||
})
|
||||
else:
|
||||
Log.warning("problem getting cardinality for {{column.name}}", column=column, cause=e)
|
||||
except Exception as e:
|
||||
Log.warning("problem in cardinality monitor", cause=e)
|
||||
|
||||
|
|
|
@ -135,9 +135,18 @@ class ColumnList(Table, jx_base.Container):
|
|||
command = wrap(command)
|
||||
eq = command.where.eq
|
||||
if eq.es_index:
|
||||
if eq.es_column and len(eq)==2:
|
||||
all_columns = self.data.get(eq.es_index, {}).values()
|
||||
if len(eq) == 1:
|
||||
# FASTEST
|
||||
with self.locker:
|
||||
columns = [
|
||||
c
|
||||
for cs in all_columns
|
||||
for c in cs
|
||||
]
|
||||
elif eq.es_column and len(eq) == 2:
|
||||
# FASTER
|
||||
with self.locker:
|
||||
all_columns = self.data.get(eq.es_index, {}).values()
|
||||
columns = [
|
||||
c
|
||||
for cs in all_columns
|
||||
|
@ -150,20 +159,25 @@ class ColumnList(Table, jx_base.Container):
|
|||
with self.locker:
|
||||
columns = [
|
||||
c
|
||||
for cs in self.data.get(eq.es_index, {}).values()
|
||||
for cs in all_columns
|
||||
for c in cs
|
||||
if all(c[k] == v for k, v in eq.items())
|
||||
if all(c[k] == v for k, v in eq.items()) # THIS LINE IS VERY SLOW
|
||||
]
|
||||
else:
|
||||
columns = list(self)
|
||||
columns = jx.filter(columns, command.where)
|
||||
|
||||
with self.locker:
|
||||
self.dirty = True
|
||||
for col in columns:
|
||||
for k in command["clear"]:
|
||||
if k == ".":
|
||||
columns.remove(col)
|
||||
lst = self.data[col.es_index]
|
||||
cols = lst[col.names['.']]
|
||||
cols.remove(col)
|
||||
if len(cols) == 0:
|
||||
del lst[col.names['.']]
|
||||
if len(lst) == 0:
|
||||
del self.data[col.es_index]
|
||||
else:
|
||||
col[k] = None
|
||||
|
||||
|
|
|
@ -34,6 +34,9 @@ class Queue(object):
|
|||
def __nonzero__(self):
|
||||
return len(self.list) > 0
|
||||
|
||||
def __contains__(self, value):
|
||||
return value in self.set
|
||||
|
||||
def __len__(self):
|
||||
return self.list.__len__()
|
||||
|
||||
|
@ -44,7 +47,8 @@ class Queue(object):
|
|||
self.list.append(value)
|
||||
|
||||
def push(self, value):
|
||||
self.add(value)
|
||||
self.set.add(value)
|
||||
self.list.insert(0, value)
|
||||
|
||||
def extend(self, values):
|
||||
for v in values:
|
||||
|
@ -58,3 +62,11 @@ class Queue(object):
|
|||
self.set.remove(output)
|
||||
return output
|
||||
|
||||
def next(self):
|
||||
if len(self.list) == 0:
|
||||
return None
|
||||
|
||||
output = self.list.pop()
|
||||
self.set.remove(output)
|
||||
return output
|
||||
|
||||
|
|
|
@ -208,7 +208,7 @@ def _all_default(d, default, seen=None):
|
|||
if default is None:
|
||||
return
|
||||
if isinstance(default, Data):
|
||||
default = object.__getattribute__(default, b"_dict") # REACH IN AND GET THE dict
|
||||
default = object.__getattribute__(default, SLOT) # REACH IN AND GET THE dict
|
||||
# Log = _late_import()
|
||||
# Log.error("strictly dict (or object) allowed: got {{type}}", type=default.__class__.__name__)
|
||||
|
||||
|
@ -421,7 +421,7 @@ def wrap(v):
|
|||
|
||||
if type_ is dict:
|
||||
m = object.__new__(Data)
|
||||
_set(m, "_dict", v)
|
||||
_set(m, SLOT, v)
|
||||
return m
|
||||
elif type_ is none_type:
|
||||
return Null
|
||||
|
@ -489,7 +489,7 @@ def _wrap_leaves(value):
|
|||
def unwrap(v):
|
||||
_type = _get(v, "__class__")
|
||||
if _type is Data:
|
||||
d = _get(v, "_dict")
|
||||
d = _get(v, SLOT)
|
||||
return d
|
||||
elif _type is FlatList:
|
||||
return v.list
|
||||
|
@ -569,6 +569,6 @@ def tuplewrap(value):
|
|||
|
||||
|
||||
from mo_dots.nones import Null, NullType
|
||||
from mo_dots.datas import Data
|
||||
from mo_dots.datas import Data, SLOT
|
||||
from mo_dots.lists import FlatList
|
||||
from mo_dots.objects import DataObject
|
||||
|
|
|
@ -20,7 +20,7 @@ from mo_future import text_type, PY2, iteritems
|
|||
_get = object.__getattribute__
|
||||
_set = object.__setattr__
|
||||
|
||||
|
||||
SLOT = str("_internal_dict")
|
||||
DEBUG = False
|
||||
|
||||
|
||||
|
@ -29,7 +29,7 @@ class Data(MutableMapping):
|
|||
Please see README.md
|
||||
"""
|
||||
|
||||
__slots__ = ["_dict"]
|
||||
__slots__ = [SLOT]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
|
@ -37,32 +37,32 @@ class Data(MutableMapping):
|
|||
IS UNLIKELY TO BE USEFUL. USE wrap() INSTEAD
|
||||
"""
|
||||
if DEBUG:
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
for k, v in kwargs.items():
|
||||
d[literal_field(k)] = unwrap(v)
|
||||
else:
|
||||
if args:
|
||||
args0 = args[0]
|
||||
if isinstance(args0, Data):
|
||||
_set(self, "_dict", _get(args0, "_dict"))
|
||||
_set(self, SLOT, _get(args0, SLOT))
|
||||
elif isinstance(args0, dict):
|
||||
_set(self, "_dict", args0)
|
||||
_set(self, SLOT, args0)
|
||||
else:
|
||||
_set(self, "_dict", dict(args0))
|
||||
_set(self, SLOT, dict(args0))
|
||||
elif kwargs:
|
||||
_set(self, "_dict", unwrap(kwargs))
|
||||
_set(self, SLOT, unwrap(kwargs))
|
||||
else:
|
||||
_set(self, "_dict", {})
|
||||
_set(self, SLOT, {})
|
||||
|
||||
def __bool__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
if isinstance(d, dict):
|
||||
return bool(d)
|
||||
else:
|
||||
return d != None
|
||||
|
||||
def __nonzero__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
if isinstance(d, dict):
|
||||
return True if d else False
|
||||
else:
|
||||
|
@ -75,22 +75,21 @@ class Data(MutableMapping):
|
|||
return False
|
||||
|
||||
def __iter__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return d.__iter__()
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key == None:
|
||||
return Null
|
||||
if key == ".":
|
||||
output = _get(self, "_dict")
|
||||
output = _get(self, SLOT)
|
||||
if isinstance(output, Mapping):
|
||||
return self
|
||||
else:
|
||||
return output
|
||||
|
||||
key = text_type(key)
|
||||
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
|
||||
if key.find(".") >= 0:
|
||||
seq = _split_field(key)
|
||||
|
@ -119,11 +118,11 @@ class Data(MutableMapping):
|
|||
# SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping;
|
||||
# HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS unwrap()
|
||||
v = unwrap(value)
|
||||
_set(self, "_dict", v)
|
||||
_set(self, SLOT, v)
|
||||
return v
|
||||
|
||||
try:
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
value = unwrap(value)
|
||||
if key.find(".") == -1:
|
||||
if value is None:
|
||||
|
@ -149,31 +148,31 @@ class Data(MutableMapping):
|
|||
raise e
|
||||
|
||||
def __getattr__(self, key):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
o = d.get(key)
|
||||
if o == None:
|
||||
return NullType(d, key)
|
||||
return wrap(o)
|
||||
|
||||
def __setattr__(self, key, value):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
value = unwrap(value)
|
||||
if value is None:
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
d.pop(key, None)
|
||||
else:
|
||||
d[key] = value
|
||||
return self
|
||||
|
||||
def __hash__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return hash_value(d)
|
||||
|
||||
def __eq__(self, other):
|
||||
if self is other:
|
||||
return True
|
||||
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
if not isinstance(d, dict):
|
||||
return d == other
|
||||
|
||||
|
@ -195,11 +194,11 @@ class Data(MutableMapping):
|
|||
return not self.__eq__(other)
|
||||
|
||||
def get(self, key, default=None):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return d.get(key, default)
|
||||
|
||||
def items(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return [(k, wrap(v)) for k, v in d.items() if v != None or isinstance(v, Mapping)]
|
||||
|
||||
def leaves(self, prefix=None):
|
||||
|
@ -210,42 +209,42 @@ class Data(MutableMapping):
|
|||
|
||||
def iteritems(self):
|
||||
# LOW LEVEL ITERATION, NO WRAPPING
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return ((k, wrap(v)) for k, v in iteritems(d))
|
||||
|
||||
def keys(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return set(d.keys())
|
||||
|
||||
def values(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return listwrap(list(d.values()))
|
||||
|
||||
def clear(self):
|
||||
get_logger().error("clear() not supported")
|
||||
|
||||
def __len__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return dict.__len__(d)
|
||||
|
||||
def copy(self):
|
||||
return Data(**self)
|
||||
|
||||
def __copy__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return Data(**d)
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return wrap(deepcopy(d, memo))
|
||||
|
||||
def __delitem__(self, key):
|
||||
if key.find(".") == -1:
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
d.pop(key, None)
|
||||
return
|
||||
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
seq = _split_field(key)
|
||||
for k in seq[:-1]:
|
||||
d = d[k]
|
||||
|
@ -253,7 +252,7 @@ class Data(MutableMapping):
|
|||
|
||||
def __delattr__(self, key):
|
||||
key = text_type(key)
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
d.pop(key, None)
|
||||
|
||||
def setdefault(self, k, d=None):
|
||||
|
@ -263,13 +262,13 @@ class Data(MutableMapping):
|
|||
|
||||
def __str__(self):
|
||||
try:
|
||||
return dict.__str__(_get(self, "_dict"))
|
||||
return dict.__str__(_get(self, SLOT))
|
||||
except Exception:
|
||||
return "{}"
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
return "Data("+dict.__repr__(_get(self, "_dict"))+")"
|
||||
return "Data("+dict.__repr__(_get(self, SLOT))+")"
|
||||
except Exception as e:
|
||||
return "Data()"
|
||||
|
||||
|
@ -450,7 +449,7 @@ class _DictUsingSelf(dict):
|
|||
get_logger().error("clear() not supported")
|
||||
|
||||
def __len__(self):
|
||||
d = _get(self, "_dict")
|
||||
d = _get(self, SLOT)
|
||||
return d.__len__()
|
||||
|
||||
def copy(self):
|
||||
|
|
|
@ -15,7 +15,7 @@ from collections import Mapping
|
|||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from mo_dots import wrap, unwrap, Data, FlatList, NullType, get_attr, set_attr
|
||||
from mo_dots import wrap, unwrap, Data, FlatList, NullType, get_attr, set_attr, SLOT
|
||||
from mo_future import text_type, binary_type, get_function_defaults, get_function_arguments, none_type, generator_types
|
||||
|
||||
_get = object.__getattribute__
|
||||
|
@ -103,7 +103,7 @@ def datawrap(v):
|
|||
|
||||
if type_ is dict:
|
||||
m = Data()
|
||||
_set(m, "_dict", v) # INJECT m.__dict__=v SO THERE IS NO COPY
|
||||
_set(m, SLOT, v) # INJECT m.__dict__=v SO THERE IS NO COPY
|
||||
return m
|
||||
elif type_ is Data:
|
||||
return v
|
||||
|
|
|
@ -11,6 +11,7 @@ from collections import Mapping
|
|||
|
||||
from mo_dots import wrap, Data, coalesce, Null
|
||||
from mo_future import urlparse, text_type, PY2, unichr
|
||||
from mo_json import value2json
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
|
|
|
@ -17,7 +17,7 @@ from collections import Mapping
|
|||
from datetime import date, timedelta, datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from mo_dots import FlatList, NullType, Data, wrap_leaves, wrap, Null
|
||||
from mo_dots import FlatList, NullType, Data, wrap_leaves, wrap, Null, SLOT
|
||||
from mo_dots.objects import DataObject
|
||||
from mo_future import text_type, none_type, long, binary_type, PY2
|
||||
from mo_logs import Except, strings, Log
|
||||
|
@ -158,7 +158,7 @@ def _scrub(value, is_done, stack, scrub_text, scrub_number):
|
|||
elif type_ is Decimal:
|
||||
return scrub_number(value)
|
||||
elif type_ is Data:
|
||||
return _scrub(_get(value, '_dict'), is_done, stack, scrub_text, scrub_number)
|
||||
return _scrub(_get(value, SLOT), is_done, stack, scrub_text, scrub_number)
|
||||
elif isinstance(value, Mapping):
|
||||
_id = id(value)
|
||||
if _id in is_done:
|
||||
|
|
|
@ -171,7 +171,7 @@ def _value2json(value, _buffer):
|
|||
_dict2json(value, _buffer)
|
||||
return
|
||||
elif type is Data:
|
||||
d = _get(value, "_dict") # MIGHT BE A VALUE NOT A DICT
|
||||
d = _get(value, SLOT) # MIGHT BE A VALUE NOT A DICT
|
||||
_value2json(d, _buffer)
|
||||
return
|
||||
elif type in (int, long, Decimal):
|
||||
|
|
|
@ -109,13 +109,19 @@ def unix(value):
|
|||
|
||||
return str(datetime2unix(value))
|
||||
|
||||
#
|
||||
# @formatter
|
||||
# def url(value):
|
||||
# """
|
||||
# convert FROM dict OR string TO URL PARAMETERS
|
||||
# """
|
||||
# return value2url_param(value)
|
||||
|
||||
value2url_param = None
|
||||
|
||||
|
||||
@formatter
|
||||
def url(value):
|
||||
"""
|
||||
convert FROM dict OR string TO URL PARAMETERS
|
||||
"""
|
||||
global value2url_param
|
||||
if not value2url_param:
|
||||
from mo_files.url import value2url_param
|
||||
return value2url_param(value)
|
||||
|
||||
|
||||
@formatter
|
||||
|
|
|
@ -159,7 +159,7 @@ def assertAlmostEqualValue(test, expected, digits=None, places=None, msg=None, d
|
|||
|
||||
if not Math.is_number(expected):
|
||||
# SOME SPECIAL CASES, EXPECTING EMPTY CONTAINERS IS THE SAME AS EXPECTING NULL
|
||||
if isinstance(expected, list) and len(expected)==0 and test == None:
|
||||
if isinstance(expected, list) and len(expected) == 0 and test == None:
|
||||
return
|
||||
if isinstance(expected, Mapping) and not expected.keys() and test == None:
|
||||
return
|
||||
|
|
|
@ -986,8 +986,8 @@ class Cluster(object):
|
|||
try:
|
||||
response = http.put(url, **kwargs)
|
||||
if response.status_code not in [200]:
|
||||
Log.error(response.reason + ": " + utf82unicode(response.all_content))
|
||||
self.debug and Log.note("response: {{response}}", response=utf82unicode(response.all_content)[0:300:])
|
||||
Log.error(response.reason + ": " + utf82unicode(response.content))
|
||||
self.debug and Log.note("response: {{response}}", response=utf82unicode(response.content)[0:300:])
|
||||
|
||||
details = json2value(utf82unicode(response.content))
|
||||
if details.error:
|
||||
|
@ -1035,7 +1035,7 @@ def _scrub(r):
|
|||
return convert.value2number(r)
|
||||
elif isinstance(r, Mapping):
|
||||
if isinstance(r, Data):
|
||||
r = object.__getattribute__(r, "_dict")
|
||||
r = object.__getattribute__(r, SLOT)
|
||||
output = {}
|
||||
for k, v in r.items():
|
||||
v = _scrub(v)
|
||||
|
@ -1487,6 +1487,8 @@ def diff_schema(A, B):
|
|||
output =[]
|
||||
def _diff_schema(path, A, B):
|
||||
for k, av in A.items():
|
||||
if k == "_id" and path == ".":
|
||||
continue # DO NOT ADD _id TO ANY SCHEMA DIFF
|
||||
bv = B[k]
|
||||
if bv == None:
|
||||
output.append((concat_field(path, k), av))
|
||||
|
|
|
@ -8,18 +8,17 @@
|
|||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import mo_threads
|
||||
from mo_logs.exceptions import suppress_exception
|
||||
from pyLibrary.meta import cache
|
||||
from mo_threads import Process
|
||||
from pyLibrary.meta import cache
|
||||
|
||||
|
||||
@cache
|
||||
def get_git_revision():
|
||||
def get_revision():
|
||||
"""
|
||||
GET THE CURRENT GIT REVISION
|
||||
"""
|
||||
|
@ -36,13 +35,12 @@ def get_git_revision():
|
|||
with suppress_exception:
|
||||
proc.join()
|
||||
|
||||
|
||||
@cache
|
||||
def get_remote_revision(url, branch):
|
||||
"""
|
||||
GET REVISION OF A REMOTE BRANCH
|
||||
"""
|
||||
|
||||
mo_threads.DEBUG = True
|
||||
proc = Process("git remote revision", ["git", "ls-remote", url, "refs/heads/" + branch])
|
||||
|
||||
try:
|
||||
|
@ -58,5 +56,22 @@ def get_remote_revision(url, branch):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
@cache
|
||||
def get_branch():
|
||||
"""
|
||||
GET THE CURRENT GIT BRANCH
|
||||
"""
|
||||
proc = Process("git status", ["git", "status"])
|
||||
|
||||
try:
|
||||
while True:
|
||||
raw_line = proc.stdout.pop()
|
||||
line = raw_line.decode('utf8').strip()
|
||||
if line.startswith("On branch "):
|
||||
return line[10:]
|
||||
finally:
|
||||
try:
|
||||
proc.join()
|
||||
except Exception:
|
||||
pass
|
||||
|
|
|
@ -11,22 +11,29 @@ from __future__ import unicode_literals
|
|||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class Graph(object):
|
||||
def __init__(self, node_type=None):
|
||||
self.nodes = []
|
||||
self.edges = []
|
||||
self.nodes = set()
|
||||
self.edges = set()
|
||||
self.node_type = node_type
|
||||
|
||||
def verticies(self):
|
||||
return self.nodes
|
||||
|
||||
def add_edge(self, edge):
|
||||
self.edges.append(edge)
|
||||
self.nodes |= {edge.parent, edge.child}
|
||||
self.edges.add(edge)
|
||||
|
||||
def remove_children(self, node):
|
||||
self.edges = [e for e in self.edges if e[0] != node]
|
||||
self.edges = [e for e in self.edges if e.parent != node]
|
||||
|
||||
def get_children(self, node):
|
||||
#FIND THE REVISION
|
||||
#
|
||||
# FIND THE REVISION
|
||||
return [c for p, c in self.edges if p == node]
|
||||
|
||||
def get_parents(self, node):
|
||||
|
@ -41,3 +48,25 @@ class Graph(object):
|
|||
"""
|
||||
return set([p if c == node else c for p, c in self.edges])
|
||||
|
||||
|
||||
Edge = namedtuple("Edge", ["parent", "child"])
|
||||
|
||||
|
||||
class Tree(Graph):
|
||||
|
||||
def get_parent(self, node):
|
||||
output = [p for p, c in self.edges if c == node]
|
||||
num = len(output)
|
||||
if num == 0:
|
||||
return None
|
||||
elif num == 1:
|
||||
return output[0]
|
||||
else:
|
||||
Log.error("not expected")
|
||||
|
||||
def get_path_to_root(self, node):
|
||||
output = []
|
||||
while node:
|
||||
output.append(node)
|
||||
node = self.get_parent(node)
|
||||
return output
|
||||
|
|
|
@ -11,7 +11,11 @@ from __future__ import unicode_literals
|
|||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
from collections import deque
|
||||
|
||||
from mo_collections.queue import Queue
|
||||
|
||||
from mo_math import INTERSECT
|
||||
from pyLibrary.graphs import Graph, Edge, Tree
|
||||
from pyLibrary.graphs.paths import Step, Path
|
||||
from mo_dots import Data
|
||||
|
||||
|
@ -99,3 +103,72 @@ def dominator(graph, head):
|
|||
bfs(graph, find_dominator, head)
|
||||
|
||||
return dom.output
|
||||
|
||||
|
||||
def dominator_tree(graph):
|
||||
"""
|
||||
RETURN DOMINATOR TREE
|
||||
ALL NODES WITH None AS DOMINATOR ARE ROOT NODES
|
||||
roots = dominator_tree(graph).get_children(None)
|
||||
"""
|
||||
todo = Queue()
|
||||
done = set()
|
||||
dominator = Tree(None)
|
||||
|
||||
def next_node():
|
||||
nodes = list(graph.nodes)
|
||||
|
||||
while True:
|
||||
if todo:
|
||||
output = todo.pop()
|
||||
elif nodes:
|
||||
output = nodes.pop()
|
||||
else:
|
||||
return
|
||||
|
||||
if output not in done:
|
||||
yield output
|
||||
|
||||
for node in next_node():
|
||||
parents = graph.get_parents(node)
|
||||
if not parents:
|
||||
# node WITHOUT parents IS A ROOT
|
||||
done.add(node)
|
||||
dominator.add_edge(Edge(None, node))
|
||||
continue
|
||||
|
||||
not_done = [p for p in parents if p not in done]
|
||||
if not_done:
|
||||
# THERE ARE MORE parents TO DO FIRST
|
||||
more_todo = [p for p in not_done if p not in todo]
|
||||
if not more_todo:
|
||||
# ALL PARENTS ARE PART OF A CYCLE, MAKE node A ROOT
|
||||
done.add(node)
|
||||
dominator.add_edge(Edge(None, node))
|
||||
else:
|
||||
# DO THE PARENTS BEFORE node
|
||||
todo.push(node)
|
||||
for p in more_todo:
|
||||
todo.push(p)
|
||||
continue
|
||||
|
||||
# WE CAN GET THE DOMINATORS FOR ALL parents
|
||||
if len(parents) == 1:
|
||||
# SHORTCUT
|
||||
dominator.add_edge(Edge(parents[0], node))
|
||||
done.add(node)
|
||||
continue
|
||||
|
||||
common_path = list(reversed(dominator.get_path_to_root(parents[0])))
|
||||
for p in parents[1:]:
|
||||
pfr = list(reversed(dominator.get_path_to_root(p)))
|
||||
# FIND COMMON PATH FROM root
|
||||
for i, (a, b) in enumerate(zip(common_path, pfr)):
|
||||
if a != b:
|
||||
common_path = common_path[:i]
|
||||
break
|
||||
dom = common_path[-1]
|
||||
dominator.add_edge(Edge(dom, node))
|
||||
done.add(node)
|
||||
|
||||
return dominator
|
||||
|
|
Загрузка…
Ссылка в новой задаче