diff --git a/README.md b/README.md index 51be17f..44f17b4 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,15 @@ set PYTHONPATH=.;vendor python -m unittest discover -v -s tests ``` +## Fixing tests + +Test runs are compared to documents found in the reference files at `tests/resources/reference`. They may need updating after changing the code. + + python -m unittest test_examples + +The output file is found in `tests/results`, and can replace the reference file. Be sure to review the `git diff`; it will show the change in the reference file, just to be sure nothing went wrong. + + ## Upgrades There may be enhancements from time to time. To get them diff --git a/bugzilla_etl/parse_bug_history.py b/bugzilla_etl/parse_bug_history.py index a34a725..e380eee 100644 --- a/bugzilla_etl/parse_bug_history.py +++ b/bugzilla_etl/parse_bug_history.py @@ -46,8 +46,9 @@ import re from bugzilla_etl.alias_analysis import AliasAnalyzer from bugzilla_etl.extract_bugzilla import MAX_TIMESTAMP from bugzilla_etl.transform_bugzilla import normalize, NUMERIC_FIELDS, MULTI_FIELDS, DIFF_FIELDS, NULL_VALUES, TIME_FIELDS, LONG_FIELDS +from jx_elasticsearch import meta_columns from jx_elasticsearch.meta import python_type_to_es_type -from jx_python import jx, meta +from jx_python import jx from mo_dots import inverse, coalesce, wrap, unwrap, literal_field, listwrap from mo_dots.datas import Data from mo_dots.lists import FlatList @@ -1241,6 +1242,9 @@ class LongField(object): return self.value + # ENSURE WE REGISTER THIS PROMISE AS A STRING +meta_columns._merge_order[ApplyDiff] = 6 +meta_columns._merge_order[LongField] = 6 python_type_to_es_type[ApplyDiff] = "string" python_type_to_es_type[LongField] = "string" diff --git a/tests/resources/config/test_etl.json b/tests/resources/config/test_etl.json index 38c1f96..be5eb2e 100644 --- a/tests/resources/config/test_etl.json +++ b/tests/resources/config/test_etl.json @@ -98,7 +98,7 @@ } }, "constants": { - "jx_elasticsearch.meta.DEBUG": true, + "jx_elasticsearch.meta.DEBUG": false, "jx_elasticsearch.meta.ENABLE_META_SCAN": false, "pyLibrary.sql.mysql.EXECUTE_TIMEOUT": 0, "pyLibrary.env.http.default_headers": { diff --git a/vendor/jx_elasticsearch/meta.py b/vendor/jx_elasticsearch/meta.py index 601a2f4..92ba816 100644 --- a/vendor/jx_elasticsearch/meta.py +++ b/vendor/jx_elasticsearch/meta.py @@ -35,7 +35,7 @@ from pyLibrary.env.elasticsearch import _get_best_type_from_mapping, es_type_to_ MAX_COLUMN_METADATA_AGE = 12 * HOUR ENABLE_META_SCAN = True -DEBUG = True +DEBUG = False TOO_OLD = 2*HOUR OLD_METADATA = MINUTE TEST_TABLE_PREFIX = "testing" # USED TO TURN OFF COMPLAINING ABOUT TEST INDEXES diff --git a/vendor/jx_elasticsearch/meta_columns.py b/vendor/jx_elasticsearch/meta_columns.py index 2e08725..1a799fe 100644 --- a/vendor/jx_elasticsearch/meta_columns.py +++ b/vendor/jx_elasticsearch/meta_columns.py @@ -40,7 +40,7 @@ from mo_math import MAX from mo_threads import Lock, Queue, Thread, Till from mo_times.dates import Date -DEBUG = True +DEBUG = False singlton = None META_INDEX_NAME = "meta.columns" META_TYPE_NAME = "column" @@ -141,14 +141,18 @@ class ColumnList(Table, jx_base.Container): self._add(c) self.last_load = MAX((self.last_load, c.last_updated)) - updates = self.todo.pop_all() - DEBUG and updates and Log.note( - "{{num}} columns to push to db", num=len(updates) - ) - self.es_index.extend( - {"value": column.__dict__()} - for column in updates - ) + while not please_stop: + updates = self.todo.pop_all() + if not updates: + break + + DEBUG and updates and Log.note( + "{{num}} columns to push to db", num=len(updates) + ) + self.es_index.extend( + {"value": column.__dict__()} + for column in updates + ) except Exception as e: Log.warning("problem updating database", cause=e) @@ -580,7 +584,7 @@ def mark_as_deleted(col): col.cardinality = 0 col.multi = 0 col.partitions = None - col.last_updated = Data.now() + col.last_updated = Date.now() SIMPLE_METADATA_COLUMNS = ( # FOR PURELY INTERNAL PYTHON LISTS, NOT MAPPING TO ANOTHER DATASTORE