more fixes

README.md

@@ -133,6 +133,15 @@ set PYTHONPATH=.;vendor
 python -m unittest discover -v -s tests
 ```
 
+## Fixing tests
+
+Test runs are compared to documents found in the reference files at `tests/resources/reference`. They may need updating after changing the code.   
+
+    python -m unittest test_examples 
+
+The output file is found in `tests/results`, and can replace the reference file. Be sure to review the `git diff`; it will show the change in the reference file, just to be sure nothing went wrong.
+
+
 ## Upgrades
 
 There may be enhancements from time to time.  To get them

bugzilla_etl/parse_bug_history.py

@@ -46,8 +46,9 @@ import re
 from bugzilla_etl.alias_analysis import AliasAnalyzer
 from bugzilla_etl.extract_bugzilla import MAX_TIMESTAMP
 from bugzilla_etl.transform_bugzilla import normalize, NUMERIC_FIELDS, MULTI_FIELDS, DIFF_FIELDS, NULL_VALUES, TIME_FIELDS, LONG_FIELDS
+from jx_elasticsearch import meta_columns
 from jx_elasticsearch.meta import python_type_to_es_type
-from jx_python import jx, meta
+from jx_python import jx
 from mo_dots import inverse, coalesce, wrap, unwrap, literal_field, listwrap
 from mo_dots.datas import Data
 from mo_dots.lists import FlatList
@@ -1241,6 +1242,9 @@ class LongField(object):
         return self.value
 
 
+
 # ENSURE WE REGISTER THIS PROMISE AS A STRING
+meta_columns._merge_order[ApplyDiff] = 6
+meta_columns._merge_order[LongField] = 6
 python_type_to_es_type[ApplyDiff] = "string"
 python_type_to_es_type[LongField] = "string"

tests/resources/config/test_etl.json

@@ -98,7 +98,7 @@
 		}
 	},
 	"constants": {
-		"jx_elasticsearch.meta.DEBUG": true,
+		"jx_elasticsearch.meta.DEBUG": false,
 		"jx_elasticsearch.meta.ENABLE_META_SCAN": false,
 		"pyLibrary.sql.mysql.EXECUTE_TIMEOUT": 0,
 		"pyLibrary.env.http.default_headers": {

vendor/jx_elasticsearch/meta.py

@@ -35,7 +35,7 @@ from pyLibrary.env.elasticsearch import _get_best_type_from_mapping, es_type_to_
 
 MAX_COLUMN_METADATA_AGE = 12 * HOUR
 ENABLE_META_SCAN = True
-DEBUG = True
+DEBUG = False
 TOO_OLD = 2*HOUR
 OLD_METADATA = MINUTE
 TEST_TABLE_PREFIX = "testing"  # USED TO TURN OFF COMPLAINING ABOUT TEST INDEXES

vendor/jx_elasticsearch/meta_columns.py

@@ -40,7 +40,7 @@ from mo_math import MAX
 from mo_threads import Lock, Queue, Thread, Till
 from mo_times.dates import Date
 
-DEBUG = True
+DEBUG = False
 singlton = None
 META_INDEX_NAME = "meta.columns"
 META_TYPE_NAME = "column"
@@ -141,14 +141,18 @@ class ColumnList(Table, jx_base.Container):
                         self._add(c)
                         self.last_load = MAX((self.last_load, c.last_updated))
 
-                updates = self.todo.pop_all()
-                DEBUG and updates and Log.note(
-                    "{{num}} columns to push to db", num=len(updates)
-                )
-                self.es_index.extend(
-                    {"value": column.__dict__()}
-                    for column in updates
-                )
+                while not please_stop:
+                    updates = self.todo.pop_all()
+                    if not updates:
+                        break
+
+                    DEBUG and updates and Log.note(
+                        "{{num}} columns to push to db", num=len(updates)
+                    )
+                    self.es_index.extend(
+                        {"value": column.__dict__()}
+                        for column in updates
+                    )
             except Exception as e:
                 Log.warning("problem updating database", cause=e)
 
@@ -580,7 +584,7 @@ def mark_as_deleted(col):
     col.cardinality = 0
     col.multi = 0
     col.partitions = None
-    col.last_updated = Data.now()
+    col.last_updated = Date.now()
 
 
 SIMPLE_METADATA_COLUMNS = (  # FOR PURELY INTERNAL PYTHON LISTS, NOT MAPPING TO ANOTHER DATASTORE