diff --git a/config.json b/config.json index 754be3a..798e120 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,8 @@ { + "es": { + "host": "http://localhost", + "port": 9200 + }, "tuid": { "database": { "name": "resources/tuid_app.db", @@ -16,8 +20,7 @@ "url": "https://hg.mozilla.org" }, "branches": { - "host": "http://localhost", - "port": 9201, + "$ref": "#es", "index": "repo-branches", "type": "branch", "typed": false, @@ -27,8 +30,7 @@ "limit_replicas": false }, "repo": { - "host": "http://localhost", - "port": 9201, + "$ref": "#es", "index": "repo", "type": "revision", "typed": false, @@ -38,8 +40,7 @@ "limit_replicas": false }, "moves": { - "host": "http://localhost", - "port": 9201, + "$ref": "#es", "index": "repo-moves", "type": "revision", "typed": false, @@ -51,8 +52,7 @@ }, "clogger": { "csetLog": { - "host": "http://localhost", - "port": 9201, + "$ref": "#es", "index": "tuid-csetlog", "type": "csetlogtype", "typed": false, @@ -101,7 +101,7 @@ "Referer": "https://github.com/mozilla/TUID", "User-Agent": "TUID Service" }, - "pyLibrary.sql.sqlite.DEBUG": false, + "jx_sqlite.sqlite.DEBUG": false, "mo_hg.hg_mozilla_org.DAEMON_HG_INTERVAL": 0, "mo_hg.hg_mozilla_org.WAIT_AFTER_CACHE_MISS":0, "mo_hg.hg_branches.BRANCH_WHITELIST": ["mozilla-central", "try"] diff --git a/resources/config/prod.json b/resources/config/prod.json index 8d45964..a21866a 100644 --- a/resources/config/prod.json +++ b/resources/config/prod.json @@ -49,8 +49,8 @@ "Referer": "https://github.com/mozilla/TUID", "User-Agent": "TUID Service" }, - "pyLibrary.sql.sqlite.DEBUG": false, - "pyLibrary.sql.sqlite.TOO_LONG_TO_HOLD_TRANSACTION": 30, + "jx_sqlite.sqlite.DEBUG": false, + "jx_sqlite.sqlite.TOO_LONG_TO_HOLD_TRANSACTION": 30, "mo_hg.hg_mozilla_org.DAEMON_HG_INTERVAL": 0, "mo_hg.hg_mozilla_org.WAIT_AFTER_CACHE_MISS": 0, "mo_hg.hg_branches.BRANCH_WHITELIST": ["mozilla-central", "try"] diff --git a/tests/config/stress_client.json b/tests/config/stress_client.json index 9eea3da..626064a 100644 --- a/tests/config/stress_client.json +++ b/tests/config/stress_client.json @@ -59,7 +59,7 @@ "tuid.service.DEBUG": false, "pyLibrary.env.http.DEBUG": true, "pyLibrary.env.http.default_headers": {"Referer": "https://github.com/mozilla/TUID"}, - "pyLibrary.sql.sqlite.DEBUG": false, + "jx_sqlite.sqlite.DEBUG": false, "mo_hg.hg_mozilla_org.DAEMON_HG_INTERVAL": 0 }, "debug": { diff --git a/tests/test_app.py b/tests/test_app.py index 0fb212b..305f140 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -14,13 +14,10 @@ import pytest from mo_dots import wrap, Null from mo_future import text_type, PY2 -from mo_json import json2value from mo_logs import Log -from mo_logs.strings import utf82unicode from mo_threads import Process from pyLibrary.env import http from tuid.app import EXPECTING_QUERY - from tuid.client import TuidClient app_process = None diff --git a/tests/test_basic.py b/tests/test_basic.py index 2ac367b..854eb2a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -11,14 +11,14 @@ from __future__ import unicode_literals import json import pytest +from mo_sql import sql_list from mo_dots import Null from mo_logs import Log, Except from mo_threads import Thread, Till from mo_times import Timer from pyLibrary.env import http -from pyLibrary.sql import sql_list -from pyLibrary.sql.sqlite import quote_value, DOUBLE_TRANSACTION_ERROR, quote_list +from jx_sqlite.sqlite import quote_value, DOUBLE_TRANSACTION_ERROR, quote_list from tuid.service import TUIDService from tuid.util import map_to_array, delete diff --git a/tests/test_cache.py b/tests/test_cache.py index 6f25c67..2f6c165 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -14,7 +14,7 @@ from mo_logs import Log from mo_threads import Till from tuid.service import TUIDService from mo_dots import Null -from pyLibrary.sql.sqlite import quote_value +from jx_sqlite.sqlite import quote_value from tuid.util import delete _service = None diff --git a/tests/test_memory.py b/tests/test_memory.py index 684f47a..92236a0 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -15,7 +15,7 @@ import pytest from mo_dots import Null from mo_logs import Log from mo_threads import Till -from pyLibrary.sql.sqlite import quote_list +from jx_sqlite.sqlite import quote_list from tuid.service import TUIDService _service = None diff --git a/tests/test_transaction.py b/tests/test_transaction.py index b2f91ea..1697399 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -8,10 +8,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals +from mo_sql import sql_iso + +from jx_sqlite import sqlite from mo_dots import Data from mo_threads import Signal, Thread -from pyLibrary.sql import sqlite, sql_iso -from pyLibrary.sql.sqlite import Sqlite, quote_value, DOUBLE_TRANSACTION_ERROR +from jx_sqlite.sqlite import Sqlite, quote_value, DOUBLE_TRANSACTION_ERROR sqlite.DEBUG = True diff --git a/tests/travis/config.json b/tests/travis/config.json index 3741252..585249e 100644 --- a/tests/travis/config.json +++ b/tests/travis/config.json @@ -1,7 +1,8 @@ { "tuid": { "database": { - "name": "resources/tuid_app.db" + "name": "resources/tuid_app.db", + "upgrade": false }, "hg": { "url": "https://hg.mozilla.org", @@ -111,7 +112,7 @@ "Referer": "https://github.com/mozilla/TUID", "User-Agent": "TUID Service" }, - "pyLibrary.sql.sqlite.DEBUG": false, + "jx_sqlite.sqlite.DEBUG": false, "mo_hg.hg_mozilla_org.DAEMON_HG_INTERVAL": 0, "mo_hg.hg_mozilla_org.WAIT_AFTER_CACHE_MISS": 0, "mo_hg.hg_branches.BRANCH_WHITELIST": ["mozilla-central", "try"] diff --git a/tuid/client.py b/tuid/client.py index 446c91e..d386b52 100644 --- a/tuid/client.py +++ b/tuid/client.py @@ -17,8 +17,7 @@ from mo_threads import Till from mo_times import Timer, Date from pyLibrary import aws from pyLibrary.env import http -from pyLibrary.sql import sql_list -from pyLibrary.sql.sqlite import Sqlite, quote_value, quote_list +from jx_sqlite.sqlite import Sqlite, quote_value, quote_list, sql_insert DEBUG = True SLEEP_ON_ERROR = 30 @@ -125,10 +124,17 @@ class TuidClient(object): ) with self.db.transaction() as transaction: - command = "INSERT INTO tuid (revision, file, tuids) VALUES " + sql_list( - quote_list((revision, r.path, value2json(r.tuids))) - for r in new_response.data - if r.tuids != None + command = sql_insert( + "tuid", + [ + { + "revision": revision, + "file": r.path, + "tuids": value2json(r.tuids), + } + for r in new_response.data + if r.tuids != None + ], ) if not command.endswith(" VALUES "): transaction.execute(command) diff --git a/vendor/jx_python/group_by.py b/vendor/jx_python/group_by.py index 66bf5b6..2b588e7 100644 --- a/vendor/jx_python/group_by.py +++ b/vendor/jx_python/group_by.py @@ -16,7 +16,7 @@ from jx_base.container import Container from jx_base.expressions import jx_expression from jx_base.language import is_expression from mo_dots import Data, FlatList, Null, listwrap -from mo_dots.lists import sequence_types +from mo_dots.lists import sequence_types, list_types from mo_future import binary_type, text from mo_logs import Log from mo_logs.exceptions import Except @@ -120,10 +120,10 @@ def chunk(data, size=0): if not size: return [data] - if data.__class__ in sequence_types + (bytearray, text, binary_type): + if data.__class__ in list_types + (tuple, bytearray, text, binary_type): # USE SLICING def _iter(): - num = int(math.ceil(len(data)/size)) + num = int(math.ceil(len(data) / size)) for i in range(num): output = (i, data[i * size:i * size + size:]) yield output diff --git a/vendor/mo_hg/cache.py b/vendor/mo_hg/cache.py index f95f6fe..8649c45 100644 --- a/vendor/mo_hg/cache.py +++ b/vendor/mo_hg/cache.py @@ -23,7 +23,7 @@ from mo_logs import Log from mo_threads import Lock, Queue, Signal, Thread, Till from mo_times import Date, MINUTE, SECOND from pyLibrary.env import http -from pyLibrary.sql.sqlite import Sqlite, quote_list, quote_value +from jx_sqlite.sqlite import Sqlite, quote_list, quote_value APP_NAME = "HG Cache" CONCURRENCY = 5