mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'master' of ssh://github.com/mozilla/treeherder-service into oauth-implementation

This commit is contained in:
Jonathan Eads 2014-01-28 14:33:54 -08:00
Родитель 8fa7524d0c 0c979369d3
Коммит ab83b849b3
14 изменённых файлов: 463 добавлений и 264 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
.gitignore поставляемый
Просмотреть файл

@ -27,6 +27,7 @@ pip-log.txt
test.log
treeherder*.log
treeherder.log.*
LOGFILE
# Unit test / coverage reports
.coverage
@ -60,5 +61,8 @@ treeherder/etl/data/
#docs build directory
docs/_build/
#supervisor
.nfs*
*.pid
*.c

8
deployment/supervisord/treeherder.conf
Просмотреть файл

@ -1,3 +1,11 @@
[supervisord]
http_port=/var/tmp/supervisor.sock ; (default is to run a UNIX domain socket server)
loglevel=info ; (logging level;default info; others: debug,warn)
nodaemon=true ; (start in foreground if true;default false)
[supervisorctl]
serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL for a unix socket
[program:gunicorn]
command=/home/vagrant/treeherder-service/bin/run_gunicorn
user=vagrant

16
docs/installation.rst
Просмотреть файл

@ -31,6 +31,18 @@ Installation
(venv)vagrant@precise32:~/treeherder-service$ ./runtests.sh
* Initialize the master database
.. code-block:: bash
(venv)vagrant@precise32:~/treeherder-service$ python manage.py init_master_db
* Populate the database with repository data sources
.. code-block:: bash
(venv)vagrant@precise32:~/treeherder-service$ python manage.py init_datasources
* And an entry to your host machine /etc/hosts so that you can point your browser to local.treeherder.mozilla.org to reach it
.. code-block:: bash
@ -71,10 +83,6 @@ Installation
* If you want to use supervisord to take care of all the services,
.. _project repo: https://github.com/mozilla/treeherder-service
.. _Vagrant: http://downloads.vagrantup.com
.. _Virtualbox: https://www.virtualbox.org

3
puppet/manifests/classes/treeherder.pp
Просмотреть файл

@ -20,7 +20,8 @@ class treeherder {
file { [
"/var/log/gunicorn",
"/var/log/celery"
"/var/log/celery",
"/var/log/socketio"
]:
ensure => "directory",
owner => "${APP_USER}",

40
tests/etl/test_buildbot.py
Просмотреть файл

@ -417,8 +417,29 @@ buildernames = [
'os': 'linux',
'os_platform': 'linux64',
'vm': False}}),
(
'Linux x86-64 mozilla-inbound leak test spidermonkey_tier_1-rootanalysis '
('Linux x86-64 mozilla-inbound leak test non-unified',
{'build_type': 'debug',
'job_type': 'build',
'name': {'group_name': 'unknown',
'group_symbol': '?',
'name': 'Non-Unified Build',
'job_symbol': 'Bn'},
'platform': {'arch': 'x86_64',
'os': 'linux',
'os_platform': 'linux64',
'vm': False}}),
('Linux x86-64 mozilla-inbound non-unified',
{'build_type': 'opt',
'job_type': 'build',
'name': {'group_name': 'unknown',
'group_symbol': '?',
'name': 'Non-Unified Build',
'job_symbol': 'Bn'},
'platform': {'arch': 'x86_64',
'os': 'linux',
'os_platform': 'linux64',
'vm': False}}),
('Linux x86-64 mozilla-inbound leak test spidermonkey_tier_1-rootanalysis '
'build',
{'build_type': 'debug',
'job_type': 'build',
@ -698,8 +719,8 @@ buildernames = [
('b2g_mozilla-b2g26_v1_2_hamachi_nightly',
{'build_type': 'opt',
'job_type': 'build',
'name': {'group_name': 'Hamachi Device Image',
'group_symbol': 'Hamachi',
'name': {'group_name': 'Buri/Hamachi Device Image',
'group_symbol': 'Buri/Hamachi',
'name': 'Hamachi Device Image Nightly',
'job_symbol': 'N'},
'platform': {'arch': 'x86',
@ -728,6 +749,17 @@ buildernames = [
'os': 'mac',
'os_platform': 'osx-10-6',
'vm': False}}),
('b2g_mozilla-inbound_wasabi_dep',
{'build_type': 'opt',
'job_type': 'build',
'name': {'group_name': 'Wasabi Device Image',
'group_symbol': 'Wasabi',
'name': 'Wasabi Device Image Build',
'job_symbol': 'B'},
'platform': {'arch': 'x86',
'os': 'b2g',
'os_platform': 'b2g-device-image',
'vm': False}}),
]

21
tests/webapp/api/test_jobs_api.py
Просмотреть файл

@ -97,21 +97,30 @@ def test_job_list(webapp, eleven_jobs_processed, jm):
"submit_timestamp",
"start_timestamp",
"result_set_id",
"product_id",
"who",
"option_collection_hash",
"reason",
"active_status",
"id",
"job_guid",
"state",
"job_type_id",
"result",
"build_platform_id",
"machine_platform_id",
"machine_id",
"job_coalesced_to_guid",
"end_timestamp"
"end_timestamp",
"build_platform",
"machine_name",
"job_group_symbol",
"job_type_name",
"job_group_name",
"job_type_description",
"build_architecture",
"job_type_symbol",
"platform",
"job_group_description",
"platform_opt",
"machine_platform_os",
"build_os",
"machine_platform_architecture"
]
for job in jobs:
assert set(job.keys()) == set(exp_keys)

283
treeherder/etl/buildbot.py
Просмотреть файл

@ -244,6 +244,7 @@ JOB_TYPE_BUILDERNAME = {
re.compile('.+ xulrunner$'),
re.compile('.+ code coverage$'),
re.compile('.*valgrind$'),
re.compile('.*non-unified'),
],
'unittest': [
re.compile('jetpack.*(opt|debug)$'),
@ -281,10 +282,12 @@ TEST_NAME_BUILDERNAME = [
{"regex": re.compile('-sh-haz'), "desc": "Static Rooting Hazard Analysis, JS Shell"},
{"regex": re.compile('xulrunner'), "desc": "XULRunner Nightly"},
{"regex": re.compile('b2g.*_hamachi_eng_nightly'), "desc": "Hamachi Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_helix_eng_nightly'), "desc": "Helix Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_inari_eng_nightly'), "desc": "Inari Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_leo_eng_nightly'), "desc": "Leo Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_unagi_eng_nightly'), "desc": "Unagi Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_eng_nightly'), "desc": "Unknown B2G Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_buri-limited-memory_nightly'), "desc": "Buri Limited Memory Device Image Nightly"},
{"regex": re.compile('b2g.*_hamachi_nightly'), "desc": "Hamachi Device Image Nightly"},
{"regex": re.compile('b2g.*_helix_nightly'), "desc": "Helix Device Image Nightly"},
{"regex": re.compile('b2g.*_inari_nightly'), "desc": "Inari Device Image Nightly"},
@ -295,17 +298,20 @@ TEST_NAME_BUILDERNAME = [
{"regex": re.compile('(?:l10n|localizer) nightly'), "desc": "L10n Nightly"},
{"regex": re.compile('nightly'), "desc": "Nightly"},
{"regex": re.compile('b2g.*_hamachi_eng_dep'), "desc": "Hamachi Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_helix_eng_dep'), "desc": "Helix Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_inari_eng_dep'), "desc": "Inari Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_leo_eng_dep'), "desc": "Leo Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_unagi_eng_dep'), "desc": "Unagi Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_eng_dep'), "desc": "Unknown B2G Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_emulator.*_dep'), "desc": "B2G Emulator Image Build"},
{"regex": re.compile('b2g.*_buri-limited-memory_dep'), "desc": "Buri Limited Memory Device Image Build"},
{"regex": re.compile('b2g.*_hamachi_dep'), "desc": "Hamachi Device Image Build"},
{"regex": re.compile('b2g.*_helix_dep'), "desc": "Helix Device Image Build"},
{"regex": re.compile('b2g.*_inari_dep'), "desc": "Inari Device Image Build"},
{"regex": re.compile('b2g.*_leo_dep'), "desc": "Leo Device Image Build"},
{"regex": re.compile('b2g.*_nexus-4_dep'), "desc": "Nexus 4 Device Image Build"},
{"regex": re.compile('b2g.*_unagi_dep'), "desc": "Unagi Device Image Build"},
{"regex": re.compile('b2g.*_wasabi_dep'), "desc": "Wasabi Device Image Build"},
{"regex": re.compile('b2g.*_dep'), "desc": "Unknown B2G Device Image Build"},
{"regex": re.compile('spidermonkey.*-dtrace'), "desc": "SpiderMonkey DTrace Build"},
{"regex": re.compile('spidermonkey.*-rootanalysis'), "desc": "SpiderMonkey --enable-root-analysis Build"},
@ -315,6 +321,7 @@ TEST_NAME_BUILDERNAME = [
#// If we start doing debug ASan tests, please kill these special build types
{"regex": re.compile('debug asan build'), "desc": "AddressSanitizer Debug Build"},
{"regex": re.compile('asan build'), "desc": "AddressSanitizer Opt Build"},
{"regex": re.compile('non-unified'), "desc": "Non-Unified Build"},
{"regex": re.compile('static analysis'), "desc": "Static Checking Build"},
{"regex": re.compile('valgrind'), "desc": "Valgrind Nightly"},
{"regex": re.compile('dxr'), "desc": "DXR Index Build"},
@ -349,112 +356,122 @@ TEST_NAME_BUILDERNAME = [
# when updating, please take care to ensure the ``testname`` AND the
# ``groupname`` exist in the ``SYMBOLS`` dict as well.
GROUP_NAMES = {
'Android x86 Test Set': 'Android x86 Test Combos',
'Crashtest': 'Reftest',
'Crashtest-IPC': 'Reftest',
'Hamachi Device Image Build': 'Hamachi Device Image',
'Hamachi Device Image Build (Engineering)': 'Hamachi Device Image',
'Hamachi Device Image Nightly': 'Hamachi Device Image',
'Hamachi Device Image Nightly (Engineering)': 'Hamachi Device Image',
'Helix Device Image Build': 'Helix Device Image',
'Helix Device Image Nightly': 'Helix Device Image',
'Inari Device Image Build': 'Inari Device Image',
'Inari Device Image Build (Engineering)': 'Inari Device Image',
'Inari Device Image Nightly': 'Inari Device Image',
'Inari Device Image Nightly (Engineering)': 'Inari Device Image',
'JSReftest': 'Reftest',
'L10n Nightly': 'L10n Repack',
'Leo Device Image Build': 'Leo Device Image',
'Leo Device Image Build (Engineering)': 'Leo Device Image',
'Leo Device Image Nightly': 'Leo Device Image',
'Leo Device Image Nightly (Engineering)': 'Leo Device Image',
'Mochitest': 'Mochitest',
'Mochitest Browser Chrome': 'Mochitest',
'Mochitest Metro Browser Chrome': 'Mochitest',
'Mochitest Other': 'Mochitest',
'Mochitest WebGL': 'Mochitest',
'Nexus 4 Device Image Build': 'Nexus 4 Device Image',
'Nexus 4 Device Image Nightly': 'Nexus 4 Device Image',
'Reftest': 'Reftest',
'Reftest Unaccelerated': 'Reftest',
'Reftest-IPC': 'Reftest',
'Robocop': 'Mochitest',
'SpiderMonkey --enable-root-analysis Build': 'SpiderMonkey',
'SpiderMonkey --enable-sm-fail-on-warnings Build': 'SpiderMonkey',
'SpiderMonkey DTrace Build': 'SpiderMonkey',
'SpiderMonkey Exact Rooting Shell Build': 'SpiderMonkey',
'SpiderMonkey GGC Shell Build': 'SpiderMonkey',
'Static Rooting Hazard Analysis, Full Browser': 'SpiderMonkey',
'Static Rooting Hazard Analysis, JS Shell': 'SpiderMonkey',
'Talos Performance': 'Talos Performance',
'Talos canvasmark': 'Talos Performance',
'Talos chrome': 'Talos Performance',
'Talos dromaeojs': 'Talos Performance',
'Talos dromaeojs Metro': 'Talos Performance',
'Talos other': 'Talos Performance',
'Talos other Metro': 'Talos Performance',
'Talos paint': 'Talos Performance',
'Talos robocheck2': 'Talos Performance',
'Talos robopan': 'Talos Performance',
'Talos roboprovider': 'Talos Performance',
'Talos svg': 'Talos Performance',
'Talos svg Metro': 'Talos Performance',
'Talos tp': 'Talos Performance',
'Talos tp Metro': 'Talos Performance',
'Talos tp nochrome': 'Talos Performance',
'Talos ts': 'Talos Performance',
'Talos tspaint': 'Talos Performance',
'Talos xperf': 'Talos Performance',
'Unagi Device Image Build': 'Unagi Device Image',
'Unagi Device Image Build (Engineering)': 'Unagi Device Image',
'Unagi Device Image Nightly': 'Unagi Device Image',
'Unagi Device Image Nightly (Engineering)': 'Unagi Device Image',
'Unknown B2G Device Image Build': 'Unknown Device Image',
'Unknown B2G Device Image Build (Engineering)': 'Unknown Device Image',
'Unknown B2G Device Image Nightly': 'Unknown Device Image',
'Unknown B2G Device Image Nightly (Engineering)': 'Unknown Device Image'
"Hamachi Device Image Build": "Buri/Hamachi Device Image",
"Hamachi Device Image Build (Engineering)": "Buri/Hamachi Device Image",
"Buri Limited Memory Device Image Build": "Buri/Hamachi Device Image",
"Hamachi Device Image Nightly": "Buri/Hamachi Device Image",
"Hamachi Device Image Nightly (Engineering)": "Buri/Hamachi Device Image",
"Buri Limited Memory Device Image Nightly": "Buri/Hamachi Device Image",
"Helix Device Image Build": "Helix Device Image",
"Helix Device Image Build (Engineering)": "Helix Device Image",
"Helix Device Image Nightly": "Helix Device Image",
"Helix Device Image Nightly (Engineering)": "Helix Device Image",
"Inari Device Image Build": "Inari Device Image",
"Inari Device Image Build (Engineering)": "Inari Device Image",
"Inari Device Image Nightly": "Inari Device Image",
"Inari Device Image Nightly (Engineering)": "Inari Device Image",
"Leo Device Image Build": "Leo Device Image",
"Leo Device Image Build (Engineering)": "Leo Device Image",
"Leo Device Image Nightly": "Leo Device Image",
"Leo Device Image Nightly (Engineering)": "Leo Device Image",
"Nexus 4 Device Image Build": "Nexus 4 Device Image",
"Nexus 4 Device Image Nightly": "Nexus 4 Device Image",
"Unagi Device Image Build": "Unagi Device Image",
"Unagi Device Image Build (Engineering)": "Unagi Device Image",
"Unagi Device Image Nightly": "Unagi Device Image",
"Unagi Device Image Nightly (Engineering)": "Unagi Device Image",
"Wasabi Device Image Build": "Wasabi Device Image",
"Unknown B2G Device Image Build": "Unknown Device Image",
"Unknown B2G Device Image Build (Engineering)": "Unknown Device Image",
"Unknown B2G Device Image Nightly": "Unknown Device Image",
"Unknown B2G Device Image Nightly (Engineering)": "Unknown Device Image",
"L10n Nightly": "L10n Repack",
"Android x86 Test Set": "Android x86 Test Combos",
"Mochitest": "Mochitest",
"Mochitest WebGL": "Mochitest",
"Mochitest Browser Chrome": "Mochitest",
"Mochitest Metro Browser Chrome": "Mochitest",
"Mochitest Other": "Mochitest",
"Robocop": "Mochitest",
"Crashtest": "Reftest",
"Crashtest-IPC": "Reftest",
"Reftest": "Reftest",
"Reftest Unaccelerated": "Reftest",
"Reftest-IPC": "Reftest",
"JSReftest": "Reftest",
"SpiderMonkey DTrace Build": "SpiderMonkey",
"SpiderMonkey --enable-root-analysis Build": "SpiderMonkey",
"SpiderMonkey --enable-sm-fail-on-warnings Build": "SpiderMonkey",
"SpiderMonkey GGC Shell Build": "SpiderMonkey",
"SpiderMonkey Exact Rooting Shell Build": "SpiderMonkey",
"Static Rooting Hazard Analysis, JS Shell": "SpiderMonkey",
"Static Rooting Hazard Analysis, Full Browser": "SpiderMonkey",
"Talos Performance": "Talos Performance",
"Talos canvasmark": "Talos Performance",
"Talos chrome": "Talos Performance",
"Talos dromaeojs": "Talos Performance",
"Talos dromaeojs Metro": "Talos Performance",
"Talos other": "Talos Performance",
"Talos other Metro": "Talos Performance",
"Talos paint": "Talos Performance",
"Talos robocheck2": "Talos Performance",
"Talos robopan": "Talos Performance",
"Talos roboprovider": "Talos Performance",
"Talos svg": "Talos Performance",
"Talos svg Metro": "Talos Performance",
"Talos tp": "Talos Performance",
"Talos tp Metro": "Talos Performance",
"Talos tp nochrome": "Talos Performance",
"Talos ts": "Talos Performance",
"Talos tspaint": "Talos Performance",
"Talos xperf": "Talos Performance"
}
# symbols displayed in the UI for all jobs and job groups
SYMBOLS = {
# builds
#// ** Dep Builds **
"Build": "B",
"Static Checking Build": "S",
"SpiderMonkey": "SM",
"SpiderMonkey DTrace Build": "d",
"SpiderMonkey --enable-root-analysis Build": "r",
"SpiderMonkey --enable-sm-fail-on-warnings Build": "e",
"SpiderMonkey GGC Shell Build": "ggc",
"SpiderMonkey Exact Rooting Shell Build": "exr",
"Static Rooting Hazard Analysis, JS Shell": "Hs",
"Static Rooting Hazard Analysis, Full Browser": "Hf",
#// ** Nightly Builds **
"Nightly": "N",
"DXR Index Build": "Dxr",
# // ** Dep Builds **
"Build" : "B",
"Non-Unified Build": "Bn",
"Static Checking Build" : "S",
"SpiderMonkey" : "SM",
"SpiderMonkey DTrace Build" : "d",
"SpiderMonkey --enable-root-analysis Build" : "r",
"SpiderMonkey --enable-sm-fail-on-warnings Build" : "e",
"SpiderMonkey GGC Shell Build" : "ggc",
"SpiderMonkey Exact Rooting Shell Build" : "exr",
"Static Rooting Hazard Analysis, JS Shell" : "Hs",
"Static Rooting Hazard Analysis, Full Browser" : "Hf",
# // ** Nightly Builds **
"Nightly" : "N",
"DXR Index Build" : "Dxr",
"Valgrind Nightly": "V",
"XULRunner Nightly": "Xr",
#// ** Special Builds **
#// If we start doing debug ASan tests, please
#// kill these special build types
"XULRunner Nightly" : "Xr",
# // ** Special Builds **
# // If we start doing debug ASan tests, please
# // kill these special build types
"AddressSanitizer Opt Build": "Bo",
"AddressSanitizer Debug Build": "Bd",
"AddressSanitizer Opt Nightly": "No",
"AddressSanitizer Debug Nightly": "Nd",
#// L10n nightlies are grouped above so they appear as N1, N2, etc.
"L10n Nightly": "N",
# // L10n nightlies are grouped above so they appear as N1, N2, etc.
"L10n Nightly" : "N",
"L10n Repack": "L10n",
"B2G Emulator Image Build": "B",
#// B2G device image builds (grouped by device in the UI)
"Hamachi Device Image": "Hamachi",
# // B2G device image builds (grouped by device in the UI)
"Buri/Hamachi Device Image": "Buri/Hamachi",
"Hamachi Device Image Build": "B",
"Hamachi Device Image Build (Engineering)": "Be",
"Buri Limited Memory Device Image Build": "Bm",
"Hamachi Device Image Nightly": "N",
"Hamachi Device Image Nightly (Engineering)": "Ne",
"Buri Limited Memory Device Image Nightly": "Nm",
"Helix Device Image": "Helix",
"Helix Device Image Build": "B",
"Helix Device Image Build (Engineering)": "Be",
"Helix Device Image Nightly": "N",
"Helix Device Image Nightly (Engineering)": "Ne",
"Inari Device Image": "Inari",
"Inari Device Image Build": "B",
"Inari Device Image Build (Engineering)": "Be",
@ -473,63 +490,64 @@ SYMBOLS = {
"Unagi Device Image Build (Engineering)": "Be",
"Unagi Device Image Nightly": "N",
"Unagi Device Image Nightly (Engineering)": "Ne",
"Wasabi Device Image": "Wasabi",
"Wasabi Device Image Build": "B",
"Unknown Device Image": "Unknown",
"Unknown B2G Device Image Build": "B",
"Unknown B2G Device Image Build (Engineering)": "Be",
"Unknown B2G Device Image Nightly": "N",
"Unknown B2G Device Image Nightly (Engineering)": "Ne",
# tests
"Mochitest": "M",
"Mochitest WebGL": "gl",
"Mochitest Browser Chrome": "bc",
"Mochitest Metro Browser Chrome": "mc",
"Mochitest Other": "oth",
"Robocop": "rc",
"Crashtest-IPC": "Cipc",
"Crashtest": "C",
"Reftest Unaccelerated": "Ru",
"Reftest-IPC": "Ripc",
"Reftest": "R",
"JSReftest": "J",
"CPP Unit Tests": "Cpp",
"JIT Tests": "Jit",
"Marionette WebAPI Tests": "Mnw",
"Marionette Framework Unit Tests": "Mn",
"Gaia Integration Test": "Gi",
"Gaia UI Test": "Gu",
"Gaia Unit Test": "G",
"XPCShellTest": "X",
"Android x86 Test Combos": "Sets",
"Android x86 Test Set": "S",
"Talos Performance": "T",
"Talos canvasmark": "cm",
"Talos chrome": "c",
"Talos dromaeojs": "d",
"Talos dromaeojs Metro": "d-m",
"Talos svg": "s",
"Talos svg Metro": "s-m",
"Talos tp": "tp",
"Talos tp Metro": "tp-m",
"Talos tp nochrome": "tpn",
"Talos other": "o",
"Talos other Metro": "o-m",
"Talos paint": "p",
"Talos robocheck2": "rck2",
"Talos robopan": "rp",
"Talos roboprovider": "rpr",
"Talos ts": "ts",
"Talos tspaint": "tsp",
"Talos xperf": "x",
"Jetpack SDK Test": "JP",
"Mozmill": "Z",
"Unknown Unit Test": "U",
"Mochitest" : "M",
"Mochitest WebGL" : "gl",
"Mochitest Browser Chrome" : "bc",
"Mochitest Metro Browser Chrome" : "mc",
"Mochitest Other" : "oth",
"Robocop" : "rc",
"Crashtest-IPC" : "Cipc",
"Crashtest" : "C",
"Reftest Unaccelerated" : "Ru",
"Reftest-IPC" : "Ripc",
"Reftest" : "R",
"JSReftest" : "J",
"CPP Unit Tests" : "Cpp",
"JIT Tests" : "Jit",
"Marionette WebAPI Tests" : "Mnw",
"Marionette Framework Unit Tests" : "Mn",
"Gaia Integration Test" : "Gi",
"Gaia UI Test" : "Gu",
"Gaia Unit Test" : "G",
"XPCShellTest" : "X",
"Android x86 Test Combos" : "Sets",
"Android x86 Test Set" : "S",
"Talos Performance" : "T",
"Talos canvasmark" : "cm",
"Talos chrome" : "c",
"Talos dromaeojs" : "d",
"Talos dromaeojs Metro" : "d-m",
"Talos svg" : "s",
"Talos svg Metro" : "s-m",
"Talos tp" : "tp",
"Talos tp Metro" : "tp-m",
"Talos tp nochrome" : "tpn",
"Talos other" : "o",
"Talos other Metro" : "o-m",
"Talos paint" : "p",
"Talos robocheck2" : "rck2",
"Talos robopan" : "rp",
"Talos roboprovider" : "rpr",
"Talos ts" : "ts",
"Talos tspaint" : "tsp",
"Talos xperf" : "x",
"Jetpack SDK Test" : "JP",
"Mozmill" : "Z",
"Unknown Unit Test" : "U",
"Unknown": "?",
"unknown": "?",
}
NUMBER_RE = re.compile(".*(?:mochitest|reftest|crashtest|robocop|androidx86-set)\-([0-9]+)", re.IGNORECASE)
NUMBER_RE = re.compile(".*(?:mochitest(?:-debug)?|reftest|crashtest|robocop|androidx86-set|browser-chrome)\-([0-9]+)", re.IGNORECASE)
def extract_platform_info(source_string):
@ -616,4 +634,3 @@ def get_symbol(name, bn):
if nummatch:
n = nummatch.group(1)
return "{0}{1}".format(s, n)

3
treeherder/events/publisher.py
Просмотреть файл

@ -50,9 +50,10 @@ class EventsPublisher(object):
class JobStatusPublisher(EventsPublisher):
def publish(self, job_id, branch, status):
def publish(self, job_id, result_set_id, branch, status):
message = {
"id": job_id,
"result_set_id": result_set_id,
"event": "job",
"branch": branch,
"status": status

12
treeherder/events/run_socketio.py
Просмотреть файл

@ -50,14 +50,14 @@ def broadcast_subscribers(body, msg):
for session_id, socket in server.sockets.iteritems():
# loop over all the open connections
# and send a message when needed
if "event" not in socket.session or "branch" not in socket.session:
if "subscriptions" not in socket.session:
continue
branch_condition = "*" in socket.session["branch"] \
or body["branch"] in socket.session["branch"]
event_condition = "*" in socket.session["event"] \
or body["event"] in socket.session["event"]
if branch_condition and event_condition:
for branch, events in socket.session['subscriptions'].items():
if branch == body["branch"] or branch == "*":
if body["event"] in events or "*" in events:
socket.send_packet(pkt)
break
msg.ack()

33
treeherder/events/sockets.py
Просмотреть файл

@ -1,5 +1,5 @@
import logging
from collections import defaultdict
from socketio.namespace import BaseNamespace
@ -9,8 +9,7 @@ class EventsNamespace(BaseNamespace):
super(EventsNamespace, self).__init__(*args, **kwargs)
self.logger = logging.getLogger("treeherder.events.socketio")
self.log("New connection")
self.session['branch'] = set()
self.session['event'] = set()
self.session['subscriptions'] = defaultdict(set)
def log(self, message):
self.logger.info("[{0}] {1}".format(self.socket.sessid, message))
@ -18,26 +17,34 @@ class EventsNamespace(BaseNamespace):
def on_subscribe(self, subscription):
"""
this method is triggered by a new client subscription.
it adds a prefix to the routing key to prevent message sniffing
subscription is a string indicating a branch or branch.event
"""
tokens = subscription.split(".")
if len(tokens) == 1:
# branch subscription
self.session['branch'].add(tokens[0])
# event is implicitly set to 'all'
self.session['event'].add("*")
self.session['subscriptions'][tokens[0]].add("*")
elif len(tokens) == 2:
# event subscription
self.session['branch'].add(tokens[0])
self.session['event'].add(tokens[1])
self.session['subscriptions'][tokens[0]].add(tokens[1])
else:
self.emit('error', 'malformed subscription')
def on_unsubscribe(self):
self.session['branch'] = set()
self.session['event'] = set()
self.log("subscription reset")
def on_unsubscribe(self, subscription=None):
"""
this method is triggered by a new client subscription.
subscription is a string indicating a branch or branch.event
if no subscription is passed, all the subscriptions are cleared
"""
if not subscription:
self.session['subscriptions'] = defaultdict(set)
else:
tokens = subscription.split(".")
if len(tokens) == 1:
del self.session['subscriptions'][tokens[0]]
else:
self.session['subscriptions'][tokens[0]].remove(tokens[1])
def recv_disconnect(self):
self.log("Disconnected")

4
treeherder/log_parser/tasks.py
Просмотреть файл

@ -18,7 +18,7 @@ from treeherder.events.publisher import JobFailurePublisher, JobStatusPublisher
@task(name='parse-log')
def parse_log(project, job_id, check_errors=False):
def parse_log(project, job_id, result_set_id, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
@ -79,7 +79,7 @@ def parse_log(project, job_id, check_errors=False):
# store the artifacts generated
jm.store_job_artifact(artifact_list)
status_publisher.publish(job_id, project, 'processed')
status_publisher.publish(job_id, result_set_id, project, 'processed')
if check_errors:
failure_publisher.publish(job_id, project)

20
treeherder/model/derived/jobs.py
Просмотреть файл

@ -75,14 +75,24 @@ class JobsModel(TreeherderModelBase):
)
return data
def get_job_list(self, offset, limit):
def get_job_list(self, offset, limit, **kwargs):
"""
Retrieve a list of jobs.
Mainly used by the restful api to list the jobs
joblist: a list of job ids to limit which jobs are returned.
"""
filter_str = ""
if "joblist" in kwargs:
filter_str += " AND j.id in ({0})".format(kwargs["joblist"])
repl = [self.refdata_model.get_db_name(), filter_str]
proc = "jobs.selects.get_job_list"
data = self.get_jobs_dhub().execute(
proc=proc,
replace=repl,
placeholders=[offset, limit],
debug_show=self.DEBUG,
)
@ -209,6 +219,9 @@ class JobsModel(TreeherderModelBase):
replace_str += " AND revision.revision = %s"
placeholders.append(kwargs["revision"])
if "resultsetlist" in kwargs:
replace_str += " AND rs.id in ({0})".format(kwargs["resultsetlist"])
# If a push doesn't have jobs we can just
# message the user, it would save us a very expensive join
# with the jobs table.
@ -867,11 +880,14 @@ class JobsModel(TreeherderModelBase):
job_guid = log_placeholders[index][0]
job_id = job_id_lookup[job_guid]['id']
result = job_results[job_guid]
result_set_id = job_id_lookup[job_guid]['result_set_id']
# Replace job_guid with id
log_placeholders[index][0] = job_id
task = dict()
task['id'] = job_id
task['result_set_id'] = result_set_id
if result != 'success':
task['check_errors'] = True
task['routing_key'] = 'parse_log.failures'
@ -888,7 +904,7 @@ class JobsModel(TreeherderModelBase):
executemany=True)
for task in tasks:
parse_log.apply_async(args=[self.project, task['id']],
parse_log.apply_async(args=[self.project, task['id'], task['result_set_id']],
kwargs={'check_errors': task['check_errors']},
routing_key=task['routing_key'])

80
treeherder/model/sql/jobs.json
Просмотреть файл

@ -158,16 +158,90 @@
},
"selects":{
"get_job":{
"sql":"SELECT j.*, m.name as machine_name
"sql":"SELECT
j.id,
j.`job_guid`,
j.`job_coalesced_to_guid`,
j.`build_platform_id`,
j.`option_collection_hash`,
mp.`platform` as platform,
mp.`os_name` as machine_platform_os,
mp.`architecture` as machine_platform_architecture,
bp.`platform` as build_platform,
bp.`os_name` as build_os,
bp.`architecture` as build_architecture,
m.`name` as machine_name,
jt.`name` as job_type_name,
jt.`symbol` as job_type_symbol,
jt.`description` as job_type_description,
jg.`name` as job_group_name,
jg.`symbol` as job_group_symbol,
jg.`description` as job_group_description,
j.`who`,
j.`result_set_id`,
j.`result`,
j.`state`,
j.`reason`,
j.`start_timestamp`,
j.`end_timestamp`,
j.`submit_timestamp`
FROM `job` as j
LEFT JOIN `REP0`.`machine` as m
ON j.`machine_id` = m.id
LEFT JOIN `REP0`.`machine_platform` as mp
ON j.`machine_platform_id` = mp.id
LEFT JOIN `REP0`.`build_platform` as bp
ON j.`build_platform_id` = bp.id
LEFT JOIN `REP0`.`job_type` as jt
ON j.`job_type_id` = jt.id
LEFT JOIN `REP0`.`job_group` as jg
ON jt.`job_group_id` = jg.id
WHERE j.id = ?",
"host":"read_host"
},
"get_job_list":{
"sql":"SELECT * FROM `job` LIMIT ?,?",
"sql":"SELECT
j.id,
j.`job_guid`,
j.`job_coalesced_to_guid`,
j.`build_platform_id`,
j.`option_collection_hash`,
mp.`platform` as platform,
mp.`os_name` as machine_platform_os,
mp.`architecture` as machine_platform_architecture,
bp.`platform` as build_platform,
bp.`os_name` as build_os,
bp.`architecture` as build_architecture,
m.`name` as machine_name,
jt.`name` as job_type_name,
jt.`symbol` as job_type_symbol,
jt.`description` as job_type_description,
jg.`name` as job_group_name,
jg.`symbol` as job_group_symbol,
jg.`description` as job_group_description,
j.`who`,
j.`result_set_id`,
j.`result`,
j.`state`,
j.`reason`,
j.`start_timestamp`,
j.`end_timestamp`,
j.`submit_timestamp`
FROM `job` as j
LEFT JOIN `REP0`.`machine` as m
ON j.`machine_id` = m.id
LEFT JOIN `REP0`.`machine_platform` as mp
ON j.`machine_platform_id` = mp.id
LEFT JOIN `REP0`.`build_platform` as bp
ON j.`build_platform_id` = bp.id
LEFT JOIN `REP0`.`job_type` as jt
ON j.`job_type_id` = jt.id
LEFT JOIN `REP0`.`job_group` as jg
ON jt.`job_group_id` = jg.id
WHERE 1
REP1
LIMIT ?,?",
"host":"read_host"
},
@ -201,7 +275,7 @@
"host": "read_host"
},
"get_job_ids_by_guids":{
"sql":"SELECT `id`, `job_guid`, `state`, `result`
"sql":"SELECT `id`, `job_guid`, `result_set_id`, `state`, `result`
FROM `job`
WHERE `active_status` = 'active' AND `job_guid` IN (REP0)",
"host": "read_host"

56
treeherder/webapp/api/views.py
Просмотреть файл

@ -266,6 +266,8 @@ class JobsViewSet(viewsets.ViewSet):
obj = jm.get_job(pk)
if obj:
job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
job["logs"] = jm.get_log_references(pk)
# make artifact ids into uris
@ -277,6 +279,9 @@ class JobsViewSet(viewsets.ViewSet):
art["resource_uri"] = ref
job["artifacts"].append(art)
option_collections = jm.refdata_model.get_all_option_collections()
option_collections[job["option_collection_hash"]]['opt']
return Response(job)
else:
return Response("No job with id: {0}".format(pk), 404)
@ -286,11 +291,26 @@ class JobsViewSet(viewsets.ViewSet):
def list(self, request, project, jm):
"""
GET method implementation for list view
"""
offset = request.QUERY_PARAMS.get('offset', 0)
count = request.QUERY_PARAMS.get('count', 10)
objs = jm.get_job_list(offset, count)
"""
filters = ["joblist"]
offset = int(request.QUERY_PARAMS.get('offset', 0))
count = int(request.QUERY_PARAMS.get('count', 10))
objs = jm.get_job_list(
offset,
count,
**dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
if k in filters)
)
if objs:
option_collections = jm.refdata_model.get_all_option_collections()
for job in objs:
job["platform_opt"] = option_collections[
job["option_collection_hash"]]['opt']
return Response(objs)
@action()
@ -348,9 +368,11 @@ class ResultSetViewSet(viewsets.ViewSet):
def list(self, request, project, jm):
"""
GET method for list of ``resultset`` records with revisions
resultsetlist - specific resultset ids to retrieve
"""
filters = ["author", "revision"]
filters = ["author", "revision", "resultsetlist"]
offset = int(request.QUERY_PARAMS.get('offset', 0))
count = int(request.QUERY_PARAMS.get('count', 10))
@ -385,12 +407,7 @@ class ResultSetViewSet(viewsets.ViewSet):
def get_resultsets_with_jobs(jm, rs_list, filter_kwargs):
"""Convert db result of resultsets in a list to JSON"""
# I think I'll just call the database in a for-loop and fetch
# the jobs for each resultset, then glue them together. Oh wait...
# I promised Jeads I wouldn't do that. I guess I'll fetch the job
# results all at once, then parse them out in memory. Jeads will
# like that better. :)
# Fetch the job results all at once, then parse them out in memory.
# organize the resultsets into an obj by key for lookups
rs_map = {}
for rs in rs_list:
@ -433,12 +450,16 @@ class ResultSetViewSet(viewsets.ViewSet):
#itertools needs the elements to be sorted by the grouper
by_platform = sorted(list(resultset_group), key=platform_grouper)
platforms = []
for platform_name, platform_group in itertools.groupby(
for platform_group_name, platform_group in itertools.groupby(
by_platform,
key=platform_grouper):
by_job_group = sorted(list(platform_group), key=job_group_grouper)
platform_name = by_job_group[0]["platform"]
platform_option = option_collections[
by_job_group[0]["option_collection_hash"]]['opt']
groups = []
for jg_symbol, jg_group in itertools.groupby(
by_job_group,
@ -454,12 +475,12 @@ class ResultSetViewSet(viewsets.ViewSet):
# build the uri ref for each job
for job in by_job_type:
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["job_id"]})
#del(job["job_group_name"])
#del(job["job_group_symbol"])
job["id"] = job["job_id"]
del(job["job_id"])
del(job["result_set_id"])
del(job["platform"])
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
if job["state"] == "completed":
result_types.append(job["result"])
@ -469,6 +490,7 @@ class ResultSetViewSet(viewsets.ViewSet):
platforms.append({
"name": platform_name,
"option": platform_option,
"groups": groups,
})