mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'master' of ssh://github.com/mozilla/treeherder-service into oauth-implementation

This commit is contained in:
Jonathan Eads 2014-01-28 14:33:54 -08:00
Родитель 8fa7524d0c 0c979369d3
Коммит ab83b849b3
14 изменённых файлов: 463 добавлений и 264 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
.gitignore поставляемый
Просмотреть файл

@ -27,6 +27,7 @@ pip-log.txt
test.log test.log
treeherder*.log treeherder*.log
treeherder.log.* treeherder.log.*
LOGFILE
# Unit test / coverage reports # Unit test / coverage reports
.coverage .coverage
@ -60,5 +61,8 @@ treeherder/etl/data/
#docs build directory #docs build directory
docs/_build/ docs/_build/
#supervisor
.nfs*
*.pid
*.c *.c

8
deployment/supervisord/treeherder.conf
Просмотреть файл

@ -1,3 +1,11 @@
[supervisord]
http_port=/var/tmp/supervisor.sock ; (default is to run a UNIX domain socket server)
loglevel=info ; (logging level;default info; others: debug,warn)
nodaemon=true ; (start in foreground if true;default false)
[supervisorctl]
serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL for a unix socket
[program:gunicorn] [program:gunicorn]
command=/home/vagrant/treeherder-service/bin/run_gunicorn command=/home/vagrant/treeherder-service/bin/run_gunicorn
user=vagrant user=vagrant

16
docs/installation.rst
Просмотреть файл

@ -31,6 +31,18 @@ Installation
(venv)vagrant@precise32:~/treeherder-service$ ./runtests.sh (venv)vagrant@precise32:~/treeherder-service$ ./runtests.sh
* Initialize the master database
.. code-block:: bash
(venv)vagrant@precise32:~/treeherder-service$ python manage.py init_master_db
* Populate the database with repository data sources
.. code-block:: bash
(venv)vagrant@precise32:~/treeherder-service$ python manage.py init_datasources
* And an entry to your host machine /etc/hosts so that you can point your browser to local.treeherder.mozilla.org to reach it * And an entry to your host machine /etc/hosts so that you can point your browser to local.treeherder.mozilla.org to reach it
.. code-block:: bash .. code-block:: bash
@ -71,10 +83,6 @@ Installation
* If you want to use supervisord to take care of all the services,
.. _project repo: https://github.com/mozilla/treeherder-service .. _project repo: https://github.com/mozilla/treeherder-service
.. _Vagrant: http://downloads.vagrantup.com .. _Vagrant: http://downloads.vagrantup.com
.. _Virtualbox: https://www.virtualbox.org .. _Virtualbox: https://www.virtualbox.org

3
puppet/manifests/classes/treeherder.pp
Просмотреть файл

@ -20,7 +20,8 @@ class treeherder {
file { [ file { [
"/var/log/gunicorn", "/var/log/gunicorn",
"/var/log/celery" "/var/log/celery",
"/var/log/socketio"
]: ]:
ensure => "directory", ensure => "directory",
owner => "${APP_USER}", owner => "${APP_USER}",

174
tests/etl/test_buildbot.py
Просмотреть файл

@ -11,7 +11,7 @@ slow = pytest.mark.slow
buildernames = [ buildernames = [
('Android 2.2 Armv6 mozilla-inbound build', ('Android 2.2 Armv6 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -22,7 +22,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test crashtest', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test crashtest',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -33,7 +33,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test jsreftest-1', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test jsreftest-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -44,7 +44,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test mochitest-1', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test mochitest-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -55,7 +55,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Tegra mozilla-inbound opt test mochitest-6', ('Android 2.2 Tegra mozilla-inbound opt test mochitest-6',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -66,7 +66,7 @@ buildernames = [
'os_platform': 'android-2-2', 'os_platform': 'android-2-2',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test plain-reftest-1', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test plain-reftest-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -77,7 +77,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test robocop-1', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test robocop-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -88,7 +88,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Armv6 Tegra mozilla-inbound opt test xpcshell', ('Android 2.2 Armv6 Tegra mozilla-inbound opt test xpcshell',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -99,7 +99,7 @@ buildernames = [
'os_platform': 'android-2-2-armv6', 'os_platform': 'android-2-2-armv6',
'vm': False}}), 'vm': False}}),
('Android 2.2 Debug mozilla-inbound build', ('Android 2.2 Debug mozilla-inbound build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -110,7 +110,7 @@ buildernames = [
'os_platform': 'android-2-2', 'os_platform': 'android-2-2',
'vm': False}}), 'vm': False}}),
('Android 2.2 mozilla-inbound build', ('Android 2.2 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -121,7 +121,7 @@ buildernames = [
'os_platform': 'android-2-2', 'os_platform': 'android-2-2',
'vm': False}}), 'vm': False}}),
('Android 2.2 Tegra mozilla-inbound opt test crashtest', ('Android 2.2 Tegra mozilla-inbound opt test crashtest',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -132,7 +132,7 @@ buildernames = [
'os_platform': 'android-2-2', 'os_platform': 'android-2-2',
'vm': False}}), 'vm': False}}),
('Android 2.2 Tegra mozilla-inbound talos remote-tcanvasmark', ('Android 2.2 Tegra mozilla-inbound talos remote-tcanvasmark',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'talos', 'job_type': 'talos',
'name': {'group_name': 'Talos Performance', 'name': {'group_name': 'Talos Performance',
'group_symbol': 'T', 'group_symbol': 'T',
@ -143,7 +143,7 @@ buildernames = [
'os_platform': 'android-2-2', 'os_platform': 'android-2-2',
'vm': False}}), 'vm': False}}),
('Android 4.0 Panda mozilla-inbound talos remote-tsvgx', ('Android 4.0 Panda mozilla-inbound talos remote-tsvgx',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'talos', 'job_type': 'talos',
'name': {'group_name': 'Talos Performance', 'name': {'group_name': 'Talos Performance',
'group_symbol': 'T', 'group_symbol': 'T',
@ -154,7 +154,7 @@ buildernames = [
'os_platform': 'android-4-0', 'os_platform': 'android-4-0',
'vm': False}}), 'vm': False}}),
('Android 4.2 x86 mozilla-inbound build', ('Android 4.2 x86 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -165,7 +165,7 @@ buildernames = [
'os_platform': 'android-4-2-x86', 'os_platform': 'android-4-2-x86',
'vm': False}}), 'vm': False}}),
('b2g_emulator mozilla-inbound opt test reftest-1', ('b2g_emulator mozilla-inbound opt test reftest-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -176,7 +176,7 @@ buildernames = [
'os_platform': 'b2g-emu-ics', 'os_platform': 'b2g-emu-ics',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_emulator_dep', ('b2g_mozilla-inbound_emulator_dep',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -187,7 +187,7 @@ buildernames = [
'os_platform': 'b2g-emu-ics', 'os_platform': 'b2g-emu-ics',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_emulator-debug_dep', ('b2g_mozilla-inbound_emulator-debug_dep',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -198,7 +198,7 @@ buildernames = [
'os_platform': 'b2g-emu-ics', 'os_platform': 'b2g-emu-ics',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_emulator-jb_dep', ('b2g_mozilla-inbound_emulator-jb_dep',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -209,7 +209,7 @@ buildernames = [
'os_platform': 'b2g-emu-jb', 'os_platform': 'b2g-emu-jb',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_emulator-jb-debug_dep', ('b2g_mozilla-inbound_emulator-jb-debug_dep',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -220,7 +220,7 @@ buildernames = [
'os_platform': 'b2g-emu-jb', 'os_platform': 'b2g-emu-jb',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_linux32_gecko build', ('b2g_mozilla-inbound_linux32_gecko build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -231,7 +231,7 @@ buildernames = [
'os_platform': 'b2g-linux32', 'os_platform': 'b2g-linux32',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_linux64_gecko build', ('b2g_mozilla-inbound_linux64_gecko build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -242,7 +242,7 @@ buildernames = [
'os_platform': 'b2g-linux64', 'os_platform': 'b2g-linux64',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_macosx64_gecko build', ('b2g_mozilla-inbound_macosx64_gecko build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -253,7 +253,7 @@ buildernames = [
'os_platform': 'b2g-osx', 'os_platform': 'b2g-osx',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_unagi_dep', ('b2g_mozilla-inbound_unagi_dep',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'Unagi Device Image', 'name': {'group_name': 'Unagi Device Image',
'group_symbol': 'Unagi', 'group_symbol': 'Unagi',
@ -264,7 +264,7 @@ buildernames = [
'os_platform': 'b2g-device-image', 'os_platform': 'b2g-device-image',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_win32_gecko build', ('b2g_mozilla-inbound_win32_gecko build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -275,7 +275,7 @@ buildernames = [
'os_platform': 'b2g-win32', 'os_platform': 'b2g-win32',
'vm': False}}), 'vm': False}}),
('b2g_ubuntu64_vm mozilla-inbound opt test gaia-ui-test', ('b2g_ubuntu64_vm mozilla-inbound opt test gaia-ui-test',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -286,7 +286,7 @@ buildernames = [
'os_platform': 'b2g-linux64', 'os_platform': 'b2g-linux64',
'vm': True}}), 'vm': True}}),
('b2g_ubuntu64_vm mozilla-inbound opt test gaia-unit', ('b2g_ubuntu64_vm mozilla-inbound opt test gaia-unit',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -297,7 +297,7 @@ buildernames = [
'os_platform': 'b2g-linux64', 'os_platform': 'b2g-linux64',
'vm': True}}), 'vm': True}}),
('b2g_ubuntu64_vm mozilla-inbound opt test mochitest-1', ('b2g_ubuntu64_vm mozilla-inbound opt test mochitest-1',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -308,7 +308,7 @@ buildernames = [
'os_platform': 'b2g-linux64', 'os_platform': 'b2g-linux64',
'vm': True}}), 'vm': True}}),
('Linux mozilla-inbound build', ('Linux mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -319,7 +319,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Linux mozilla-inbound leak test build', ('Linux mozilla-inbound leak test build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -330,7 +330,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Linux mozilla-inbound leak test spidermonkey_info-warnaserrdebug build', ('Linux mozilla-inbound leak test spidermonkey_info-warnaserrdebug build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'SpiderMonkey', 'name': {'group_name': 'SpiderMonkey',
'group_symbol': 'SM', 'group_symbol': 'SM',
@ -341,7 +341,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Linux mozilla-inbound pgo-build', ('Linux mozilla-inbound pgo-build',
{'build_type': 'pgo', {'build_type': 'pgo',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -352,7 +352,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Linux mozilla-inbound spidermonkey_info-warnaserr build', ('Linux mozilla-inbound spidermonkey_info-warnaserr build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'SpiderMonkey', 'name': {'group_name': 'SpiderMonkey',
'group_symbol': 'SM', 'group_symbol': 'SM',
@ -363,7 +363,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound asan build', ('Linux x86-64 mozilla-inbound asan build',
{'build_type': 'asan', {'build_type': 'asan',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -374,7 +374,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound build', ('Linux x86-64 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -385,7 +385,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound debug asan build', ('Linux x86-64 mozilla-inbound debug asan build',
{'build_type': 'asan', {'build_type': 'asan',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -396,7 +396,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound debug static analysis build', ('Linux x86-64 mozilla-inbound debug static analysis build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -407,7 +407,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound leak test build', ('Linux x86-64 mozilla-inbound leak test build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -417,21 +417,42 @@ buildernames = [
'os': 'linux', 'os': 'linux',
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
( ('Linux x86-64 mozilla-inbound leak test non-unified',
'Linux x86-64 mozilla-inbound leak test spidermonkey_tier_1-rootanalysis ' {'build_type': 'debug',
'job_type': 'build',
'name': {'group_name': 'unknown',
'group_symbol': '?',
'name': 'Non-Unified Build',
'job_symbol': 'Bn'},
'platform': {'arch': 'x86_64',
'os': 'linux',
'os_platform': 'linux64',
'vm': False}}),
('Linux x86-64 mozilla-inbound non-unified',
{'build_type': 'opt',
'job_type': 'build',
'name': {'group_name': 'unknown',
'group_symbol': '?',
'name': 'Non-Unified Build',
'job_symbol': 'Bn'},
'platform': {'arch': 'x86_64',
'os': 'linux',
'os_platform': 'linux64',
'vm': False}}),
('Linux x86-64 mozilla-inbound leak test spidermonkey_tier_1-rootanalysis '
'build', 'build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'SpiderMonkey', 'name': {'group_name': 'SpiderMonkey',
'group_symbol': 'SM', 'group_symbol': 'SM',
'name': 'SpiderMonkey --enable-root-analysis Build', 'name': 'SpiderMonkey --enable-root-analysis Build',
'job_symbol': 'r'}, 'job_symbol': 'r'},
'platform': {'arch': 'x86_64', 'platform': {'arch': 'x86_64',
'os': 'linux', 'os': 'linux',
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound pgo-build', ('Linux x86-64 mozilla-inbound pgo-build',
{'build_type': 'pgo', {'build_type': 'pgo',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -442,7 +463,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Linux x86-64 mozilla-inbound spidermonkey_info-warnaserr build', ('Linux x86-64 mozilla-inbound spidermonkey_info-warnaserr build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'SpiderMonkey', 'name': {'group_name': 'SpiderMonkey',
'group_symbol': 'SM', 'group_symbol': 'SM',
@ -453,7 +474,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('OS X 10.7 64-bit mozilla-inbound leak test build', ('OS X 10.7 64-bit mozilla-inbound leak test build',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -464,7 +485,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('OS X 10.7 mozilla-inbound build', ('OS X 10.7 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -475,7 +496,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('Rev3 Fedora 12 mozilla-inbound debug test mochitest-browser-chrome', ('Rev3 Fedora 12 mozilla-inbound debug test mochitest-browser-chrome',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -486,7 +507,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Rev3 Fedora 12x64 mozilla-inbound debug test mochitest-browser-chrome', ('Rev3 Fedora 12x64 mozilla-inbound debug test mochitest-browser-chrome',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Mochitest', 'name': {'group_name': 'Mochitest',
'group_symbol': 'M', 'group_symbol': 'M',
@ -497,7 +518,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test jetpack', ('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test jetpack',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -508,7 +529,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test marionette', ('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test marionette',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -519,7 +540,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test reftest', ('Rev4 MacOSX Lion 10.7 mozilla-inbound debug test reftest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -530,7 +551,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('Rev4 MacOSX Lion 10.7 mozilla-inbound talos dromaeojs', ('Rev4 MacOSX Lion 10.7 mozilla-inbound talos dromaeojs',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'talos', 'job_type': 'talos',
'name': {'group_name': 'Talos Performance', 'name': {'group_name': 'Talos Performance',
'group_symbol': 'T', 'group_symbol': 'T',
@ -541,7 +562,7 @@ buildernames = [
'os_platform': 'osx-10-7', 'os_platform': 'osx-10-7',
'vm': False}}), 'vm': False}}),
('Rev4 MacOSX Snow Leopard 10.6 mozilla-inbound debug test crashtest', ('Rev4 MacOSX Snow Leopard 10.6 mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -552,7 +573,7 @@ buildernames = [
'os_platform': 'osx-10-6', 'os_platform': 'osx-10-6',
'vm': False}}), 'vm': False}}),
('Rev5 MacOSX Mountain Lion 10.8 mozilla-inbound debug test crashtest', ('Rev5 MacOSX Mountain Lion 10.8 mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -563,7 +584,7 @@ buildernames = [
'os_platform': 'osx-10-8', 'os_platform': 'osx-10-8',
'vm': False}}), 'vm': False}}),
('Ubuntu ASAN VM 12.04 x64 mozilla-inbound opt test crashtest', ('Ubuntu ASAN VM 12.04 x64 mozilla-inbound opt test crashtest',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -574,7 +595,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': True}}), 'vm': True}}),
('Ubuntu VM 12.04 x64 mozilla-inbound debug test marionette', ('Ubuntu VM 12.04 x64 mozilla-inbound debug test marionette',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -585,7 +606,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': True}}), 'vm': True}}),
('Ubuntu HW 12.04 mozilla-inbound pgo talos chromez', ('Ubuntu HW 12.04 mozilla-inbound pgo talos chromez',
{'build_type': 'pgo', {'build_type': 'pgo',
'job_type': 'talos', 'job_type': 'talos',
'name': {'group_name': 'Talos Performance', 'name': {'group_name': 'Talos Performance',
'group_symbol': 'T', 'group_symbol': 'T',
@ -596,7 +617,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': False}}), 'vm': False}}),
('Ubuntu HW 12.04 x64 mozilla-inbound pgo talos chromez', ('Ubuntu HW 12.04 x64 mozilla-inbound pgo talos chromez',
{'build_type': 'pgo', {'build_type': 'pgo',
'job_type': 'talos', 'job_type': 'talos',
'name': {'group_name': 'Talos Performance', 'name': {'group_name': 'Talos Performance',
'group_symbol': 'T', 'group_symbol': 'T',
@ -607,7 +628,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('Ubuntu VM 12.04 mozilla-inbound debug test crashtest', ('Ubuntu VM 12.04 mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -618,7 +639,7 @@ buildernames = [
'os_platform': 'linux32', 'os_platform': 'linux32',
'vm': True}}), 'vm': True}}),
('Ubuntu VM 12.04 x64 mozilla-inbound debug test crashtest', ('Ubuntu VM 12.04 x64 mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -629,7 +650,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': True}}), 'vm': True}}),
('Windows 7 32-bit mozilla-inbound debug test crashtest', ('Windows 7 32-bit mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -640,7 +661,7 @@ buildernames = [
'os_platform': 'windows7-32', 'os_platform': 'windows7-32',
'vm': False}}), 'vm': False}}),
('Windows XP 32-bit mozilla-inbound debug test crashtest', ('Windows XP 32-bit mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -651,7 +672,7 @@ buildernames = [
'os_platform': 'windowsxp', 'os_platform': 'windowsxp',
'vm': False}}), 'vm': False}}),
('WINNT 5.2 mozilla-inbound build', ('WINNT 5.2 mozilla-inbound build',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -662,7 +683,7 @@ buildernames = [
'os_platform': 'windowsxp', 'os_platform': 'windowsxp',
'vm': False}}), 'vm': False}}),
('WINNT 6.2 mozilla-inbound debug test crashtest', ('WINNT 6.2 mozilla-inbound debug test crashtest',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'Reftest', 'name': {'group_name': 'Reftest',
'group_symbol': 'R', 'group_symbol': 'R',
@ -674,7 +695,7 @@ buildernames = [
'vm': False}}), 'vm': False}}),
('Linux x86-64 b2g-inbound valgrind', ('Linux x86-64 b2g-inbound valgrind',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -685,7 +706,7 @@ buildernames = [
'os_platform': 'linux64', 'os_platform': 'linux64',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-b2g18_leo_eng_nightly', ('b2g_mozilla-b2g18_leo_eng_nightly',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'Leo Device Image', 'name': {'group_name': 'Leo Device Image',
'group_symbol': 'Leo', 'group_symbol': 'Leo',
@ -696,10 +717,10 @@ buildernames = [
'os_platform': 'b2g-device-image', 'os_platform': 'b2g-device-image',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-b2g26_v1_2_hamachi_nightly', ('b2g_mozilla-b2g26_v1_2_hamachi_nightly',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'build', 'job_type': 'build',
'name': {'group_name': 'Hamachi Device Image', 'name': {'group_name': 'Buri/Hamachi Device Image',
'group_symbol': 'Hamachi', 'group_symbol': 'Buri/Hamachi',
'name': 'Hamachi Device Image Nightly', 'name': 'Hamachi Device Image Nightly',
'job_symbol': 'N'}, 'job_symbol': 'N'},
'platform': {'arch': 'x86', 'platform': {'arch': 'x86',
@ -707,7 +728,7 @@ buildernames = [
'os_platform': 'b2g-device-image', 'os_platform': 'b2g-device-image',
'vm': False}}), 'vm': False}}),
('jetpack-fx-team-snowleopard-debug', ('jetpack-fx-team-snowleopard-debug',
{'build_type': 'debug', {'build_type': 'debug',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -718,7 +739,7 @@ buildernames = [
'os_platform': 'osx-10-6', 'os_platform': 'osx-10-6',
'vm': False}}), 'vm': False}}),
('jetpack-fx-team-snowleopard-opt', ('jetpack-fx-team-snowleopard-opt',
{'build_type': 'opt', {'build_type': 'opt',
'job_type': 'unittest', 'job_type': 'unittest',
'name': {'group_name': 'unknown', 'name': {'group_name': 'unknown',
'group_symbol': '?', 'group_symbol': '?',
@ -728,6 +749,17 @@ buildernames = [
'os': 'mac', 'os': 'mac',
'os_platform': 'osx-10-6', 'os_platform': 'osx-10-6',
'vm': False}}), 'vm': False}}),
('b2g_mozilla-inbound_wasabi_dep',
{'build_type': 'opt',
'job_type': 'build',
'name': {'group_name': 'Wasabi Device Image',
'group_symbol': 'Wasabi',
'name': 'Wasabi Device Image Build',
'job_symbol': 'B'},
'platform': {'arch': 'x86',
'os': 'b2g',
'os_platform': 'b2g-device-image',
'vm': False}}),
] ]

21
tests/webapp/api/test_jobs_api.py
Просмотреть файл

@ -97,21 +97,30 @@ def test_job_list(webapp, eleven_jobs_processed, jm):
"submit_timestamp", "submit_timestamp",
"start_timestamp", "start_timestamp",
"result_set_id", "result_set_id",
"product_id",
"who", "who",
"option_collection_hash", "option_collection_hash",
"reason", "reason",
"active_status",
"id", "id",
"job_guid", "job_guid",
"state", "state",
"job_type_id",
"result", "result",
"build_platform_id", "build_platform_id",
"machine_platform_id",
"machine_id",
"job_coalesced_to_guid", "job_coalesced_to_guid",
"end_timestamp" "end_timestamp",
"build_platform",
"machine_name",
"job_group_symbol",
"job_type_name",
"job_group_name",
"job_type_description",
"build_architecture",
"job_type_symbol",
"platform",
"job_group_description",
"platform_opt",
"machine_platform_os",
"build_os",
"machine_platform_architecture"
] ]
for job in jobs: for job in jobs:
assert set(job.keys()) == set(exp_keys) assert set(job.keys()) == set(exp_keys)

283
treeherder/etl/buildbot.py
Просмотреть файл

@ -244,6 +244,7 @@ JOB_TYPE_BUILDERNAME = {
re.compile('.+ xulrunner$'), re.compile('.+ xulrunner$'),
re.compile('.+ code coverage$'), re.compile('.+ code coverage$'),
re.compile('.*valgrind$'), re.compile('.*valgrind$'),
re.compile('.*non-unified'),
], ],
'unittest': [ 'unittest': [
re.compile('jetpack.*(opt|debug)$'), re.compile('jetpack.*(opt|debug)$'),
@ -281,10 +282,12 @@ TEST_NAME_BUILDERNAME = [
{"regex": re.compile('-sh-haz'), "desc": "Static Rooting Hazard Analysis, JS Shell"}, {"regex": re.compile('-sh-haz'), "desc": "Static Rooting Hazard Analysis, JS Shell"},
{"regex": re.compile('xulrunner'), "desc": "XULRunner Nightly"}, {"regex": re.compile('xulrunner'), "desc": "XULRunner Nightly"},
{"regex": re.compile('b2g.*_hamachi_eng_nightly'), "desc": "Hamachi Device Image Nightly (Engineering)"}, {"regex": re.compile('b2g.*_hamachi_eng_nightly'), "desc": "Hamachi Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_helix_eng_nightly'), "desc": "Helix Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_inari_eng_nightly'), "desc": "Inari Device Image Nightly (Engineering)"}, {"regex": re.compile('b2g.*_inari_eng_nightly'), "desc": "Inari Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_leo_eng_nightly'), "desc": "Leo Device Image Nightly (Engineering)"}, {"regex": re.compile('b2g.*_leo_eng_nightly'), "desc": "Leo Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_unagi_eng_nightly'), "desc": "Unagi Device Image Nightly (Engineering)"}, {"regex": re.compile('b2g.*_unagi_eng_nightly'), "desc": "Unagi Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_eng_nightly'), "desc": "Unknown B2G Device Image Nightly (Engineering)"}, {"regex": re.compile('b2g.*_eng_nightly'), "desc": "Unknown B2G Device Image Nightly (Engineering)"},
{"regex": re.compile('b2g.*_buri-limited-memory_nightly'), "desc": "Buri Limited Memory Device Image Nightly"},
{"regex": re.compile('b2g.*_hamachi_nightly'), "desc": "Hamachi Device Image Nightly"}, {"regex": re.compile('b2g.*_hamachi_nightly'), "desc": "Hamachi Device Image Nightly"},
{"regex": re.compile('b2g.*_helix_nightly'), "desc": "Helix Device Image Nightly"}, {"regex": re.compile('b2g.*_helix_nightly'), "desc": "Helix Device Image Nightly"},
{"regex": re.compile('b2g.*_inari_nightly'), "desc": "Inari Device Image Nightly"}, {"regex": re.compile('b2g.*_inari_nightly'), "desc": "Inari Device Image Nightly"},
@ -295,17 +298,20 @@ TEST_NAME_BUILDERNAME = [
{"regex": re.compile('(?:l10n|localizer) nightly'), "desc": "L10n Nightly"}, {"regex": re.compile('(?:l10n|localizer) nightly'), "desc": "L10n Nightly"},
{"regex": re.compile('nightly'), "desc": "Nightly"}, {"regex": re.compile('nightly'), "desc": "Nightly"},
{"regex": re.compile('b2g.*_hamachi_eng_dep'), "desc": "Hamachi Device Image Build (Engineering)"}, {"regex": re.compile('b2g.*_hamachi_eng_dep'), "desc": "Hamachi Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_helix_eng_dep'), "desc": "Helix Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_inari_eng_dep'), "desc": "Inari Device Image Build (Engineering)"}, {"regex": re.compile('b2g.*_inari_eng_dep'), "desc": "Inari Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_leo_eng_dep'), "desc": "Leo Device Image Build (Engineering)"}, {"regex": re.compile('b2g.*_leo_eng_dep'), "desc": "Leo Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_unagi_eng_dep'), "desc": "Unagi Device Image Build (Engineering)"}, {"regex": re.compile('b2g.*_unagi_eng_dep'), "desc": "Unagi Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_eng_dep'), "desc": "Unknown B2G Device Image Build (Engineering)"}, {"regex": re.compile('b2g.*_eng_dep'), "desc": "Unknown B2G Device Image Build (Engineering)"},
{"regex": re.compile('b2g.*_emulator.*_dep'), "desc": "B2G Emulator Image Build"}, {"regex": re.compile('b2g.*_emulator.*_dep'), "desc": "B2G Emulator Image Build"},
{"regex": re.compile('b2g.*_buri-limited-memory_dep'), "desc": "Buri Limited Memory Device Image Build"},
{"regex": re.compile('b2g.*_hamachi_dep'), "desc": "Hamachi Device Image Build"}, {"regex": re.compile('b2g.*_hamachi_dep'), "desc": "Hamachi Device Image Build"},
{"regex": re.compile('b2g.*_helix_dep'), "desc": "Helix Device Image Build"}, {"regex": re.compile('b2g.*_helix_dep'), "desc": "Helix Device Image Build"},
{"regex": re.compile('b2g.*_inari_dep'), "desc": "Inari Device Image Build"}, {"regex": re.compile('b2g.*_inari_dep'), "desc": "Inari Device Image Build"},
{"regex": re.compile('b2g.*_leo_dep'), "desc": "Leo Device Image Build"}, {"regex": re.compile('b2g.*_leo_dep'), "desc": "Leo Device Image Build"},
{"regex": re.compile('b2g.*_nexus-4_dep'), "desc": "Nexus 4 Device Image Build"}, {"regex": re.compile('b2g.*_nexus-4_dep'), "desc": "Nexus 4 Device Image Build"},
{"regex": re.compile('b2g.*_unagi_dep'), "desc": "Unagi Device Image Build"}, {"regex": re.compile('b2g.*_unagi_dep'), "desc": "Unagi Device Image Build"},
{"regex": re.compile('b2g.*_wasabi_dep'), "desc": "Wasabi Device Image Build"},
{"regex": re.compile('b2g.*_dep'), "desc": "Unknown B2G Device Image Build"}, {"regex": re.compile('b2g.*_dep'), "desc": "Unknown B2G Device Image Build"},
{"regex": re.compile('spidermonkey.*-dtrace'), "desc": "SpiderMonkey DTrace Build"}, {"regex": re.compile('spidermonkey.*-dtrace'), "desc": "SpiderMonkey DTrace Build"},
{"regex": re.compile('spidermonkey.*-rootanalysis'), "desc": "SpiderMonkey --enable-root-analysis Build"}, {"regex": re.compile('spidermonkey.*-rootanalysis'), "desc": "SpiderMonkey --enable-root-analysis Build"},
@ -315,6 +321,7 @@ TEST_NAME_BUILDERNAME = [
#// If we start doing debug ASan tests, please kill these special build types #// If we start doing debug ASan tests, please kill these special build types
{"regex": re.compile('debug asan build'), "desc": "AddressSanitizer Debug Build"}, {"regex": re.compile('debug asan build'), "desc": "AddressSanitizer Debug Build"},
{"regex": re.compile('asan build'), "desc": "AddressSanitizer Opt Build"}, {"regex": re.compile('asan build'), "desc": "AddressSanitizer Opt Build"},
{"regex": re.compile('non-unified'), "desc": "Non-Unified Build"},
{"regex": re.compile('static analysis'), "desc": "Static Checking Build"}, {"regex": re.compile('static analysis'), "desc": "Static Checking Build"},
{"regex": re.compile('valgrind'), "desc": "Valgrind Nightly"}, {"regex": re.compile('valgrind'), "desc": "Valgrind Nightly"},
{"regex": re.compile('dxr'), "desc": "DXR Index Build"}, {"regex": re.compile('dxr'), "desc": "DXR Index Build"},
@ -349,112 +356,122 @@ TEST_NAME_BUILDERNAME = [
# when updating, please take care to ensure the ``testname`` AND the # when updating, please take care to ensure the ``testname`` AND the
# ``groupname`` exist in the ``SYMBOLS`` dict as well. # ``groupname`` exist in the ``SYMBOLS`` dict as well.
GROUP_NAMES = { GROUP_NAMES = {
'Android x86 Test Set': 'Android x86 Test Combos', "Hamachi Device Image Build": "Buri/Hamachi Device Image",
'Crashtest': 'Reftest', "Hamachi Device Image Build (Engineering)": "Buri/Hamachi Device Image",
'Crashtest-IPC': 'Reftest', "Buri Limited Memory Device Image Build": "Buri/Hamachi Device Image",
'Hamachi Device Image Build': 'Hamachi Device Image', "Hamachi Device Image Nightly": "Buri/Hamachi Device Image",
'Hamachi Device Image Build (Engineering)': 'Hamachi Device Image', "Hamachi Device Image Nightly (Engineering)": "Buri/Hamachi Device Image",
'Hamachi Device Image Nightly': 'Hamachi Device Image', "Buri Limited Memory Device Image Nightly": "Buri/Hamachi Device Image",
'Hamachi Device Image Nightly (Engineering)': 'Hamachi Device Image', "Helix Device Image Build": "Helix Device Image",
'Helix Device Image Build': 'Helix Device Image', "Helix Device Image Build (Engineering)": "Helix Device Image",
'Helix Device Image Nightly': 'Helix Device Image', "Helix Device Image Nightly": "Helix Device Image",
'Inari Device Image Build': 'Inari Device Image', "Helix Device Image Nightly (Engineering)": "Helix Device Image",
'Inari Device Image Build (Engineering)': 'Inari Device Image', "Inari Device Image Build": "Inari Device Image",
'Inari Device Image Nightly': 'Inari Device Image', "Inari Device Image Build (Engineering)": "Inari Device Image",
'Inari Device Image Nightly (Engineering)': 'Inari Device Image', "Inari Device Image Nightly": "Inari Device Image",
'JSReftest': 'Reftest', "Inari Device Image Nightly (Engineering)": "Inari Device Image",
'L10n Nightly': 'L10n Repack', "Leo Device Image Build": "Leo Device Image",
'Leo Device Image Build': 'Leo Device Image', "Leo Device Image Build (Engineering)": "Leo Device Image",
'Leo Device Image Build (Engineering)': 'Leo Device Image', "Leo Device Image Nightly": "Leo Device Image",
'Leo Device Image Nightly': 'Leo Device Image', "Leo Device Image Nightly (Engineering)": "Leo Device Image",
'Leo Device Image Nightly (Engineering)': 'Leo Device Image', "Nexus 4 Device Image Build": "Nexus 4 Device Image",
'Mochitest': 'Mochitest', "Nexus 4 Device Image Nightly": "Nexus 4 Device Image",
'Mochitest Browser Chrome': 'Mochitest', "Unagi Device Image Build": "Unagi Device Image",
'Mochitest Metro Browser Chrome': 'Mochitest', "Unagi Device Image Build (Engineering)": "Unagi Device Image",
'Mochitest Other': 'Mochitest', "Unagi Device Image Nightly": "Unagi Device Image",
'Mochitest WebGL': 'Mochitest', "Unagi Device Image Nightly (Engineering)": "Unagi Device Image",
'Nexus 4 Device Image Build': 'Nexus 4 Device Image', "Wasabi Device Image Build": "Wasabi Device Image",
'Nexus 4 Device Image Nightly': 'Nexus 4 Device Image', "Unknown B2G Device Image Build": "Unknown Device Image",
'Reftest': 'Reftest', "Unknown B2G Device Image Build (Engineering)": "Unknown Device Image",
'Reftest Unaccelerated': 'Reftest', "Unknown B2G Device Image Nightly": "Unknown Device Image",
'Reftest-IPC': 'Reftest', "Unknown B2G Device Image Nightly (Engineering)": "Unknown Device Image",
'Robocop': 'Mochitest', "L10n Nightly": "L10n Repack",
'SpiderMonkey --enable-root-analysis Build': 'SpiderMonkey', "Android x86 Test Set": "Android x86 Test Combos",
'SpiderMonkey --enable-sm-fail-on-warnings Build': 'SpiderMonkey', "Mochitest": "Mochitest",
'SpiderMonkey DTrace Build': 'SpiderMonkey', "Mochitest WebGL": "Mochitest",
'SpiderMonkey Exact Rooting Shell Build': 'SpiderMonkey', "Mochitest Browser Chrome": "Mochitest",
'SpiderMonkey GGC Shell Build': 'SpiderMonkey', "Mochitest Metro Browser Chrome": "Mochitest",
'Static Rooting Hazard Analysis, Full Browser': 'SpiderMonkey', "Mochitest Other": "Mochitest",
'Static Rooting Hazard Analysis, JS Shell': 'SpiderMonkey', "Robocop": "Mochitest",
'Talos Performance': 'Talos Performance', "Crashtest": "Reftest",
'Talos canvasmark': 'Talos Performance', "Crashtest-IPC": "Reftest",
'Talos chrome': 'Talos Performance', "Reftest": "Reftest",
'Talos dromaeojs': 'Talos Performance', "Reftest Unaccelerated": "Reftest",
'Talos dromaeojs Metro': 'Talos Performance', "Reftest-IPC": "Reftest",
'Talos other': 'Talos Performance', "JSReftest": "Reftest",
'Talos other Metro': 'Talos Performance', "SpiderMonkey DTrace Build": "SpiderMonkey",
'Talos paint': 'Talos Performance', "SpiderMonkey --enable-root-analysis Build": "SpiderMonkey",
'Talos robocheck2': 'Talos Performance', "SpiderMonkey --enable-sm-fail-on-warnings Build": "SpiderMonkey",
'Talos robopan': 'Talos Performance', "SpiderMonkey GGC Shell Build": "SpiderMonkey",
'Talos roboprovider': 'Talos Performance', "SpiderMonkey Exact Rooting Shell Build": "SpiderMonkey",
'Talos svg': 'Talos Performance', "Static Rooting Hazard Analysis, JS Shell": "SpiderMonkey",
'Talos svg Metro': 'Talos Performance', "Static Rooting Hazard Analysis, Full Browser": "SpiderMonkey",
'Talos tp': 'Talos Performance', "Talos Performance": "Talos Performance",
'Talos tp Metro': 'Talos Performance', "Talos canvasmark": "Talos Performance",
'Talos tp nochrome': 'Talos Performance', "Talos chrome": "Talos Performance",
'Talos ts': 'Talos Performance', "Talos dromaeojs": "Talos Performance",
'Talos tspaint': 'Talos Performance', "Talos dromaeojs Metro": "Talos Performance",
'Talos xperf': 'Talos Performance', "Talos other": "Talos Performance",
'Unagi Device Image Build': 'Unagi Device Image', "Talos other Metro": "Talos Performance",
'Unagi Device Image Build (Engineering)': 'Unagi Device Image', "Talos paint": "Talos Performance",
'Unagi Device Image Nightly': 'Unagi Device Image', "Talos robocheck2": "Talos Performance",
'Unagi Device Image Nightly (Engineering)': 'Unagi Device Image', "Talos robopan": "Talos Performance",
'Unknown B2G Device Image Build': 'Unknown Device Image', "Talos roboprovider": "Talos Performance",
'Unknown B2G Device Image Build (Engineering)': 'Unknown Device Image', "Talos svg": "Talos Performance",
'Unknown B2G Device Image Nightly': 'Unknown Device Image', "Talos svg Metro": "Talos Performance",
'Unknown B2G Device Image Nightly (Engineering)': 'Unknown Device Image' "Talos tp": "Talos Performance",
"Talos tp Metro": "Talos Performance",
"Talos tp nochrome": "Talos Performance",
"Talos ts": "Talos Performance",
"Talos tspaint": "Talos Performance",
"Talos xperf": "Talos Performance"
} }
# symbols displayed in the UI for all jobs and job groups # symbols displayed in the UI for all jobs and job groups
SYMBOLS = { SYMBOLS = {
# builds # builds
#// ** Dep Builds ** # // ** Dep Builds **
"Build": "B", "Build" : "B",
"Static Checking Build": "S", "Non-Unified Build": "Bn",
"SpiderMonkey": "SM", "Static Checking Build" : "S",
"SpiderMonkey DTrace Build": "d", "SpiderMonkey" : "SM",
"SpiderMonkey --enable-root-analysis Build": "r", "SpiderMonkey DTrace Build" : "d",
"SpiderMonkey --enable-sm-fail-on-warnings Build": "e", "SpiderMonkey --enable-root-analysis Build" : "r",
"SpiderMonkey GGC Shell Build": "ggc", "SpiderMonkey --enable-sm-fail-on-warnings Build" : "e",
"SpiderMonkey Exact Rooting Shell Build": "exr", "SpiderMonkey GGC Shell Build" : "ggc",
"Static Rooting Hazard Analysis, JS Shell": "Hs", "SpiderMonkey Exact Rooting Shell Build" : "exr",
"Static Rooting Hazard Analysis, Full Browser": "Hf", "Static Rooting Hazard Analysis, JS Shell" : "Hs",
#// ** Nightly Builds ** "Static Rooting Hazard Analysis, Full Browser" : "Hf",
"Nightly": "N", # // ** Nightly Builds **
"DXR Index Build": "Dxr", "Nightly" : "N",
"DXR Index Build" : "Dxr",
"Valgrind Nightly": "V", "Valgrind Nightly": "V",
"XULRunner Nightly": "Xr", "XULRunner Nightly" : "Xr",
#// ** Special Builds ** # // ** Special Builds **
#// If we start doing debug ASan tests, please # // If we start doing debug ASan tests, please
#// kill these special build types # // kill these special build types
"AddressSanitizer Opt Build": "Bo", "AddressSanitizer Opt Build": "Bo",
"AddressSanitizer Debug Build": "Bd", "AddressSanitizer Debug Build": "Bd",
"AddressSanitizer Opt Nightly": "No", "AddressSanitizer Opt Nightly": "No",
"AddressSanitizer Debug Nightly": "Nd", "AddressSanitizer Debug Nightly": "Nd",
#// L10n nightlies are grouped above so they appear as N1, N2, etc. # // L10n nightlies are grouped above so they appear as N1, N2, etc.
"L10n Nightly": "N", "L10n Nightly" : "N",
"L10n Repack": "L10n", "L10n Repack": "L10n",
"B2G Emulator Image Build": "B", "B2G Emulator Image Build": "B",
#// B2G device image builds (grouped by device in the UI) # // B2G device image builds (grouped by device in the UI)
"Hamachi Device Image": "Hamachi", "Buri/Hamachi Device Image": "Buri/Hamachi",
"Hamachi Device Image Build": "B", "Hamachi Device Image Build": "B",
"Hamachi Device Image Build (Engineering)": "Be", "Hamachi Device Image Build (Engineering)": "Be",
"Buri Limited Memory Device Image Build": "Bm",
"Hamachi Device Image Nightly": "N", "Hamachi Device Image Nightly": "N",
"Hamachi Device Image Nightly (Engineering)": "Ne", "Hamachi Device Image Nightly (Engineering)": "Ne",
"Buri Limited Memory Device Image Nightly": "Nm",
"Helix Device Image": "Helix", "Helix Device Image": "Helix",
"Helix Device Image Build": "B", "Helix Device Image Build": "B",
"Helix Device Image Build (Engineering)": "Be",
"Helix Device Image Nightly": "N", "Helix Device Image Nightly": "N",
"Helix Device Image Nightly (Engineering)": "Ne",
"Inari Device Image": "Inari", "Inari Device Image": "Inari",
"Inari Device Image Build": "B", "Inari Device Image Build": "B",
"Inari Device Image Build (Engineering)": "Be", "Inari Device Image Build (Engineering)": "Be",
@ -473,63 +490,64 @@ SYMBOLS = {
"Unagi Device Image Build (Engineering)": "Be", "Unagi Device Image Build (Engineering)": "Be",
"Unagi Device Image Nightly": "N", "Unagi Device Image Nightly": "N",
"Unagi Device Image Nightly (Engineering)": "Ne", "Unagi Device Image Nightly (Engineering)": "Ne",
"Wasabi Device Image": "Wasabi",
"Wasabi Device Image Build": "B",
"Unknown Device Image": "Unknown", "Unknown Device Image": "Unknown",
"Unknown B2G Device Image Build": "B", "Unknown B2G Device Image Build": "B",
"Unknown B2G Device Image Build (Engineering)": "Be", "Unknown B2G Device Image Build (Engineering)": "Be",
"Unknown B2G Device Image Nightly": "N", "Unknown B2G Device Image Nightly": "N",
"Unknown B2G Device Image Nightly (Engineering)": "Ne", "Unknown B2G Device Image Nightly (Engineering)": "Ne",
# tests # tests
"Mochitest": "M", "Mochitest" : "M",
"Mochitest WebGL": "gl", "Mochitest WebGL" : "gl",
"Mochitest Browser Chrome": "bc", "Mochitest Browser Chrome" : "bc",
"Mochitest Metro Browser Chrome": "mc", "Mochitest Metro Browser Chrome" : "mc",
"Mochitest Other": "oth", "Mochitest Other" : "oth",
"Robocop": "rc", "Robocop" : "rc",
"Crashtest-IPC": "Cipc", "Crashtest-IPC" : "Cipc",
"Crashtest": "C", "Crashtest" : "C",
"Reftest Unaccelerated": "Ru", "Reftest Unaccelerated" : "Ru",
"Reftest-IPC": "Ripc", "Reftest-IPC" : "Ripc",
"Reftest": "R", "Reftest" : "R",
"JSReftest": "J", "JSReftest" : "J",
"CPP Unit Tests": "Cpp", "CPP Unit Tests" : "Cpp",
"JIT Tests": "Jit", "JIT Tests" : "Jit",
"Marionette WebAPI Tests": "Mnw", "Marionette WebAPI Tests" : "Mnw",
"Marionette Framework Unit Tests": "Mn", "Marionette Framework Unit Tests" : "Mn",
"Gaia Integration Test": "Gi", "Gaia Integration Test" : "Gi",
"Gaia UI Test": "Gu", "Gaia UI Test" : "Gu",
"Gaia Unit Test": "G", "Gaia Unit Test" : "G",
"XPCShellTest": "X", "XPCShellTest" : "X",
"Android x86 Test Combos": "Sets", "Android x86 Test Combos" : "Sets",
"Android x86 Test Set": "S", "Android x86 Test Set" : "S",
"Talos Performance": "T", "Talos Performance" : "T",
"Talos canvasmark": "cm", "Talos canvasmark" : "cm",
"Talos chrome": "c", "Talos chrome" : "c",
"Talos dromaeojs": "d", "Talos dromaeojs" : "d",
"Talos dromaeojs Metro": "d-m", "Talos dromaeojs Metro" : "d-m",
"Talos svg": "s", "Talos svg" : "s",
"Talos svg Metro": "s-m", "Talos svg Metro" : "s-m",
"Talos tp": "tp", "Talos tp" : "tp",
"Talos tp Metro": "tp-m", "Talos tp Metro" : "tp-m",
"Talos tp nochrome": "tpn", "Talos tp nochrome" : "tpn",
"Talos other": "o", "Talos other" : "o",
"Talos other Metro": "o-m", "Talos other Metro" : "o-m",
"Talos paint": "p", "Talos paint" : "p",
"Talos robocheck2": "rck2", "Talos robocheck2" : "rck2",
"Talos robopan": "rp", "Talos robopan" : "rp",
"Talos roboprovider": "rpr", "Talos roboprovider" : "rpr",
"Talos ts": "ts", "Talos ts" : "ts",
"Talos tspaint": "tsp", "Talos tspaint" : "tsp",
"Talos xperf": "x", "Talos xperf" : "x",
"Jetpack SDK Test": "JP", "Jetpack SDK Test" : "JP",
"Mozmill": "Z", "Mozmill" : "Z",
"Unknown Unit Test": "U", "Unknown Unit Test" : "U",
"Unknown": "?", "Unknown": "?",
"unknown": "?", "unknown": "?",
} }
NUMBER_RE = re.compile(".*(?:mochitest|reftest|crashtest|robocop|androidx86-set)\-([0-9]+)", re.IGNORECASE) NUMBER_RE = re.compile(".*(?:mochitest(?:-debug)?|reftest|crashtest|robocop|androidx86-set|browser-chrome)\-([0-9]+)", re.IGNORECASE)
def extract_platform_info(source_string): def extract_platform_info(source_string):
@ -616,4 +634,3 @@ def get_symbol(name, bn):
if nummatch: if nummatch:
n = nummatch.group(1) n = nummatch.group(1)
return "{0}{1}".format(s, n) return "{0}{1}".format(s, n)

3
treeherder/events/publisher.py
Просмотреть файл

@ -50,9 +50,10 @@ class EventsPublisher(object):
class JobStatusPublisher(EventsPublisher): class JobStatusPublisher(EventsPublisher):
def publish(self, job_id, branch, status): def publish(self, job_id, result_set_id, branch, status):
message = { message = {
"id": job_id, "id": job_id,
"result_set_id": result_set_id,
"event": "job", "event": "job",
"branch": branch, "branch": branch,
"status": status "status": status

14
treeherder/events/run_socketio.py
Просмотреть файл

@ -50,14 +50,14 @@ def broadcast_subscribers(body, msg):
for session_id, socket in server.sockets.iteritems(): for session_id, socket in server.sockets.iteritems():
# loop over all the open connections # loop over all the open connections
# and send a message when needed # and send a message when needed
if "event" not in socket.session or "branch" not in socket.session: if "subscriptions" not in socket.session:
continue continue
branch_condition = "*" in socket.session["branch"] \
or body["branch"] in socket.session["branch"] for branch, events in socket.session['subscriptions'].items():
event_condition = "*" in socket.session["event"] \ if branch == body["branch"] or branch == "*":
or body["event"] in socket.session["event"] if body["event"] in events or "*" in events:
if branch_condition and event_condition: socket.send_packet(pkt)
socket.send_packet(pkt) break
msg.ack() msg.ack()

33
treeherder/events/sockets.py
Просмотреть файл

@ -1,5 +1,5 @@
import logging import logging
from collections import defaultdict
from socketio.namespace import BaseNamespace from socketio.namespace import BaseNamespace
@ -9,8 +9,7 @@ class EventsNamespace(BaseNamespace):
super(EventsNamespace, self).__init__(*args, **kwargs) super(EventsNamespace, self).__init__(*args, **kwargs)
self.logger = logging.getLogger("treeherder.events.socketio") self.logger = logging.getLogger("treeherder.events.socketio")
self.log("New connection") self.log("New connection")
self.session['branch'] = set() self.session['subscriptions'] = defaultdict(set)
self.session['event'] = set()
def log(self, message): def log(self, message):
self.logger.info("[{0}] {1}".format(self.socket.sessid, message)) self.logger.info("[{0}] {1}".format(self.socket.sessid, message))
@ -18,26 +17,34 @@ class EventsNamespace(BaseNamespace):
def on_subscribe(self, subscription): def on_subscribe(self, subscription):
""" """
this method is triggered by a new client subscription. this method is triggered by a new client subscription.
it adds a prefix to the routing key to prevent message sniffing subscription is a string indicating a branch or branch.event
""" """
tokens = subscription.split(".") tokens = subscription.split(".")
if len(tokens) == 1: if len(tokens) == 1:
# branch subscription
self.session['branch'].add(tokens[0])
# event is implicitly set to 'all' # event is implicitly set to 'all'
self.session['event'].add("*") self.session['subscriptions'][tokens[0]].add("*")
elif len(tokens) == 2: elif len(tokens) == 2:
# event subscription # event subscription
self.session['branch'].add(tokens[0]) self.session['subscriptions'][tokens[0]].add(tokens[1])
self.session['event'].add(tokens[1])
else: else:
self.emit('error', 'malformed subscription') self.emit('error', 'malformed subscription')
def on_unsubscribe(self): def on_unsubscribe(self, subscription=None):
self.session['branch'] = set() """
self.session['event'] = set() this method is triggered by a new client subscription.
self.log("subscription reset") subscription is a string indicating a branch or branch.event
if no subscription is passed, all the subscriptions are cleared
"""
if not subscription:
self.session['subscriptions'] = defaultdict(set)
else:
tokens = subscription.split(".")
if len(tokens) == 1:
del self.session['subscriptions'][tokens[0]]
else:
self.session['subscriptions'][tokens[0]].remove(tokens[1])
def recv_disconnect(self): def recv_disconnect(self):
self.log("Disconnected") self.log("Disconnected")

4
treeherder/log_parser/tasks.py
Просмотреть файл

@ -18,7 +18,7 @@ from treeherder.events.publisher import JobFailurePublisher, JobStatusPublisher
@task(name='parse-log') @task(name='parse-log')
def parse_log(project, job_id, check_errors=False): def parse_log(project, job_id, result_set_id, check_errors=False):
""" """
Call ArtifactBuilderCollection on the given job. Call ArtifactBuilderCollection on the given job.
""" """
@ -79,7 +79,7 @@ def parse_log(project, job_id, check_errors=False):
# store the artifacts generated # store the artifacts generated
jm.store_job_artifact(artifact_list) jm.store_job_artifact(artifact_list)
status_publisher.publish(job_id, project, 'processed') status_publisher.publish(job_id, result_set_id, project, 'processed')
if check_errors: if check_errors:
failure_publisher.publish(job_id, project) failure_publisher.publish(job_id, project)

20
treeherder/model/derived/jobs.py
Просмотреть файл

@ -75,14 +75,24 @@ class JobsModel(TreeherderModelBase):
) )
return data return data
def get_job_list(self, offset, limit): def get_job_list(self, offset, limit, **kwargs):
""" """
Retrieve a list of jobs. Retrieve a list of jobs.
Mainly used by the restful api to list the jobs Mainly used by the restful api to list the jobs
joblist: a list of job ids to limit which jobs are returned.
""" """
filter_str = ""
if "joblist" in kwargs:
filter_str += " AND j.id in ({0})".format(kwargs["joblist"])
repl = [self.refdata_model.get_db_name(), filter_str]
proc = "jobs.selects.get_job_list" proc = "jobs.selects.get_job_list"
data = self.get_jobs_dhub().execute( data = self.get_jobs_dhub().execute(
proc=proc, proc=proc,
replace=repl,
placeholders=[offset, limit], placeholders=[offset, limit],
debug_show=self.DEBUG, debug_show=self.DEBUG,
) )
@ -209,6 +219,9 @@ class JobsModel(TreeherderModelBase):
replace_str += " AND revision.revision = %s" replace_str += " AND revision.revision = %s"
placeholders.append(kwargs["revision"]) placeholders.append(kwargs["revision"])
if "resultsetlist" in kwargs:
replace_str += " AND rs.id in ({0})".format(kwargs["resultsetlist"])
# If a push doesn't have jobs we can just # If a push doesn't have jobs we can just
# message the user, it would save us a very expensive join # message the user, it would save us a very expensive join
# with the jobs table. # with the jobs table.
@ -867,11 +880,14 @@ class JobsModel(TreeherderModelBase):
job_guid = log_placeholders[index][0] job_guid = log_placeholders[index][0]
job_id = job_id_lookup[job_guid]['id'] job_id = job_id_lookup[job_guid]['id']
result = job_results[job_guid] result = job_results[job_guid]
result_set_id = job_id_lookup[job_guid]['result_set_id']
# Replace job_guid with id # Replace job_guid with id
log_placeholders[index][0] = job_id log_placeholders[index][0] = job_id
task = dict() task = dict()
task['id'] = job_id task['id'] = job_id
task['result_set_id'] = result_set_id
if result != 'success': if result != 'success':
task['check_errors'] = True task['check_errors'] = True
task['routing_key'] = 'parse_log.failures' task['routing_key'] = 'parse_log.failures'
@ -888,7 +904,7 @@ class JobsModel(TreeherderModelBase):
executemany=True) executemany=True)
for task in tasks: for task in tasks:
parse_log.apply_async(args=[self.project, task['id']], parse_log.apply_async(args=[self.project, task['id'], task['result_set_id']],
kwargs={'check_errors': task['check_errors']}, kwargs={'check_errors': task['check_errors']},
routing_key=task['routing_key']) routing_key=task['routing_key'])

88
treeherder/model/sql/jobs.json
Просмотреть файл

@ -158,16 +158,90 @@
}, },
"selects":{ "selects":{
"get_job":{ "get_job":{
"sql":"SELECT j.*, m.name as machine_name "sql":"SELECT
FROM `job` as j j.id,
LEFT JOIN `REP0`.`machine` as m j.`job_guid`,
ON j.`machine_id` = m.id j.`job_coalesced_to_guid`,
WHERE j.id = ?", j.`build_platform_id`,
j.`option_collection_hash`,
mp.`platform` as platform,
mp.`os_name` as machine_platform_os,
mp.`architecture` as machine_platform_architecture,
bp.`platform` as build_platform,
bp.`os_name` as build_os,
bp.`architecture` as build_architecture,
m.`name` as machine_name,
jt.`name` as job_type_name,
jt.`symbol` as job_type_symbol,
jt.`description` as job_type_description,
jg.`name` as job_group_name,
jg.`symbol` as job_group_symbol,
jg.`description` as job_group_description,
j.`who`,
j.`result_set_id`,
j.`result`,
j.`state`,
j.`reason`,
j.`start_timestamp`,
j.`end_timestamp`,
j.`submit_timestamp`
FROM `job` as j
LEFT JOIN `REP0`.`machine` as m
ON j.`machine_id` = m.id
LEFT JOIN `REP0`.`machine_platform` as mp
ON j.`machine_platform_id` = mp.id
LEFT JOIN `REP0`.`build_platform` as bp
ON j.`build_platform_id` = bp.id
LEFT JOIN `REP0`.`job_type` as jt
ON j.`job_type_id` = jt.id
LEFT JOIN `REP0`.`job_group` as jg
ON jt.`job_group_id` = jg.id
WHERE j.id = ?",
"host":"read_host" "host":"read_host"
}, },
"get_job_list":{ "get_job_list":{
"sql":"SELECT * FROM `job` LIMIT ?,?", "sql":"SELECT
j.id,
j.`job_guid`,
j.`job_coalesced_to_guid`,
j.`build_platform_id`,
j.`option_collection_hash`,
mp.`platform` as platform,
mp.`os_name` as machine_platform_os,
mp.`architecture` as machine_platform_architecture,
bp.`platform` as build_platform,
bp.`os_name` as build_os,
bp.`architecture` as build_architecture,
m.`name` as machine_name,
jt.`name` as job_type_name,
jt.`symbol` as job_type_symbol,
jt.`description` as job_type_description,
jg.`name` as job_group_name,
jg.`symbol` as job_group_symbol,
jg.`description` as job_group_description,
j.`who`,
j.`result_set_id`,
j.`result`,
j.`state`,
j.`reason`,
j.`start_timestamp`,
j.`end_timestamp`,
j.`submit_timestamp`
FROM `job` as j
LEFT JOIN `REP0`.`machine` as m
ON j.`machine_id` = m.id
LEFT JOIN `REP0`.`machine_platform` as mp
ON j.`machine_platform_id` = mp.id
LEFT JOIN `REP0`.`build_platform` as bp
ON j.`build_platform_id` = bp.id
LEFT JOIN `REP0`.`job_type` as jt
ON j.`job_type_id` = jt.id
LEFT JOIN `REP0`.`job_group` as jg
ON jt.`job_group_id` = jg.id
WHERE 1
REP1
LIMIT ?,?",
"host":"read_host" "host":"read_host"
}, },
@ -201,7 +275,7 @@
"host": "read_host" "host": "read_host"
}, },
"get_job_ids_by_guids":{ "get_job_ids_by_guids":{
"sql":"SELECT `id`, `job_guid`, `state`, `result` "sql":"SELECT `id`, `job_guid`, `result_set_id`, `state`, `result`
FROM `job` FROM `job`
WHERE `active_status` = 'active' AND `job_guid` IN (REP0)", WHERE `active_status` = 'active' AND `job_guid` IN (REP0)",
"host": "read_host" "host": "read_host"

56
treeherder/webapp/api/views.py
Просмотреть файл

@ -266,6 +266,8 @@ class JobsViewSet(viewsets.ViewSet):
obj = jm.get_job(pk) obj = jm.get_job(pk)
if obj: if obj:
job = obj[0] job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
job["logs"] = jm.get_log_references(pk) job["logs"] = jm.get_log_references(pk)
# make artifact ids into uris # make artifact ids into uris
@ -277,6 +279,9 @@ class JobsViewSet(viewsets.ViewSet):
art["resource_uri"] = ref art["resource_uri"] = ref
job["artifacts"].append(art) job["artifacts"].append(art)
option_collections = jm.refdata_model.get_all_option_collections()
option_collections[job["option_collection_hash"]]['opt']
return Response(job) return Response(job)
else: else:
return Response("No job with id: {0}".format(pk), 404) return Response("No job with id: {0}".format(pk), 404)
@ -286,11 +291,26 @@ class JobsViewSet(viewsets.ViewSet):
def list(self, request, project, jm): def list(self, request, project, jm):
""" """
GET method implementation for list view GET method implementation for list view
"""
offset = request.QUERY_PARAMS.get('offset', 0)
count = request.QUERY_PARAMS.get('count', 10)
objs = jm.get_job_list(offset, count) """
filters = ["joblist"]
offset = int(request.QUERY_PARAMS.get('offset', 0))
count = int(request.QUERY_PARAMS.get('count', 10))
objs = jm.get_job_list(
offset,
count,
**dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
if k in filters)
)
if objs:
option_collections = jm.refdata_model.get_all_option_collections()
for job in objs:
job["platform_opt"] = option_collections[
job["option_collection_hash"]]['opt']
return Response(objs) return Response(objs)
@action() @action()
@ -348,9 +368,11 @@ class ResultSetViewSet(viewsets.ViewSet):
def list(self, request, project, jm): def list(self, request, project, jm):
""" """
GET method for list of ``resultset`` records with revisions GET method for list of ``resultset`` records with revisions
resultsetlist - specific resultset ids to retrieve
""" """
filters = ["author", "revision"] filters = ["author", "revision", "resultsetlist"]
offset = int(request.QUERY_PARAMS.get('offset', 0)) offset = int(request.QUERY_PARAMS.get('offset', 0))
count = int(request.QUERY_PARAMS.get('count', 10)) count = int(request.QUERY_PARAMS.get('count', 10))
@ -385,12 +407,7 @@ class ResultSetViewSet(viewsets.ViewSet):
def get_resultsets_with_jobs(jm, rs_list, filter_kwargs): def get_resultsets_with_jobs(jm, rs_list, filter_kwargs):
"""Convert db result of resultsets in a list to JSON""" """Convert db result of resultsets in a list to JSON"""
# I think I'll just call the database in a for-loop and fetch # Fetch the job results all at once, then parse them out in memory.
# the jobs for each resultset, then glue them together. Oh wait...
# I promised Jeads I wouldn't do that. I guess I'll fetch the job
# results all at once, then parse them out in memory. Jeads will
# like that better. :)
# organize the resultsets into an obj by key for lookups # organize the resultsets into an obj by key for lookups
rs_map = {} rs_map = {}
for rs in rs_list: for rs in rs_list:
@ -433,12 +450,16 @@ class ResultSetViewSet(viewsets.ViewSet):
#itertools needs the elements to be sorted by the grouper #itertools needs the elements to be sorted by the grouper
by_platform = sorted(list(resultset_group), key=platform_grouper) by_platform = sorted(list(resultset_group), key=platform_grouper)
platforms = [] platforms = []
for platform_name, platform_group in itertools.groupby( for platform_group_name, platform_group in itertools.groupby(
by_platform, by_platform,
key=platform_grouper): key=platform_grouper):
by_job_group = sorted(list(platform_group), key=job_group_grouper) by_job_group = sorted(list(platform_group), key=job_group_grouper)
platform_name = by_job_group[0]["platform"]
platform_option = option_collections[
by_job_group[0]["option_collection_hash"]]['opt']
groups = [] groups = []
for jg_symbol, jg_group in itertools.groupby( for jg_symbol, jg_group in itertools.groupby(
by_job_group, by_job_group,
@ -454,12 +475,12 @@ class ResultSetViewSet(viewsets.ViewSet):
# build the uri ref for each job # build the uri ref for each job
for job in by_job_type: for job in by_job_type:
job["resource_uri"] = reverse("jobs-detail", job["id"] = job["job_id"]
kwargs={"project": jm.project, "pk": job["job_id"]}) del(job["job_id"])
#del(job["job_group_name"])
#del(job["job_group_symbol"])
del(job["result_set_id"]) del(job["result_set_id"])
del(job["platform"])
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
if job["state"] == "completed": if job["state"] == "completed":
result_types.append(job["result"]) result_types.append(job["result"])
@ -469,6 +490,7 @@ class ResultSetViewSet(viewsets.ViewSet):
platforms.append({ platforms.append({
"name": platform_name, "name": platform_name,
"option": platform_option,
"groups": groups, "groups": groups,
}) })