mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1077136 - fix builds4hr tests and improve tasks per feedback

This commit is contained in:
Cameron Dawson 2014-10-08 12:25:05 -07:00
Родитель 9cf09d8372
Коммит 89639f96ae
15 изменённых файлов: 2112 добавлений и 5484 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

214
tests/etl/test_buildapi.py
Просмотреть файл

@ -12,20 +12,7 @@ def mock_buildapi_pending_url(monkeypatch):
path = os.path.join(
tests_folder,
"sample_data",
"builds-pending.js"
)
monkeypatch.setattr(settings,
'BUILDAPI_PENDING_URL',
"file://{0}".format(path))
@pytest.fixture
def mock_buildapi_pending_missing1_url(monkeypatch):
tests_folder = os.path.dirname(os.path.dirname(__file__))
path = os.path.join(
tests_folder,
"sample_data",
"builds-pending-missing1.js"
"builds-pending.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_PENDING_URL',
@ -38,7 +25,7 @@ def mock_buildapi_running_url(monkeypatch):
path = os.path.join(
tests_folder,
"sample_data",
"builds-running.js"
"builds-running.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_RUNNING_URL',
@ -51,7 +38,46 @@ def mock_buildapi_builds4h_url(monkeypatch):
path = os.path.join(
tests_folder,
"sample_data",
"builds-4h.js"
"builds-4h.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_BUILDS4H_URL',
"file://{0}".format(path))
@pytest.fixture
def mock_buildapi_pending_missing1_url(monkeypatch):
tests_folder = os.path.dirname(os.path.dirname(__file__))
path = os.path.join(
tests_folder,
"sample_data",
"builds-pending-missing1.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_PENDING_URL',
"file://{0}".format(path))
@pytest.fixture
def mock_buildapi_running_missing1_url(monkeypatch):
tests_folder = os.path.dirname(os.path.dirname(__file__))
path = os.path.join(
tests_folder,
"sample_data",
"builds-running-missing1.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_RUNNING_URL',
"file://{0}".format(path))
@pytest.fixture
def mock_buildapi_builds4h_missing1_url(monkeypatch):
tests_folder = os.path.dirname(os.path.dirname(__file__))
path = os.path.join(
tests_folder,
"sample_data",
"builds-4h-missing1.json"
)
monkeypatch.setattr(settings,
'BUILDAPI_BUILDS4H_URL',
@ -79,16 +105,112 @@ def test_ingest_pending_jobs(jm, initial_data,
assert len(stored_obj) == 1
def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, sample_resultset,
test_repository,
mock_buildapi_pending_missing1_url,
def test_ingest_running_jobs(jm, initial_data,
mock_buildapi_running_url,
mock_post_json_data,
mock_log_parser,
mock_get_resultset,
mock_get_remote_content,
activate_responses):
mock_get_remote_content):
"""
Ensure the job with the missing resultset is queued for refetching
a new buildapi running job creates a new obj in the job table
"""
from treeherder.etl.buildapi import RunningJobsProcess
etl_process = RunningJobsProcess()
etl_process.run()
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
jm.disconnect()
assert len(stored_obj) == 1
def test_ingest_builds4h_jobs(jm, initial_data,
mock_buildapi_builds4h_url,
mock_post_json_data,
mock_log_parser,
mock_get_resultset,
mock_get_remote_content):
"""
a new buildapi completed job creates a new obj in the job table
"""
from treeherder.etl.buildapi import Builds4hJobsProcess
etl_process = Builds4hJobsProcess()
etl_process.run()
jm.process_objects(20)
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
jm.disconnect()
assert len(stored_obj) == 20
def test_ingest_running_job_fields(jm, initial_data,
mock_buildapi_running_url,
mock_post_json_data,
mock_log_parser,
mock_get_resultset,
mock_get_remote_content):
"""
a new buildapi running job creates a new obj in the job table
"""
from treeherder.etl.buildapi import RunningJobsProcess
etl_process = RunningJobsProcess()
etl_process.run()
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
jm.disconnect()
assert len(stored_obj) == 1
assert stored_obj[0]["start_timestamp"] is not 0
#####################
# MISSING RESULTSETS
#####################
def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data,
sample_resultset, test_repository, mock_buildapi_pending_missing1_url,
mock_post_json_data, mock_get_resultset, mock_get_remote_content,
activate_responses):
"""
Ensure the pending job with the missing resultset is queued for refetching
"""
from treeherder.etl.buildapi import PendingJobsProcess
etl_process = PendingJobsProcess()
_do_missing_resultset_test(jm, etl_process)
def test_ingest_running_jobs_1_missing_resultset(jm, initial_data,
sample_resultset, test_repository, mock_buildapi_running_missing1_url,
mock_post_json_data, mock_get_resultset, mock_get_remote_content,
activate_responses):
"""
Ensure the running job with the missing resultset is queued for refetching
"""
from treeherder.etl.buildapi import RunningJobsProcess
etl_process = RunningJobsProcess()
_do_missing_resultset_test(jm, etl_process)
def test_ingest_builds4h_jobs_1_missing_resultset(jm, initial_data,
sample_resultset, test_repository, mock_buildapi_builds4h_missing1_url,
mock_post_json_data, mock_get_resultset, mock_get_remote_content,
activate_responses):
"""
Ensure the builds4h job with the missing resultset is queued for refetching
"""
from treeherder.etl.buildapi import Builds4hJobsProcess
etl_process = Builds4hJobsProcess()
_do_missing_resultset_test(jm, etl_process)
def _do_missing_resultset_test(jm, etl_process):
new_revision = '222222222222'
pushlog_content = json.dumps(
{"33270": {
@ -114,9 +236,8 @@ def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, sample_result
match_querystring=True,
content_type='application/json')
from treeherder.etl.buildapi import PendingJobsProcess
etl_process = PendingJobsProcess()
etl_process.run()
jm.process_objects(2)
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
@ -135,48 +256,3 @@ def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, sample_result
was_stored = True
assert was_stored
jm.disconnect()
def test_ingest_running_jobs(jm, initial_data,
mock_buildapi_running_url,
mock_post_json_data,
mock_log_parser,
mock_get_resultset,
mock_get_remote_content):
"""
a new buildapi running job creates a new obj in the job table
"""
from treeherder.etl.buildapi import RunningJobsProcess
etl_process = RunningJobsProcess()
etl_process.run()
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
jm.disconnect()
assert len(stored_obj) == 1
def test_ingest_running_job_fields(jm, initial_data,
mock_buildapi_running_url,
mock_post_json_data,
mock_log_parser,
mock_get_resultset,
mock_get_remote_content):
"""
a new buildapi running job creates a new obj in the job table
"""
from treeherder.etl.buildapi import RunningJobsProcess
etl_process = RunningJobsProcess()
etl_process.run()
stored_obj = jm.get_jobs_dhub().execute(
proc="jobs_test.selects.jobs")
jm.disconnect()
assert len(stored_obj) == 1
assert stored_obj[0]["start_timestamp"] is not 0

144
tests/sample_data/builds-4h-missing1.json Normal file
Просмотреть файл

@ -0,0 +1,144 @@
{
"builds": [
{
"builder_id": 274259,
"buildnumber": 42,
"endtime": 1412776103,
"id": 49799397,
"master_id": 132,
"properties": {
"base_bundle_urls": [
"https://ftp-ssl.mozilla.org/pub/mozilla.org/firefox/bundles"
],
"base_mirror_urls": null,
"basedir": "/builds/slave/b2g_fx-team_emu-kk-d_nu-000000",
"branch": "test_treeherder",
"buildername": "b2g_fx-team_emulator-kk-debug_nonunified",
"buildid": "20141008023001",
"buildnumber": 42,
"builduid": "28d3ea4c5aa144b1b160670dc37d8d39",
"compare_locales_revision": "1fc4e9bc8287",
"gaia_revision": "0bc74ce502672cf0265b24cf3a25d117c3de5e71",
"gecko_revision": "9637293b166a",
"hgurl": "https://hg.mozilla.org/",
"log_url": "http://ftp.mozilla.org/pub/mozilla.org/b2g/tinderbox-builds/fx-team-emulator-kk-debug/1412760601/b2g_fx-team_emulator-kk-debug_nonunified-bm94-build1-build42.txt.gz",
"master": "http://buildbot-master94.srv.releng.use1.mozilla.com:8001/",
"mock_target": null,
"platform": "emulator-kk-debug",
"product": "b2g",
"project": "",
"repo_path": "integration/fx-team",
"repository": "",
"request_ids": [
52163682
],
"request_times": {
"52163682": 1412760601
},
"revision": "45f8637cb9f7",
"scheduler": "b2g_fx-team periodic",
"script_repo_revision": "1b5e35f7a0ec",
"slavename": "bld-linux64-spot-007",
"tools_revision": "2469042323a6",
"tooltool_url_list": [
"http://runtime-binaries.pvt.build.mozilla.org/tooltool"
],
"upload_ssh_key": "ffxbld_dsa",
"upload_ssh_server": "stage.mozilla.org",
"upload_ssh_user": "ffxbld"
},
"reason": "The Nightly scheduler named 'b2g_fx-team periodic' triggered this build",
"request_ids": [
52163682
],
"requesttime": 1412760601,
"result": 0,
"slave_id": 6971,
"starttime": 1412761403
},
{
"builder_id": 202754,
"buildnumber": 76,
"endtime": 1412775820,
"id": 49799157,
"master_id": 106,
"properties": {
"appName": "Firefox",
"appVersion": "35.0a1",
"basedir": "c:/builds/moz2_slave/m-cen-w32-ntly-000000000000000",
"branch": "test_treeherder",
"builddir": "m-cen-w32-ntly-000000000000000",
"buildername": "WINNT 5.2 mozilla-central nightly",
"buildid": "20141008030202",
"buildnumber": 76,
"builduid": "4c3bcb4280a146869a709cd9c588fb6e",
"comments": "",
"completeMarFilename": "firefox-35.0a1.en-US.win32.complete.mar",
"completeMarHash": "8ed585ec103aec121af4f81633b1201994c9d8e9bb797cbdad1323f84f176c8086cb522d801cbb7c8dc8e18b6339fbb80211d9d0a8b9118c3a92969f23f59501",
"completeMarSize": "48921988",
"completeMarUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2014/10/2014-10-08-03-02-02-mozilla-central/firefox-35.0a1.en-US.win32.complete.mar",
"completesnippetFilename": "build/obj-firefox/dist/update/complete.update.snippet",
"filepath": null,
"forced_clobber": false,
"hashType": "sha512",
"installerFilename": "firefox-35.0a1.en-US.win32.installer.exe",
"installerHash": "97d6149ff2649ac3ee63429376d4a0d0cfd6c4017071c16aedbaf1b73a6c181715e4b18779f19c5896de64d84b56cca7eac2f67b1c958b623707115ce9d10e74",
"installerSize": "40424984",
"jsshellUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-win32/1412762522/jsshell-win32.zip",
"log_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2014/10/2014-10-08-03-02-02-mozilla-central/mozilla-central-win32-nightly-bm85-build1-build76.txt.gz",
"master": "http://buildbot-master85.srv.releng.scl3.mozilla.com:8001/",
"nightly_build": true,
"packageFilename": "firefox-35.0a1.en-US.win32.zip",
"packageHash": "9f3f0b59d33b719c9c6b8d029c557e396f120b471db31b523289eacfee7268a2c8f50aca937dd2faf20fa6af7ba562e5d9c0e724bd361648be40e8e39147e19b",
"packageSize": "50571473",
"packageUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-win32/1412762522/firefox-35.0a1.en-US.win32.zip",
"partialInfo": [
{
"from_buildid": "20141007030202",
"hash": "97b74d6163fa92503dfeb4f6f116975b92ca815fa8e3bcedaef97f37868782062d801b5cdad656ff5c2eed08ee0739f4ad946036b92778b0edb3ef3f82beba55",
"size": "4203960",
"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2014/10/2014-10-08-03-02-02-mozilla-central/firefox-35.0a1.en-US.win32.partial.20141007030202-20141008030202.mar"
}
],
"partialMarFilename": "firefox-35.0a1.en-US.win32.partial.20141007030202-20141008030202.mar",
"partialMarHash": "97b74d6163fa92503dfeb4f6f116975b92ca815fa8e3bcedaef97f37868782062d801b5cdad656ff5c2eed08ee0739f4ad946036b92778b0edb3ef3f82beba55",
"partialMarSize": "4203960",
"partialMarUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2014/10/2014-10-08-03-02-02-mozilla-central/firefox-35.0a1.en-US.win32.partial.20141007030202-20141008030202.mar",
"partialsnippetFilename": "build/obj-firefox/dist/update/partial.update.snippet",
"periodic_clobber": false,
"platform": "win32",
"previousMarFilename": "firefox-35.0a1.en-US.win32.complete.mar",
"previous_buildid": "20141007030202",
"previous_inipath": "previous/application.ini",
"product": "firefox",
"project": "",
"purge_actual": "60.69GB",
"purge_target": "12GB",
"purged_clobber": true,
"repository": "",
"request_ids": [
52167028
],
"request_times": {
"52167028": 1412762523
},
"revision": "222222222222902057b6f698a5ca9f78aea25d06",
"scheduler": "mozilla-central nightly",
"slavebuilddir": "m-cen-w32-ntly-000000000000000",
"slavename": "b-2008-ix-0140",
"stage_platform": "win32",
"symbolsUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-win32/1412762522/firefox-35.0a1.en-US.win32.crashreporter-symbols.zip",
"testsUrl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-win32/1412762522/firefox-35.0a1.en-US.win32.tests.zip",
"toolsdir": "c:/builds/moz2_slave/m-cen-w32-ntly-000000000000000/tools"
},
"reason": "The Nightly scheduler named 'mozilla-central nightly' triggered this build",
"request_ids": [
52167028
],
"requesttime": 1412762523,
"result": 0,
"slave_id": 9671,
"starttime": 1412762525
}
]
}

5270
tests/sample_data/builds-4h.js
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1627
tests/sample_data/builds-4h.json Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

0
tests/sample_data/builds-pending-missing1.js → tests/sample_data/builds-pending-missing1.json
Просмотреть файл

0
tests/sample_data/builds-pending.js → tests/sample_data/builds-pending.json
Просмотреть файл

36
tests/sample_data/builds-running-missing1.json Normal file
Просмотреть файл

@ -0,0 +1,36 @@
{
"running": {
"test_treeherder": {
"45f8637cb9f7": [
{
"submitted_at": 1369231311,
"buildername": "WINNT 5.2 profiling build",
"start_time": 1369231311,
"number": 3,
"claimed_by_name": "buildbot-master66.srv.releng.usw2.mozilla.com:/builds/buildbot/build1/master",
"request_ids": [
24526180
],
"last_heartbeat": 1369231939,
"id": 24767134,
"revision": "45f8637cb9f7"
}
],
"222222222222": [
{
"submitted_at": 1369231312,
"buildername": "WINNT 5.2 profiling build",
"start_time": 1369231312,
"number": 3,
"claimed_by_name": "buildbot-master66.srv.releng.usw2.mozilla.com:/builds/buildbot/build1/master",
"request_ids": [
24526181
],
"last_heartbeat": 1369231940,
"id": 24767134,
"revision": "222222222222aa71f463867b15d956a972ae7574"
}
]
}
}
}

0
tests/sample_data/builds-running.js → tests/sample_data/builds-running.json
Просмотреть файл

17
treeherder/etl/buildapi.py
Просмотреть файл

@ -12,7 +12,7 @@ from thclient import TreeherderRequest, TreeherderJobCollection
from treeherder.etl import common, buildbot
from treeherder.etl.mixins import JsonExtractorMixin, OAuthLoaderMixin
from treeherder.model.models import Datasource
from .cleanup_tasks import fetch_missing_push_logs
from treeherder.etl.tasks.cleanup_tasks import fetch_missing_push_logs
logger = logging.getLogger(__name__)
@ -110,8 +110,8 @@ class Builds4hTransformerMixin(object):
continue
prop['revision'] = prop.get('revision',
prop.get('got_revision',
prop.get('sourcestamp', None)))
prop.get('got_revision',
prop.get('sourcestamp', None)))
if not prop['revision']:
logger.warning("property 'revision' not found in build4h")
@ -127,24 +127,26 @@ class Builds4hTransformerMixin(object):
for build in data['builds']:
prop = build['properties']
project = prop['branch']
artifact_build = copy.deepcopy(build)
try:
branch = revisions_lookup[prop['branch']]
branch = revisions_lookup[project]
try:
resultset = branch[prop['revision']]
print "found " + prop['revision']
except KeyError:
# we don't have the resultset for this build/job yet
# we need to queue fetching that resultset
missing_revisions[prop['branch']].append(prop['revision'])
missing_revisions[project].append(prop['revision'])
print "missing " + prop['revision']
continue
except KeyError:
# this branch is not one of those we care about
continue
project = prop['branch']
treeherder_data = {
'revision_hash': resultset['revision_hash'],
'resultset_id': resultset['id'],
@ -385,7 +387,6 @@ class PendingTransformerMixin(object):
}
treeherder_data['job'] = new_job
print project
if project not in th_collections:
th_collections[project] = TreeherderJobCollection(
job_type='update'

136
treeherder/etl/tasks.py
Просмотреть файл

@ -1,136 +0,0 @@
"""
This module contains
"""
import urllib
from celery import task, group
from treeherder.model.derived import RefDataManager
from .buildapi import (RunningJobsProcess,
PendingJobsProcess,
Builds4hJobsProcess,
Builds4hAnalyzer)
from .bugzilla import BzApiBugProcess
from .tbpl import OrangeFactorBugRequest, TbplBugRequest, BugzillaBugRequest
from .pushlog import HgPushlogProcess
@task(name='fetch-buildapi-pending', time_limit=3*60)
def fetch_buildapi_pending():
"""
Fetches the buildapi pending jobs api and load them to
the objectstore ingestion endpoint
"""
PendingJobsProcess().run()
@task(name='fetch-buildapi-running', time_limit=3*60)
def fetch_buildapi_running():
"""
Fetches the buildapi running jobs api and load them to
the objectstore ingestion endpoint
"""
RunningJobsProcess().run()
@task(name='fetch-buildapi-build4h', time_limit=3*60)
def fetch_buildapi_build4h():
"""
Fetches the buildapi running jobs api and load them to
the objectstore ingestion endpoint
"""
Builds4hJobsProcess().run()
@task(name='fetch-push-logs')
def fetch_push_logs():
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg':
fetch_hg_push_log.apply_async(
args=(repo['name'], repo['url']),
routing_key='pushlog'
)
finally:
rdm.disconnect()
@task(name='fetch-hg-push-logs', time_limit=3*60)
def fetch_hg_push_log(repo_name, repo_url):
"""
Run a HgPushlog etl process
"""
process = HgPushlogProcess()
process.run(repo_url + '/json-pushes/?full=1', repo_name)
@task(name='fetch-bugs', time_limit=10 * 60)
def fetch_bugs():
"""
Run a BzApiBug process
"""
process = BzApiBugProcess()
process.run()
@task(name='run-builds4h-analyzer')
def run_builds4h_analyzer():
"""
Run a Builds4h Analysis process
"""
process = Builds4hAnalyzer()
process.run()
@task(name="submit-star-comment", max_retries=10, time_limit=30)
def submit_star_comment(project, job_id, bug_id, submit_timestamp, who):
"""
Send a post request to tbpl's starcomment.php containing a bug association.
starcomment.php proxies then the request to orange factor
"""
try:
req = OrangeFactorBugRequest(project, job_id, bug_id, submit_timestamp, who)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_star_comment.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise
@task(name="submit-build-star", max_retries=10, time_limit=30)
def submit_build_star(project, job_id, who, bug_id=None, classification_id=None, note=None):
"""
Send a post request to tbpl's submitBuildStar.php to mirror sheriff's activity
from treeherder to tbpl. It can be used for both bug association and classification
"""
try:
req = TbplBugRequest(project, job_id, who, bug_id=bug_id, classification_id=classification_id, note=note)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_build_star.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise
@task(name="submit-bug-comment", max_retries=10, time_limit=30)
def submit_bug_comment(project, job_id, bug_id):
"""
Send a post request to tbpl's submitBugzillaComment.php
to add a new comment to the associated bug on bugzilla.
"""
try:
req = BugzillaBugRequest(project, job_id, bug_id)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_bug_comment.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise

4
treeherder/etl/tasks/__init__.py Normal file
Просмотреть файл

@ -0,0 +1,4 @@
from .buildapi_tasks import *
from .cleanup_tasks import *
from .tbpl_tasks import *
from .tasks import *

76
treeherder/etl/tasks/buildapi_tasks.py Normal file
Просмотреть файл

@ -0,0 +1,76 @@
"""
This module contains
"""
from celery import task, group
from treeherder.model.derived import RefDataManager
from treeherder.etl.buildapi import (RunningJobsProcess,
PendingJobsProcess,
Builds4hJobsProcess,
Builds4hAnalyzer)
from treeherder.etl.pushlog import HgPushlogProcess
@task(name='fetch-buildapi-pending', time_limit=3*60)
def fetch_buildapi_pending():
"""
Fetches the buildapi pending jobs api and load them to
the objectstore ingestion endpoint
"""
PendingJobsProcess().run()
@task(name='fetch-buildapi-running', time_limit=3*60)
def fetch_buildapi_running():
"""
Fetches the buildapi running jobs api and load them to
the objectstore ingestion endpoint
"""
RunningJobsProcess().run()
@task(name='fetch-buildapi-build4h', time_limit=3*60)
def fetch_buildapi_build4h():
"""
Fetches the buildapi running jobs api and load them to
the objectstore ingestion endpoint
"""
Builds4hJobsProcess().run()
@task(name='fetch-push-logs')
def fetch_push_logs():
"""
Run several fetch_hg_push_log subtasks, one per repository
"""
rdm = RefDataManager()
try:
repos = filter(lambda x: x['url'], rdm.get_all_repository_info())
for repo in repos:
if repo['dvcs_type'] == 'hg':
fetch_hg_push_log.apply_async(
args=(repo['name'], repo['url']),
routing_key='pushlog'
)
finally:
rdm.disconnect()
@task(name='fetch-hg-push-logs', time_limit=3*60)
def fetch_hg_push_log(repo_name, repo_url):
"""
Run a HgPushlog etl process
"""
process = HgPushlogProcess()
process.run(repo_url + '/json-pushes/?full=1', repo_name)
@task(name='run-builds4h-analyzer')
def run_builds4h_analyzer():
"""
Run a Builds4h Analysis process
"""
process = Builds4hAnalyzer()
process.run()

2
treeherder/etl/cleanup_tasks.py → treeherder/etl/tasks/cleanup_tasks.py
Просмотреть файл

@ -1,7 +1,7 @@
import urllib
from celery import task, group
from treeherder.model.derived import RefDataManager
from .pushlog import MissingHgPushlogProcess
from treeherder.etl.pushlog import MissingHgPushlogProcess
@task(name='fetch-missing-push-logs')

14
treeherder/etl/tasks/tasks.py Normal file
Просмотреть файл

@ -0,0 +1,14 @@
"""
This module contains
"""
from celery import task, group
from treeherder.etl.bugzilla import BzApiBugProcess
@task(name='fetch-bugs', time_limit=10 * 60)
def fetch_bugs():
"""
Run a BzApiBug process
"""
process = BzApiBugProcess()
process.run()

56
treeherder/etl/tasks/tbpl_tasks.py Normal file
Просмотреть файл

@ -0,0 +1,56 @@
"""
This module contains
"""
from celery import task, group
from treeherder.etl.tbpl import OrangeFactorBugRequest, TbplBugRequest, BugzillaBugRequest
@task(name="submit-star-comment", max_retries=10, time_limit=30)
def submit_star_comment(project, job_id, bug_id, submit_timestamp, who):
"""
Send a post request to tbpl's starcomment.php containing a bug association.
starcomment.php proxies then the request to orange factor
"""
try:
req = OrangeFactorBugRequest(project, job_id, bug_id, submit_timestamp, who)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_star_comment.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise
@task(name="submit-build-star", max_retries=10, time_limit=30)
def submit_build_star(project, job_id, who, bug_id=None, classification_id=None, note=None):
"""
Send a post request to tbpl's submitBuildStar.php to mirror sheriff's activity
from treeherder to tbpl. It can be used for both bug association and classification
"""
try:
req = TbplBugRequest(project, job_id, who, bug_id=bug_id, classification_id=classification_id, note=note)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_build_star.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise
@task(name="submit-bug-comment", max_retries=10, time_limit=30)
def submit_bug_comment(project, job_id, bug_id):
"""
Send a post request to tbpl's submitBugzillaComment.php
to add a new comment to the associated bug on bugzilla.
"""
try:
req = BugzillaBugRequest(project, job_id, bug_id)
req.generate_request_body()
req.send_request()
except Exception, e:
submit_bug_comment.retry(exc=e)
# this exception will be raised once the number of retries
# exceeds max_retries
raise