From 1f286055e23b769ec6cdb185741324f5e1cb40bf Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 28 Mar 2018 11:34:20 -0700 Subject: [PATCH] Bug 1447460: Teach `mach taskgraph cron` to handle configuration in sub-repositories; r=dustin Differential Revision: https://phabricator.services.mozilla.com/D815 --HG-- extra : rebase_source : f0153c78bf4ffe0dbf520e0fb4d0683c82173f8b --- taskcluster/mach_commands.py | 3 +++ taskcluster/taskgraph/cron/__init__.py | 27 +++++++++++++------------- taskcluster/taskgraph/cron/decision.py | 11 ++++++----- taskcluster/taskgraph/cron/util.py | 4 ++-- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py index 12e2e179e421..630fb168fe16 100644 --- a/taskcluster/mach_commands.py +++ b/taskcluster/mach_commands.py @@ -196,6 +196,9 @@ class MachCommands(MachCommandBase): required=False, action='store_true', help='Do not actually create tasks') + @CommandArgument('--root', '-r', + required=False, + help="root of the repository to get cron task definitions from") def taskgraph_cron(self, **options): """Run the cron task; this task creates zero or more decision tasks. It is run from the hooks service on a regular basis.""" diff --git a/taskcluster/taskgraph/cron/__init__.py b/taskcluster/taskgraph/cron/__init__.py index e4e31dd29213..3c0e9e99f0ea 100644 --- a/taskcluster/taskgraph/cron/__init__.py +++ b/taskcluster/taskgraph/cron/__init__.py @@ -36,8 +36,8 @@ JOB_TYPES = { logger = logging.getLogger(__name__) -def load_jobs(params): - with open(os.path.join(GECKO, '.cron.yml'), 'rb') as f: +def load_jobs(params, root): + with open(os.path.join(root, '.cron.yml'), 'rb') as f: cron_yml = yaml.load(f) schema.validate(cron_yml) @@ -61,13 +61,14 @@ def should_run(job, params): return True -def run_job(job_name, job, params): +def run_job(job_name, job, params, root): + params = params.copy() params['job_name'] = job_name try: job_type = job['job']['type'] if job_type in JOB_TYPES: - tasks = JOB_TYPES[job_type](job['job'], params) + tasks = JOB_TYPES[job_type](job['job'], params, root=root) else: raise Exception("job type {} not recognized".format(job_type)) if params['no_create']: @@ -76,7 +77,7 @@ def run_job(job_name, job, params): json.dumps(task, sort_keys=True, indent=4, separators=(',', ': '))) else: for task_id, task in tasks: - create_task(get_session(), task_id, params['job_name'], task) + create_task(get_session(), task_id, job_name, task) except Exception: # report the exception, but don't fail the whole cron task, as that @@ -95,6 +96,7 @@ def calculate_time(options): logger.warning("setting params['time'] based on $CRON_TIME") time = datetime.datetime.utcfromtimestamp( int(os.environ['CRON_TIME'])) + print(time) else: logger.warning("using current time for params['time']; try setting $CRON_TIME " "to a timestamp") @@ -122,16 +124,15 @@ def calculate_time(options): def taskgraph_cron(options): - params = { - # name of this cron job (set per job below) - 'job_name': '..', + root = options.get('root') or GECKO + params = { # repositories - 'head_repository': options['head_repository'], + 'repository_url': options['head_repository'], # *calculated* head_rev; this is based on the current meaning of this # reference in the working copy - 'head_rev': calculate_head_rev(options), + 'head_rev': calculate_head_rev(root), # the project (short name for the repository) and its SCM level 'project': options['project'], @@ -144,17 +145,17 @@ def taskgraph_cron(options): 'time': calculate_time(options), } - jobs = load_jobs(params) + jobs = load_jobs(params, root=root) if options['force_run']: job_name = options['force_run'] logger.info("force-running cron job {}".format(job_name)) - run_job(job_name, jobs[job_name], params) + run_job(job_name, jobs[job_name], params, root) return for job_name, job in sorted(jobs.items()): if should_run(job, params): logger.info("running cron job {}".format(job_name)) - run_job(job_name, job, params) + run_job(job_name, job, params, root) else: logger.info("not running cron job {}".format(job_name)) diff --git a/taskcluster/taskgraph/cron/decision.py b/taskcluster/taskgraph/cron/decision.py index 3652e36fef1b..0f0d0909bcee 100644 --- a/taskcluster/taskgraph/cron/decision.py +++ b/taskcluster/taskgraph/cron/decision.py @@ -16,7 +16,7 @@ import slugid from taskgraph.util.time import current_json_time -def run_decision_task(job, params): +def run_decision_task(job, params, root): arguments = [] if 'target-tasks-method' in job: arguments.append('--target-tasks-method={}'.format(job['target-tasks-method'])) @@ -24,13 +24,14 @@ def run_decision_task(job, params): make_decision_task( params, symbol=job['treeherder-symbol'], - arguments=arguments), + arguments=arguments, + root=root), ] -def make_decision_task(params, symbol, arguments=[], head_rev=None): +def make_decision_task(params, root, symbol, arguments=[], head_rev=None): """Generate a basic decision task, based on the root .taskcluster.yml""" - with open('.taskcluster.yml') as f: + with open(os.path.join(root, '.taskcluster.yml'), 'rb') as f: taskcluster_yml = yaml.load(f) if not head_rev: @@ -51,7 +52,7 @@ def make_decision_task(params, symbol, arguments=[], head_rev=None): context = { 'tasks_for': 'cron', 'repository': { - 'url': params['head_repository'], + 'url': params['repository_url'], 'project': params['project'], 'level': params['level'], }, diff --git a/taskcluster/taskgraph/cron/util.py b/taskcluster/taskgraph/cron/util.py index 70d2d60d9175..79b7eb568ac3 100644 --- a/taskcluster/taskgraph/cron/util.py +++ b/taskcluster/taskgraph/cron/util.py @@ -25,8 +25,8 @@ def match_utc(params, hour=None, minute=None): return True -def calculate_head_rev(options): +def calculate_head_rev(root): # we assume that run-task has correctly checked out the revision indicated by # GECKO_HEAD_REF, so all that remains is to see what the current revision is. # Mercurial refers to that as `.`. - return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}']) + return subprocess.check_output(['hg', 'log', '-r', '.', '-T', '{node}'], cwd=root)