diff --git a/taskcluster/gecko_taskgraph/generator.py b/taskcluster/gecko_taskgraph/generator.py index 8ae6ed79c7e4..3ccb91ee2e4d 100644 --- a/taskcluster/gecko_taskgraph/generator.py +++ b/taskcluster/gecko_taskgraph/generator.py @@ -358,7 +358,20 @@ class TaskGraphGenerator: for t in full_task_graph.tasks.values() if t.attributes["kind"] == "docker-image" } - requested_tasks = target_tasks | docker_image_tasks + # include all tasks with `always_target` set + if parameters["tasks_for"] == "hg-push": + always_target_tasks = { + t.label + for t in full_task_graph.tasks.values() + if t.attributes.get("always_target") + } + else: + always_target_tasks = set() + logger.info( + "Adding %d tasks with `always_target` attribute" + % (len(always_target_tasks) - len(always_target_tasks & target_tasks)) + ) + requested_tasks = target_tasks | docker_image_tasks | always_target_tasks target_graph = full_task_graph.graph.transitive_closure(requested_tasks) target_task_graph = TaskGraph( {l: all_tasks[l] for l in target_graph.nodes}, target_graph diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index 30ee77411e0c..1da1a9df91db 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -4,10 +4,9 @@ import copy -import logging +from datetime import datetime, timedelta import os import re -from datetime import datetime, timedelta from redo import retry from taskgraph.parameters import Parameters @@ -22,34 +21,6 @@ from gecko_taskgraph.util.attributes import ( from gecko_taskgraph.util.platforms import platform_family from gecko_taskgraph.util.hg import find_hg_revision_push_info, get_hg_commit_message -logger = logging.getLogger(__name__) - - -def register_target_task(name, enable_always_target=True): - """Wrapper around `taskgraph._target_task` which supports the `always_target` attribute.""" - - def wrap(func): - def always_target_wrapper(full_task_graph, parameters, graph_config): - target_tasks = func(full_task_graph, parameters, graph_config) - if not enable_always_target or parameters["tasks_for"] != "hg-push": - return target_tasks - - target_tasks = set(target_tasks) - always_target_tasks = { - t.label - for t in full_task_graph.tasks.values() - if t.attributes.get("always_target") - } - logger.info( - "Adding %d tasks with `always_target` attribute" - % (len(always_target_tasks) - len(always_target_tasks & target_tasks)) - ) - return list(target_tasks | always_target_tasks) - - return _target_task(name)(always_target_wrapper) - - return wrap - # Some tasks show up in the target task set, but are possibly special cases, # uncommon tasks, or tasks running against limited hardware set that they @@ -362,7 +333,7 @@ def _try_option_syntax(full_task_graph, parameters, graph_config): return target_tasks_labels -@register_target_task("try_tasks") +@_target_task("try_tasks") def target_tasks_try(full_task_graph, parameters, graph_config): try_mode = parameters["try_mode"] if try_mode == "try_task_config": @@ -375,13 +346,13 @@ def target_tasks_try(full_task_graph, parameters, graph_config): return [] -@register_target_task("try_select_tasks") +@_target_task("try_select_tasks") def target_tasks_try_select(full_task_graph, parameters, graph_config): tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config) return [l for l in tasks if filter_by_uncommon_try_tasks(l)] -@register_target_task("try_select_tasks_uncommon") +@_target_task("try_select_tasks_uncommon") def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS @@ -407,7 +378,7 @@ def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): return sorted(tasks) -@register_target_task("try_auto") +@_target_task("try_auto") def target_tasks_try_auto(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on autoland (rather than try) via the `run_on_projects` attributes. @@ -438,7 +409,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config): ] -@register_target_task("default") +@_target_task("default") def target_tasks_default(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on this project via the `run_on_projects` attributes.""" @@ -451,7 +422,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config): ] -@register_target_task("autoland_tasks") +@_target_task("autoland_tasks") def target_tasks_autoland(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against shippable builds @@ -477,7 +448,7 @@ def target_tasks_autoland(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@register_target_task("mozilla_central_tasks") +@_target_task("mozilla_central_tasks") def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against regular (aka not shippable, @@ -517,7 +488,7 @@ def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@register_target_task("graphics_tasks") +@_target_task("graphics_tasks") def target_tasks_graphics(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove artifact builds because we have csets on @@ -535,7 +506,7 @@ def target_tasks_graphics(full_task_graph, parameters, graph_config): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@register_target_task("mozilla_beta_tasks") +@_target_task("mozilla_beta_tasks") def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -548,7 +519,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): ] -@register_target_task("mozilla_release_tasks") +@_target_task("mozilla_release_tasks") def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -561,7 +532,7 @@ def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): ] -@register_target_task("mozilla_esr102_tasks") +@_target_task("mozilla_esr102_tasks") def target_tasks_mozilla_esr102(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, without android CI. The candidates build process involves a pipeline @@ -585,7 +556,7 @@ def target_tasks_mozilla_esr102(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("promote_desktop") +@_target_task("promote_desktop") def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): """Select the superset of tasks required to promote a beta or release build of a desktop product. This should include all non-android @@ -623,7 +594,7 @@ def is_geckoview(task, parameters): ) -@register_target_task("push_desktop") +@_target_task("push_desktop") def target_tasks_push_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to push a build of desktop to cdns. Previous build deps will be optimized out via action task.""" @@ -649,7 +620,7 @@ def target_tasks_push_desktop(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("ship_desktop") +@_target_task("ship_desktop") def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to ship desktop. Previous build deps will be optimized out via action task.""" @@ -691,7 +662,7 @@ def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("pine_tasks") +@_target_task("pine_tasks") def target_tasks_pine(full_task_graph, parameters, graph_config): """Bug 1339179 - no mobile automation needed on pine""" @@ -712,7 +683,7 @@ def target_tasks_pine(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("kaios_tasks") +@_target_task("kaios_tasks") def target_tasks_kaios(full_task_graph, parameters, graph_config): """The set of tasks to run for kaios integration""" @@ -723,7 +694,7 @@ def target_tasks_kaios(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("ship_geckoview") +@_target_task("ship_geckoview") def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config): """Select the set of tasks required to ship geckoview nightly. The nightly build process involves a pipeline of builds and an upload to @@ -752,7 +723,7 @@ def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("general_perf_testing") +@_target_task("general_perf_testing") def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): """ Select tasks required for running performance tests 3 times a week. @@ -855,7 +826,7 @@ def make_desktop_nightly_filter(platforms): return filter -@register_target_task("nightly_linux") +@_target_task("nightly_linux") def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux. The nightly build process involves a pipeline of builds, signing, @@ -864,7 +835,7 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("nightly_macosx") +@_target_task("nightly_macosx") def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of macosx. The nightly build process involves a pipeline of builds, signing, @@ -873,7 +844,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("nightly_win32") +@_target_task("nightly_win32") def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -882,7 +853,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("nightly_win64") +@_target_task("nightly_win64") def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -891,7 +862,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("nightly_win64_aarch64") +@_target_task("nightly_win64_aarch64") def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -900,7 +871,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("nightly_asan") +@_target_task("nightly_asan") def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of asan. The nightly build process involves a pipeline of builds, signing, @@ -911,7 +882,7 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@register_target_task("daily_releases") +@_target_task("daily_releases") def target_tasks_daily_releases(full_task_graph, parameters, graph_config): """Select the set of tasks required to identify if we should release. If we determine that we should the task will communicate to ship-it to @@ -923,7 +894,7 @@ def target_tasks_daily_releases(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("nightly_desktop") +@_target_task("nightly_desktop") def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux, mac, windows.""" @@ -962,7 +933,7 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): # Run Searchfox analysis once daily. -@register_target_task("searchfox_index") +@_target_task("searchfox_index") def target_tasks_searchfox(full_task_graph, parameters, graph_config): """Select tasks required for indexing Firefox for Searchfox web site each day""" return [ @@ -977,7 +948,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config): # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests -@register_target_task("linux64_clang_trunk_perf") +@_target_task("linux64_clang_trunk_perf") def target_tasks_build_linux64_clang_trunk_perf( full_task_graph, parameters, graph_config ): @@ -993,19 +964,19 @@ def target_tasks_build_linux64_clang_trunk_perf( # Run Updatebot's cron job 4 times daily. -@register_target_task("updatebot_cron") +@_target_task("updatebot_cron") def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config): """Select tasks required to run Updatebot's cron job""" return ["updatebot-cron"] -@register_target_task("customv8_update") +@_target_task("customv8_update") def target_tasks_customv8_update(full_task_graph, parameters, graph_config): """Select tasks required for building latest d8/v8 version.""" return ["toolchain-linux64-custom-v8"] -@register_target_task("chromium_update") +@_target_task("chromium_update") def target_tasks_chromium_update(full_task_graph, parameters, graph_config): """Select tasks required for building latest chromium versions.""" return [ @@ -1016,7 +987,7 @@ def target_tasks_chromium_update(full_task_graph, parameters, graph_config): ] -@register_target_task("file_update") +@_target_task("file_update") def target_tasks_file_update(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform nightly in-tree file updates""" @@ -1027,7 +998,7 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("l10n_bump") +@_target_task("l10n_bump") def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform l10n bumping.""" @@ -1038,7 +1009,7 @@ def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("merge_automation") +@_target_task("merge_automation") def target_tasks_merge_automation(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform repository merges.""" @@ -1049,7 +1020,7 @@ def target_tasks_merge_automation(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("scriptworker_canary") +@_target_task("scriptworker_canary") def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): """Select the set of tasks required to run scriptworker canaries.""" @@ -1060,7 +1031,7 @@ def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("cron_bouncer_check") +@_target_task("cron_bouncer_check") def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform bouncer version verification.""" @@ -1073,7 +1044,7 @@ def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("staging_release_builds") +@_target_task("staging_release_builds") def target_tasks_staging_release(full_task_graph, parameters, graph_config): """ Select all builds that are part of releases. @@ -1097,7 +1068,7 @@ def target_tasks_staging_release(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("release_simulation") +@_target_task("release_simulation") def target_tasks_release_simulation(full_task_graph, parameters, graph_config): """ Select builds that would run on push on a release branch. @@ -1134,7 +1105,7 @@ def target_tasks_release_simulation(full_task_graph, parameters, graph_config): ] -@register_target_task("codereview") +@_target_task("codereview") def target_tasks_codereview(full_task_graph, parameters, graph_config): """Select all code review tasks needed to produce a report""" @@ -1152,13 +1123,13 @@ def target_tasks_codereview(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("nothing") +@_target_task("nothing") def target_tasks_nothing(full_task_graph, parameters, graph_config): """Select nothing, for DONTBUILD pushes""" return [] -@register_target_task("daily_beta_perf") +@_target_task("daily_beta_perf") def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the beta branch to be run daily @@ -1209,7 +1180,7 @@ def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("weekly_release_perf") +@_target_task("weekly_release_perf") def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the release branch to be run weekly @@ -1250,7 +1221,7 @@ def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("raptor_tp6m") +@_target_task("raptor_tp6m") def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): """ Select tasks required for running raptor cold page-load tests on fenix and refbrow @@ -1278,7 +1249,7 @@ def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("backfill_all_browsertime") +@_target_task("backfill_all_browsertime") def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config): """ Search for revisions that contains patches that were reviewed by perftest reviewers @@ -1349,7 +1320,7 @@ def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_con return [] -@register_target_task("condprof") +@_target_task("condprof") def target_tasks_condprof(full_task_graph, parameters, graph_config): """ Select tasks required for building conditioned profiles. @@ -1360,7 +1331,7 @@ def target_tasks_condprof(full_task_graph, parameters, graph_config): yield name -@register_target_task("system_symbols") +@_target_task("system_symbols") def target_tasks_system_symbols(full_task_graph, parameters, graph_config): """ Select tasks for scraping and uploading system symbols. @@ -1374,7 +1345,7 @@ def target_tasks_system_symbols(full_task_graph, parameters, graph_config): yield name -@register_target_task("perftest") +@_target_task("perftest") def target_tasks_perftest(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1386,7 +1357,7 @@ def target_tasks_perftest(full_task_graph, parameters, graph_config): yield name -@register_target_task("perftest-on-autoland") +@_target_task("perftest-on-autoland") def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1400,7 +1371,7 @@ def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): yield name -@register_target_task("l10n-cross-channel") +@_target_task("l10n-cross-channel") def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): """Select the set of tasks required to run l10n cross-channel.""" @@ -1410,7 +1381,7 @@ def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@register_target_task("are-we-esmified-yet") +@_target_task("are-we-esmified-yet") def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config): """ select the task to track the progress of the esmification project diff --git a/taskcluster/gecko_taskgraph/test/test_generator.py b/taskcluster/gecko_taskgraph/test/test_generator.py index e74860e2b87f..35c62e2479c1 100644 --- a/taskcluster/gecko_taskgraph/test/test_generator.py +++ b/taskcluster/gecko_taskgraph/test/test_generator.py @@ -74,6 +74,36 @@ def test_target_task_graph(maketgg): ) +def test_always_target_tasks(maketgg): + "The target_task_graph includes tasks with 'always_target'" + tgg_args = { + "target_tasks": ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"], + "kinds": [ + ("_fake", {"job-defaults": {"optimization": {"odd": None}}}), + ( + "_ignore", + { + "job-defaults": { + "attributes": {"always_target": True}, + "optimization": {"even": None}, + } + }, + ), + ], + "params": {"optimize_target_tasks": False}, + } + tgg = maketgg(**tgg_args) + assert sorted(tgg.target_task_set.tasks.keys()) == sorted( + ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"] + ) + assert sorted(tgg.target_task_graph.tasks.keys()) == sorted( + ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1", "_ignore-t-2"] + ) + assert sorted(t.label for t in tgg.optimized_task_graph.tasks.values()) == sorted( + ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"] + ) + + def test_optimized_task_graph(maketgg): "The optimized task graph contains task ids" tgg = maketgg(["_fake-t-2"]) diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py index a9edb3aeca3a..80fa380a907c 100644 --- a/taskcluster/gecko_taskgraph/test/test_target_tasks.py +++ b/taskcluster/gecko_taskgraph/test/test_target_tasks.py @@ -41,7 +41,6 @@ class TestTargetTasks(unittest.TestCase): parameters={ "project": project, "hg_branch": "default", - "tasks_for": "hg-push", }, ) @@ -55,7 +54,6 @@ class TestTargetTasks(unittest.TestCase): parameters={ "project": "mozilla-central", "hg_branch": hg_branch, - "tasks_for": "hg-push", }, ) @@ -156,7 +154,6 @@ class TestTargetTasks(unittest.TestCase): "try_mode": None, "project": "try", "message": "", - "tasks_for": "hg-push", } # only runs the task with run_on_projects: try self.assertEqual(method(tg, params, {}), []) @@ -169,7 +166,6 @@ class TestTargetTasks(unittest.TestCase): params = { "try_mode": "try_option_syntax", "message": "try: -p all", - "tasks_for": "hg-push", } self.assertEqual(method(tg, params, {}), ["b"]) @@ -180,21 +176,9 @@ class TestTargetTasks(unittest.TestCase): params = { "try_mode": "try_task_config", "try_task_config": {"tasks": ["a"]}, - "tasks_for": "hg-push", } self.assertEqual(method(tg, params, {}), ["a"]) - def test_always_target(self): - "The target_task_graph includes tasks with 'always_target'" - tg = self.make_task_graph() - tg.tasks["a"].attributes["always_target"] = True - method = target_tasks.get_method("nothing") - params = {"tasks_for": "hg-push"} - self.assertEqual(method(tg, params, {}), ["a"]) - - params = {"tasks_for": "release"} - self.assertEqual(method(tg, params, {}), []) - # tests for specific filters