From 3f73a3103ec8fe4b332d388c8db2d562302b6a80 Mon Sep 17 00:00:00 2001 From: ionutgoldan Date: Tue, 14 Apr 2020 13:02:44 +0300 Subject: [PATCH] Bug 1612547 - Define component for doing backfills using existing reports --- requirements/common.txt | 3 +- tests/perfalert/conftest.py | 11 + tests/perfalert/test_backfill_tool.py | 14 + .../perf_sheriff_bot/backfilltask.json | 77 ++ .../perf_sheriff_bot/initialActions.json | 1032 +++++++++++++++++ .../perf_sheriff_bot/matchingTagSetList.json | 8 + .../perf_sheriff_bot/matchingTaskTags.json | 9 + .../mismatchingTagSetList.json | 11 + .../perf_sheriff_bot/mismatchingTaskTags.json | 9 + .../perf_sheriff_bot/originalTask.json | 127 ++ .../perf_sheriff_bot/reducedActions.json | 355 ++++++ tests/services/test_taskcluster.py | 60 + tests/utils/test_taskcluster_lib_scopes.py | 87 ++ treeherder/model/models.py | 10 +- treeherder/perf/backfill_tool.py | 46 + treeherder/perf/exceptions.py | 4 + .../management/commands/backfill_perf_jobs.py | 40 + treeherder/services/taskcluster.py | 156 +++ treeherder/utils/taskcluster_lib_scopes.py | 30 + 19 files changed, 2087 insertions(+), 2 deletions(-) create mode 100644 tests/perfalert/conftest.py create mode 100644 tests/perfalert/test_backfill_tool.py create mode 100644 tests/sample_data/perf_sheriff_bot/backfilltask.json create mode 100644 tests/sample_data/perf_sheriff_bot/initialActions.json create mode 100644 tests/sample_data/perf_sheriff_bot/matchingTagSetList.json create mode 100644 tests/sample_data/perf_sheriff_bot/matchingTaskTags.json create mode 100644 tests/sample_data/perf_sheriff_bot/mismatchingTagSetList.json create mode 100644 tests/sample_data/perf_sheriff_bot/mismatchingTaskTags.json create mode 100644 tests/sample_data/perf_sheriff_bot/originalTask.json create mode 100644 tests/sample_data/perf_sheriff_bot/reducedActions.json create mode 100644 tests/services/test_taskcluster.py create mode 100644 tests/utils/test_taskcluster_lib_scopes.py create mode 100644 treeherder/perf/backfill_tool.py create mode 100644 treeherder/perf/management/commands/backfill_perf_jobs.py create mode 100644 treeherder/services/taskcluster.py create mode 100644 treeherder/utils/taskcluster_lib_scopes.py diff --git a/requirements/common.txt b/requirements/common.txt index b95cbe4e0..1f1bcde3a 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -512,7 +512,8 @@ zipp==3.1.0 \ --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \ --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \ # via importlib-metadata - +json-e==4.0.0 \ + --hash=sha256:c1add008dbfbb4b7c48d30b42e11e26db4883394989e013b603a63b6c59b8530 # WARNING: The following packages were not pinned, but pip requires them to be # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag. # setuptools diff --git a/tests/perfalert/conftest.py b/tests/perfalert/conftest.py new file mode 100644 index 000000000..34d89b914 --- /dev/null +++ b/tests/perfalert/conftest.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.fixture +def job_from_try(eleven_job_blobs, create_jobs): + job_blob = eleven_job_blobs[0] + job = create_jobs([job_blob])[0] + + job.repository.is_try_repo = True + job.repository.save() + return job diff --git a/tests/perfalert/test_backfill_tool.py b/tests/perfalert/test_backfill_tool.py new file mode 100644 index 000000000..397e6cbed --- /dev/null +++ b/tests/perfalert/test_backfill_tool.py @@ -0,0 +1,14 @@ +import pytest + +from treeherder.perf.backfill_tool import BackfillTool +from treeherder.perf.exceptions import CannotBackfill +from treeherder.services.taskcluster import TaskclusterModel + + +# BackfillTool +def test_backfilling_job_from_try_repo_raises_exception(job_from_try): + backfill_tool = BackfillTool( + TaskclusterModel('https://fakerooturl.org', 'FAKE_CLIENT_ID', 'FAKE_ACCESS_TOKEN')) + + with pytest.raises(CannotBackfill): + backfill_tool.backfill_job(job_from_try.id) diff --git a/tests/sample_data/perf_sheriff_bot/backfilltask.json b/tests/sample_data/perf_sheriff_bot/backfilltask.json new file mode 100644 index 000000000..f7322eb0b --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/backfilltask.json @@ -0,0 +1,77 @@ +{ + "context": [{}], + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "backfill", + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "name": "backfill", + "symbol": "Bk", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Backfill" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "backfill", + "schema": { + "additionalProperties": false, + "properties": { + "depth": { + "default": 9, + "description": "The number of previous pushes before the current push to attempt to trigger this task on.", + "maximum": 25, + "minimum": 1, + "title": "Depth", + "type": "integer" + }, + "inclusive": { + "default": false, + "description": "If true, the backfill will also retrigger the task on the selected push.", + "title": "Inclusive Range", + "type": "boolean" + }, + "testPath": { + "title": "Test Path", + "type": "string" + }, + "times": { + "default": 1, + "description": "The number of times to execute each job you are backfilling.", + "maximum": 10, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Backfill" +} diff --git a/tests/sample_data/perf_sheriff_bot/initialActions.json b/tests/sample_data/perf_sheriff_bot/initialActions.json new file mode 100644 index 000000000..0bda88149 --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/initialActions.json @@ -0,0 +1,1032 @@ +{ + "actions": [ + { + "context": [ + { + "test-type": "mochitest" + }, + { + "test-type": "reftest" + } + ], + "description": "Retriggers the specified task with custom environment and parameters", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-custom", + "description": "Retriggers the specified task with custom environment and parameters", + "name": "retrigger-custom", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger task with custom parameters" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger-custom", + "schema": { + "additionalProperties": false, + "properties": { + "environment": { + "additionalProperties": { + "type": "string" + }, + "default": { + "MOZ_LOG": "" + }, + "description": "Extra environment variables to use for this run", + "title": "Extra environment variables", + "type": "object" + }, + "logLevel": { + "default": "debug", + "description": "Log level for output (default is DEBUG, which is highest)", + "enum": ["debug", "info", "warning", "error", "critical"], + "title": "Log level", + "type": "string" + }, + "path": { + "default": "", + "description": "Path of test to retrigger", + "maxLength": 255, + "title": "Path name", + "type": "string" + }, + "preferences": { + "additionalProperties": { + "type": "string" + }, + "default": { + "mygeckopreferences.pref": "myvalue2" + }, + "description": "Extra gecko (about:config) preferences to use for this run", + "title": "Extra gecko (about:config) preferences", + "type": "object" + }, + "repeat": { + "default": 30, + "description": "Run tests repeatedly (usually used in conjunction with runUntilFail)", + "minimum": 1, + "title": "Run tests N times", + "type": "integer" + }, + "runUntilFail": { + "default": true, + "description": "Runs the specified set of tests repeatedly until failure (or 30 times)", + "title": "Run until failure", + "type": "boolean" + } + }, + "required": ["path"], + "type": "object" + }, + "title": "Retrigger task with custom parameters" + }, + { + "context": [ + { + "kind": "decision-task" + }, + { + "kind": "action-callback" + }, + { + "kind": "cron-task" + } + ], + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-decision", + "description": "Create a clone of the task (retriggering decision, action, and cron tasks requires\nspecial scopes).", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "title": "Retrigger" + }, + { + "context": [ + { + "retrigger": "true" + } + ], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger", + "description": "Create a clone of the task.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger" + }, + { + "context": [{}], + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-disabled", + "description": "Create a clone of the task.\n\nThis type of task should typically be re-run instead of re-triggered.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger (disabled)" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "force": { + "default": false, + "description": "This task should not be re-triggered. This can be overridden by passing `true` here.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger (disabled)" + }, + { + "context": [ + { + "kind": "test", + "worker-implementation": "docker-worker" + } + ], + "description": "Create a a copy of the task that you can interact with", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "create-interactive", + "description": "Create a a copy of the task that you can interact with", + "name": "create-interactive", + "symbol": "create-inter", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Create Interactive Task" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "create-interactive", + "schema": { + "additionalProperties": false, + "properties": { + "notify": { + "default": "noreply@noreply.mozilla.org", + "description": "Enter your email here to get an email containing a link to interact with the task", + "format": "email", + "title": "Who to notify of the pending interactive task", + "type": "string" + } + }, + "type": "object" + }, + "title": "Create Interactive Task" + }, + { + "context": [], + "description": "Add new jobs using task labels.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "add-new-jobs", + "description": "Add new jobs using task labels.", + "name": "add-new-jobs", + "symbol": "add-new", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Add new jobs" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "add-new-jobs", + "schema": { + "properties": { + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Add new jobs" + }, + { + "context": [], + "description": "Add all Talos tasks to a push.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "run-all-talos", + "description": "Add all Talos tasks to a push.", + "name": "run-all-talos", + "symbol": "raT", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Run All Talos Tests" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "run-all-talos", + "schema": { + "additionalProperties": false, + "properties": { + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 6, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Run All Talos Tests" + }, + { + "context": [ + { + "kind": "test" + } + ], + "description": "Re-run Tests for original manifest, directories and tests for failing tests.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "isolate-test-failures", + "description": "Re-run Tests for original manifest, directories and tests for failing tests.", + "name": "isolate-test-failures", + "symbol": "it", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Isolate test failures in job" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "isolate-test-failures", + "schema": { + "additionalProperties": false, + "properties": { + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Isolate test failures in job" + }, + { + "context": [{}], + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "backfill", + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "name": "backfill", + "symbol": "Bk", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Backfill" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "backfill", + "schema": { + "additionalProperties": false, + "properties": { + "depth": { + "default": 9, + "description": "The number of previous pushes before the current push to attempt to trigger this task on.", + "maximum": 25, + "minimum": 1, + "title": "Depth", + "type": "integer" + }, + "inclusive": { + "default": false, + "description": "If true, the backfill will also retrigger the task on the selected push.", + "title": "Inclusive Range", + "type": "boolean" + }, + "testPath": { + "title": "Test Path", + "type": "string" + }, + "times": { + "default": 1, + "description": "The number of times to execute each job you are backfilling.", + "maximum": 10, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Backfill" + }, + { + "context": [ + { + "test-type": "talos" + }, + { + "test-type": "raptor" + } + ], + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project while adding the --geckoProfile cmd arg.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "geckoprofile", + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project while adding the --geckoProfile cmd arg.", + "name": "geckoprofile", + "symbol": "Gp", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "GeckoProfile" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "geckoprofile", + "title": "GeckoProfile" + }, + { + "context": [], + "description": "Run tests in the selected push that were optimized away, usually by SETA.\nThis action is for use on pushes that will be merged into another branch,to check that optimization hasn't hidden any failures.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "run-missing-tests", + "description": "Run tests in the selected push that were optimized away, usually by SETA.\nThis action is for use on pushes that will be merged into another branch,to check that optimization hasn't hidden any failures.", + "name": "run-missing-tests", + "symbol": "rmt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Run Missing Tests" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "run-missing-tests", + "title": "Run Missing Tests" + }, + { + "context": [{}], + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "rerun", + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "name": "rerun", + "symbol": "rr", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Rerun" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "rerun", + "schema": { + "properties": {}, + "type": "object" + }, + "title": "Rerun" + }, + { + "context": [{}], + "description": "Cancel the given task", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel", + "description": "Cancel the given task", + "name": "cancel", + "symbol": "cx", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Cancel Task" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel", + "title": "Cancel Task" + }, + { + "context": [], + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel-all", + "description": "Cancel all running and pending tasks created by the decision task this action task is associated with.", + "name": "cancel-all", + "symbol": "cAll", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Cancel All" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel-all", + "title": "Cancel All" + }, + { + "context": [ + { + "worker-implementation": "docker-worker" + } + ], + "description": "Purge any caches associated with this task across all workers of the same workertype as the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "purge-cache", + "description": "Purge any caches associated with this task across all workers of the same workertype as the task.", + "name": "purge-cache", + "symbol": "purge-cache", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Purge Worker Caches" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "purge-cache", + "title": "Purge Worker Caches" + }, + { + "context": [], + "description": "Action to prepare openh264 binaries for shipping", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "openh264", + "description": "Action to prepare openh264 binaries for shipping", + "name": "openh264", + "symbol": "h264", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "OpenH264 Binaries" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "openh264", + "title": "OpenH264 Binaries" + }, + { + "context": [], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger-multiple", + "description": "Create a clone of the task.", + "name": "retrigger-multiple", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger-multiple", + "schema": { + "properties": { + "additionalProperties": false, + "requests": { + "items": { + "additionalProperties": false, + "tasks": { + "description": "An array of task labels", + "items": { + "type": "string" + }, + "type": "array" + }, + "times": { + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "array" + } + }, + "type": "object" + }, + "title": "Retrigger" + } + ], + "variables": {}, + "version": 1 +} diff --git a/tests/sample_data/perf_sheriff_bot/matchingTagSetList.json b/tests/sample_data/perf_sheriff_bot/matchingTagSetList.json new file mode 100644 index 000000000..315283b4d --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/matchingTagSetList.json @@ -0,0 +1,8 @@ +[ + { + "test-type": "talos" + }, + { + "test-type": "raptor" + } +] diff --git a/tests/sample_data/perf_sheriff_bot/matchingTaskTags.json b/tests/sample_data/perf_sheriff_bot/matchingTaskTags.json new file mode 100644 index 000000000..61b2e434a --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/matchingTaskTags.json @@ -0,0 +1,9 @@ +{ + "kind": "test", + "os": "linux", + "createdForUser": "tnikkel@mozilla.com", + "retrigger": "true", + "label": "test-linux64-shippable-qr/opt-raptor-assorted-dom-firefox-e10s", + "test-type": "raptor", + "worker-implementation": "generic-worker" +} diff --git a/tests/sample_data/perf_sheriff_bot/mismatchingTagSetList.json b/tests/sample_data/perf_sheriff_bot/mismatchingTagSetList.json new file mode 100644 index 000000000..03c2df809 --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/mismatchingTagSetList.json @@ -0,0 +1,11 @@ +[ + { + "kind": "decision-task" + }, + { + "kind": "action-callback" + }, + { + "kind": "cron-task" + } +] diff --git a/tests/sample_data/perf_sheriff_bot/mismatchingTaskTags.json b/tests/sample_data/perf_sheriff_bot/mismatchingTaskTags.json new file mode 100644 index 000000000..61b2e434a --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/mismatchingTaskTags.json @@ -0,0 +1,9 @@ +{ + "kind": "test", + "os": "linux", + "createdForUser": "tnikkel@mozilla.com", + "retrigger": "true", + "label": "test-linux64-shippable-qr/opt-raptor-assorted-dom-firefox-e10s", + "test-type": "raptor", + "worker-implementation": "generic-worker" +} diff --git a/tests/sample_data/perf_sheriff_bot/originalTask.json b/tests/sample_data/perf_sheriff_bot/originalTask.json new file mode 100644 index 000000000..cccc681f7 --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/originalTask.json @@ -0,0 +1,127 @@ +{ + "provisionerId": "releng-hardware", + "workerType": "gecko-t-linux-talos", + "schedulerId": "gecko-level-3", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "dependencies": [ + "UHhU74OiQ5mpwdWVqkFPPQ", + "e0LnTgZNQcSg9w2FctDyTw", + "eFUvia2tSZaEnveSffSN3w" + ], + "requires": "all-completed", + "routes": [ + "tc-treeherder.v2.autoland.c7766d0b4a121985a8b07e6721d66ccab57bbf76.109591" + ], + "priority": "low", + "retries": 5, + "created": "2020-03-10T04:40:43.718Z", + "deadline": "2020-03-11T04:40:43.718Z", + "expires": "2021-03-10T04:40:43.718Z", + "scopes": [], + "payload": { + "onExitStatus": { + "retry": [4] + }, + "maxRunTime": 2100, + "artifacts": [ + { + "path": "logs", + "type": "directory", + "name": "public/logs" + }, + { + "path": "build/blobber_upload_dir", + "type": "directory", + "name": "public/test_info" + } + ], + "command": [ + ["chmod", "+x", "run-task"], + [ + "./run-task", + "--", + "/usr/bin/python2.7", + "-u", + "mozharness/scripts/raptor_script.py", + "--cfg", + "mozharness/configs/raptor/linux_config.py", + "--test=raptor-assorted-dom", + "--enable-webrender", + "--download-symbols", + "ondemand" + ] + ], + "env": { + "SCCACHE_DISABLE": "1", + "GECKO_HEAD_REV": "c7766d0b4a121985a8b07e6721d66ccab57bbf76", + "MOZ_SCM_LEVEL": "3", + "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/integration/autoland", + "EXTRA_MOZHARNESS_CONFIG": "{\"test_packages_url\": \"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/eFUvia2tSZaEnveSffSN3w/artifacts/public/build/target.test_packages.json\", \"installer_url\": \"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/eFUvia2tSZaEnveSffSN3w/artifacts/public/build/target.tar.bz2\"}", + "MOZ_FETCHES": "[{\"artifact\": \"public/build/minidump_stackwalk.tar.xz\", \"extract\": true, \"task\": \"UHhU74OiQ5mpwdWVqkFPPQ\"}, {\"artifact\": \"public/assorted-dom-4befd28725c6.zip\", \"extract\": true, \"task\": \"e0LnTgZNQcSg9w2FctDyTw\"}]", + "MOZ_FETCHES_DIR": "fetches", + "MOZ_AUTOMATION": "1" + }, + "mounts": [ + { + "directory": ".", + "content": { + "taskId": "eFUvia2tSZaEnveSffSN3w", + "artifact": "public/build/mozharness.zip" + }, + "format": "zip" + }, + { + "content": { + "url": "https://hg.mozilla.org/integration/autoland/raw-file/c7766d0b4a121985a8b07e6721d66ccab57bbf76/taskcluster/scripts/run-task" + }, + "file": "./run-task" + }, + { + "content": { + "url": "https://hg.mozilla.org/integration/autoland/raw-file/c7766d0b4a121985a8b07e6721d66ccab57bbf76/taskcluster/scripts/misc/fetch-content" + }, + "file": "./fetch-content" + } + ] + }, + "metadata": { + "owner": "sikeda.birchill@mozilla.com", + "source": "https://hg.mozilla.org/integration/autoland/file/c7766d0b4a121985a8b07e6721d66ccab57bbf76/taskcluster/ci/test", + "description": "Raptor Assorted-Dom on Firefox ([Treeherder push](https://treeherder.mozilla.org/#/jobs?repo=autoland&revision=c7766d0b4a121985a8b07e6721d66ccab57bbf76))", + "name": "test-linux64-shippable-qr/opt-raptor-assorted-dom-firefox-e10s" + }, + "tags": { + "kind": "test", + "os": "linux", + "createdForUser": "sikeda.birchill@mozilla.com", + "retrigger": "true", + "label": "test-linux64-shippable-qr/opt-raptor-assorted-dom-firefox-e10s", + "test-type": "raptor", + "worker-implementation": "generic-worker" + }, + "extra": { + "index": { + "rank": 1583815073 + }, + "parent": "f7Jj_h6MTEKr5Ln_7aFqbw", + "chunks": { + "current": 1, + "total": 1 + }, + "suite": "raptor", + "treeherder": { + "jobKind": "test", + "groupSymbol": "Rap", + "collection": { + "opt": true + }, + "machine": { + "platform": "linux64-shippable-qr" + }, + "groupName": "Raptor performance tests on Firefox", + "tier": 1, + "symbol": "dom" + }, + "treeherder-platform": "linux64-shippable-qr/opt" + } +} diff --git a/tests/sample_data/perf_sheriff_bot/reducedActions.json b/tests/sample_data/perf_sheriff_bot/reducedActions.json new file mode 100644 index 000000000..89cc5012d --- /dev/null +++ b/tests/sample_data/perf_sheriff_bot/reducedActions.json @@ -0,0 +1,355 @@ +[ + { + "context": [ + { + "retrigger": "true" + } + ], + "description": "Create a clone of the task.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "retrigger", + "description": "Create a clone of the task.", + "name": "retrigger", + "symbol": "rt", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Retrigger" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "retrigger", + "schema": { + "properties": { + "downstream": { + "default": false, + "description": "If true, downstream tasks from this one will be cloned as well. The dependencies will be updated to work with the new task at the root.", + "type": "boolean" + }, + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Retrigger" + }, + { + "context": [ + { + "kind": "test" + } + ], + "description": "Re-run Tests for original manifest, directories and tests for failing tests.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "isolate-test-failures", + "description": "Re-run Tests for original manifest, directories and tests for failing tests.", + "name": "isolate-test-failures", + "symbol": "it", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Isolate test failures in job" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "isolate-test-failures", + "schema": { + "additionalProperties": false, + "properties": { + "times": { + "default": 1, + "description": "How many times to run each task.", + "maximum": 100, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Isolate test failures in job" + }, + { + "context": [{}], + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "backfill", + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project.", + "name": "backfill", + "symbol": "Bk", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Backfill" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "backfill", + "schema": { + "additionalProperties": false, + "properties": { + "depth": { + "default": 9, + "description": "The number of previous pushes before the current push to attempt to trigger this task on.", + "maximum": 25, + "minimum": 1, + "title": "Depth", + "type": "integer" + }, + "inclusive": { + "default": false, + "description": "If true, the backfill will also retrigger the task on the selected push.", + "title": "Inclusive Range", + "type": "boolean" + }, + "testPath": { + "title": "Test Path", + "type": "string" + }, + "times": { + "default": 1, + "description": "The number of times to execute each job you are backfilling.", + "maximum": 10, + "minimum": 1, + "title": "Times", + "type": "integer" + } + }, + "type": "object" + }, + "title": "Backfill" + }, + { + "context": [ + { + "test-type": "talos" + }, + { + "test-type": "raptor" + } + ], + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project while adding the --geckoProfile cmd arg.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "geckoprofile", + "description": "Take the label of the current task, and trigger the task with that label on previous pushes in the same project while adding the --geckoProfile cmd arg.", + "name": "geckoprofile", + "symbol": "Gp", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "GeckoProfile" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "geckoprofile", + "title": "GeckoProfile" + }, + { + "context": [{}], + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "rerun", + "description": "Rerun a task.\n\nThis only works on failed or exception tasks in the original taskgraph, and is CoT friendly.", + "name": "rerun", + "symbol": "rr", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Rerun" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "rerun", + "schema": { + "properties": {}, + "type": "object" + }, + "title": "Rerun" + }, + { + "context": [{}], + "description": "Cancel the given task", + "extra": { + "actionPerm": "generic" + }, + "hookGroupId": "project-gecko", + "hookId": "in-tree-action-3-generic/9353e8f146", + "hookPayload": { + "decision": { + "action": { + "cb_name": "cancel", + "description": "Cancel the given task", + "name": "cancel", + "symbol": "cx", + "taskGroupId": "f7Jj_h6MTEKr5Ln_7aFqbw", + "title": "Cancel Task" + }, + "push": { + "owner": "mozilla-taskcluster-maintenance@mozilla.com", + "pushlog_id": "109591", + "revision": "c7766d0b4a121985a8b07e6721d66ccab57bbf76" + }, + "repository": { + "level": "3", + "project": "autoland", + "url": "https://hg.mozilla.org/integration/autoland" + } + }, + "user": { + "input": { + "$eval": "input" + }, + "taskGroupId": { + "$eval": "taskGroupId" + }, + "taskId": { + "$eval": "taskId" + } + } + }, + "kind": "hook", + "name": "cancel", + "title": "Cancel Task" + } +] diff --git a/tests/services/test_taskcluster.py b/tests/services/test_taskcluster.py new file mode 100644 index 000000000..338d383a5 --- /dev/null +++ b/tests/services/test_taskcluster.py @@ -0,0 +1,60 @@ +import json +from os.path import (dirname, + join) + +import pytest + +from treeherder.services.taskcluster import TaskclusterModel + +SAMPLE_DATA_PATH = join( + dirname(dirname(__file__)), + 'sample_data') + + +def load_json_fixture(from_file): + fixture_path = join(SAMPLE_DATA_PATH, 'perf_sheriff_bot', from_file) + with open(fixture_path, 'r') as f: + return json.load(f) + + +@pytest.fixture(scope="module") +def actions_json(): return load_json_fixture('initialActions.json') + + +@pytest.fixture(scope="module") +def expected_actions_json(): return load_json_fixture('reducedActions.json') + + +@pytest.fixture(scope="module") +def original_task(): return load_json_fixture('originalTask.json') + + +@pytest.fixture(scope="module") +def expected_backfill_task(): return load_json_fixture('backfilltask.json') + + +# TaskclusterModel +def test_filter_relevant_actions(actions_json, original_task, expected_actions_json): + reduced_actions_json = TaskclusterModel._filter_relevant_actions(actions_json, + original_task) + + assert reduced_actions_json == expected_actions_json + + +def test_task_in_context(): + # match + tag_set_list, task_tags = [load_json_fixture(f) + for f in ("matchingTagSetList.json", "matchingTaskTags.json")] + assert TaskclusterModel._task_in_context(tag_set_list, task_tags) is True + + # mismatch + tag_set_list, task_tags = [load_json_fixture(f) + for f in ("mismatchingTagSetList.json", "mismatchingTaskTags.json")] + assert TaskclusterModel._task_in_context(tag_set_list, task_tags) is False + + +def test_get_action(actions_json, expected_backfill_task): + action_array = actions_json["actions"] + + backfill_task = TaskclusterModel._get_action(action_array, "backfill") + assert backfill_task == expected_backfill_task diff --git a/tests/utils/test_taskcluster_lib_scopes.py b/tests/utils/test_taskcluster_lib_scopes.py new file mode 100644 index 000000000..c9018ae2f --- /dev/null +++ b/tests/utils/test_taskcluster_lib_scopes.py @@ -0,0 +1,87 @@ +import pytest + +from treeherder.utils.taskcluster_lib_scopes import (patternMatch, + satisfiesExpression) + + +# satisfiesExpression() +@pytest.mark.parametrize('scopeset, expression', [ + [[], {'AllOf': []}], + [['A'], {'AllOf': ['A']}], + [['A', 'B'], 'A'], + [['a*', 'b*', 'c*'], 'abc'], + [['abc'], {'AnyOf': ['abc', 'def']}], + [['def'], {'AnyOf': ['abc', 'def']}], + [['abc', 'def'], {'AnyOf': ['abc', 'def']}], + [['abc*'], {'AnyOf': ['abc', 'def']}], + [['abc*'], {'AnyOf': ['abc']}], + [['abc*', 'def*'], {'AnyOf': ['abc', 'def']}], + [['foo'], {'AllOf': [{'AnyOf': [{'AllOf': ['foo']}, {'AllOf': ['bar']}]}]}], + [['a*', 'b*', 'c*'], {'AnyOf': ['cfoo', 'dfoo']}], + [['a*', 'b*', 'c*'], {'AnyOf': ['bx', 'by']}], + [['a*', 'b*', 'c*'], {'AllOf': ['bx', 'cx']}], + # complex expression with only + # some AnyOf branches matching + [ + ['a*', 'b*', 'c*'], + {'AnyOf': [ + {'AllOf': ['ax', 'jx']}, # doesn't match + {'AllOf': ['bx', 'cx']}, # does match + 'bbb', + ]}, + ], +]) +def test_expression_is_satisfied(scopeset, expression): + assert satisfiesExpression(scopeset, expression) is True + + +@pytest.mark.parametrize('scopeset, expression', [ + [[], {'AnyOf': []}], + [[], 'missing-scope'], + [['wrong-scope'], 'missing-scope'], + [['ghi'], {'AnyOf': ['abc', 'def']}], + [['ghi*'], {'AnyOf': ['abc', 'def']}], + [['ghi', 'fff'], {'AnyOf': ['abc', 'def']}], + [['ghi*', 'fff*'], {'AnyOf': ['abc', 'def']}], + [['abc'], {'AnyOf': ['ghi']}], + [['abc*'], {'AllOf': ['abc', 'ghi']}], + [[''], {'AnyOf': ['abc', 'def']}], + [['abc:def'], {'AnyOf': ['abc', 'def']}], + [['xyz', 'abc'], {'AllOf': [{'AnyOf': [{'AllOf': ['foo']}, {'AllOf': ['bar']}]}]}], + [['a*', 'b*', 'c*'], {'AllOf': ['bx', 'cx', {'AnyOf': ['xxx', 'yyyy']}]}], +]) +def test_expression_is_not_satisfied(scopeset, expression): + assert not satisfiesExpression(scopeset, expression) + + +@pytest.mark.parametrize('scopeset', [ + None, + 'scopeset_argument', + ('scopeset', 'argument'), + {'scopeset', 'argument'}, +]) +def test_wrong_scopeset_type_raises_exception(scopeset): + with pytest.raises(TypeError): + satisfiesExpression(scopeset, 'in-tree:hook-action:{hook_group_id}/{hook_id}') + + +# patternMatch() +def test_identical_scope_and_pattern_are_matching(): + assert patternMatch('mock:scope', 'mock:scope') is True + + +@pytest.mark.parametrize('pattern, scope', [ + ('matching*', 'matching'), + ('matching*', 'matching/scope') +]) +def test_starred_patterns_are_matching(pattern, scope): + assert patternMatch(pattern, scope) is True + + +@pytest.mark.parametrize('pattern, scope', [ + ('matching*', 'mismatching'), + ('match*ing', 'matching'), + ('*matching', 'matching') +]) +def test_starred_patterns_dont_matching(pattern, scope): + assert not patternMatch(pattern, scope) diff --git a/treeherder/model/models.py b/treeherder/model/models.py index 95749e2ea..25aa60597 100644 --- a/treeherder/model/models.py +++ b/treeherder/model/models.py @@ -15,7 +15,8 @@ from django.db import (models, from django.db.models import (Count, Max, Min, - Q) + Q, + Subquery) from django.db.utils import ProgrammingError from django.forms import model_to_dict from django.utils import timezone @@ -647,6 +648,13 @@ class Job(models.Model): return text_log_error + def fetch_associated_decision_job(self): + decision_type = JobType.objects.filter(name="Gecko Decision Task", + symbol="D") + return Job.objects.get(repository_id=self.repository_id, + job_type_id=Subquery(decision_type.values('id')[:1]), + push_id=self.push_id) + @staticmethod def get_duration(submit_time, start_time, end_time): endtime = end_time if to_timestamp(end_time) else datetime.datetime.now() diff --git a/treeherder/perf/backfill_tool.py b/treeherder/perf/backfill_tool.py new file mode 100644 index 000000000..824c5a0e2 --- /dev/null +++ b/treeherder/perf/backfill_tool.py @@ -0,0 +1,46 @@ +import logging + +from django.core.exceptions import ObjectDoesNotExist + +from treeherder.model.models import Job +from treeherder.perf.exceptions import CannotBackfill +from treeherder.services.taskcluster import TaskclusterModel + +logger = logging.getLogger(__name__) + + +class BackfillTool: + + def __init__(self, taskcluster_model: TaskclusterModel): + self.tc_model = taskcluster_model + + def backfill_job(self, job_id: str) -> str: + job = self._fetch_job(job_id) + + self.assert_backfill_ability(job) + + logger.debug(f"Fetching decision task of job {job.id}...") + task_id_to_backfill = job.taskcluster_metadata.task_id + decision_job = job.fetch_associated_decision_job() + decision_task_id = decision_job.taskcluster_metadata.task_id + + logger.debug(f"Requesting backfill for task {task_id_to_backfill}...") + task_id = self.tc_model.trigger_action( + action='backfill', + task_id=task_id_to_backfill, + decision_task_id=decision_task_id, + input={}, + root_url=job.repository.tc_root_url + ) + return task_id + + def assert_backfill_ability(self, over_job: Job): + if over_job.repository.is_try_repo: + raise CannotBackfill("Try repository isn't suited for backfilling.") + + @staticmethod + def _fetch_job(job_id: str) -> Job: + try: + return Job.objects.get(id=job_id) + except ObjectDoesNotExist: + raise LookupError(f"Job {job_id} not found.") diff --git a/treeherder/perf/exceptions.py b/treeherder/perf/exceptions.py index 4c66c2dc5..24a8c3154 100644 --- a/treeherder/perf/exceptions.py +++ b/treeherder/perf/exceptions.py @@ -8,3 +8,7 @@ class MaxRuntimeExceeded(Exception): class MissingRecords(Exception): pass + + +class CannotBackfill(Exception): + pass diff --git a/treeherder/perf/management/commands/backfill_perf_jobs.py b/treeherder/perf/management/commands/backfill_perf_jobs.py new file mode 100644 index 000000000..89b943118 --- /dev/null +++ b/treeherder/perf/management/commands/backfill_perf_jobs.py @@ -0,0 +1,40 @@ +"""IMPORTANT! +This subcommand isn't intended for use on any non-local +dev environments. Any attempt to configure it otherwise +is considered a known and unapproved risk. + +The subcommand's sole purpose is to act as a smoke test +harness that quickly does an end-to-end check over the +functionality of the `BackfillTool`. +""" +from django.conf import settings +from django.core.management.base import BaseCommand + +from treeherder.perf.backfill_tool import BackfillTool +from treeherder.services.taskcluster import DEFAULT_ROOT_URL as root_url +from treeherder.services.taskcluster import TaskclusterModel + + +class Command(BaseCommand): + help = "Backfill missing performance jobs" + + def add_arguments(self, parser): + parser.add_argument( + 'job', + action='store', + type=str, + help="Performance job to backfill from", + metavar='JOB_ID', + ) + + def handle(self, *args, **options): + job_id = options['job'] + client_id = settings.PERF_SHERIFF_BOT_CLIENT_ID + access_token = settings.PERF_SHERIFF_BOT_ACCESS_TOKEN + + taskcluster_model = TaskclusterModel(root_url, client_id, access_token) + backfill_tool = BackfillTool(taskcluster_model) + + task_id = backfill_tool.backfill_job(job_id) + + print(f"Task id {task_id} created when backfilled job id {job_id}") diff --git a/treeherder/services/taskcluster.py b/treeherder/services/taskcluster.py new file mode 100644 index 000000000..d74f4fe47 --- /dev/null +++ b/treeherder/services/taskcluster.py @@ -0,0 +1,156 @@ +import logging +from typing import List + +import jsone +import taskcluster + +from treeherder.utils.taskcluster_lib_scopes import satisfiesExpression + +logger = logging.getLogger(__name__) + +DEFAULT_ROOT_URL = 'https://firefox-ci-tc.services.mozilla.com' + + +class TaskclusterModel: + """Javascript -> Python rewrite of frontend's TaskclusterModel""" + + def __init__(self, root_url, client_id=None, access_token=None): + options = {'rootUrl': root_url} + credentials = {} + + if client_id: + credentials['clientId'] = client_id + if access_token: + credentials['accessToken'] = access_token + + # Taskcluster APIs + self.hooks = taskcluster.Hooks({**options, 'credentials': credentials}) + + # Following least-privilege principle, as services + # bellow don't really need authorization credentials. + self.queue = taskcluster.Queue(options) + self.auth = taskcluster.Auth(options) + + def set_root_url(self, root_url): + for service in (self.hooks, self.queue, self.auth): + service.options['rootUrl'] = root_url + + def trigger_action(self, action, task_id, decision_task_id, input, root_url=None) -> str: + if root_url is not None: + self.set_root_url(root_url) + + actions_context = self._load(decision_task_id, task_id) + action_to_trigger = self._get_action(actions_context['actions'], action) + + return self._submit( + action=action_to_trigger, + decision_task_id=decision_task_id, + task_id=task_id, + input=input, + static_action_variables=actions_context['staticActionVariables'] + ) + + def _load(self, decision_task_id: str, task_id: str) -> dict: + if not decision_task_id: + raise ValueError("No decision task, can't find taskcluster actions") + + # fetch + logger.debug('Fetching actions.json...') + actions_json = self.queue.getLatestArtifact(decision_task_id, 'public/actions.json') + task_definition = self.queue.task(task_id) + + if actions_json['version'] != 1: + raise RuntimeError('Wrong version of actions.json, unable to continue') + + return { + 'staticActionVariables': actions_json['variables'], + 'actions': self._filter_relevant_actions(actions_json, task_definition), + } + + def _submit(self, + action=None, + decision_task_id=None, + task_id=None, + input=None, + static_action_variables=None) -> str: + context = { + "taskGroupId": decision_task_id, + "taskId": task_id or None, + "input": input, + } + context.update(static_action_variables) + action_kind = action["kind"] + + if action_kind == "hook": + hook_payload = jsone.render(action["hookPayload"], context) + hook_id, hook_group_id = action["hookId"], action["hookGroupId"] + + decision_task = self.queue.task(decision_task_id) + expansion = self.auth.expandScopes({"scopes": decision_task["scopes"]}) + expression = f"in-tree:hook-action:{hook_group_id}/{hook_id}" + + if not satisfiesExpression(expansion["scopes"], expression): + raise RuntimeError(f"Action is misconfigured: decision task's scopes do not satisfy {expression}") + + result = self.hooks.triggerHook(hook_group_id, hook_id, hook_payload) + return result["status"]["taskId"] + + raise NotImplementedError(f"Unable to submit actions with '{action_kind}' kind.") + + @classmethod + def _filter_relevant_actions(cls, actions_json: dict, original_task: dict) -> list: + relevant_actions = {} + + for action in actions_json['actions']: + action_name = action['name'] + if action_name in relevant_actions: + continue + + no_context_or_task_to_check = (not len(action['context'])) and (not original_task) + task_is_in_context = (original_task and original_task.get('tags') and + cls._task_in_context(action['context'], original_task['tags'])) + + if no_context_or_task_to_check or task_is_in_context: + relevant_actions[action_name] = action + + return list(relevant_actions.values()) + + @staticmethod + def _get_action(action_array: list, action_name: str) -> str: + """ + Each action entry (from action array) must define a name, title and description. + The order of the array of actions is **significant**: actions should be displayed + in this order, and when multiple actions apply, **the first takes precedence**. + + More updated details: https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#action-metadata + + @return: most relevant action entry + """ + try: + return [a for a in action_array if a["name"] == action_name][0] + except IndexError: + available_actions = ", ".join(sorted( + {a["name"] for a in action_array} + )) + raise LookupError(f"{action_name} action is not available for this task. Available: {available_actions}") + + @classmethod + def _task_in_context(cls, context: List[dict], task_tags: dict) -> bool: + """ + A task (as defined by its tags) is said to match a tag-set if its + tags are a super-set of the tag-set. A tag-set is a set of key-value pairs. + + An action (as defined by its context) is said to be relevant for + a given task, if that task's tags match one of the tag-sets given + in the context property for the action. + + More updated details: https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#action-context + + @param context: list of tag-sets + @param task_tags: task's tags + """ + return any( + all(tag in task_tags and task_tags[tag] == tag_set[tag] + for tag in tag_set.keys()) + for tag_set in context + ) diff --git a/treeherder/utils/taskcluster_lib_scopes.py b/treeherder/utils/taskcluster_lib_scopes.py new file mode 100644 index 000000000..a3b27ee35 --- /dev/null +++ b/treeherder/utils/taskcluster_lib_scopes.py @@ -0,0 +1,30 @@ +""" +TODO: Extract this module into a dedicated PyPI package, acting as the + Python variant of https://github.com/taskcluster/taskcluster-lib-scopes +""" + + +def satisfiesExpression(scopeset, expression): + if not isinstance(scopeset, list): + raise TypeError("Scopeset must be an array.") + + def isSatisfied(expr): + if isinstance(expr, str): + return any([patternMatch(s, expr) for s in scopeset]) + + return ( + "AllOf" in expr and all([isSatisfied(e) for e in expr["AllOf"]]) or + "AnyOf" in expr and any([isSatisfied(e) for e in expr["AnyOf"]]) + ) + + return isSatisfied(expression) + + +def patternMatch(pattern: str, scope): + if scope == pattern: + return True + + if pattern.endswith('*'): + return scope.startswith(pattern[:-1]) + + return False