зеркало из https://github.com/mozilla/treeherder.git
Revert "Bug 1509181 - Fix ingestion of GitHub pushes + manual ingestion support (#6019)"
This reverts commit b41f68286d.
This commit is contained in:
Armen Zambrano G
Родитель
c702d2b548
Коммит
15b4d4d92b
|
|
@ -141,31 +141,21 @@ Ingestion tasks populate the database with version control push logs, queued/run
|
||||||
|
|
||||||
`NOTE`; You have to include `--root-url https://community-tc.services.mozilla.com` in order to ingest from the [Taskcluster Community instance](https://community-tc.services.mozilla.com), otherwise, it will default to the Firefox CI.
|
`NOTE`; You have to include `--root-url https://community-tc.services.mozilla.com` in order to ingest from the [Taskcluster Community instance](https://community-tc.services.mozilla.com), otherwise, it will default to the Firefox CI.
|
||||||
|
|
||||||
Open a terminal window and run `docker-compose up`. All following sections assume this step.
|
#### Ingesting pushes & tasks
|
||||||
|
|
||||||
#### Ingesting pushes
|
|
||||||
|
|
||||||
Mercurial pushes:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker-compose exec backend ./manage.py ingest push -p autoland -r 63f8a47cfdf5
|
docker-compose run backend ./manage.py ingest push -p autoland -r 63f8a47cfdf5
|
||||||
```
|
```
|
||||||
|
|
||||||
`NOTE`: You can ingest all tasks for a push. Check the help output for the script to determine the
|
You can ingest all tasks for a push. Check the help output for the script to determine the
|
||||||
parameters needed.
|
parameters needed.
|
||||||
|
|
||||||
Github pushes:
|
### Ingesting a range of pushes
|
||||||
|
|
||||||
```bash
|
|
||||||
docker-compose exec backend ./manage.py ingest git-push -p servo-try -c 92fc94588f3b6987082923c0003012fd696b1a2d
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Ingesting a range of pushes
|
|
||||||
|
|
||||||
It is also possible to ingest the last N pushes for a repository:
|
It is also possible to ingest the last N pushes for a repository:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker-compose exec backend ./manage.py ingest_push mozilla-central --last-n-pushes 100
|
docker-compose run backend ./manage.py ingest_push mozilla-central --last-n-pushes 100
|
||||||
```
|
```
|
||||||
|
|
||||||
In this mode, only the push information will be ingested: tasks
|
In this mode, only the push information will be ingested: tasks
|
||||||
|
|
@ -178,7 +168,7 @@ changesets from the web interface into subsequent commands to ingest all tasks.
|
||||||
`NOTE`: This will only ingest the commits if there's an active Github PRs project. It will only ingest the commits.
|
`NOTE`: This will only ingest the commits if there's an active Github PRs project. It will only ingest the commits.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker-compose exec backend ./manage.py ingest pr --pr-url https://github.com/mozilla-mobile/android-components/pull/4821
|
docker-compose run backend ./manage.py ingest pr --pr-url https://github.com/mozilla-mobile/android-components/pull/4821
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Ingesting individual task
|
#### Ingesting individual task
|
||||||
|
|
@ -187,7 +177,7 @@ This will work if the push associated to the task exists in the database.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Make sure to ingest 1bd9d4f431c4c9f93388bd04a6368cb07398f646 for autoland first
|
# Make sure to ingest 1bd9d4f431c4c9f93388bd04a6368cb07398f646 for autoland first
|
||||||
docker-compose exec backend ./manage.py ingest task --task-id KQ5h1BVYTBy_XT21wFpLog
|
docker-compose run backend ./manage.py ingest task --task-id KQ5h1BVYTBy_XT21wFpLog
|
||||||
```
|
```
|
||||||
|
|
||||||
## Learn more
|
## Learn more
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,7 @@ def test_ingest_github_pull_request(test_repository, github_pr, transformed_gith
|
||||||
|
|
||||||
def test_ingest_github_push(test_repository, github_push, transformed_github_push,
|
def test_ingest_github_push(test_repository, github_push, transformed_github_push,
|
||||||
mock_github_push_compare):
|
mock_github_push_compare):
|
||||||
xformer = GithubPushTransformer(github_push["payload"])
|
xformer = GithubPushTransformer(github_push)
|
||||||
push = xformer.transform(test_repository.name)
|
push = xformer.transform(test_repository.name)
|
||||||
assert transformed_github_push == push
|
assert transformed_github_push == push
|
||||||
|
|
||||||
|
|
@ -145,8 +145,8 @@ def test_ingest_hg_push_bad_repo(hg_push):
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_ingest_github_push_bad_repo(github_push):
|
def test_ingest_github_push_bad_repo(github_push):
|
||||||
"""Test graceful handling of an unknown GH repo"""
|
"""Test graceful handling of an unknown GH repo"""
|
||||||
github_push["payload"]["details"]["event.head.repo.url"] = "https://bad.repo.com"
|
github_push["details"]["event.head.repo.url"] = "https://bad.repo.com"
|
||||||
PushLoader().process(github_push["payload"], "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||||
assert Push.objects.count() == 0
|
assert Push.objects.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -160,10 +160,10 @@ def test_ingest_github_push_bad_repo(github_push):
|
||||||
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes, github_push,
|
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes, github_push,
|
||||||
test_repository, mock_github_push_compare):
|
test_repository, mock_github_push_compare):
|
||||||
"""Test a repository accepting pushes for multiple branches"""
|
"""Test a repository accepting pushes for multiple branches"""
|
||||||
test_repository.url = "https://github.com/mozilla-mobile/android-components"
|
test_repository.url = "https://github.com/mozilla/test_treeherder"
|
||||||
test_repository.branch = "master,foo,bar"
|
test_repository.branch = "master,foo,bar"
|
||||||
test_repository.save()
|
test_repository.save()
|
||||||
github_push["payload"]["details"]["event.base.repo.branch"] = branch
|
github_push["details"]["event.base.repo.branch"] = branch
|
||||||
assert Push.objects.count() == 0
|
assert Push.objects.count() == 0
|
||||||
PushLoader().process(github_push["payload"], "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||||
assert Push.objects.count() == expected_pushes
|
assert Push.objects.count() == expected_pushes
|
||||||
|
|
|
||||||
|
|
@ -1,263 +1,16 @@
|
||||||
{
|
{
|
||||||
"payload": {
|
"organization": "mozilla-services",
|
||||||
"body": {
|
"details": {
|
||||||
"ref": "refs/heads/master",
|
"event.type": "push",
|
||||||
"before": "7285afe57ae6207fdb5d6db45133dac2053b7820",
|
"event.base.repo.branch": "master",
|
||||||
"after": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
"event.base.sha": "71c02dbb26cb60beee94bac433086bc540c9d6d4",
|
||||||
"repository": {
|
"event.head.repo.branch": "master",
|
||||||
"id": 126199585,
|
"event.head.user.login": "mozilla-test_treeherder",
|
||||||
"node_id": "MDEwOlJlcG9zaXRvcnkxMjYxOTk1ODU=",
|
"event.head.repo.url": "https://github.com/mozilla/test_treeherder.git",
|
||||||
"name": "android-components",
|
"event.head.sha": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
|
||||||
"full_name": "mozilla-mobile/android-components",
|
"event.head.ref": "refs/heads/master",
|
||||||
"private": false,
|
"event.head.user.email": "mozilla-test_treeherder@noreply.github.com"
|
||||||
"owner": {
|
|
||||||
"name": "mozilla-mobile",
|
|
||||||
"email": null,
|
|
||||||
"login": "mozilla-mobile",
|
|
||||||
"id": 22351667,
|
|
||||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjIyMzUxNjY3",
|
|
||||||
"avatar_url": "https://avatars3.githubusercontent.com/u/22351667?v=4",
|
|
||||||
"gravatar_id": "",
|
|
||||||
"url": "https://api.github.com/users/mozilla-mobile",
|
|
||||||
"html_url": "https://github.com/mozilla-mobile",
|
|
||||||
"followers_url": "https://api.github.com/users/mozilla-mobile/followers",
|
|
||||||
"following_url": "https://api.github.com/users/mozilla-mobile/following{/other_user}",
|
|
||||||
"gists_url": "https://api.github.com/users/mozilla-mobile/gists{/gist_id}",
|
|
||||||
"starred_url": "https://api.github.com/users/mozilla-mobile/starred{/owner}{/repo}",
|
|
||||||
"subscriptions_url": "https://api.github.com/users/mozilla-mobile/subscriptions",
|
|
||||||
"organizations_url": "https://api.github.com/users/mozilla-mobile/orgs",
|
|
||||||
"repos_url": "https://api.github.com/users/mozilla-mobile/repos",
|
|
||||||
"events_url": "https://api.github.com/users/mozilla-mobile/events{/privacy}",
|
|
||||||
"received_events_url": "https://api.github.com/users/mozilla-mobile/received_events",
|
|
||||||
"type": "Organization",
|
|
||||||
"site_admin": false
|
|
||||||
},
|
|
||||||
"html_url": "https://github.com/mozilla-mobile/android-components",
|
|
||||||
"description": "A collection of Android libraries to build browsers or browser-like applications.",
|
|
||||||
"fork": false,
|
|
||||||
"url": "https://github.com/mozilla-mobile/android-components",
|
|
||||||
"forks_url": "https://api.github.com/repos/mozilla-mobile/android-components/forks",
|
|
||||||
"keys_url": "https://api.github.com/repos/mozilla-mobile/android-components/keys{/key_id}",
|
|
||||||
"collaborators_url": "https://api.github.com/repos/mozilla-mobile/android-components/collaborators{/collaborator}",
|
|
||||||
"teams_url": "https://api.github.com/repos/mozilla-mobile/android-components/teams",
|
|
||||||
"hooks_url": "https://api.github.com/repos/mozilla-mobile/android-components/hooks",
|
|
||||||
"issue_events_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues/events{/number}",
|
|
||||||
"events_url": "https://api.github.com/repos/mozilla-mobile/android-components/events",
|
|
||||||
"assignees_url": "https://api.github.com/repos/mozilla-mobile/android-components/assignees{/user}",
|
|
||||||
"branches_url": "https://api.github.com/repos/mozilla-mobile/android-components/branches{/branch}",
|
|
||||||
"tags_url": "https://api.github.com/repos/mozilla-mobile/android-components/tags",
|
|
||||||
"blobs_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/blobs{/sha}",
|
|
||||||
"git_tags_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/tags{/sha}",
|
|
||||||
"git_refs_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/refs{/sha}",
|
|
||||||
"trees_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/trees{/sha}",
|
|
||||||
"statuses_url": "https://api.github.com/repos/mozilla-mobile/android-components/statuses/{sha}",
|
|
||||||
"languages_url": "https://api.github.com/repos/mozilla-mobile/android-components/languages",
|
|
||||||
"stargazers_url": "https://api.github.com/repos/mozilla-mobile/android-components/stargazers",
|
|
||||||
"contributors_url": "https://api.github.com/repos/mozilla-mobile/android-components/contributors",
|
|
||||||
"subscribers_url": "https://api.github.com/repos/mozilla-mobile/android-components/subscribers",
|
|
||||||
"subscription_url": "https://api.github.com/repos/mozilla-mobile/android-components/subscription",
|
|
||||||
"commits_url": "https://api.github.com/repos/mozilla-mobile/android-components/commits{/sha}",
|
|
||||||
"git_commits_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/commits{/sha}",
|
|
||||||
"comments_url": "https://api.github.com/repos/mozilla-mobile/android-components/comments{/number}",
|
|
||||||
"issue_comment_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues/comments{/number}",
|
|
||||||
"contents_url": "https://api.github.com/repos/mozilla-mobile/android-components/contents/{+path}",
|
|
||||||
"compare_url": "https://api.github.com/repos/mozilla-mobile/android-components/compare/{base}...{head}",
|
|
||||||
"merges_url": "https://api.github.com/repos/mozilla-mobile/android-components/merges",
|
|
||||||
"archive_url": "https://api.github.com/repos/mozilla-mobile/android-components/{archive_format}{/ref}",
|
|
||||||
"downloads_url": "https://api.github.com/repos/mozilla-mobile/android-components/downloads",
|
|
||||||
"issues_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues{/number}",
|
|
||||||
"pulls_url": "https://api.github.com/repos/mozilla-mobile/android-components/pulls{/number}",
|
|
||||||
"milestones_url": "https://api.github.com/repos/mozilla-mobile/android-components/milestones{/number}",
|
|
||||||
"notifications_url": "https://api.github.com/repos/mozilla-mobile/android-components/notifications{?since,all,participating}",
|
|
||||||
"labels_url": "https://api.github.com/repos/mozilla-mobile/android-components/labels{/name}",
|
|
||||||
"releases_url": "https://api.github.com/repos/mozilla-mobile/android-components/releases{/id}",
|
|
||||||
"deployments_url": "https://api.github.com/repos/mozilla-mobile/android-components/deployments",
|
|
||||||
"created_at": 1521645843,
|
|
||||||
"updated_at": "2020-02-12T15:21:24Z",
|
|
||||||
"pushed_at": 1581521353,
|
|
||||||
"git_url": "git://github.com/mozilla-mobile/android-components.git",
|
|
||||||
"ssh_url": "git@github.com:mozilla-mobile/android-components.git",
|
|
||||||
"clone_url": "https://github.com/mozilla-mobile/android-components.git",
|
|
||||||
"svn_url": "https://github.com/mozilla-mobile/android-components",
|
|
||||||
"homepage": "https://mozac.org",
|
|
||||||
"size": 61634,
|
|
||||||
"stargazers_count": 1214,
|
|
||||||
"watchers_count": 1214,
|
|
||||||
"language": "Kotlin",
|
|
||||||
"has_issues": true,
|
|
||||||
"has_projects": false,
|
|
||||||
"has_downloads": true,
|
|
||||||
"has_wiki": false,
|
|
||||||
"has_pages": true,
|
|
||||||
"forks_count": 268,
|
|
||||||
"mirror_url": null,
|
|
||||||
"archived": false,
|
|
||||||
"disabled": false,
|
|
||||||
"open_issues_count": 684,
|
|
||||||
"license": {
|
|
||||||
"key": "mpl-2.0",
|
|
||||||
"name": "Mozilla Public License 2.0",
|
|
||||||
"spdx_id": "MPL-2.0",
|
|
||||||
"url": "https://api.github.com/licenses/mpl-2.0",
|
|
||||||
"node_id": "MDc6TGljZW5zZTE0"
|
|
||||||
},
|
|
||||||
"forks": 268,
|
|
||||||
"open_issues": 684,
|
|
||||||
"watchers": 1214,
|
|
||||||
"default_branch": "master",
|
|
||||||
"stargazers": 1214,
|
|
||||||
"master_branch": "master",
|
|
||||||
"organization": "mozilla-mobile"
|
|
||||||
},
|
|
||||||
"pusher": {
|
|
||||||
"name": "bors[bot]",
|
|
||||||
"email": null
|
|
||||||
},
|
|
||||||
"organization": {
|
|
||||||
"login": "mozilla-mobile",
|
|
||||||
"id": 22351667,
|
|
||||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjIyMzUxNjY3",
|
|
||||||
"url": "https://api.github.com/orgs/mozilla-mobile",
|
|
||||||
"repos_url": "https://api.github.com/orgs/mozilla-mobile/repos",
|
|
||||||
"events_url": "https://api.github.com/orgs/mozilla-mobile/events",
|
|
||||||
"hooks_url": "https://api.github.com/orgs/mozilla-mobile/hooks",
|
|
||||||
"issues_url": "https://api.github.com/orgs/mozilla-mobile/issues",
|
|
||||||
"members_url": "https://api.github.com/orgs/mozilla-mobile/members{/member}",
|
|
||||||
"public_members_url": "https://api.github.com/orgs/mozilla-mobile/public_members{/member}",
|
|
||||||
"avatar_url": "https://avatars3.githubusercontent.com/u/22351667?v=4",
|
|
||||||
"description": "Mozilla Mobile Applications"
|
|
||||||
},
|
|
||||||
"sender": {
|
|
||||||
"login": "bors[bot]",
|
|
||||||
"id": 26634292,
|
|
||||||
"node_id": "MDM6Qm90MjY2MzQyOTI=",
|
|
||||||
"avatar_url": "https://avatars3.githubusercontent.com/in/1847?v=4",
|
|
||||||
"gravatar_id": "",
|
|
||||||
"url": "https://api.github.com/users/bors%5Bbot%5D",
|
|
||||||
"html_url": "https://github.com/apps/bors",
|
|
||||||
"followers_url": "https://api.github.com/users/bors%5Bbot%5D/followers",
|
|
||||||
"following_url": "https://api.github.com/users/bors%5Bbot%5D/following{/other_user}",
|
|
||||||
"gists_url": "https://api.github.com/users/bors%5Bbot%5D/gists{/gist_id}",
|
|
||||||
"starred_url": "https://api.github.com/users/bors%5Bbot%5D/starred{/owner}{/repo}",
|
|
||||||
"subscriptions_url": "https://api.github.com/users/bors%5Bbot%5D/subscriptions",
|
|
||||||
"organizations_url": "https://api.github.com/users/bors%5Bbot%5D/orgs",
|
|
||||||
"repos_url": "https://api.github.com/users/bors%5Bbot%5D/repos",
|
|
||||||
"events_url": "https://api.github.com/users/bors%5Bbot%5D/events{/privacy}",
|
|
||||||
"received_events_url": "https://api.github.com/users/bors%5Bbot%5D/received_events",
|
|
||||||
"type": "Bot",
|
|
||||||
"site_admin": false
|
|
||||||
},
|
|
||||||
"installation": {
|
|
||||||
"id": 5110595,
|
|
||||||
"node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTExMDU5NQ=="
|
|
||||||
},
|
|
||||||
"created": false,
|
|
||||||
"deleted": false,
|
|
||||||
"forced": false,
|
|
||||||
"base_ref": null,
|
|
||||||
"compare": "https://github.com/mozilla-mobile/android-components/compare/7285afe57ae6...5fdb785b28b3",
|
|
||||||
"commits": [
|
|
||||||
{
|
|
||||||
"id": "8a4cb15deb60e490ec7dfae03ca350bde3688dcb",
|
|
||||||
"tree_id": "f7071289b2d2e51edd7ecf525132ef60e19e554e",
|
|
||||||
"distinct": true,
|
|
||||||
"message": "Closes #4779 - Fix `ExperimentsDebugActivity` command to change server",
|
|
||||||
"timestamp": "2020-02-06T13:22:20-06:00",
|
|
||||||
"url": "https://github.com/mozilla-mobile/android-components/commit/8a4cb15deb60e490ec7dfae03ca350bde3688dcb",
|
|
||||||
"author": {
|
|
||||||
"name": "User",
|
|
||||||
"email": "user@mozilla.com",
|
|
||||||
"username": "user"
|
|
||||||
},
|
|
||||||
"committer": {
|
|
||||||
"name": "User",
|
|
||||||
"email": "user@mozilla.com",
|
|
||||||
"username": "user"
|
|
||||||
},
|
|
||||||
"added": [],
|
|
||||||
"removed": [],
|
|
||||||
"modified": [
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
|
||||||
"tree_id": "410cf16b218c00bf743585d1d9cce5d89f418b97",
|
|
||||||
"distinct": true,
|
|
||||||
"message": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
|
||||||
"timestamp": "2020-02-12T15:29:12Z",
|
|
||||||
"url": "https://github.com/mozilla-mobile/android-components/commit/5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
|
||||||
"author": {
|
|
||||||
"name": "bors[bot]",
|
|
||||||
"email": "26634292+bors[bot]@users.noreply.github.com",
|
|
||||||
"username": "bors[bot]"
|
|
||||||
},
|
|
||||||
"committer": {
|
|
||||||
"name": "GitHub",
|
|
||||||
"email": "noreply@github.com",
|
|
||||||
"username": "web-flow"
|
|
||||||
},
|
|
||||||
"added": [],
|
|
||||||
"removed": [],
|
|
||||||
"modified": [
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"head_commit": {
|
|
||||||
"id": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
|
||||||
"tree_id": "410cf16b218c00bf743585d1d9cce5d89f418b97",
|
|
||||||
"distinct": true,
|
|
||||||
"message": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
|
||||||
"timestamp": "2020-02-12T15:29:12Z",
|
|
||||||
"url": "https://github.com/mozilla-mobile/android-components/commit/5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
|
||||||
"author": {
|
|
||||||
"name": "bors[bot]",
|
|
||||||
"email": "26634292+bors[bot]@users.noreply.github.com",
|
|
||||||
"username": "bors[bot]"
|
|
||||||
},
|
|
||||||
"committer": {
|
|
||||||
"name": "GitHub",
|
|
||||||
"email": "noreply@github.com",
|
|
||||||
"username": "web-flow"
|
|
||||||
},
|
|
||||||
"added": [],
|
|
||||||
"removed": [],
|
|
||||||
"modified": [
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
|
||||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"organization": "mozilla-mobile",
|
|
||||||
"details": {
|
|
||||||
"event.base.ref": "refs/heads/master",
|
|
||||||
"event.base.repo.name": "android-components",
|
|
||||||
"event.base.repo.url": "https://github.com/mozilla-mobile/android-components.git",
|
|
||||||
"event.base.sha": "7285afe57ae6207fdb5d6db45133dac2053b7820",
|
|
||||||
"event.base.user.login": "bors[bot]",
|
|
||||||
"event.head.ref": "refs/heads/master",
|
|
||||||
"event.head.repo.name": "android-components",
|
|
||||||
"event.head.repo.url": "https://github.com/mozilla-mobile/android-components.git",
|
|
||||||
"event.head.sha": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
|
||||||
"event.head.user.login": "bors[bot]",
|
|
||||||
"event.head.user.id": 26634292,
|
|
||||||
"event.type": "push",
|
|
||||||
"event.base.repo.branch": "master",
|
|
||||||
"event.head.repo.branch": "master",
|
|
||||||
"event.head.user.email": "bors@users.noreply.github.com"
|
|
||||||
},
|
|
||||||
"installationId": 5110595,
|
|
||||||
"tasks_for": "github-push",
|
|
||||||
"branch": "master",
|
|
||||||
"repository": "android-components",
|
|
||||||
"eventId": "6c722d22-4dac-11ea-8d13-04e94ccdb80b",
|
|
||||||
"version": 1
|
|
||||||
},
|
},
|
||||||
"exchange": "exchange/taskcluster-github/v1/push",
|
"repository": "test_treeherder",
|
||||||
"routingKey": "primary.mozilla-mobile.android-components",
|
"version": 1
|
||||||
"redelivered": false,
|
|
||||||
"cc": []
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,22 @@
|
||||||
{
|
{
|
||||||
"author": "26634292+bors[bot]@users.noreply.github.com",
|
"author": "kwierso@users.noreply.github.com",
|
||||||
"push_timestamp": 1581521352,
|
"push_timestamp": 1469726966,
|
||||||
"revision": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
"revision": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
|
||||||
"revisions": [
|
"revisions": [
|
||||||
{
|
{
|
||||||
"author": "User <user@mozilla.com>",
|
"author": "KWierso <kwierso@users.noreply.github.com>",
|
||||||
"comment": "Closes #4779 - Fix `ExperimentsDebugActivity` command to change server",
|
"comment": "Bug 1287911 - Add the ability to tag revisions with metadata (#1706) r=camd\n\n* Bug 1287911 - Add the ability to tag revisions with metadata\r\n\r\n* Bug 1287911 - Make backout commit text be red in the list of revisions",
|
||||||
"revision": "8a4cb15deb60e490ec7dfae03ca350bde3688dcb"
|
"revision": "09ef55394535acd453eb7728ef8981a96aa0aef6"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"author": "bors[bot] <26634292+bors[bot]@users.noreply.github.com>",
|
"author": "Roy C <crosscent@gmail.com>",
|
||||||
"comment": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
"comment": "Bug 1288530 - Update classifier unconditionally, and move classifier tooltip in AlertView (#1742)",
|
||||||
"revision": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1"
|
"revision": "ef46fa00eb0919e92f11ffdfe6a6926541ab409c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"author": "KWierso <kwierso@users.noreply.github.com>",
|
||||||
|
"comment": "Bug 1289651 - Only scroll job into view if it's not already visible (#1735) r=camd",
|
||||||
|
"revision": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ import taskcluster_urls as liburls
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
|
|
||||||
from treeherder.etl.common import fetch_json
|
|
||||||
from treeherder.etl.db_semaphore import (acquire_connection,
|
from treeherder.etl.db_semaphore import (acquire_connection,
|
||||||
release_connection)
|
release_connection)
|
||||||
from treeherder.etl.job_loader import JobLoader
|
from treeherder.etl.job_loader import JobLoader
|
||||||
|
|
@ -154,87 +153,6 @@ def get_decision_task_id(project, revision, root_url):
|
||||||
return find_task_id(index_path, root_url)
|
return find_task_id(index_path, root_url)
|
||||||
|
|
||||||
|
|
||||||
def ingestGitPush(options, root_url):
|
|
||||||
project = options["project"]
|
|
||||||
commit = options["commit"]
|
|
||||||
branch = None
|
|
||||||
# Ingesting pushes out of band from the ingestion pipeline would require
|
|
||||||
# a lot of work (not impossible) because the way Servo uses the "auto"
|
|
||||||
# and "try-" branches. A commit can temporarily belong to those branches.
|
|
||||||
# We need to imply the original branch directly from the project name
|
|
||||||
if project.startswith("servo"):
|
|
||||||
branch = project.split("-")[-1]
|
|
||||||
assert branch in ["auto", "try", "master"], \
|
|
||||||
"Valid servo projects are: servo-auto, servo-try, servo-master."
|
|
||||||
|
|
||||||
repository = Repository.objects.filter(name=project)
|
|
||||||
url = repository[0].url
|
|
||||||
splitUrl = url.split('/')
|
|
||||||
owner = splitUrl[3]
|
|
||||||
repo = splitUrl[4]
|
|
||||||
githubApi = "https://api.github.com"
|
|
||||||
baseUrl = "{}/repos/{}/{}".format(githubApi, owner, repo)
|
|
||||||
defaultBranch = fetch_json(baseUrl)["default_branch"]
|
|
||||||
# e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e
|
|
||||||
compareUrl = "{}/compare/{}...{}"
|
|
||||||
compareResponse = fetch_json(compareUrl.format(baseUrl, defaultBranch, commit))
|
|
||||||
headCommit = None
|
|
||||||
mergeBaseCommit = compareResponse["merge_base_commit"]
|
|
||||||
if mergeBaseCommit:
|
|
||||||
# Since we don't use PushEvents that contain the "before" field [1]
|
|
||||||
# we need to discover the right parent. A merge commit has two parents
|
|
||||||
# [1] https://github.com/taskcluster/taskcluster/blob/3dda0adf85619d18c5dcf255259f3e274d2be346/services/github/src/api.js#L55
|
|
||||||
parents = compareResponse["merge_base_commit"]["parents"]
|
|
||||||
eventBaseSha = None
|
|
||||||
for parent in parents:
|
|
||||||
_commit = fetch_json(parent["url"])
|
|
||||||
if _commit["parents"] and len(_commit["parents"]) > 1:
|
|
||||||
eventBaseSha = parent["sha"]
|
|
||||||
logger.info("We have a new base: %s", eventBaseSha)
|
|
||||||
break
|
|
||||||
# When using the correct sha the "commits" field will have information
|
|
||||||
compareResponse = fetch_json(compareUrl.format(baseUrl, eventBaseSha, commit))
|
|
||||||
|
|
||||||
headCommit = compareResponse["commits"][-1]
|
|
||||||
assert headCommit["sha"] == commit
|
|
||||||
|
|
||||||
commits = []
|
|
||||||
for c in compareResponse["commits"]:
|
|
||||||
commits.append({
|
|
||||||
"message": c["commit"]["message"],
|
|
||||||
"author": {
|
|
||||||
"name": c["commit"]["committer"]["name"],
|
|
||||||
"email": c["commit"]["committer"]["email"],
|
|
||||||
},
|
|
||||||
"id": c["sha"],
|
|
||||||
})
|
|
||||||
|
|
||||||
pulse = {
|
|
||||||
"exchange": "exchange/taskcluster-github/v1/push",
|
|
||||||
"routingKey": "primary.{}.{}".format(owner, repo),
|
|
||||||
"payload": {
|
|
||||||
"organization": owner,
|
|
||||||
"details": {
|
|
||||||
"event.head.repo.url": "https://github.com/{}/{}.git".format(owner, repo),
|
|
||||||
"event.base.repo.branch": branch
|
|
||||||
},
|
|
||||||
"repository": repo,
|
|
||||||
"body": {
|
|
||||||
"commits": commits,
|
|
||||||
"head_commit": {
|
|
||||||
"id": headCommit["sha"],
|
|
||||||
"author": {
|
|
||||||
"name": headCommit["committer"]["login"],
|
|
||||||
"email": headCommit["commit"]["committer"]["email"],
|
|
||||||
},
|
|
||||||
"timestamp": headCommit["commit"]["committer"]["date"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
|
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
"""Management command to ingest data from a single push."""
|
"""Management command to ingest data from a single push."""
|
||||||
help = "Ingests a single push and tasks into Treeherder"
|
help = "Ingests a single push and tasks into Treeherder"
|
||||||
|
|
@ -310,7 +228,7 @@ class Command(BaseCommand):
|
||||||
}
|
}
|
||||||
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
|
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
|
||||||
elif typeOfIngestion == "git-push":
|
elif typeOfIngestion == "git-push":
|
||||||
ingestGitPush(options, root_url)
|
raise Exception("This is not yet implemented")
|
||||||
elif typeOfIngestion == "push":
|
elif typeOfIngestion == "push":
|
||||||
if not options["enable_eager_celery"]:
|
if not options["enable_eager_celery"]:
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
|
||||||
|
|
@ -119,52 +119,35 @@ class GithubTransformer:
|
||||||
|
|
||||||
|
|
||||||
class GithubPushTransformer(GithubTransformer):
|
class GithubPushTransformer(GithubTransformer):
|
||||||
# https://github.com/taskcluster/taskcluster/blob/master/services/github/src/api.js#L254-L260
|
|
||||||
# {
|
# {
|
||||||
# exchange: exchange/taskcluster-github/v1/push
|
# organization:mozilla - services
|
||||||
# routingKey: primary.mozilla-mobile.android-components
|
# details:{
|
||||||
# payload: {
|
# event.type:push
|
||||||
# organization: mozilla-mobile
|
# event.base.repo.branch:master
|
||||||
# details: {
|
# event.head.repo.branch:master
|
||||||
# event.head.repo.url: https://github.com/mozilla-mobile/android-components.git
|
# event.head.user.login:mozilla-cloudops-deploy
|
||||||
# event.base.repo.branch: staging.tmp
|
# event.head.repo.url:https://github.com/mozilla-services/cloudops-jenkins.git
|
||||||
# }
|
# event.head.sha:845aa1c93726af92accd9b748ea361a37d5238b6
|
||||||
# repository: android-components
|
# event.head.ref:refs/heads/master
|
||||||
# body: {
|
# event.head.user.email:mozilla-cloudops-deploy@noreply.github.com
|
||||||
# commits: [{
|
|
||||||
# message: [ci skip][skip ci][skip netlify] -bors-staging-tmp-5835
|
|
||||||
# author:
|
|
||||||
# name: bors[bot]
|
|
||||||
# email: 26634292+bors[bot]@users.noreply.github.com
|
|
||||||
# }]
|
|
||||||
# head_commit: {
|
|
||||||
# id: 5fdb785b28b356f50fc1d9cb180d401bb03fc1f1
|
|
||||||
# author: {
|
|
||||||
# name: bors[bot]
|
|
||||||
# email: 26634292+bors[bot]@users.noreply.github.com
|
|
||||||
# }
|
|
||||||
# timestamp: 2020-02-12T15:29:12Z
|
|
||||||
# }
|
|
||||||
# }
|
# }
|
||||||
|
# repository:cloudops-jenkins
|
||||||
|
# version:1
|
||||||
# }
|
# }
|
||||||
|
|
||||||
|
URL_BASE = "https://api.github.com/repos/{}/{}/compare/{}...{}"
|
||||||
|
|
||||||
def transform(self, repository):
|
def transform(self, repository):
|
||||||
head_commit = self.message_body["body"]["head_commit"]
|
push_url = self.URL_BASE.format(
|
||||||
push = {
|
self.message_body["organization"],
|
||||||
"revision": head_commit["id"],
|
self.message_body["repository"],
|
||||||
"push_timestamp": to_timestamp(head_commit["timestamp"]),
|
self.message_body["details"]["event.base.sha"],
|
||||||
"author": head_commit["author"]["email"],
|
self.message_body["details"]["event.head.sha"],
|
||||||
"revisions": [],
|
)
|
||||||
}
|
return self.fetch_push(push_url, repository)
|
||||||
for commit in self.message_body["body"]["commits"]:
|
|
||||||
push["revisions"].append({
|
def get_cleaned_commits(self, compare):
|
||||||
"comment": commit["message"],
|
return compare["commits"]
|
||||||
"author": u"{} <{}>".format(
|
|
||||||
commit["author"]["name"],
|
|
||||||
commit["author"]["email"]),
|
|
||||||
"revision": commit["id"]
|
|
||||||
})
|
|
||||||
return push
|
|
||||||
|
|
||||||
def get_repo(self):
|
def get_repo(self):
|
||||||
return self.message_body["details"]["event.head.repo.url"].replace(".git", "")
|
return self.message_body["details"]["event.head.repo.url"].replace(".git", "")
|
||||||
|
|
|
||||||
Загрузка…
Ссылка в новой задаче