зеркало из https://github.com/mozilla/treeherder.git
Revert "Bug 1509181 - Fix ingestion of GitHub pushes + manual ingestion support (#6019)"
This reverts commit b41f68286d.
This commit is contained in:
Armen Zambrano G
Родитель
c702d2b548
Коммит
15b4d4d92b
|
|
@ -141,31 +141,21 @@ Ingestion tasks populate the database with version control push logs, queued/run
|
|||
|
||||
`NOTE`; You have to include `--root-url https://community-tc.services.mozilla.com` in order to ingest from the [Taskcluster Community instance](https://community-tc.services.mozilla.com), otherwise, it will default to the Firefox CI.
|
||||
|
||||
Open a terminal window and run `docker-compose up`. All following sections assume this step.
|
||||
|
||||
#### Ingesting pushes
|
||||
|
||||
Mercurial pushes:
|
||||
#### Ingesting pushes & tasks
|
||||
|
||||
```bash
|
||||
docker-compose exec backend ./manage.py ingest push -p autoland -r 63f8a47cfdf5
|
||||
docker-compose run backend ./manage.py ingest push -p autoland -r 63f8a47cfdf5
|
||||
```
|
||||
|
||||
`NOTE`: You can ingest all tasks for a push. Check the help output for the script to determine the
|
||||
You can ingest all tasks for a push. Check the help output for the script to determine the
|
||||
parameters needed.
|
||||
|
||||
Github pushes:
|
||||
|
||||
```bash
|
||||
docker-compose exec backend ./manage.py ingest git-push -p servo-try -c 92fc94588f3b6987082923c0003012fd696b1a2d
|
||||
```
|
||||
|
||||
#### Ingesting a range of pushes
|
||||
### Ingesting a range of pushes
|
||||
|
||||
It is also possible to ingest the last N pushes for a repository:
|
||||
|
||||
```bash
|
||||
docker-compose exec backend ./manage.py ingest_push mozilla-central --last-n-pushes 100
|
||||
docker-compose run backend ./manage.py ingest_push mozilla-central --last-n-pushes 100
|
||||
```
|
||||
|
||||
In this mode, only the push information will be ingested: tasks
|
||||
|
|
@ -178,7 +168,7 @@ changesets from the web interface into subsequent commands to ingest all tasks.
|
|||
`NOTE`: This will only ingest the commits if there's an active Github PRs project. It will only ingest the commits.
|
||||
|
||||
```bash
|
||||
docker-compose exec backend ./manage.py ingest pr --pr-url https://github.com/mozilla-mobile/android-components/pull/4821
|
||||
docker-compose run backend ./manage.py ingest pr --pr-url https://github.com/mozilla-mobile/android-components/pull/4821
|
||||
```
|
||||
|
||||
#### Ingesting individual task
|
||||
|
|
@ -187,7 +177,7 @@ This will work if the push associated to the task exists in the database.
|
|||
|
||||
```bash
|
||||
# Make sure to ingest 1bd9d4f431c4c9f93388bd04a6368cb07398f646 for autoland first
|
||||
docker-compose exec backend ./manage.py ingest task --task-id KQ5h1BVYTBy_XT21wFpLog
|
||||
docker-compose run backend ./manage.py ingest task --task-id KQ5h1BVYTBy_XT21wFpLog
|
||||
```
|
||||
|
||||
## Learn more
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ def test_ingest_github_pull_request(test_repository, github_pr, transformed_gith
|
|||
|
||||
def test_ingest_github_push(test_repository, github_push, transformed_github_push,
|
||||
mock_github_push_compare):
|
||||
xformer = GithubPushTransformer(github_push["payload"])
|
||||
xformer = GithubPushTransformer(github_push)
|
||||
push = xformer.transform(test_repository.name)
|
||||
assert transformed_github_push == push
|
||||
|
||||
|
|
@ -145,8 +145,8 @@ def test_ingest_hg_push_bad_repo(hg_push):
|
|||
@pytest.mark.django_db
|
||||
def test_ingest_github_push_bad_repo(github_push):
|
||||
"""Test graceful handling of an unknown GH repo"""
|
||||
github_push["payload"]["details"]["event.head.repo.url"] = "https://bad.repo.com"
|
||||
PushLoader().process(github_push["payload"], "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||
github_push["details"]["event.head.repo.url"] = "https://bad.repo.com"
|
||||
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||
assert Push.objects.count() == 0
|
||||
|
||||
|
||||
|
|
@ -160,10 +160,10 @@ def test_ingest_github_push_bad_repo(github_push):
|
|||
def test_ingest_github_push_comma_separated_branches(branch, expected_pushes, github_push,
|
||||
test_repository, mock_github_push_compare):
|
||||
"""Test a repository accepting pushes for multiple branches"""
|
||||
test_repository.url = "https://github.com/mozilla-mobile/android-components"
|
||||
test_repository.url = "https://github.com/mozilla/test_treeherder"
|
||||
test_repository.branch = "master,foo,bar"
|
||||
test_repository.save()
|
||||
github_push["payload"]["details"]["event.base.repo.branch"] = branch
|
||||
github_push["details"]["event.base.repo.branch"] = branch
|
||||
assert Push.objects.count() == 0
|
||||
PushLoader().process(github_push["payload"], "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||
PushLoader().process(github_push, "exchange/taskcluster-github/v1/push", "https://tc.example.com")
|
||||
assert Push.objects.count() == expected_pushes
|
||||
|
|
|
|||
|
|
@ -1,263 +1,16 @@
|
|||
{
|
||||
"payload": {
|
||||
"body": {
|
||||
"ref": "refs/heads/master",
|
||||
"before": "7285afe57ae6207fdb5d6db45133dac2053b7820",
|
||||
"after": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"repository": {
|
||||
"id": 126199585,
|
||||
"node_id": "MDEwOlJlcG9zaXRvcnkxMjYxOTk1ODU=",
|
||||
"name": "android-components",
|
||||
"full_name": "mozilla-mobile/android-components",
|
||||
"private": false,
|
||||
"owner": {
|
||||
"name": "mozilla-mobile",
|
||||
"email": null,
|
||||
"login": "mozilla-mobile",
|
||||
"id": 22351667,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjIyMzUxNjY3",
|
||||
"avatar_url": "https://avatars3.githubusercontent.com/u/22351667?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/mozilla-mobile",
|
||||
"html_url": "https://github.com/mozilla-mobile",
|
||||
"followers_url": "https://api.github.com/users/mozilla-mobile/followers",
|
||||
"following_url": "https://api.github.com/users/mozilla-mobile/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/mozilla-mobile/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/mozilla-mobile/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/mozilla-mobile/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/mozilla-mobile/orgs",
|
||||
"repos_url": "https://api.github.com/users/mozilla-mobile/repos",
|
||||
"events_url": "https://api.github.com/users/mozilla-mobile/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/mozilla-mobile/received_events",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"html_url": "https://github.com/mozilla-mobile/android-components",
|
||||
"description": "A collection of Android libraries to build browsers or browser-like applications.",
|
||||
"fork": false,
|
||||
"url": "https://github.com/mozilla-mobile/android-components",
|
||||
"forks_url": "https://api.github.com/repos/mozilla-mobile/android-components/forks",
|
||||
"keys_url": "https://api.github.com/repos/mozilla-mobile/android-components/keys{/key_id}",
|
||||
"collaborators_url": "https://api.github.com/repos/mozilla-mobile/android-components/collaborators{/collaborator}",
|
||||
"teams_url": "https://api.github.com/repos/mozilla-mobile/android-components/teams",
|
||||
"hooks_url": "https://api.github.com/repos/mozilla-mobile/android-components/hooks",
|
||||
"issue_events_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues/events{/number}",
|
||||
"events_url": "https://api.github.com/repos/mozilla-mobile/android-components/events",
|
||||
"assignees_url": "https://api.github.com/repos/mozilla-mobile/android-components/assignees{/user}",
|
||||
"branches_url": "https://api.github.com/repos/mozilla-mobile/android-components/branches{/branch}",
|
||||
"tags_url": "https://api.github.com/repos/mozilla-mobile/android-components/tags",
|
||||
"blobs_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/blobs{/sha}",
|
||||
"git_tags_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/tags{/sha}",
|
||||
"git_refs_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/refs{/sha}",
|
||||
"trees_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/trees{/sha}",
|
||||
"statuses_url": "https://api.github.com/repos/mozilla-mobile/android-components/statuses/{sha}",
|
||||
"languages_url": "https://api.github.com/repos/mozilla-mobile/android-components/languages",
|
||||
"stargazers_url": "https://api.github.com/repos/mozilla-mobile/android-components/stargazers",
|
||||
"contributors_url": "https://api.github.com/repos/mozilla-mobile/android-components/contributors",
|
||||
"subscribers_url": "https://api.github.com/repos/mozilla-mobile/android-components/subscribers",
|
||||
"subscription_url": "https://api.github.com/repos/mozilla-mobile/android-components/subscription",
|
||||
"commits_url": "https://api.github.com/repos/mozilla-mobile/android-components/commits{/sha}",
|
||||
"git_commits_url": "https://api.github.com/repos/mozilla-mobile/android-components/git/commits{/sha}",
|
||||
"comments_url": "https://api.github.com/repos/mozilla-mobile/android-components/comments{/number}",
|
||||
"issue_comment_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues/comments{/number}",
|
||||
"contents_url": "https://api.github.com/repos/mozilla-mobile/android-components/contents/{+path}",
|
||||
"compare_url": "https://api.github.com/repos/mozilla-mobile/android-components/compare/{base}...{head}",
|
||||
"merges_url": "https://api.github.com/repos/mozilla-mobile/android-components/merges",
|
||||
"archive_url": "https://api.github.com/repos/mozilla-mobile/android-components/{archive_format}{/ref}",
|
||||
"downloads_url": "https://api.github.com/repos/mozilla-mobile/android-components/downloads",
|
||||
"issues_url": "https://api.github.com/repos/mozilla-mobile/android-components/issues{/number}",
|
||||
"pulls_url": "https://api.github.com/repos/mozilla-mobile/android-components/pulls{/number}",
|
||||
"milestones_url": "https://api.github.com/repos/mozilla-mobile/android-components/milestones{/number}",
|
||||
"notifications_url": "https://api.github.com/repos/mozilla-mobile/android-components/notifications{?since,all,participating}",
|
||||
"labels_url": "https://api.github.com/repos/mozilla-mobile/android-components/labels{/name}",
|
||||
"releases_url": "https://api.github.com/repos/mozilla-mobile/android-components/releases{/id}",
|
||||
"deployments_url": "https://api.github.com/repos/mozilla-mobile/android-components/deployments",
|
||||
"created_at": 1521645843,
|
||||
"updated_at": "2020-02-12T15:21:24Z",
|
||||
"pushed_at": 1581521353,
|
||||
"git_url": "git://github.com/mozilla-mobile/android-components.git",
|
||||
"ssh_url": "git@github.com:mozilla-mobile/android-components.git",
|
||||
"clone_url": "https://github.com/mozilla-mobile/android-components.git",
|
||||
"svn_url": "https://github.com/mozilla-mobile/android-components",
|
||||
"homepage": "https://mozac.org",
|
||||
"size": 61634,
|
||||
"stargazers_count": 1214,
|
||||
"watchers_count": 1214,
|
||||
"language": "Kotlin",
|
||||
"has_issues": true,
|
||||
"has_projects": false,
|
||||
"has_downloads": true,
|
||||
"has_wiki": false,
|
||||
"has_pages": true,
|
||||
"forks_count": 268,
|
||||
"mirror_url": null,
|
||||
"archived": false,
|
||||
"disabled": false,
|
||||
"open_issues_count": 684,
|
||||
"license": {
|
||||
"key": "mpl-2.0",
|
||||
"name": "Mozilla Public License 2.0",
|
||||
"spdx_id": "MPL-2.0",
|
||||
"url": "https://api.github.com/licenses/mpl-2.0",
|
||||
"node_id": "MDc6TGljZW5zZTE0"
|
||||
},
|
||||
"forks": 268,
|
||||
"open_issues": 684,
|
||||
"watchers": 1214,
|
||||
"default_branch": "master",
|
||||
"stargazers": 1214,
|
||||
"master_branch": "master",
|
||||
"organization": "mozilla-mobile"
|
||||
},
|
||||
"pusher": {
|
||||
"name": "bors[bot]",
|
||||
"email": null
|
||||
},
|
||||
"organization": {
|
||||
"login": "mozilla-mobile",
|
||||
"id": 22351667,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjIyMzUxNjY3",
|
||||
"url": "https://api.github.com/orgs/mozilla-mobile",
|
||||
"repos_url": "https://api.github.com/orgs/mozilla-mobile/repos",
|
||||
"events_url": "https://api.github.com/orgs/mozilla-mobile/events",
|
||||
"hooks_url": "https://api.github.com/orgs/mozilla-mobile/hooks",
|
||||
"issues_url": "https://api.github.com/orgs/mozilla-mobile/issues",
|
||||
"members_url": "https://api.github.com/orgs/mozilla-mobile/members{/member}",
|
||||
"public_members_url": "https://api.github.com/orgs/mozilla-mobile/public_members{/member}",
|
||||
"avatar_url": "https://avatars3.githubusercontent.com/u/22351667?v=4",
|
||||
"description": "Mozilla Mobile Applications"
|
||||
},
|
||||
"sender": {
|
||||
"login": "bors[bot]",
|
||||
"id": 26634292,
|
||||
"node_id": "MDM6Qm90MjY2MzQyOTI=",
|
||||
"avatar_url": "https://avatars3.githubusercontent.com/in/1847?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/bors%5Bbot%5D",
|
||||
"html_url": "https://github.com/apps/bors",
|
||||
"followers_url": "https://api.github.com/users/bors%5Bbot%5D/followers",
|
||||
"following_url": "https://api.github.com/users/bors%5Bbot%5D/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/bors%5Bbot%5D/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/bors%5Bbot%5D/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/bors%5Bbot%5D/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/bors%5Bbot%5D/orgs",
|
||||
"repos_url": "https://api.github.com/users/bors%5Bbot%5D/repos",
|
||||
"events_url": "https://api.github.com/users/bors%5Bbot%5D/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/bors%5Bbot%5D/received_events",
|
||||
"type": "Bot",
|
||||
"site_admin": false
|
||||
},
|
||||
"installation": {
|
||||
"id": 5110595,
|
||||
"node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNTExMDU5NQ=="
|
||||
},
|
||||
"created": false,
|
||||
"deleted": false,
|
||||
"forced": false,
|
||||
"base_ref": null,
|
||||
"compare": "https://github.com/mozilla-mobile/android-components/compare/7285afe57ae6...5fdb785b28b3",
|
||||
"commits": [
|
||||
{
|
||||
"id": "8a4cb15deb60e490ec7dfae03ca350bde3688dcb",
|
||||
"tree_id": "f7071289b2d2e51edd7ecf525132ef60e19e554e",
|
||||
"distinct": true,
|
||||
"message": "Closes #4779 - Fix `ExperimentsDebugActivity` command to change server",
|
||||
"timestamp": "2020-02-06T13:22:20-06:00",
|
||||
"url": "https://github.com/mozilla-mobile/android-components/commit/8a4cb15deb60e490ec7dfae03ca350bde3688dcb",
|
||||
"author": {
|
||||
"name": "User",
|
||||
"email": "user@mozilla.com",
|
||||
"username": "user"
|
||||
},
|
||||
"committer": {
|
||||
"name": "User",
|
||||
"email": "user@mozilla.com",
|
||||
"username": "user"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [],
|
||||
"modified": [
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"tree_id": "410cf16b218c00bf743585d1d9cce5d89f418b97",
|
||||
"distinct": true,
|
||||
"message": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
||||
"timestamp": "2020-02-12T15:29:12Z",
|
||||
"url": "https://github.com/mozilla-mobile/android-components/commit/5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"author": {
|
||||
"name": "bors[bot]",
|
||||
"email": "26634292+bors[bot]@users.noreply.github.com",
|
||||
"username": "bors[bot]"
|
||||
},
|
||||
"committer": {
|
||||
"name": "GitHub",
|
||||
"email": "noreply@github.com",
|
||||
"username": "web-flow"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [],
|
||||
"modified": [
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
||||
]
|
||||
}
|
||||
],
|
||||
"head_commit": {
|
||||
"id": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"tree_id": "410cf16b218c00bf743585d1d9cce5d89f418b97",
|
||||
"distinct": true,
|
||||
"message": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
||||
"timestamp": "2020-02-12T15:29:12Z",
|
||||
"url": "https://github.com/mozilla-mobile/android-components/commit/5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"author": {
|
||||
"name": "bors[bot]",
|
||||
"email": "26634292+bors[bot]@users.noreply.github.com",
|
||||
"username": "bors[bot]"
|
||||
},
|
||||
"committer": {
|
||||
"name": "GitHub",
|
||||
"email": "noreply@github.com",
|
||||
"username": "web-flow"
|
||||
},
|
||||
"added": [],
|
||||
"removed": [],
|
||||
"modified": [
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/ExperimentsUpdater.kt",
|
||||
"components/service/experiments/src/main/java/mozilla/components/service/experiments/debug/ExperimentsDebugActivity.kt"
|
||||
]
|
||||
}
|
||||
},
|
||||
"organization": "mozilla-mobile",
|
||||
"organization": "mozilla-services",
|
||||
"details": {
|
||||
"event.base.ref": "refs/heads/master",
|
||||
"event.base.repo.name": "android-components",
|
||||
"event.base.repo.url": "https://github.com/mozilla-mobile/android-components.git",
|
||||
"event.base.sha": "7285afe57ae6207fdb5d6db45133dac2053b7820",
|
||||
"event.base.user.login": "bors[bot]",
|
||||
"event.head.ref": "refs/heads/master",
|
||||
"event.head.repo.name": "android-components",
|
||||
"event.head.repo.url": "https://github.com/mozilla-mobile/android-components.git",
|
||||
"event.head.sha": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"event.head.user.login": "bors[bot]",
|
||||
"event.head.user.id": 26634292,
|
||||
"event.type": "push",
|
||||
"event.base.repo.branch": "master",
|
||||
"event.base.sha": "71c02dbb26cb60beee94bac433086bc540c9d6d4",
|
||||
"event.head.repo.branch": "master",
|
||||
"event.head.user.email": "bors@users.noreply.github.com"
|
||||
"event.head.user.login": "mozilla-test_treeherder",
|
||||
"event.head.repo.url": "https://github.com/mozilla/test_treeherder.git",
|
||||
"event.head.sha": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
|
||||
"event.head.ref": "refs/heads/master",
|
||||
"event.head.user.email": "mozilla-test_treeherder@noreply.github.com"
|
||||
},
|
||||
"installationId": 5110595,
|
||||
"tasks_for": "github-push",
|
||||
"branch": "master",
|
||||
"repository": "android-components",
|
||||
"eventId": "6c722d22-4dac-11ea-8d13-04e94ccdb80b",
|
||||
"repository": "test_treeherder",
|
||||
"version": 1
|
||||
},
|
||||
"exchange": "exchange/taskcluster-github/v1/push",
|
||||
"routingKey": "primary.mozilla-mobile.android-components",
|
||||
"redelivered": false,
|
||||
"cc": []
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,22 @@
|
|||
{
|
||||
"author": "26634292+bors[bot]@users.noreply.github.com",
|
||||
"push_timestamp": 1581521352,
|
||||
"revision": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1",
|
||||
"author": "kwierso@users.noreply.github.com",
|
||||
"push_timestamp": 1469726966,
|
||||
"revision": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
|
||||
"revisions": [
|
||||
{
|
||||
"author": "User <user@mozilla.com>",
|
||||
"comment": "Closes #4779 - Fix `ExperimentsDebugActivity` command to change server",
|
||||
"revision": "8a4cb15deb60e490ec7dfae03ca350bde3688dcb"
|
||||
"author": "KWierso <kwierso@users.noreply.github.com>",
|
||||
"comment": "Bug 1287911 - Add the ability to tag revisions with metadata (#1706) r=camd\n\n* Bug 1287911 - Add the ability to tag revisions with metadata\r\n\r\n* Bug 1287911 - Make backout commit text be red in the list of revisions",
|
||||
"revision": "09ef55394535acd453eb7728ef8981a96aa0aef6"
|
||||
},
|
||||
{
|
||||
"author": "bors[bot] <26634292+bors[bot]@users.noreply.github.com>",
|
||||
"comment": "[ci skip][skip ci][skip netlify] -bors-staging-tmp-5835",
|
||||
"revision": "5fdb785b28b356f50fc1d9cb180d401bb03fc1f1"
|
||||
"author": "Roy C <crosscent@gmail.com>",
|
||||
"comment": "Bug 1288530 - Update classifier unconditionally, and move classifier tooltip in AlertView (#1742)",
|
||||
"revision": "ef46fa00eb0919e92f11ffdfe6a6926541ab409c"
|
||||
},
|
||||
{
|
||||
"author": "KWierso <kwierso@users.noreply.github.com>",
|
||||
"comment": "Bug 1289651 - Only scroll job into view if it's not already visible (#1735) r=camd",
|
||||
"revision": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import taskcluster_urls as liburls
|
|||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from treeherder.etl.common import fetch_json
|
||||
from treeherder.etl.db_semaphore import (acquire_connection,
|
||||
release_connection)
|
||||
from treeherder.etl.job_loader import JobLoader
|
||||
|
|
@ -154,87 +153,6 @@ def get_decision_task_id(project, revision, root_url):
|
|||
return find_task_id(index_path, root_url)
|
||||
|
||||
|
||||
def ingestGitPush(options, root_url):
|
||||
project = options["project"]
|
||||
commit = options["commit"]
|
||||
branch = None
|
||||
# Ingesting pushes out of band from the ingestion pipeline would require
|
||||
# a lot of work (not impossible) because the way Servo uses the "auto"
|
||||
# and "try-" branches. A commit can temporarily belong to those branches.
|
||||
# We need to imply the original branch directly from the project name
|
||||
if project.startswith("servo"):
|
||||
branch = project.split("-")[-1]
|
||||
assert branch in ["auto", "try", "master"], \
|
||||
"Valid servo projects are: servo-auto, servo-try, servo-master."
|
||||
|
||||
repository = Repository.objects.filter(name=project)
|
||||
url = repository[0].url
|
||||
splitUrl = url.split('/')
|
||||
owner = splitUrl[3]
|
||||
repo = splitUrl[4]
|
||||
githubApi = "https://api.github.com"
|
||||
baseUrl = "{}/repos/{}/{}".format(githubApi, owner, repo)
|
||||
defaultBranch = fetch_json(baseUrl)["default_branch"]
|
||||
# e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e
|
||||
compareUrl = "{}/compare/{}...{}"
|
||||
compareResponse = fetch_json(compareUrl.format(baseUrl, defaultBranch, commit))
|
||||
headCommit = None
|
||||
mergeBaseCommit = compareResponse["merge_base_commit"]
|
||||
if mergeBaseCommit:
|
||||
# Since we don't use PushEvents that contain the "before" field [1]
|
||||
# we need to discover the right parent. A merge commit has two parents
|
||||
# [1] https://github.com/taskcluster/taskcluster/blob/3dda0adf85619d18c5dcf255259f3e274d2be346/services/github/src/api.js#L55
|
||||
parents = compareResponse["merge_base_commit"]["parents"]
|
||||
eventBaseSha = None
|
||||
for parent in parents:
|
||||
_commit = fetch_json(parent["url"])
|
||||
if _commit["parents"] and len(_commit["parents"]) > 1:
|
||||
eventBaseSha = parent["sha"]
|
||||
logger.info("We have a new base: %s", eventBaseSha)
|
||||
break
|
||||
# When using the correct sha the "commits" field will have information
|
||||
compareResponse = fetch_json(compareUrl.format(baseUrl, eventBaseSha, commit))
|
||||
|
||||
headCommit = compareResponse["commits"][-1]
|
||||
assert headCommit["sha"] == commit
|
||||
|
||||
commits = []
|
||||
for c in compareResponse["commits"]:
|
||||
commits.append({
|
||||
"message": c["commit"]["message"],
|
||||
"author": {
|
||||
"name": c["commit"]["committer"]["name"],
|
||||
"email": c["commit"]["committer"]["email"],
|
||||
},
|
||||
"id": c["sha"],
|
||||
})
|
||||
|
||||
pulse = {
|
||||
"exchange": "exchange/taskcluster-github/v1/push",
|
||||
"routingKey": "primary.{}.{}".format(owner, repo),
|
||||
"payload": {
|
||||
"organization": owner,
|
||||
"details": {
|
||||
"event.head.repo.url": "https://github.com/{}/{}.git".format(owner, repo),
|
||||
"event.base.repo.branch": branch
|
||||
},
|
||||
"repository": repo,
|
||||
"body": {
|
||||
"commits": commits,
|
||||
"head_commit": {
|
||||
"id": headCommit["sha"],
|
||||
"author": {
|
||||
"name": headCommit["committer"]["login"],
|
||||
"email": headCommit["commit"]["committer"]["email"],
|
||||
},
|
||||
"timestamp": headCommit["commit"]["committer"]["date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""Management command to ingest data from a single push."""
|
||||
help = "Ingests a single push and tasks into Treeherder"
|
||||
|
|
@ -310,7 +228,7 @@ class Command(BaseCommand):
|
|||
}
|
||||
PushLoader().process(pulse["payload"], pulse["exchange"], root_url)
|
||||
elif typeOfIngestion == "git-push":
|
||||
ingestGitPush(options, root_url)
|
||||
raise Exception("This is not yet implemented")
|
||||
elif typeOfIngestion == "push":
|
||||
if not options["enable_eager_celery"]:
|
||||
logger.info(
|
||||
|
|
|
|||
|
|
@ -119,52 +119,35 @@ class GithubTransformer:
|
|||
|
||||
|
||||
class GithubPushTransformer(GithubTransformer):
|
||||
# https://github.com/taskcluster/taskcluster/blob/master/services/github/src/api.js#L254-L260
|
||||
# {
|
||||
# exchange: exchange/taskcluster-github/v1/push
|
||||
# routingKey: primary.mozilla-mobile.android-components
|
||||
# payload: {
|
||||
# organization: mozilla-mobile
|
||||
# details: {
|
||||
# event.head.repo.url: https://github.com/mozilla-mobile/android-components.git
|
||||
# event.base.repo.branch: staging.tmp
|
||||
# }
|
||||
# repository: android-components
|
||||
# body: {
|
||||
# commits: [{
|
||||
# message: [ci skip][skip ci][skip netlify] -bors-staging-tmp-5835
|
||||
# author:
|
||||
# name: bors[bot]
|
||||
# email: 26634292+bors[bot]@users.noreply.github.com
|
||||
# }]
|
||||
# head_commit: {
|
||||
# id: 5fdb785b28b356f50fc1d9cb180d401bb03fc1f1
|
||||
# author: {
|
||||
# name: bors[bot]
|
||||
# email: 26634292+bors[bot]@users.noreply.github.com
|
||||
# }
|
||||
# timestamp: 2020-02-12T15:29:12Z
|
||||
# }
|
||||
# organization:mozilla - services
|
||||
# details:{
|
||||
# event.type:push
|
||||
# event.base.repo.branch:master
|
||||
# event.head.repo.branch:master
|
||||
# event.head.user.login:mozilla-cloudops-deploy
|
||||
# event.head.repo.url:https://github.com/mozilla-services/cloudops-jenkins.git
|
||||
# event.head.sha:845aa1c93726af92accd9b748ea361a37d5238b6
|
||||
# event.head.ref:refs/heads/master
|
||||
# event.head.user.email:mozilla-cloudops-deploy@noreply.github.com
|
||||
# }
|
||||
# repository:cloudops-jenkins
|
||||
# version:1
|
||||
# }
|
||||
|
||||
URL_BASE = "https://api.github.com/repos/{}/{}/compare/{}...{}"
|
||||
|
||||
def transform(self, repository):
|
||||
head_commit = self.message_body["body"]["head_commit"]
|
||||
push = {
|
||||
"revision": head_commit["id"],
|
||||
"push_timestamp": to_timestamp(head_commit["timestamp"]),
|
||||
"author": head_commit["author"]["email"],
|
||||
"revisions": [],
|
||||
}
|
||||
for commit in self.message_body["body"]["commits"]:
|
||||
push["revisions"].append({
|
||||
"comment": commit["message"],
|
||||
"author": u"{} <{}>".format(
|
||||
commit["author"]["name"],
|
||||
commit["author"]["email"]),
|
||||
"revision": commit["id"]
|
||||
})
|
||||
return push
|
||||
push_url = self.URL_BASE.format(
|
||||
self.message_body["organization"],
|
||||
self.message_body["repository"],
|
||||
self.message_body["details"]["event.base.sha"],
|
||||
self.message_body["details"]["event.head.sha"],
|
||||
)
|
||||
return self.fetch_push(push_url, repository)
|
||||
|
||||
def get_cleaned_commits(self, compare):
|
||||
return compare["commits"]
|
||||
|
||||
def get_repo(self):
|
||||
return self.message_body["details"]["event.head.repo.url"].replace(".git", "")
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче