Fix import of force-pushes from github (#4284)

When `git push -f` is used, the "base" commit of the github push event
is whatever was previously in that branch.
That commit is likely not part of the history of the new branch,
so using the "List commits" API will not allow to find it
as a starting point.
Instead, if the API did not force pagination, we’d get the entire
repository history!

Using the "Compare two commits" API instead lets GitHub’s server
figure out which commits reachable from the new branch
that were not before, which is exactly the set of commits we need
to be "part of a push".

tests/etl/test_push_loader.py

@@ -58,16 +58,19 @@ def mock_github_pr_commits(activate_responses):
 
 
 @pytest.fixture
-def mock_github_push_commits(activate_responses):
+def mock_github_push_compare(activate_responses):
     tests_folder = os.path.dirname(os.path.dirname(__file__))
     path = os.path.join(
         tests_folder,
         "sample_data/pulse_consumer",
-        "github_push_commits.json"
+        "github_push_compare.json"
     )
     with open(path) as f:
         mocked_content = f.read()
-    responses.add(responses.GET, "https://api.github.com/repos/mozilla-services/test_treeherder/commits",
+    responses.add(responses.GET,
+                  "https://api.github.com/repos/mozilla-services/test_treeherder/compare/"
+                  "71c02dbb26cb60beee94bac433086bc540c9d6d4..."
+                  "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
                   body=mocked_content, status=200, match_querystring=False,
                   content_type='application/json')
 
@@ -109,7 +112,7 @@ def test_ingest_github_pull_request(test_repository, github_pr, transformed_gith
 
 
 def test_ingest_github_push(test_repository, github_push, transformed_github_push,
-                            mock_github_push_commits):
+                            mock_github_push_compare):
     xformer = GithubPushTransformer(github_push)
     push = xformer.transform(test_repository.name)
     assert transformed_github_push == push

tests/sample_data/pulse_consumer/github_push_compare.json

@@ -0,0 +1,470 @@
+{
+  "url": "https://api.github.com/repos/mozilla/treeherder/compare/71c02dbb26cb60beee94bac433086bc540c9d6d4...5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+  "html_url": "https://github.com/mozilla/treeherder/compare/71c02dbb26cb60beee94bac433086bc540c9d6d4...5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+  "permalink_url": "https://github.com/mozilla/treeherder/compare/mozilla:71c02db...mozilla:5219f00",
+  "diff_url": "https://github.com/mozilla/treeherder/compare/71c02dbb26cb60beee94bac433086bc540c9d6d4...5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa.diff",
+  "patch_url": "https://github.com/mozilla/treeherder/compare/71c02dbb26cb60beee94bac433086bc540c9d6d4...5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa.patch",
+  "base_commit": {
+    "sha": "71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "node_id": "MDY6Q29tbWl0ODQ2MzI5NDo3MWMwMmRiYjI2Y2I2MGJlZWU5NGJhYzQzMzA4NmJjNTQwYzlkNmQ0",
+    "commit": {
+      "author": {
+        "name": "Shruti Jasoria",
+        "email": "SJasoria@users.noreply.github.com",
+        "date": "2016-07-28T15:45:24Z"
+      },
+      "committer": {
+        "name": "William Lachance",
+        "email": "wrlach@gmail.com",
+        "date": "2016-07-28T15:45:24Z"
+      },
+      "message": "Bug 1269825 - Auto load mozilla-aurora data for mozilla-beta alerts (#1737)",
+      "tree": {
+        "sha": "15b61e31f3927b35431432338a0b302b11fe7535",
+        "url": "https://api.github.com/repos/mozilla/treeherder/git/trees/15b61e31f3927b35431432338a0b302b11fe7535"
+      },
+      "url": "https://api.github.com/repos/mozilla/treeherder/git/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+      "comment_count": 0,
+      "verification": {
+        "verified": false,
+        "reason": "unsigned",
+        "signature": null,
+        "payload": null
+      }
+    },
+    "url": "https://api.github.com/repos/mozilla/treeherder/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "html_url": "https://github.com/mozilla/treeherder/commit/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "comments_url": "https://api.github.com/repos/mozilla/treeherder/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4/comments",
+    "author": {
+      "login": "SJasoria",
+      "id": 11815849,
+      "node_id": "MDQ6VXNlcjExODE1ODQ5",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/11815849?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/SJasoria",
+      "html_url": "https://github.com/SJasoria",
+      "followers_url": "https://api.github.com/users/SJasoria/followers",
+      "following_url": "https://api.github.com/users/SJasoria/following{/other_user}",
+      "gists_url": "https://api.github.com/users/SJasoria/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/SJasoria/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/SJasoria/subscriptions",
+      "organizations_url": "https://api.github.com/users/SJasoria/orgs",
+      "repos_url": "https://api.github.com/users/SJasoria/repos",
+      "events_url": "https://api.github.com/users/SJasoria/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/SJasoria/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "committer": {
+      "login": "wlach",
+      "id": 20569,
+      "node_id": "MDQ6VXNlcjIwNTY5",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/20569?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/wlach",
+      "html_url": "https://github.com/wlach",
+      "followers_url": "https://api.github.com/users/wlach/followers",
+      "following_url": "https://api.github.com/users/wlach/following{/other_user}",
+      "gists_url": "https://api.github.com/users/wlach/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/wlach/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/wlach/subscriptions",
+      "organizations_url": "https://api.github.com/users/wlach/orgs",
+      "repos_url": "https://api.github.com/users/wlach/repos",
+      "events_url": "https://api.github.com/users/wlach/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/wlach/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "parents": [
+      {
+        "sha": "7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2",
+        "url": "https://api.github.com/repos/mozilla/treeherder/commits/7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2",
+        "html_url": "https://github.com/mozilla/treeherder/commit/7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2"
+      }
+    ]
+  },
+  "merge_base_commit": {
+    "sha": "71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "node_id": "MDY6Q29tbWl0ODQ2MzI5NDo3MWMwMmRiYjI2Y2I2MGJlZWU5NGJhYzQzMzA4NmJjNTQwYzlkNmQ0",
+    "commit": {
+      "author": {
+        "name": "Shruti Jasoria",
+        "email": "SJasoria@users.noreply.github.com",
+        "date": "2016-07-28T15:45:24Z"
+      },
+      "committer": {
+        "name": "William Lachance",
+        "email": "wrlach@gmail.com",
+        "date": "2016-07-28T15:45:24Z"
+      },
+      "message": "Bug 1269825 - Auto load mozilla-aurora data for mozilla-beta alerts (#1737)",
+      "tree": {
+        "sha": "15b61e31f3927b35431432338a0b302b11fe7535",
+        "url": "https://api.github.com/repos/mozilla/treeherder/git/trees/15b61e31f3927b35431432338a0b302b11fe7535"
+      },
+      "url": "https://api.github.com/repos/mozilla/treeherder/git/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+      "comment_count": 0,
+      "verification": {
+        "verified": false,
+        "reason": "unsigned",
+        "signature": null,
+        "payload": null
+      }
+    },
+    "url": "https://api.github.com/repos/mozilla/treeherder/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "html_url": "https://github.com/mozilla/treeherder/commit/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+    "comments_url": "https://api.github.com/repos/mozilla/treeherder/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4/comments",
+    "author": {
+      "login": "SJasoria",
+      "id": 11815849,
+      "node_id": "MDQ6VXNlcjExODE1ODQ5",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/11815849?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/SJasoria",
+      "html_url": "https://github.com/SJasoria",
+      "followers_url": "https://api.github.com/users/SJasoria/followers",
+      "following_url": "https://api.github.com/users/SJasoria/following{/other_user}",
+      "gists_url": "https://api.github.com/users/SJasoria/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/SJasoria/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/SJasoria/subscriptions",
+      "organizations_url": "https://api.github.com/users/SJasoria/orgs",
+      "repos_url": "https://api.github.com/users/SJasoria/repos",
+      "events_url": "https://api.github.com/users/SJasoria/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/SJasoria/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "committer": {
+      "login": "wlach",
+      "id": 20569,
+      "node_id": "MDQ6VXNlcjIwNTY5",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/20569?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/wlach",
+      "html_url": "https://github.com/wlach",
+      "followers_url": "https://api.github.com/users/wlach/followers",
+      "following_url": "https://api.github.com/users/wlach/following{/other_user}",
+      "gists_url": "https://api.github.com/users/wlach/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/wlach/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/wlach/subscriptions",
+      "organizations_url": "https://api.github.com/users/wlach/orgs",
+      "repos_url": "https://api.github.com/users/wlach/repos",
+      "events_url": "https://api.github.com/users/wlach/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/wlach/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "parents": [
+      {
+        "sha": "7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2",
+        "url": "https://api.github.com/repos/mozilla/treeherder/commits/7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2",
+        "html_url": "https://github.com/mozilla/treeherder/commit/7f81eb32ce9a0b2de0939311f10e81fc7e23ccf2"
+      }
+    ]
+  },
+  "status": "ahead",
+  "ahead_by": 3,
+  "behind_by": 0,
+  "total_commits": 3,
+  "commits": [
+    {
+      "sha": "09ef55394535acd453eb7728ef8981a96aa0aef6",
+      "node_id": "MDY6Q29tbWl0ODQ2MzI5NDowOWVmNTUzOTQ1MzVhY2Q0NTNlYjc3MjhlZjg5ODFhOTZhYTBhZWY2",
+      "commit": {
+        "author": {
+          "name": "KWierso",
+          "email": "kwierso@users.noreply.github.com",
+          "date": "2016-07-28T16:55:43Z"
+        },
+        "committer": {
+          "name": "GitHub",
+          "email": "noreply@github.com",
+          "date": "2016-07-28T16:55:43Z"
+        },
+        "message": "Bug 1287911 - Add the ability to tag revisions with metadata (#1706) r=camd\n\n* Bug 1287911 - Add the ability to tag revisions with metadata\r\n\r\n* Bug 1287911 - Make backout commit text be red in the list of revisions",
+        "tree": {
+          "sha": "9e6c3fbec461ad55df307bbb7598416d2207000d",
+          "url": "https://api.github.com/repos/mozilla/treeherder/git/trees/9e6c3fbec461ad55df307bbb7598416d2207000d"
+        },
+        "url": "https://api.github.com/repos/mozilla/treeherder/git/commits/09ef55394535acd453eb7728ef8981a96aa0aef6",
+        "comment_count": 0,
+        "verification": {
+          "verified": false,
+          "reason": "unsigned",
+          "signature": null,
+          "payload": null
+        }
+      },
+      "url": "https://api.github.com/repos/mozilla/treeherder/commits/09ef55394535acd453eb7728ef8981a96aa0aef6",
+      "html_url": "https://github.com/mozilla/treeherder/commit/09ef55394535acd453eb7728ef8981a96aa0aef6",
+      "comments_url": "https://api.github.com/repos/mozilla/treeherder/commits/09ef55394535acd453eb7728ef8981a96aa0aef6/comments",
+      "author": {
+        "login": "KWierso",
+        "id": 172215,
+        "node_id": "MDQ6VXNlcjE3MjIxNQ==",
+        "avatar_url": "https://avatars1.githubusercontent.com/u/172215?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/KWierso",
+        "html_url": "https://github.com/KWierso",
+        "followers_url": "https://api.github.com/users/KWierso/followers",
+        "following_url": "https://api.github.com/users/KWierso/following{/other_user}",
+        "gists_url": "https://api.github.com/users/KWierso/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/KWierso/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/KWierso/subscriptions",
+        "organizations_url": "https://api.github.com/users/KWierso/orgs",
+        "repos_url": "https://api.github.com/users/KWierso/repos",
+        "events_url": "https://api.github.com/users/KWierso/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/KWierso/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "committer": {
+        "login": "web-flow",
+        "id": 19864447,
+        "node_id": "MDQ6VXNlcjE5ODY0NDQ3",
+        "avatar_url": "https://avatars3.githubusercontent.com/u/19864447?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/web-flow",
+        "html_url": "https://github.com/web-flow",
+        "followers_url": "https://api.github.com/users/web-flow/followers",
+        "following_url": "https://api.github.com/users/web-flow/following{/other_user}",
+        "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions",
+        "organizations_url": "https://api.github.com/users/web-flow/orgs",
+        "repos_url": "https://api.github.com/users/web-flow/repos",
+        "events_url": "https://api.github.com/users/web-flow/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/web-flow/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "parents": [
+        {
+          "sha": "71c02dbb26cb60beee94bac433086bc540c9d6d4",
+          "url": "https://api.github.com/repos/mozilla/treeherder/commits/71c02dbb26cb60beee94bac433086bc540c9d6d4",
+          "html_url": "https://github.com/mozilla/treeherder/commit/71c02dbb26cb60beee94bac433086bc540c9d6d4"
+        }
+      ]
+    },
+    {
+      "sha": "ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+      "node_id": "MDY6Q29tbWl0ODQ2MzI5NDplZjQ2ZmEwMGViMDkxOWU5MmYxMWZmZGZlNmE2OTI2NTQxYWI0MDlj",
+      "commit": {
+        "author": {
+          "name": "Roy C",
+          "email": "crosscent@gmail.com",
+          "date": "2016-07-28T16:59:27Z"
+        },
+        "committer": {
+          "name": "William Lachance",
+          "email": "wrlach@gmail.com",
+          "date": "2016-07-28T16:59:27Z"
+        },
+        "message": "Bug 1288530 - Update classifier unconditionally, and move classifier tooltip in AlertView (#1742)",
+        "tree": {
+          "sha": "1dd394b90e6e4413996acaac3845144d0220ba65",
+          "url": "https://api.github.com/repos/mozilla/treeherder/git/trees/1dd394b90e6e4413996acaac3845144d0220ba65"
+        },
+        "url": "https://api.github.com/repos/mozilla/treeherder/git/commits/ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+        "comment_count": 0,
+        "verification": {
+          "verified": false,
+          "reason": "unsigned",
+          "signature": null,
+          "payload": null
+        }
+      },
+      "url": "https://api.github.com/repos/mozilla/treeherder/commits/ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+      "html_url": "https://github.com/mozilla/treeherder/commit/ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+      "comments_url": "https://api.github.com/repos/mozilla/treeherder/commits/ef46fa00eb0919e92f11ffdfe6a6926541ab409c/comments",
+      "author": {
+        "login": "roystchiang",
+        "id": 3461710,
+        "node_id": "MDQ6VXNlcjM0NjE3MTA=",
+        "avatar_url": "https://avatars3.githubusercontent.com/u/3461710?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/roystchiang",
+        "html_url": "https://github.com/roystchiang",
+        "followers_url": "https://api.github.com/users/roystchiang/followers",
+        "following_url": "https://api.github.com/users/roystchiang/following{/other_user}",
+        "gists_url": "https://api.github.com/users/roystchiang/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/roystchiang/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/roystchiang/subscriptions",
+        "organizations_url": "https://api.github.com/users/roystchiang/orgs",
+        "repos_url": "https://api.github.com/users/roystchiang/repos",
+        "events_url": "https://api.github.com/users/roystchiang/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/roystchiang/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "committer": {
+        "login": "wlach",
+        "id": 20569,
+        "node_id": "MDQ6VXNlcjIwNTY5",
+        "avatar_url": "https://avatars3.githubusercontent.com/u/20569?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/wlach",
+        "html_url": "https://github.com/wlach",
+        "followers_url": "https://api.github.com/users/wlach/followers",
+        "following_url": "https://api.github.com/users/wlach/following{/other_user}",
+        "gists_url": "https://api.github.com/users/wlach/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/wlach/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/wlach/subscriptions",
+        "organizations_url": "https://api.github.com/users/wlach/orgs",
+        "repos_url": "https://api.github.com/users/wlach/repos",
+        "events_url": "https://api.github.com/users/wlach/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/wlach/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "parents": [
+        {
+          "sha": "09ef55394535acd453eb7728ef8981a96aa0aef6",
+          "url": "https://api.github.com/repos/mozilla/treeherder/commits/09ef55394535acd453eb7728ef8981a96aa0aef6",
+          "html_url": "https://github.com/mozilla/treeherder/commit/09ef55394535acd453eb7728ef8981a96aa0aef6"
+        }
+      ]
+    },
+    {
+      "sha": "5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "node_id": "MDY6Q29tbWl0ODQ2MzI5NDo1MjE5ZjAwZTdhZjdiNTJlNjZlMzYyZDIwYmI1ZDRiMGNlYjg0YmZh",
+      "commit": {
+        "author": {
+          "name": "KWierso",
+          "email": "kwierso@users.noreply.github.com",
+          "date": "2016-07-28T17:29:26Z"
+        },
+        "committer": {
+          "name": "GitHub",
+          "email": "noreply@github.com",
+          "date": "2016-07-28T17:29:26Z"
+        },
+        "message": "Bug 1289651 - Only scroll job into view if it's not already visible (#1735) r=camd",
+        "tree": {
+          "sha": "53101534e4e2e9726f1a4c2a36d7eefc7485c4ea",
+          "url": "https://api.github.com/repos/mozilla/treeherder/git/trees/53101534e4e2e9726f1a4c2a36d7eefc7485c4ea"
+        },
+        "url": "https://api.github.com/repos/mozilla/treeherder/git/commits/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+        "comment_count": 0,
+        "verification": {
+          "verified": false,
+          "reason": "unsigned",
+          "signature": null,
+          "payload": null
+        }
+      },
+      "url": "https://api.github.com/repos/mozilla/treeherder/commits/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "html_url": "https://github.com/mozilla/treeherder/commit/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "comments_url": "https://api.github.com/repos/mozilla/treeherder/commits/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/comments",
+      "author": {
+        "login": "KWierso",
+        "id": 172215,
+        "node_id": "MDQ6VXNlcjE3MjIxNQ==",
+        "avatar_url": "https://avatars1.githubusercontent.com/u/172215?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/KWierso",
+        "html_url": "https://github.com/KWierso",
+        "followers_url": "https://api.github.com/users/KWierso/followers",
+        "following_url": "https://api.github.com/users/KWierso/following{/other_user}",
+        "gists_url": "https://api.github.com/users/KWierso/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/KWierso/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/KWierso/subscriptions",
+        "organizations_url": "https://api.github.com/users/KWierso/orgs",
+        "repos_url": "https://api.github.com/users/KWierso/repos",
+        "events_url": "https://api.github.com/users/KWierso/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/KWierso/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "committer": {
+        "login": "web-flow",
+        "id": 19864447,
+        "node_id": "MDQ6VXNlcjE5ODY0NDQ3",
+        "avatar_url": "https://avatars3.githubusercontent.com/u/19864447?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/web-flow",
+        "html_url": "https://github.com/web-flow",
+        "followers_url": "https://api.github.com/users/web-flow/followers",
+        "following_url": "https://api.github.com/users/web-flow/following{/other_user}",
+        "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions",
+        "organizations_url": "https://api.github.com/users/web-flow/orgs",
+        "repos_url": "https://api.github.com/users/web-flow/repos",
+        "events_url": "https://api.github.com/users/web-flow/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/web-flow/received_events",
+        "type": "User",
+        "site_admin": false
+      },
+      "parents": [
+        {
+          "sha": "ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+          "url": "https://api.github.com/repos/mozilla/treeherder/commits/ef46fa00eb0919e92f11ffdfe6a6926541ab409c",
+          "html_url": "https://github.com/mozilla/treeherder/commit/ef46fa00eb0919e92f11ffdfe6a6926541ab409c"
+        }
+      ]
+    }
+  ],
+  "files": [
+    {
+      "sha": "18c57e1cb50521be87e30e66f0fa1e35bb979722",
+      "filename": "treeherder/webapp/api/performance_data.py",
+      "status": "modified",
+      "additions": 1,
+      "deletions": 2,
+      "changes": 3,
+      "blob_url": "https://github.com/mozilla/treeherder/blob/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/treeherder/webapp/api/performance_data.py",
+      "raw_url": "https://github.com/mozilla/treeherder/raw/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/treeherder/webapp/api/performance_data.py",
+      "contents_url": "https://api.github.com/repos/mozilla/treeherder/contents/treeherder/webapp/api/performance_data.py?ref=5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "patch": "@@ -254,8 +254,7 @@ class AlertPagination(pagination.CursorPagination):\n     pagination_class = AlertPagination\n \n     def update(self, request, *args, **kwargs):\n-        if 'related_summary_id' in request.data:\n-            request.data['classifier'] = request.user.email\n+        request.data['classifier'] = request.user.email\n         return super(PerformanceAlertViewSet, self).update(request, *args, **kwargs)\n \n     def create(self, request, *args, **kwargs):"
+    },
+    {
+      "sha": "f7352f24db7b97314617cd6a6c2c2a30210b2db4",
+      "filename": "ui/css/treeherder-resultsets.css",
+      "status": "modified",
+      "additions": 4,
+      "deletions": 0,
+      "changes": 4,
+      "blob_url": "https://github.com/mozilla/treeherder/blob/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/css/treeherder-resultsets.css",
+      "raw_url": "https://github.com/mozilla/treeherder/raw/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/css/treeherder-resultsets.css",
+      "contents_url": "https://api.github.com/repos/mozilla/treeherder/contents/ui/css/treeherder-resultsets.css?ref=5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "patch": "@@ -121,6 +121,10 @@ fieldset[disabled] .btn-resultset:hover {\n   overflow: auto;\n }\n \n+.revision[data-tags~=\"backout\"] .revision-comment{\n+  color: red;\n+}\n+\n .revision-holder > a {\n   padding-top: 2px;\n   font: 11px \"Lucida Console\",Monaco,monospace;"
+    },
+    {
+      "sha": "a067aa1c1936560c143a1c560e7d66579e37a8ec",
+      "filename": "ui/index.html",
+      "status": "modified",
+      "additions": 1,
+      "deletions": 1,
+      "changes": 2,
+      "blob_url": "https://github.com/mozilla/treeherder/blob/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/index.html",
+      "raw_url": "https://github.com/mozilla/treeherder/raw/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/index.html",
+      "contents_url": "https://api.github.com/repos/mozilla/treeherder/contents/ui/index.html?ref=5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "patch": "@@ -147,7 +147,7 @@\n         <script type=\"'text/ng-template'\" id=\"revisionsClone.html\">\n             <div class=\"clearfix\"></div>\n             <li>\n-                <span class=\"revision\">\n+                <span class=\"revision\" data-tags=\"{{comment_tags}}\">\n                     <span class=\"revision-holder\">\n                         <a href=\"{{currentRepo.getRevisionHref(revision)}}\"\n                            title=\"Open revision {{revision}} on {{currentRepo.url}}\""
+    },
+    {
+      "sha": "7bec7bcca40a273ef4373639035d4d1c59444ee3",
+      "filename": "ui/js/directives/treeherder/clonejobs.js",
+      "status": "modified",
+      "additions": 22,
+      "deletions": 1,
+      "changes": 23,
+      "blob_url": "https://github.com/mozilla/treeherder/blob/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/js/directives/treeherder/clonejobs.js",
+      "raw_url": "https://github.com/mozilla/treeherder/raw/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/js/directives/treeherder/clonejobs.js",
+      "contents_url": "https://api.github.com/repos/mozilla/treeherder/contents/ui/js/directives/treeherder/clonejobs.js?ref=5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "patch": "@@ -460,6 +460,15 @@ treeherder.directive('thCloneJobs', [\n                     // Only use the first line of the full commit message.\n                     revision.escaped_comment = _.escape(revision.comments.split('\\n')[0]);\n                     revision.escaped_comment_linkified = linkifyBugsFilter(revision.escaped_comment);\n+\n+                    // Parse the comment so we can tag things like backouts\n+                    var tags = \"\";\n+                    if(revision.escaped_comment.search(\"Backed out\") >= 0 ||\n+                       revision.escaped_comment.search(\"Back out\") >= 0) {\n+                        tags += \"backout \";\n+                    }\n+                    revision.comment_tags = tags.trim();\n+\n                     revisionHtml = revisionInterpolator(revision);\n                     ulEl.append(revisionHtml);\n                 }\n@@ -788,11 +797,23 @@ treeherder.directive('thCloneJobs', [\n                 } else if (window.innerHeight >= 1000) {\n                     scrollOffset = -200;\n                 }\n-                $('.th-global-content').scrollTo(el, duration, {offset: scrollOffset});\n+                if(!isOnScreen(el)) {\n+                    $('.th-global-content').scrollTo(el, duration, {offset: scrollOffset});\n+                }\n             }\n \n         };\n \n+        var isOnScreen = function(el){\n+            var viewport = {};\n+            viewport.top = $(window).scrollTop();\n+            viewport.bottom = viewport.top + $(window).height() - $(\"#info-panel\").height();\n+            var bounds = {};\n+            bounds.top = el.offset().top;\n+            bounds.bottom = bounds.top + el.outerHeight();\n+            return ((bounds.top <= viewport.bottom) && (bounds.bottom >= viewport.top));\n+        };\n+\n         var registerCustomEventCallbacks = function(scope, element, attrs){\n \n             //Register rootScope custom event listeners that require"
+    },
+    {
+      "sha": "e9025ed0d51d0ca9edfe332d61c4dc916e2a7ebe",
+      "filename": "ui/partials/perf/alertsctrl.html",
+      "status": "modified",
+      "additions": 4,
+      "deletions": 7,
+      "changes": 11,
+      "blob_url": "https://github.com/mozilla/treeherder/blob/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/partials/perf/alertsctrl.html",
+      "raw_url": "https://github.com/mozilla/treeherder/raw/5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa/ui/partials/perf/alertsctrl.html",
+      "contents_url": "https://api.github.com/repos/mozilla/treeherder/contents/ui/partials/perf/alertsctrl.html?ref=5219f00e7af7b52e66e362d20bb5d4b0ceb84bfa",
+      "patch": "@@ -109,21 +109,18 @@\n           <input type=\"checkbox\" ng-disabled=\"!user.is_staff\" ng-model=\"alert.selected\" ng-change=\"alertSelected(alertSummary)\"/>\n         </td>\n         <td class=\"alert-title\">\n-          <span ng-class=\"{'alert-strike': alert.isInvalid() || (alert.related_summary_id && alert.related_summary_id !== alertSummary.id)}\">\n+          <span ng-class=\"{'alert-strike': alert.isInvalid() || (alert.related_summary_id && alert.related_summary_id !== alertSummary.id)}\"\n+                uib-tooltip=\"{{ !alert.isUntriaged() ? (alert.classifier ?  'Classified by ' + alert.classifier : 'Classified automatically') : ''}}\">\n             {{alert.title}}\n           </span>\n           &nbsp;(<span ng-class=\"{'alert-invalid': alert.isInvalid(), 'alert-untriaged': alert.isUntriaged()}\">{{alert.getStatusText()}}</span><span ng-show=\"alert.related_summary_id\">\n             <!-- Reassigned or downstream *to* another alert -->\n             <span ng-if=\"alert.related_summary_id !== alertSummary.id\">\n-              to <a href=\"#/alerts?id={{alert.related_summary_id}}\" target=\"_blank\"\n-                    uib-tooltip=\"{{alert.classifier ? 'Classified by ' + alert.classifier : 'Classified automatically'}}\"\n-                  >alert #{{alert.related_summary_id}}</a>\n+              to <a href=\"#/alerts?id={{alert.related_summary_id}}\" target=\"_blank\">alert #{{alert.related_summary_id}}</a>\n             </span>\n             <!-- Reassigned or downstream *from* the another alert -->\n             <span ng-if=\"alert.related_summary_id === alertSummary.id\">\n-              from <a href=\"#/alerts?id={{alert.summary_id}}\" target=\"_blank\"\n-                    uib-tooltip=\"{{alert.classifier ? 'Classified by ' + alert.classifier : 'Classified automatically'}}\"\n-                  >alert #{{alert.summary_id}}</a>\n+              from <a href=\"#/alerts?id={{alert.summary_id}}\" target=\"_blank\" >alert #{{alert.summary_id}}</a>\n             </span>\n           </span>)&nbsp;&nbsp;\n           <span class=\"result-links\">"
+    }
+  ]
+}

tests/sample_data/pulse_consumer/transformed_gh_push.json

@@ -5,7 +5,7 @@
                {"author": "KWierso <kwierso@users.noreply.github.com>",
                 "comment": "Bug 1287911 - Add the ability to tag revisions with metadata (#1706) r=camd\n\n* Bug 1287911 - Add the ability to tag revisions with metadata\r\n\r\n* Bug 1287911 - Make backout commit text be red in the list of revisions",
                 "revision": "09ef55394535acd453eb7728ef8981a96aa0aef6"},
-               {"author": "Fausto Núñez Alberro <fausto.nunez@outlook.com>",
+               {"author": "Roy C <crosscent@gmail.com>",
                 "comment": "Bug 1288530 - Update classifier unconditionally, and move classifier tooltip in AlertView (#1742)",
                 "revision": "ef46fa00eb0919e92f11ffdfe6a6926541ab409c"},
                {"author": "KWierso <kwierso@users.noreply.github.com>",

treeherder/etl/push_loader.py

@@ -83,12 +83,11 @@ class GithubTransformer(object):
         })
         return info
 
-    def fetch_push(self, url, repository, sha=None):
+    def fetch_push(self, url, repository):
-        params = {"sha": sha} if sha else {}
+        params = {}
         params.update(self.CREDENTIALS)
 
         logger.info("Fetching push details: %s", url)
-        newrelic.agent.add_custom_parameter("sha", sha)
 
         commits = self.get_cleaned_commits(fetch_json(url, params))
         head_commit = commits[-1]
@@ -134,24 +133,19 @@ class GithubPushTransformer(GithubTransformer):
     #     version:1
     # }
 
-    URL_BASE = "https://api.github.com/repos/{}/{}/commits"
+    URL_BASE = "https://api.github.com/repos/{}/{}/compare/{}...{}"
 
     def transform(self, repository):
-        commit = self.message_body["details"]["event.head.sha"]
         push_url = self.URL_BASE.format(
             self.message_body["organization"],
-            self.message_body["repository"]
+            self.message_body["repository"],
+            self.message_body["details"]["event.base.sha"],
+            self.message_body["details"]["event.head.sha"],
         )
-        return self.fetch_push(push_url, repository, sha=commit)
+        return self.fetch_push(push_url, repository)
 
-    def get_cleaned_commits(self, commits):
+    def get_cleaned_commits(self, compare):
-        # The list of commits will include ones not in the push.  we
+        return compare["commits"]
-        # need to trim the list
-        base_sha = self.message_body["details"]["event.base.sha"]
-        for idx, commit in enumerate(commits):
-            if commit["sha"] == base_sha:
-                commits = commits[:idx]
-        return list(reversed(commits))
 
 
 class GithubPullRequestTransformer(GithubTransformer):