bug 1077136 - test passing for ingestion

Vagrantfile

@@ -8,7 +8,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # Every Vagrant virtual environment requires a box to build off of.
   config.vm.box = "precise32"
   config.vm.box_url = "http://files.vagrantup.com/precise32.box"
-  
+
   config.vm.network "private_network", ip: "192.168.33.10"
 
   config.vm.synced_folder ".", "/home/vagrant/treeherder-service", type: "nfs"
@@ -21,7 +21,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision :puppet do |puppet|
     puppet.manifests_path = "puppet/manifests"
     puppet.manifest_file = "vagrant.pp"
-    
+
     #uncomment in production to serve treeherder-ui from dist directory
     #puppet.manifest_file = "production.pp"
 

tests/etl/test_buildapi.py

@@ -1,6 +1,7 @@
 import os
 import pytest
 import responses
+import json
 
 from django.conf import settings
 
@@ -78,38 +79,39 @@ def test_ingest_pending_jobs(jm, initial_data,
     assert len(stored_obj) == 1
 
 
-def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data,
+def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data, sample_resultset,
                                 test_repository,
-                                test_base_dir,
                                 mock_buildapi_pending_missing1_url,
                                 mock_post_json_data,
-                                mock_log_parser,
                                 mock_get_resultset,
                                 mock_get_remote_content,
                                 activate_responses):
     """
     Ensure the job with the missing resultset is queued for refetching
     """
-    pushlog_content = {"33270": {
-        "date": 1378288232,
-        "changesets": [
-            {
-                "node": "4b40022e5c4cb344655ed7be9a408d2970a736c4",
-                "files": [
-                    "browser/base/content/browser.js"
-                ],
-                "tags": [],
-                "author": "John Doe <jdoe@mozilla.com>",
-                "branch": "default",
-                "desc": "bug 909264 - control characters in the location bar "
-                "should be %-encoded for visibility. r=gavin"
-            }
-        ],
-        "user": "jdoe@mozilla.com"
-    }}
-    pushlog_fake_url = "https://hg.mozilla.org/mozilla-central/json-pushes/?full=1&changeset=4b40022e5c4c"
+    new_revision = '222222222222'
+    pushlog_content = json.dumps(
+        {"33270": {
+            "date": 1378288232,
+            "changesets": [
+                {
+                    "node": new_revision + "b344655ed7be9a408d2970a736c4",
+                    "files": [
+                        "browser/base/content/browser.js"
+                    ],
+                    "tags": [],
+                    "author": "John Doe <jdoe@mozilla.com>",
+                    "branch": "default",
+                    "desc": "bug 909264 - control characters"
+                }
+            ],
+            "user": "jdoe@mozilla.com"
+        }}
+    )
+    pushlog_fake_url = "https://hg.mozilla.org/mozilla-central/json-pushes/?full=1&changeset=" + new_revision
     responses.add(responses.GET, pushlog_fake_url,
                   body=pushlog_content, status=200,
+                  match_querystring=True,
                   content_type='application/json')
 
     from treeherder.etl.buildapi import PendingJobsProcess
@@ -126,7 +128,12 @@ def test_ingest_pending_jobs_1_missing_resultset(jm, initial_data,
         return_type='tuple'
     )
 
-    assert len(revisions_stored) == 15
+    assert len(revisions_stored) == 20
+    was_stored = False
+    for rs in revisions_stored:
+        if str(rs['revision']) == new_revision:
+            was_stored = True
+    assert was_stored
 
     jm.disconnect()
 

tests/sample_data/builds-pending-missing1.js

@@ -1,7 +1,7 @@
 {
     "pending":{
         "test_treeherder": {
-            "4b40022e5c4c": [
+            "45f8637cb9f7": [
                 {"submitted_at": 1369304814, "id": 24575179, "buildername": "WINNT 6.2 try debug test mochitest-4"}
             ],
             "222222222222": [

treeherder/etl/cleanup_tasks.py

@@ -36,10 +36,6 @@ def fetch_missing_hg_push_logs(repo_name, repo_url, revisions):
     changesetParam = urllib.urlencode({"changeset": revisions}, True)
     urlStr = repo_url + '/json-pushes/?full=1&' + changesetParam
 
-    print "<><>"
-    print revisions
-    print urlStr
-
     process.run(urlStr, repo_name)