Bug 1172052 - handle text_log_summary artifact blobs that are both dicts and json strings

tests/e2e/test_client_job_ingestion.py

@@ -15,7 +15,7 @@ from treeherder.model import error_summary
 
 
 @pytest.fixture
-def text_log_summary_blob():
+def text_log_summary_dict():
     return {
         "header": {},
         "step_data": {
@@ -121,7 +121,7 @@ def test_post_job_with_text_log_summary_artifact_parsed(
         result_set_stored,
         mock_post_collection,
         mock_error_summary,
-        text_log_summary_blob,
+        text_log_summary_dict,
         ):
     """
     test submitting a job with a pre-parsed log gets parse_status of
@@ -146,7 +146,56 @@ def test_post_job_with_text_log_summary_artifact_parsed(
                 'parse_status': 'parsed'
             }],
             'artifacts': [{
-                "blob": json.dumps(text_log_summary_blob),
+                "blob": json.dumps(text_log_summary_dict),
+                "type": "json",
+                "name": "text_log_summary",
+                "job_guid": job_guid
+            }]
+        }
+    })
+    tjc.add(tj)
+
+    do_post_collection(test_project, tjc)
+
+    check_artifacts(test_project, job_guid, 'parsed', 2,
+                    {'Bug suggestions', 'text_log_summary'}, mock_error_summary)
+
+    # ensure the parsing didn't happen
+    assert mock_parse.called is False
+
+
+def test_post_job_with_text_log_summary_artifact_parsed_dict_blob(
+        test_project,
+        monkeypatch,
+        result_set_stored,
+        mock_post_collection,
+        mock_error_summary,
+        text_log_summary_dict,
+        ):
+    """
+    test submitting a job with a pre-parsed log gets parse_status of
+    "parsed" and doesn't parse the log, but still generates
+    the bug suggestions.
+    """
+
+    mock_parse = MagicMock(name="parse_line")
+    monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
+
+    job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
+    tjc = client.TreeherderJobCollection()
+    tj = client.TreeherderJob({
+        'project': test_project,
+        'revision_hash': result_set_stored[0]['revision_hash'],
+        'job': {
+            'job_guid': job_guid,
+            'state': 'completed',
+            'log_references': [{
+                'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
+                'name': 'buildbot_text',
+                'parse_status': 'parsed'
+            }],
+            'artifacts': [{
+                "blob": text_log_summary_dict,
                 "type": "json",
                 "name": "text_log_summary",
                 "job_guid": job_guid
@@ -171,7 +220,7 @@ def test_post_job_with_text_log_summary_artifact_pending(
         mock_post_collection,
         mock_error_summary,
         mock_update_parse_status,
-        text_log_summary_blob,
+        text_log_summary_dict,
         ):
     """
     test submitting a job with a log set to pending, but with a text_log_summary.
@@ -197,7 +246,7 @@ def test_post_job_with_text_log_summary_artifact_pending(
                 'parse_status': 'pending'
             }],
             'artifacts': [{
-                "blob": json.dumps(text_log_summary_blob),
+                "blob": json.dumps(text_log_summary_dict),
                 "type": "json",
                 "name": "text_log_summary",
                 "job_guid": job_guid
@@ -222,7 +271,7 @@ def test_post_job_with_text_log_summary_and_bug_suggestions_artifact(
         result_set_stored,
         mock_post_collection,
         mock_error_summary,
-        text_log_summary_blob,
+        text_log_summary_dict,
         ):
     """
     test submitting a job with a pre-parsed log and both artifacts
@@ -252,7 +301,7 @@ def test_post_job_with_text_log_summary_and_bug_suggestions_artifact(
             }],
             'artifacts': [
                 {
-                    "blob": json.dumps(text_log_summary_blob),
+                    "blob": json.dumps(text_log_summary_dict),
                     "type": "json",
                     "name": "text_log_summary",
                     "job_guid": job_guid
@@ -284,7 +333,6 @@ def test_post_job_artifacts_by_add_artifact(
         result_set_stored,
         mock_post_collection,
         mock_error_summary,
-        text_log_summary_blob,
         ):
     """
     test submitting a job with artifacts added by ``add_artifact``
@@ -332,7 +380,7 @@ def test_post_job_artifacts_by_add_artifact(
     bapi_blob = json.dumps({"buildername": "merd"})
     pb_blob = json.dumps({"build_url": "feh", "chunk": 1, "config_file": "mah"})
 
-    tj.add_artifact("text_log_summary", "json", json.dumps(tls_blob))
+    tj.add_artifact("text_log_summary", "json", tls_blob)
     tj.add_artifact("Job Info", "json", ji_blob)
     tj.add_artifact("buildapi", "json", bapi_blob)
     tj.add_artifact("privatebuild", "json", pb_blob)

tests/webapp/api/test_artifact_api.py

@@ -127,8 +127,6 @@ def test_artifact_create_text_log_summary_and_bug_suggestions(
     submitted.
     """
 
-    credentials = OAuthCredentials.get_credentials(test_project)
-
     with JobsModel(test_project) as jobs_model:
         job = jobs_model.get_job_list(0, 1)[0]
     tls = sample_data.text_log_summary
@@ -144,7 +142,7 @@ def test_artifact_create_text_log_summary_and_bug_suggestions(
     tac.add(client.TreeherderArtifact({
         'type': 'json',
         'name': 'Bug suggestions',
-        'blob': json.dumps(bs_blob),
+        'blob': bs_blob,
         'job_guid': job['job_guid']
     }))
 

treeherder/model/derived/artifacts.py

@@ -343,15 +343,26 @@ class ArtifactsModel(TreeherderModelBase):
 
         return job_data
 
+    @staticmethod
+    def serialize_artifact_json_blobs(artifacts):
+        """
+        Ensure that JSON artifact blobs passed as dicts are converted to JSON
+        """
+        for artifact in artifacts:
+            blob = artifact['blob']
+            if (artifact['type'].lower() == 'json' and
+                    not isinstance(blob, str) and
+                    not isinstance(blob, unicode)):
+                artifact['blob'] = json.dumps(blob)
+
+        return artifacts
+
     @staticmethod
     def populate_placeholders(artifacts, artifact_placeholders, job_guid):
         for artifact in artifacts:
             name = artifact.get('name')
             artifact_type = artifact.get('type')
-
             blob = artifact.get('blob')
-            if (artifact_type == 'json') and (not isinstance(blob, str)):
-                blob = json.dumps(blob)
 
             if name and artifact_type and blob:
                 artifact_placeholders.append(

treeherder/model/derived/jobs.py

@@ -1591,6 +1591,7 @@ into chunks of chunk_size size. Returns the number of result sets deleted"""
 
         text_log_summary = False
         if artifacts:
+            artifacts = ArtifactsModel.serialize_artifact_json_blobs(artifacts)
             # the artifacts in this list could be ones that should have
             # bug suggestions generated for them.  If so, queue them to be
             # scheduled for asynchronous generation.

treeherder/webapp/api/artifact.py

@@ -56,7 +56,7 @@ class ArtifactViewSet(viewsets.ViewSet):
 
     @oauth_required
     def create(self, request, project):
-        artifacts = request.DATA
+        artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA)
 
         job_guids = [x['job_guid'] for x in artifacts]
         with JobsModel(project) as jobs_model, ArtifactsModel(project) as artifacts_model: