diff --git a/treeherder/log_parser/artifactbuilders.pyx b/treeherder/log_parser/artifactbuilders.pyx index 3b142ff01..dc72bad2a 100644 --- a/treeherder/log_parser/artifactbuilders.pyx +++ b/treeherder/log_parser/artifactbuilders.pyx @@ -32,7 +32,12 @@ class ArtifactBuilderBase(object): def parse_line(self, line): """Parse a single line of the log.""" - # truncate the line to the max line-length + """ + Talos data is stored in a json structure contained in + a single line, if the MAX_LINE_LENGTH is applied the + data structure could be truncated preventing it from + being ingested. + """ if "TALOSDATA" not in line: line = line[:self.MAX_LINE_LENGTH] diff --git a/treeherder/log_parser/parsers.pyx b/treeherder/log_parser/parsers.pyx index 08bc5abf2..03c89b664 100644 --- a/treeherder/log_parser/parsers.pyx +++ b/treeherder/log_parser/parsers.pyx @@ -364,7 +364,8 @@ class ErrorParser(ParserBase): ## # Using $ in the regex as an end of line bounds causes the -# regex to fail on windows logs +# regex to fail on windows logs. This is likely due to the +# ^M character representation of the windows end of line. ## RE_TALOSDATA = re.compile('.*?TALOSDATA:\s+(\[.*\])') diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py index e804f5e0f..47d3f4e7a 100644 --- a/treeherder/model/derived/jobs.py +++ b/treeherder/model/derived/jobs.py @@ -2266,24 +2266,41 @@ class JobsModel(TreeherderModelBase): # Determine what type of artifact we have received if artifact: + job_id = None + job_guid = None + if type(artifact) is list: + job_guid = artifact[0] + job_id = job_id_lookup.get(job_guid, {}).get('id', None) + self._adapt_job_artifact_placeholders( - artifact, artifact_placeholders_list, job_id_lookup) + artifact, artifact_placeholders_list, job_id) else: artifact_name = artifact['name'] + job_guid = artifact.get('job_guid', None) + job_id = job_id_lookup.get( + artifact['job_guid'], {} + ).get('id', None) if artifact_name in PerformanceDataAdapter.performance_types: - self._adapt_performance_artifact_collection( artifact, performance_artifact_list, - performance_artifact_job_id_list, job_id_lookup) - + performance_artifact_job_id_list, job_id) else: - self._adapt_job_artifact_collection( - artifact, job_artifact_list, job_id_lookup) + artifact, job_artifact_list, job_id) + + if not job_id: + logger.error( + 'load_job_artifacts: No job_id for {0} job_guid {1}' + ).format(self.project, job_guid) + + else: + logger.error( + 'load_job_artifacts: artifact not defined for {0}' + ).format(self.project) # Store the various artifact types if we collected them if artifact_placeholders_list: @@ -2297,10 +2314,7 @@ class JobsModel(TreeherderModelBase): performance_artifact_job_id_list, performance_artifact_list) def _adapt_job_artifact_placeholders( - self, artifact, artifact_placeholders_list, job_id_lookup): - - job_guid = artifact[0] - job_id = job_id_lookup.get(job_guid, {}).get('id', None) + self, artifact, artifact_placeholders_list, job_id): if job_id: # Replace job_guid with id in artifact data @@ -2310,11 +2324,7 @@ class JobsModel(TreeherderModelBase): artifact_placeholders_list.append(artifact) def _adapt_job_artifact_collection( - self, artifact, artifact_data, job_id_lookup): - - job_id = job_id_lookup.get( - artifact['job_guid'], {} - ).get('id', None) + self, artifact, artifact_data, job_id): if job_id: artifact_data.append(( @@ -2327,11 +2337,7 @@ class JobsModel(TreeherderModelBase): )) def _adapt_performance_artifact_collection( - self, artifact, artifact_data, job_id_list, job_id_lookup): - - job_id = job_id_lookup.get( - artifact['job_guid'], {} - ).get('id', None) + self, artifact, artifact_data, job_id_list, job_id): if job_id: job_id_list.append(job_id)