From cf06199a67bde8cbaa626cc5c8fc83d8d00b2be8 Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Fri, 13 Feb 2015 13:09:08 +0000 Subject: [PATCH] Bug 1132893 - Move the Builds4hAnalyzer blacklist inline A json file is overkill, and it is currently in a directory intended for production-specific files, so move the blacklist inline. We should probably move the whole analyser to another file, but that can be done later. Also store the analyser output in the root of the media directory, since a subdirectory is unnecessary. The media directory is now empty, so we must use .gitkeep to ensure it is created. --- .gitignore | 5 +- treeherder/etl/buildapi.py | 95 +++++++++++++------ treeherder/webapp/media/.gitkeep | 0 .../media/builds4hanalysis/blacklist.json | 63 ------------ 4 files changed, 70 insertions(+), 93 deletions(-) create mode 100644 treeherder/webapp/media/.gitkeep delete mode 100644 treeherder/webapp/media/builds4hanalysis/blacklist.json diff --git a/.gitignore b/.gitignore index 5984f6f58..af475f452 100644 --- a/.gitignore +++ b/.gitignore @@ -54,8 +54,8 @@ htmlcov/ treeherder/webapp/static/ #media files in deployment -treeherder/webapp/media/builds4hanalysis/builds4h_analysis.json -treeherder/webapp/media/builds4hanalysis/builds4h_report.txt +treeherder/webapp/media/builds4h_analysis.json +treeherder/webapp/media/builds4h_report.txt #etl data files treeherder/etl/data/credentials.json @@ -71,3 +71,4 @@ docs/_build/ treeherder/webapp/log_cache celerybeat-schedule +!.gitkeep diff --git a/treeherder/etl/buildapi.py b/treeherder/etl/buildapi.py index 1f6dca7dc..5f96c98cb 100644 --- a/treeherder/etl/buildapi.py +++ b/treeherder/etl/buildapi.py @@ -572,13 +572,75 @@ class RunningJobsProcess(JsonExtractorMixin, filter_to_project=filter_to_project) ) + class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): def __init__(self): - # builds4h deserialized json self.raw_data = [] - self.blacklist = set() + self.blacklist = set([ + "fuzzer-linux", + "fuzzer-macosx64-lion", + "fuzzer-win64", + "release-comm-esr24-final_verification", + "release-comm-esr24-push_to_mirrors", + "release-comm-esr24-ready_for_releasetest_testing", + "release-comm-esr24-start_uptake_monitoring", + "release-mozilla-beta-almost_ready_for_release", + "release-mozilla-beta-android_repack_1/10", + "release-mozilla-beta-android_repack_10/10", + "release-mozilla-beta-android_repack_2/10", + "release-mozilla-beta-android_repack_3/10", + "release-mozilla-beta-android_repack_4/10", + "release-mozilla-beta-android_repack_5/10", + "release-mozilla-beta-android_repack_6/10", + "release-mozilla-beta-android_repack_7/10", + "release-mozilla-beta-android_repack_8/10", + "release-mozilla-beta-android_repack_9/10", + "release-mozilla-beta-antivirus", + "release-mozilla-beta-fennec_source", + "release-mozilla-beta-final_verification", + "release-mozilla-beta-linux64_update_verify_3/6", + "release-mozilla-beta-linux_update_verify_3/6", + "release-mozilla-beta-linux_update_verify_6/6", + "release-mozilla-beta-macosx64_update_verify_1/6", + "release-mozilla-beta-macosx64_update_verify_2/6", + "release-mozilla-beta-macosx64_update_verify_3/6", + "release-mozilla-beta-macosx64_update_verify_4/6", + "release-mozilla-beta-macosx64_update_verify_5/6", + "release-mozilla-beta-macosx64_update_verify_6/6", + "release-mozilla-beta-push_to_mirrors", + "release-mozilla-beta-ready_for_releasetest_testing", + "release-mozilla-beta-start_uptake_monitoring", + "release-mozilla-beta-win32_update_verify_1/6", + "release-mozilla-beta-win32_update_verify_2/6", + "release-mozilla-beta-win32_update_verify_3/6", + "release-mozilla-beta-win32_update_verify_4/6", + "release-mozilla-beta-win32_update_verify_5/6", + "release-mozilla-beta-win32_update_verify_6/6", + "release-mozilla-esr24-postrelease", + "Firefox mozilla-aurora linux l10n dep", + "Firefox mozilla-aurora linux64 l10n dep", + "Firefox mozilla-aurora macosx64 l10n dep", + "Firefox mozilla-aurora win32 l10n dep", + "Firefox mozilla-central linux l10n dep", + "Firefox mozilla-central linux64 l10n dep", + "Firefox mozilla-central macosx64 l10n dep", + "Firefox mozilla-central win32 l10n dep", + "Thunderbird comm-aurora linux l10n dep", + "Thunderbird comm-aurora linux64 l10n dep", + "Thunderbird comm-aurora macosx64 l10n dep", + "Thunderbird comm-aurora win32 l10n dep", + "Thunderbird comm-central linux l10n dep", + "Thunderbird comm-central linux64 l10n dep", + "Thunderbird comm-central macosx64 l10n dep", + "Thunderbird comm-central win32 l10n dep", + "Firefox birch win32 l10n nightly", + "release-mozilla-beta-android-armv6_build", + "release-mozilla-beta-android-x86_build", + "release-mozilla-beta-android_build", + "Firefox mozilla-central win32 l10n nightly" + ]) self.t_stamp = int(time.time()) self.readable_time = datetime.datetime.fromtimestamp( @@ -645,18 +707,15 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): 'get_func':self.get_objects_missing_buildernames, }, }, - 'guids': {} - - } + } # load last analysis data self.data_path = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'treeherder', - settings.MEDIA_ROOT, - 'builds4hanalysis' - ) + settings.MEDIA_ROOT + ) # file that stores the json data self.builds4h_analysis_file_path = os.path.join( @@ -671,20 +730,15 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): self.data_path, 'blacklist.json') def run(self): - - self.get_blacklist() self.get_analysis_file() - self.raw_data = self.extract(settings.BUILDAPI_BUILDS4H_URL) for build in self.raw_data['builds']: - buildername = build['properties'].get('buildername', None) if buildername in self.blacklist: continue - job_guid_data = self.find_job_guid(build) for analysis_type in self.report_obj['analyzers']: @@ -696,7 +750,6 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): self._increment_buildername_total_count( analysis_type, buildername, job_guid_data['job_guid']) - if job_guid_data['job_guid']: self.report_obj['guids'][job_guid_data['job_guid']] = True @@ -709,7 +762,6 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): self.write_report() def get_analysis_file(self): - if os.path.isfile(self.builds4h_analysis_file_path): # Set the data attribute in the report object to whats # found in the analysis file @@ -727,20 +779,7 @@ class Builds4hAnalyzer(JsonExtractorMixin, Builds4hTransformerMixin): self.report_obj['analyzers'][analysis_type]['data'] = \ deserialized_data['analyzers'][analysis_type] - def get_blacklist(self): - - if os.path.isfile(self.builds4h_blacklist_file_path): - with open(self.builds4h_blacklist_file_path) as f: - data = f.read() - - deserialized_data = [] - if data: - deserialized_data = json.loads(data) - - self.blacklist = set(deserialized_data) - def write_report(self): - report_fh = open(self.builds4h_report_file_path, 'w') divider = "------------------------------------------------------\n" diff --git a/treeherder/webapp/media/.gitkeep b/treeherder/webapp/media/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/treeherder/webapp/media/builds4hanalysis/blacklist.json b/treeherder/webapp/media/builds4hanalysis/blacklist.json deleted file mode 100644 index bb75ef060..000000000 --- a/treeherder/webapp/media/builds4hanalysis/blacklist.json +++ /dev/null @@ -1,63 +0,0 @@ -[ - "fuzzer-linux", - "fuzzer-macosx64-lion", - "fuzzer-win64", - "release-comm-esr24-final_verification", - "release-comm-esr24-push_to_mirrors", - "release-comm-esr24-ready_for_releasetest_testing", - "release-comm-esr24-start_uptake_monitoring", - "release-mozilla-beta-almost_ready_for_release", - "release-mozilla-beta-android_repack_1/10", - "release-mozilla-beta-android_repack_10/10", - "release-mozilla-beta-android_repack_2/10", - "release-mozilla-beta-android_repack_3/10", - "release-mozilla-beta-android_repack_4/10", - "release-mozilla-beta-android_repack_5/10", - "release-mozilla-beta-android_repack_6/10", - "release-mozilla-beta-android_repack_7/10", - "release-mozilla-beta-android_repack_8/10", - "release-mozilla-beta-android_repack_9/10", - "release-mozilla-beta-antivirus", - "release-mozilla-beta-fennec_source", - "release-mozilla-beta-final_verification", - "release-mozilla-beta-linux64_update_verify_3/6", - "release-mozilla-beta-linux_update_verify_3/6", - "release-mozilla-beta-linux_update_verify_6/6", - "release-mozilla-beta-macosx64_update_verify_1/6", - "release-mozilla-beta-macosx64_update_verify_2/6", - "release-mozilla-beta-macosx64_update_verify_3/6", - "release-mozilla-beta-macosx64_update_verify_4/6", - "release-mozilla-beta-macosx64_update_verify_5/6", - "release-mozilla-beta-macosx64_update_verify_6/6", - "release-mozilla-beta-push_to_mirrors", - "release-mozilla-beta-ready_for_releasetest_testing", - "release-mozilla-beta-start_uptake_monitoring", - "release-mozilla-beta-win32_update_verify_1/6", - "release-mozilla-beta-win32_update_verify_2/6", - "release-mozilla-beta-win32_update_verify_3/6", - "release-mozilla-beta-win32_update_verify_4/6", - "release-mozilla-beta-win32_update_verify_5/6", - "release-mozilla-beta-win32_update_verify_6/6", - "release-mozilla-esr24-postrelease", - "Firefox mozilla-aurora linux l10n dep", - "Firefox mozilla-aurora linux64 l10n dep", - "Firefox mozilla-aurora macosx64 l10n dep", - "Firefox mozilla-aurora win32 l10n dep", - "Firefox mozilla-central linux l10n dep", - "Firefox mozilla-central linux64 l10n dep", - "Firefox mozilla-central macosx64 l10n dep", - "Firefox mozilla-central win32 l10n dep", - "Thunderbird comm-aurora linux l10n dep", - "Thunderbird comm-aurora linux64 l10n dep", - "Thunderbird comm-aurora macosx64 l10n dep", - "Thunderbird comm-aurora win32 l10n dep", - "Thunderbird comm-central linux l10n dep", - "Thunderbird comm-central linux64 l10n dep", - "Thunderbird comm-central macosx64 l10n dep", - "Thunderbird comm-central win32 l10n dep", - "Firefox birch win32 l10n nightly", - "release-mozilla-beta-android-armv6_build", - "release-mozilla-beta-android-x86_build", - "release-mozilla-beta-android_build", - "Firefox mozilla-central win32 l10n nightly" -] \ No newline at end of file