(Bug 1071577) - remove pushlog last fetched cache; also increase the pushlog time range to the last 24hr

treeherder/etl/pushlog.py

@@ -13,12 +13,6 @@ class HgPushlogTransformerMixin(object):
         # this contain the whole list of transformed pushes
         result_sets = []
 
-        # last push available
-        if pushlog:
-            last_push = max(pushlog.keys())
-        else:
-            last_push = None
-
         th_collections = {}
 
         # iterate over the pushes
@@ -59,10 +53,6 @@ class HgPushlogTransformerMixin(object):
             th_resultset = th_collections[ repository ].get_resultset(result_set)
             th_collections[ repository ].add(th_resultset)
 
-        # cache the last push seen
-        if last_push:
-            cache.set("{0}:last_push".format(repository), last_push)
-
         return th_collections
 
 
@@ -71,13 +61,6 @@ class HgPushlogProcess(JsonExtractorMixin,
                        OAuthLoaderMixin):
 
     def run(self, source_url, repository):
-
-        # get the last object seen from cache. this will
-        # reduce the number of pushes processed every time
-        last_object = cache.get("{0}:last_push".format(repository))
-        if last_object:
-            source_url += "&startID=" + last_object
-
         extracted_content = self.extract(source_url)
         if extracted_content:
             self.load(

treeherder/etl/tasks.py

@@ -65,7 +65,7 @@ def fetch_hg_push_log(repo_name, repo_url):
     Run a HgPushlog etl process
     """
     process = HgPushlogProcess()
-    process.run(repo_url+'/json-pushes/?full=1', repo_name)
+    process.run(repo_url+'/json-pushes/?full=1&maxhours=24', repo_name)
 
 
 @task(name='fetch-bugs', time_limit=60*5)

treeherder/webapp/admin.py

@@ -23,20 +23,9 @@ class ReferenceDataSignatureAdmin(admin.ModelAdmin):
                     "job_type_name", "job_type_symbol", "option_collection_hash", "build_system_type"]
 
 
-class RepositoryAdmin(admin.ModelAdmin):
-    actions = ['clear_repo_cache']
-
-    def clear_repo_cache(self, request, queryset):
-        repo_list = queryset.values_list("name", flat=True)
-        cache_key_list = ["{0}:last_push".format(repo) for repo in repo_list]
-        cache.delete_many(cache_key_list)
-        self.message_user(request, "Pushlog cache successfully deleted.")
-    clear_repo_cache.short_description = "Clear the pushlog cache for selected repositories"
-
-
 # custom admin classes
 browserid_admin.register(JobType, JobTypeAdmin)
-browserid_admin.register(Repository, RepositoryAdmin)
+browserid_admin.register(Repository)
 browserid_admin.register(ReferenceDataSignatures, ReferenceDataSignatureAdmin)
 # default admin classes
 browserid_admin.register(Product)