Retrieve push data in chunks when there are a lot of pushes to retrieve

Fixes #6

ci_info/push.py

@@ -405,19 +405,29 @@ class Push:
 
 
 def make_push_objects(**kwargs):
-    data = run_query("push_revisions", Namespace(**kwargs))["data"]
+    push_min, push_max = run_query("push_revision_count", Namespace(**kwargs))["data"][0]
+
+    CHUNK_SIZE = 10000
+    pushes_groups = [(i, min(i+CHUNK_SIZE-1, push_max)) for i in range(push_min, push_max, CHUNK_SIZE)]
 
     pushes = []
     cur = prev = None
-    for pushid, revs, parents in data:
-        topmost = list(set(revs) - set(parents))[0]
 
-        cur = Push(topmost)
-        if prev:
-            # avoids the need to query hgmo to find parent pushes
-            cur._parent = prev
+    for pushes_group in pushes_groups:
+        kwargs["from_push"] = pushes_group[0]
+        kwargs["to_push"] = pushes_group[1]
 
-        pushes.append(cur)
-        prev = cur
+        data = run_query("push_revisions", Namespace(**kwargs))["data"]
+
+        for pushid, revs, parents in data:
+            topmost = list(set(revs) - set(parents))[0]
+
+            cur = Push(topmost)
+            if prev:
+                # avoids the need to query hgmo to find parent pushes
+                cur._parent = prev
+
+            pushes.append(cur)
+            prev = cur
 
     return pushes

recipes/queries/push_revision_count.query

@@ -0,0 +1,10 @@
+from: repo
+select:
+    - {name: push_min, value: push.id, aggregate: min}
+    - {name: push_max, value: push.id, aggregate: max}
+where:
+    and:
+        - eq: {branch.name: {$eval: branch}}
+        - gte: [push.date, {date: {$eval: from_date}}]
+        - lte: [push.date, {date: {$eval: to_date}}]
+format: table

recipes/queries/push_revisions.query

@@ -8,7 +8,7 @@ sort: push.id
 where:
     and:
         - eq: {branch.name: {$eval: branch}}
-        - gte: [push.date, {date: {$eval: from_date}}]
-        - lte: [push.date, {date: {$eval: to_date}}]
+        - gte: [push.id, {$eval: from_push}]
+        - lte: [push.id, {$eval: to_push}]
 limit: 10000
 format: table