make the crawl loop more resilient

lib/crawler.js

@@ -137,14 +137,26 @@ class Crawler {
 
   _startNext(name, request) {
     const now = Date.now();
-    const requestGate = now + (request.shouldDelay() ? 1000 : 0);
-    const delayGate = request.nextRequestTime || now;
-    const nextRequestTime = Math.max(requestGate, delayGate, now);
-    const delay = Math.max(0, nextRequestTime - now);
+    let delay = 0;
+    if (request) {
+      const requestGate = now + (request.shouldDelay() ? 1000 : 0);
+      const delayGate = request.nextRequestTime || now;
+      const nextRequestTime = Math.max(requestGate, delayGate, now);
+      delay = Math.max(0, nextRequestTime - now);
+    }
     if (delay) {
       this.logger.verbose(`Crawler: ${name} waiting for ${delay}ms`);
     }
-    setTimeout(this.start.bind(this, name), delay);
+    setTimeout(() => {
+      try {
+        this.start(name);
+      } catch (error) {
+        // If for some reason we throw all the way out of start, log and restart the loop
+        this.logger.error(new Error('PANIC! Crawl loop exited unexpectedly'));
+        this.logger.error(error);
+        this._startNext(name, null);
+      }
+    }, delay);
   }
 
   _filter(request) {