From e21187b275d5eaa7f0ff2dc9e8f7232e4c060ffb Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Sun, 23 Sep 2012 21:00:01 +0200 Subject: [PATCH] Put in-progress URLs back near the front of URL queue, not at end. --- linkcheck/cache/urlqueue.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/linkcheck/cache/urlqueue.py b/linkcheck/cache/urlqueue.py index 12db72eb..26ffe5c9 100644 --- a/linkcheck/cache/urlqueue.py +++ b/linkcheck/cache/urlqueue.py @@ -113,7 +113,14 @@ class UrlQueue (object): url_data.copy_from_cache(self.checked[key]) elif key in self.in_progress: # It's being checked currently; put it back in the queue. - self.queue.append(url_data) + if len(self.queue) < 100: + # queue is small - put at end is ok + self.queue.append(url_data) + else: + # queue is large - put near front + self.queue.rotate(-10) + self.queue.appendleft(url_data) + self.queue.rotate(10) url_data = None else: self.in_progress[key] = url_data