Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-05 15:30:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Put in-progress URLs back near the front of URL queue, not at end.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-09-23 21:00:01 +02:00
parent 1f3034b5f5
commit e21187b275
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
linkcheck/cache/urlqueue.py vendored
View file

@ -113,7 +113,14 @@ class UrlQueue (object):
url_data.copy_from_cache(self.checked[key])
elif key in self.in_progress:
# It's being checked currently; put it back in the queue.
self.queue.append(url_data)
if len(self.queue) < 100:
# queue is small - put at end is ok
self.queue.append(url_data)
else:
# queue is large - put near front
self.queue.rotate(-10)
self.queue.appendleft(url_data)
self.queue.rotate(10)
url_data = None
else:
self.in_progress[key] = url_data