less aggressive thread aqcuiring

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@1182 e7d03fd6-7b0d-0410-9947-9c21f3af8025

linkcheck/Threader.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+import time
 try:
     import threading as _threading
 except ImportError:
@@ -34,12 +35,11 @@ class Threader (object):
         "Wait until we are allowed to start a new thread"
         while self.active_threads() >= self.threads_max:
             self._reduce_threads()
+            time.sleep(0.1)
 
 
     def _reduce_threads (self):
-        for t in self.threads:
-            if not t.isAlive():
-                self.threads.remove(t)
+        self.threads = [ t for t in self.threads if t.isAlive() ]
 
 
     def active_threads (self):

linkcheck/__init__.py

@@ -16,9 +16,11 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
+
 class LinkCheckerError (Exception):
     pass
 
+
 import time, re, i18n
 def getLinkPat (arg, strict=False):
     """get a link pattern matcher for intern/extern links"""
@@ -47,6 +49,7 @@ import UrlData
 from debug import *
 from linkcheck.log import strduration
 
+
 # main check function
 def checkUrls (config):
     """ checkUrls gets a complete configuration object as parameter where all
@@ -61,9 +64,11 @@ def checkUrls (config):
     try:
         start_time = time.time()
         status_time = start_time
-        while not config.finished():
+        while True:
             if config.hasMoreUrls():
                 config.checkUrl(config.getUrl())
+            elif config.finished():
+                break
             else:
                 # active connections are downloading/parsing, so
                 # wait a little