documentation added

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3292 e7d03fd6-7b0d-0410-9947-9c21f3af8025

linkcheck/director/__init__.py

@@ -58,6 +58,9 @@ def check_urls (aggregate):
 
 
 def get_aggregate (config):
+    """
+    Get an aggregator instance with given configuration.
+    """
     urlqueue = linkcheck.cache.urlqueue.UrlQueue()
     connections = linkcheck.cache.connection.ConnectionPool(wait=config["wait"])
     cookies = linkcheck.cache.cookie.CookieJar()

linkcheck/director/aggregator.py

@@ -27,6 +27,10 @@ import linkcheck.director
 
 
 def check_target (target, args):
+    """
+    Wrapper function calling target() while catching keyboard
+    interrupt and errors.
+    """
     try:
         target(*args)
     except KeyboardInterrupt:
@@ -38,6 +42,9 @@ def check_target (target, args):
 
 
 def start_thread (target, *args):
+    """
+    Spawn a new subthread executing target().
+    """
     t = threading.Thread(target=lambda: check_target(target, args))
     t.setDaemon(True)
     t.start()
@@ -45,6 +52,9 @@ def start_thread (target, *args):
 
 
 class Aggregate (object):
+    """
+    Store thread-safe data collections for checker threads.
+    """
 
     def __init__ (self, config, urlqueue, connections, cookies, robots_txt):
         self.config = config
@@ -56,6 +66,9 @@ class Aggregate (object):
         self.threads = []
 
     def start_threads (self):
+        """
+        Spawn threads for URL checking and status printing.
+        """
         if self.config["status"]:
             t = start_thread(status.do_status, self.urlqueue)
             self.threads.append(t)
@@ -68,6 +81,9 @@ class Aggregate (object):
             self.worker()
 
     def worker (self):
+        """
+        Check URLs from queue until finished.
+        """
         name = threading.currentThread().getName()
         while True:
             self.check_url()
@@ -76,6 +92,9 @@ class Aggregate (object):
                 break
 
     def check_url (self):
+        """
+        Try to get URL data from queue and check it.
+        """
         try:
             url_data = self.urlqueue.get(timeout=1)
         except Queue.Empty:
@@ -85,6 +104,9 @@ class Aggregate (object):
             self.check_url_data(url_data)
 
     def check_url_data (self, url_data):
+        """
+        Check one URL data instance.
+        """
         try:
             url = url_data.url.encode("ascii", "replace")
             threading.currentThread().setName("Thread-%s" % url)
@@ -95,6 +117,9 @@ class Aggregate (object):
             self.urlqueue.task_done(url_data)
 
     def abort (self):
+        """
+        Empty the URL queue.
+        """
         self.urlqueue.do_shutdown()
         try:
             self.urlqueue.join(timeout=self.config["timeout"])
@@ -102,6 +127,9 @@ class Aggregate (object):
             linkcheck.log.warn(linkcheck.LOG_CHECK, "Abort timed out")
 
     def finish (self):
+        """
+        Wait for checker threads to finish.
+        """
         assert self.urlqueue.empty()
         if self.config["status"]:
             status.disable_status()

linkcheck/director/logger.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+"""Logger for aggregator instances"""
 import threading
 from linkcheck.decorators import synchronized
 
@@ -21,6 +22,9 @@ _lock = threading.Lock()
 
 
 class Logger (object):
+    """
+    Thread safe multi-logger class used by aggregator instances.
+    """
 
     def __init__ (self, config):
         self.logs = [config['logger']]

linkcheck/director/status.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+"""Status message handling"""
 import threading
 import time
 import sys
@@ -23,6 +24,8 @@ import os
 import linkcheck.i18n
 from linkcheck.decorators import synchronized
 
+# All output goes to stderr here, making sure the console gets correct
+# encoded messages.
 _encoding = linkcheck.i18n.default_encoding
 stderr = codecs.getwriter(_encoding)(sys.stderr, errors="ignore")
 
@@ -69,19 +72,29 @@ def print_app_info ():
 
 # lock for status thread
 _status_lock = threading.Lock()
+# flag to tell when status thread should stop
 status_flag = True
 
 @synchronized(_status_lock)
 def status_is_active ():
+    """
+    Check status control flag.
+    """
     return status_flag
 
 @synchronized(_status_lock)
 def disable_status ():
+    """
+    Set status control flag in order to stop the status thread.
+    """
     global status_flag
     status_flag = False
 
 
 def do_status (urlqueue):
+    """
+    Print periodic status messages.
+    """
     start_time = time.time()
     threading.currentThread().setName("Status")
     while True:
@@ -93,6 +106,9 @@ def do_status (urlqueue):
 
 
 def print_status (urlqueue, start_time):
+    """
+    Print a status message.
+    """
     duration = time.time() - start_time
     checked, in_progress, queue = urlqueue.status()
     msg = _n("%2d URL active,", "%2d URLs active,", in_progress) % in_progress