Refactor logging configuration.

linkcheck/__init__.py

@@ -39,7 +39,16 @@ import re
 import signal
 import traceback
 
-from . import i18n
+from . import i18n, log
+from .logconf import (
+    LOG_ROOT,
+    LOG_CMDLINE,
+    LOG_CHECK,
+    LOG_CACHE,
+    LOG_GUI,
+    LOG_THREAD,
+    LOG_PLUGIN,
+)
 import _LinkChecker_configdata as configdata
 
 
@@ -56,51 +65,6 @@ def get_install_data ():
     return configdata.install_data
 
 
-# application log areas
-LOG_ROOT = "linkcheck"
-LOG_CMDLINE = "linkcheck.cmdline"
-LOG_CHECK = "linkcheck.check"
-LOG_CACHE = "linkcheck.cache"
-LOG_GUI = "linkcheck.gui"
-LOG_THREAD = "linkcheck.thread"
-LOG_PLUGIN = "linkcheck.plugin"
-lognames = {
-    "cmdline": LOG_CMDLINE,
-    "checking": LOG_CHECK,
-    "cache": LOG_CACHE,
-    "gui": LOG_GUI,
-    "thread": LOG_THREAD,
-    "plugin": LOG_PLUGIN,
-    "all": LOG_ROOT,
-}
-# XXX debug httplib
-#import httplib
-#httplib.HTTPConnection.debuglevel = 1
-
-lognamelist = ", ".join(repr(name) for name in lognames)
-
-# logging configuration
-configdict = {
-    'version': 1,
-    'loggers': {
-    },
-    'root': {
-      'level': 'DEBUG',
-    },
-}
-
-def init_log_config():
-    """Configure the application loggers."""
-    for applog in lognames.values():
-        # propagate except for root app logger 'linkcheck'
-        propagate = (applog != LOG_ROOT)
-        configdict['loggers'][applog] = dict(level='INFO', propagate=propagate)
-
-init_log_config()
-
-from . import log
-
-
 class LinkCheckerError (StandardError):
     """Exception to be raised on linkchecker-specific check errors."""
     pass
@@ -162,6 +126,7 @@ def init_i18n (loc=None):
     logging.addLevelName(logging.DEBUG, _('DEBUG'))
     logging.addLevelName(logging.NOTSET, _('NOTSET'))
 
+
 # initialize i18n, puts _() and _n() function into global namespace
 init_i18n()
 

linkcheck/checker/fileurl.py

@@ -26,7 +26,7 @@ import urllib2
 from datetime import datetime
 
 from . import urlbase, get_index_html
-from .. import log, LOG_CHECK, fileutil, LinkCheckerError, url as urlutil
+from .. import log, LOG_CHECK, fileutil, mimeutil, LinkCheckerError, url as urlutil
 from ..bookmarks import firefox
 from .const import WARN_FILE_MISSING_SLASH, WARN_FILE_SYSTEM_PATH
 
@@ -251,7 +251,7 @@ class FileUrl (urlbase.UrlBase):
         """Return URL content type, or an empty string if content
         type could not be found."""
         if self.url:
-            self.content_type = fileutil.guess_mimetype(self.url, read=self.get_content)
+            self.content_type = mimeutil.guess_mimetype(self.url, read=self.get_content)
         else:
             self.content_type = u""
 

linkcheck/checker/ftpurl.py

@@ -21,7 +21,7 @@ Handle FTP links.
 import ftplib
 from cStringIO import StringIO
 
-from .. import log, LOG_CHECK, LinkCheckerError, fileutil
+from .. import log, LOG_CHECK, LinkCheckerError, mimeutil
 from . import proxysupport, httpurl, internpaturl, get_index_html
 from .const import WARN_FTP_MISSING_SLASH
 
@@ -179,7 +179,7 @@ class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
     def set_content_type (self):
         """Set URL content type, or an empty string if content
         type could not be found."""
-        self.content_type = fileutil.guess_mimetype(self.url, read=self.get_content)
+        self.content_type = mimeutil.guess_mimetype(self.url, read=self.get_content)
 
     def read_content (self):
         """Return URL target content, or in case of directories a dummy HTML

linkcheck/checker/httpurl.py

@@ -21,7 +21,7 @@ Handle http links.
 import requests
 from cStringIO import StringIO
 
-from .. import (log, LOG_CHECK, strformat, fileutil,
+from .. import (log, LOG_CHECK, strformat, mimeutil,
     url as urlutil, LinkCheckerError, httputil)
 from . import (internpaturl, proxysupport)
 from ..HtmlParser import htmlsax
@@ -314,7 +314,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
             return False
         # some content types must be validated with the page content
         if self.content_type in ("application/xml", "text/xml"):
-            rtype = fileutil.guess_mimetype_read(self.get_content)
+            rtype = mimeutil.guess_mimetype_read(self.get_content)
             if rtype is not None:
                 # XXX side effect
                 self.content_type = rtype

linkcheck/configuration/__init__.py

@@ -18,17 +18,14 @@
 Store metadata and options.
 """
 
-import sys
 import os
 import re
-import logging.config
 import urllib
 import urlparse
 import shutil
 import socket
 import _LinkChecker_configdata as configdata
-from .. import (log, LOG_CHECK, LOG_ROOT, ansicolor, lognames,
+from .. import (log, LOG_CHECK, get_install_data, fileutil)
-    get_install_data, fileutil, configdict)
 from . import confparse
 from ..decorators import memoized
 
@@ -182,57 +179,10 @@ class Configuration (dict):
             self[key] = {}
             self.loggers[key] = c
 
-    def init_logging (self, status_logger, debug=None, handler=None):
+    def set_status_logger(self, status_logger):
-        """
+        """Set the status logger."""
-        Set up the application logging (not to be confused with check
-        loggers). When debug is not None it is expected to be a list of
-        logger names for which debugging will be enabled.
-
-        If no thread debugging is enabled, threading will be disabled.
-        """
-        logging.config.dictConfig(configdict)
-        if handler is None:
-            handler = ansicolor.ColoredStreamHandler(strm=sys.stderr)
-        self.add_loghandler(handler, debug)
-        self.set_debug(debug)
         self.status_logger = status_logger
 
-    def set_debug (self, debug):
-        """Set debugging levels for configured loggers. The argument
-        is a list of logger names to enable debug for."""
-        self.set_loglevel(debug, logging.DEBUG)
-
-    def add_loghandler (self, handler, debug):
-        """Add log handler to root logger LOG_ROOT and set formatting."""
-        logging.getLogger(LOG_ROOT).addHandler(handler)
-        format = "%(levelname)s "
-        if debug:
-            format += "%(asctime)s "
-        if self['threads'] > 0:
-            format += "%(threadName)s "
-        format += "%(message)s"
-        handler.setFormatter(logging.Formatter(format))
-
-    def remove_loghandler (self, handler):
-        """Remove log handler from root logger LOG_ROOT."""
-        logging.getLogger(LOG_ROOT).removeHandler(handler)
-
-    def reset_loglevel (self):
-        """Reset log level to display only warnings and errors."""
-        self.set_loglevel(['all'], logging.WARN)
-
-    def set_loglevel (self, loggers, level):
-        """Set logging levels for given loggers."""
-        if not loggers:
-            return
-        if 'all' in loggers:
-            loggers = lognames.keys()
-        # disable threading if no thread debugging
-        if "thread" not in loggers and level == logging.DEBUG:
-            self['threads'] = 0
-        for key in loggers:
-            logging.getLogger(lognames[key]).setLevel(level)
-
     def logger_new (self, loggername, **kwargs):
         """Instantiate new logger and return it."""
         args = self[loggername]

linkcheck/configuration/confparse.py

@@ -18,7 +18,7 @@
 
 import ConfigParser
 import os
-from .. import LinkCheckerError, get_link_pat, LOG_CHECK, log, fileutil, plugins
+from .. import LinkCheckerError, get_link_pat, LOG_CHECK, log, fileutil, plugins, logconf
 
 
 def read_multiline (value):
@@ -113,7 +113,7 @@ class LCConfigParser (ConfigParser.RawConfigParser, object):
         if self.has_option(section, "debug"):
             val = self.get(section, "debug")
             parts = [f.strip().lower() for f in val.split(',')]
-            self.config.set_debug(parts)
+            logconf.set_debug(parts)
         self.read_boolean_option(section, "status")
         if self.has_option(section, "log"):
             val = self.get(section, "log").strip().lower()

linkcheck/gui/__init__.py

@@ -37,7 +37,7 @@ from .settings import Settings
 from .recentdocs import RecentDocumentModel
 from .projects import openproject, saveproject, loadproject, ProjectExt
 from .. import configuration, checker, director, get_link_pat, \
-    strformat, fileutil, LinkCheckerError, i18n, httputil
+    strformat, mimeutil, LinkCheckerError, i18n, httputil, logconf
 from ..containers import enum
 from .. import url as urlutil
 
@@ -111,6 +111,7 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow):
         self.label_busy.setText(u"")
         self.label_busy.setMovie(self.movie)
         # init the rest
+        self.init_logging()
         self.init_url(url)
         self.init_treeview()
         self.connect_widgets()
@@ -121,6 +122,11 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow):
         self.init_drop()
         self.init_app(project)
 
+    def init_logging(self):
+        """Initialize logging."""
+        self.handler = GuiLogHandler(self.debug.log_msg_signal)
+        logconf.init_log_config(handler=self.handler)
+
     def init_url (self, url):
         """Initialize URL input."""
         documents = self.settings.read_recent_documents()
@@ -228,6 +234,8 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow):
     def init_config (self):
         """Create a configuration object."""
         self.config = configuration.Configuration()
+        status = StatusLogger(self.log_status_signal)
+        self.config.set_status_logger(status)
         # dictionary holding overwritten values
         self.config_backup = {}
         # set standard GUI configuration values
@@ -236,9 +244,6 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow):
             signal=self.log_url_signal, stats=self.log_stats_signal)
         self.config["status"] = True
         self.config["status_wait_seconds"] = 2
-        self.handler = GuiLogHandler(self.debug.log_msg_signal)
-        status = StatusLogger(self.log_status_signal)
-        self.config.init_logging(status, handler=self.handler)
 
     def read_config (self, filename=None):
         """Read user and system configuration file."""
@@ -365,7 +370,7 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow):
         self.settings.save_recent_documents(self.recent.get_documents())
         self.settings.save_misc(dict(saveresultas=self.saveresultas))
         self.settings.sync()
-        self.config.remove_loghandler(self.handler)
+        logconf.remove_loghandler(self.handler)
         if e is not None:
             e.accept()
 
@@ -537,7 +542,7 @@ Version 2 or later.
             if not content_type:
                 # read function for content type guessing
                 read = lambda: data
-                content_type = fileutil.guess_mimetype(url, read=read)
+                content_type = mimeutil.guess_mimetype(url, read=read)
             self.editor.setContentType(content_type)
             self.editor.setText(data, line=line, col=col)
         self.editor.show()

linkcheck/logconf.py

@@ -0,0 +1,101 @@
+# -*- coding: iso-8859-1 -*-
+# Copyright (C) 2000-2014 Bastian Kleineidam
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""
+Logging configuration
+"""
+import logging.config
+import sys
+from . import ansicolor
+
+# application log areas
+LOG_ROOT = "linkcheck"
+LOG_CMDLINE = "linkcheck.cmdline"
+LOG_CHECK = "linkcheck.check"
+LOG_CACHE = "linkcheck.cache"
+LOG_GUI = "linkcheck.gui"
+LOG_THREAD = "linkcheck.thread"
+LOG_PLUGIN = "linkcheck.plugin"
+lognames = {
+    "cmdline": LOG_CMDLINE,
+    "checking": LOG_CHECK,
+    "cache": LOG_CACHE,
+    "gui": LOG_GUI,
+    "thread": LOG_THREAD,
+    "plugin": LOG_PLUGIN,
+    "all": LOG_ROOT,
+}
+
+lognamelist = ", ".join(repr(name) for name in lognames)
+
+# logging configuration
+configdict = {
+    'version': 1,
+    'loggers': {
+    },
+    'root': {
+      'level': 'DEBUG',
+    },
+}
+
+def init_log_config(handler=None):
+    """
+    Set up the application logging (not to be confused with check
+    loggers). When debug is not None it is expected to be a list of
+    logger names for which debugging will be enabled.
+
+    """
+    for applog in lognames.values():
+        # propagate except for root app logger 'linkcheck'
+        propagate = (applog != LOG_ROOT)
+        configdict['loggers'][applog] = dict(level='INFO', propagate=propagate)
+
+    logging.config.dictConfig(configdict)
+    if handler is None:
+        handler = ansicolor.ColoredStreamHandler(strm=sys.stderr)
+    add_loghandler(handler)
+
+
+def add_loghandler (handler):
+    """Add log handler to root logger LOG_ROOT and set formatting."""
+    logging.getLogger(LOG_ROOT).addHandler(handler)
+    format = "%(levelname)s %(asctime)s %(threadName)s %(message)s"
+    handler.setFormatter(logging.Formatter(format))
+
+
+def remove_loghandler (handler):
+    """Remove log handler from root logger LOG_ROOT."""
+    logging.getLogger(LOG_ROOT).removeHandler(handler)
+
+
+def reset_loglevel():
+    """Reset log level to display only warnings and errors."""
+    set_loglevel(['all'], logging.WARN)
+
+
+def set_debug(loggers):
+    """Set debugging log level."""
+    set_loglevel(loggers, logging.DEBUG)
+
+
+def set_loglevel(loggers, level):
+    """Set logging levels for given loggers."""
+    if not loggers:
+        return
+    if 'all' in loggers:
+        loggers = lognames.keys()
+    for key in loggers:
+        logging.getLogger(lognames[key]).setLevel(level)

linkchecker

@@ -28,12 +28,14 @@ import argparse
 import getpass
 # installs _() and _n() gettext functions into global namespace
 import linkcheck
+from linkcheck import logconf, LOG_CMDLINE
+logconf.init_log_config()
 # override argparse gettext method with the one from linkcheck.init_i18n()
 #argparse._ = _
 # now import the rest of the linkchecker gang
 from linkcheck.cmdline import print_version, print_usage, aggregate_url, \
   LCArgumentParser, print_plugins
-from linkcheck import log, LOG_CMDLINE, i18n, strformat
+from linkcheck import log, i18n, strformat
 import linkcheck.checker
 import linkcheck.configuration
 import linkcheck.fileutil
@@ -212,6 +214,7 @@ argparser = LCArgumentParser(
 
 # build a config object for this check session
 config = linkcheck.configuration.Configuration()
+config.set_status_logger(console.StatusLogger())
 
 ################# general options ##################
 group = argparser.add_argument_group(_("General options"))
@@ -253,7 +256,7 @@ The option can be given multiple times to debug with more
 than one logger.
 
 For accurate results, threading will be disabled during debug runs.""") % \
-{"lognamelist": linkcheck.lognamelist})
+{"lognamelist": logconf.lognamelist})
 group.add_argument("-F", "--file-output", action="append",
                  dest="fileoutput", metavar="TYPE[/ENCODING[/FILENAME]]",
                  help=_(
@@ -391,11 +394,11 @@ options = argparser.parse_args()
 
 # initialize logging
 if options.debug:
-    allowed_debugs = linkcheck.lognames.keys()
+    allowed_debugs = logconf.lognames.keys()
     for _name in options.debug:
         if _name not in allowed_debugs:
             print_usage(_("Invalid debug level %(level)r") % {'level': _name})
-config.init_logging(console.StatusLogger(), debug=options.debug)
+    # XXX set logging debug level
 log.debug(LOG_CMDLINE, _("Python %(version)s on %(platform)s") % \
    {"version": sys.version, "platform": sys.platform})
 # read configuration files