diff --git a/linkcheck/__init__.py b/linkcheck/__init__.py index 40f292b6..56248556 100644 --- a/linkcheck/__init__.py +++ b/linkcheck/__init__.py @@ -39,7 +39,16 @@ import re import signal import traceback -from . import i18n +from . import i18n, log +from .logconf import ( + LOG_ROOT, + LOG_CMDLINE, + LOG_CHECK, + LOG_CACHE, + LOG_GUI, + LOG_THREAD, + LOG_PLUGIN, +) import _LinkChecker_configdata as configdata @@ -56,51 +65,6 @@ def get_install_data (): return configdata.install_data -# application log areas -LOG_ROOT = "linkcheck" -LOG_CMDLINE = "linkcheck.cmdline" -LOG_CHECK = "linkcheck.check" -LOG_CACHE = "linkcheck.cache" -LOG_GUI = "linkcheck.gui" -LOG_THREAD = "linkcheck.thread" -LOG_PLUGIN = "linkcheck.plugin" -lognames = { - "cmdline": LOG_CMDLINE, - "checking": LOG_CHECK, - "cache": LOG_CACHE, - "gui": LOG_GUI, - "thread": LOG_THREAD, - "plugin": LOG_PLUGIN, - "all": LOG_ROOT, -} -# XXX debug httplib -#import httplib -#httplib.HTTPConnection.debuglevel = 1 - -lognamelist = ", ".join(repr(name) for name in lognames) - -# logging configuration -configdict = { - 'version': 1, - 'loggers': { - }, - 'root': { - 'level': 'DEBUG', - }, -} - -def init_log_config(): - """Configure the application loggers.""" - for applog in lognames.values(): - # propagate except for root app logger 'linkcheck' - propagate = (applog != LOG_ROOT) - configdict['loggers'][applog] = dict(level='INFO', propagate=propagate) - -init_log_config() - -from . import log - - class LinkCheckerError (StandardError): """Exception to be raised on linkchecker-specific check errors.""" pass @@ -162,6 +126,7 @@ def init_i18n (loc=None): logging.addLevelName(logging.DEBUG, _('DEBUG')) logging.addLevelName(logging.NOTSET, _('NOTSET')) + # initialize i18n, puts _() and _n() function into global namespace init_i18n() diff --git a/linkcheck/checker/fileurl.py b/linkcheck/checker/fileurl.py index 8cac89e5..00645bbe 100644 --- a/linkcheck/checker/fileurl.py +++ b/linkcheck/checker/fileurl.py @@ -26,7 +26,7 @@ import urllib2 from datetime import datetime from . import urlbase, get_index_html -from .. import log, LOG_CHECK, fileutil, LinkCheckerError, url as urlutil +from .. import log, LOG_CHECK, fileutil, mimeutil, LinkCheckerError, url as urlutil from ..bookmarks import firefox from .const import WARN_FILE_MISSING_SLASH, WARN_FILE_SYSTEM_PATH @@ -251,7 +251,7 @@ class FileUrl (urlbase.UrlBase): """Return URL content type, or an empty string if content type could not be found.""" if self.url: - self.content_type = fileutil.guess_mimetype(self.url, read=self.get_content) + self.content_type = mimeutil.guess_mimetype(self.url, read=self.get_content) else: self.content_type = u"" diff --git a/linkcheck/checker/ftpurl.py b/linkcheck/checker/ftpurl.py index 36455463..067310c1 100644 --- a/linkcheck/checker/ftpurl.py +++ b/linkcheck/checker/ftpurl.py @@ -21,7 +21,7 @@ Handle FTP links. import ftplib from cStringIO import StringIO -from .. import log, LOG_CHECK, LinkCheckerError, fileutil +from .. import log, LOG_CHECK, LinkCheckerError, mimeutil from . import proxysupport, httpurl, internpaturl, get_index_html from .const import WARN_FTP_MISSING_SLASH @@ -179,7 +179,7 @@ class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport): def set_content_type (self): """Set URL content type, or an empty string if content type could not be found.""" - self.content_type = fileutil.guess_mimetype(self.url, read=self.get_content) + self.content_type = mimeutil.guess_mimetype(self.url, read=self.get_content) def read_content (self): """Return URL target content, or in case of directories a dummy HTML diff --git a/linkcheck/checker/httpurl.py b/linkcheck/checker/httpurl.py index e604f300..10919f39 100644 --- a/linkcheck/checker/httpurl.py +++ b/linkcheck/checker/httpurl.py @@ -21,7 +21,7 @@ Handle http links. import requests from cStringIO import StringIO -from .. import (log, LOG_CHECK, strformat, fileutil, +from .. import (log, LOG_CHECK, strformat, mimeutil, url as urlutil, LinkCheckerError, httputil) from . import (internpaturl, proxysupport) from ..HtmlParser import htmlsax @@ -314,7 +314,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport): return False # some content types must be validated with the page content if self.content_type in ("application/xml", "text/xml"): - rtype = fileutil.guess_mimetype_read(self.get_content) + rtype = mimeutil.guess_mimetype_read(self.get_content) if rtype is not None: # XXX side effect self.content_type = rtype diff --git a/linkcheck/configuration/__init__.py b/linkcheck/configuration/__init__.py index 1b4c783b..ec0ca2e5 100644 --- a/linkcheck/configuration/__init__.py +++ b/linkcheck/configuration/__init__.py @@ -18,17 +18,14 @@ Store metadata and options. """ -import sys import os import re -import logging.config import urllib import urlparse import shutil import socket import _LinkChecker_configdata as configdata -from .. import (log, LOG_CHECK, LOG_ROOT, ansicolor, lognames, - get_install_data, fileutil, configdict) +from .. import (log, LOG_CHECK, get_install_data, fileutil) from . import confparse from ..decorators import memoized @@ -182,57 +179,10 @@ class Configuration (dict): self[key] = {} self.loggers[key] = c - def init_logging (self, status_logger, debug=None, handler=None): - """ - Set up the application logging (not to be confused with check - loggers). When debug is not None it is expected to be a list of - logger names for which debugging will be enabled. - - If no thread debugging is enabled, threading will be disabled. - """ - logging.config.dictConfig(configdict) - if handler is None: - handler = ansicolor.ColoredStreamHandler(strm=sys.stderr) - self.add_loghandler(handler, debug) - self.set_debug(debug) + def set_status_logger(self, status_logger): + """Set the status logger.""" self.status_logger = status_logger - def set_debug (self, debug): - """Set debugging levels for configured loggers. The argument - is a list of logger names to enable debug for.""" - self.set_loglevel(debug, logging.DEBUG) - - def add_loghandler (self, handler, debug): - """Add log handler to root logger LOG_ROOT and set formatting.""" - logging.getLogger(LOG_ROOT).addHandler(handler) - format = "%(levelname)s " - if debug: - format += "%(asctime)s " - if self['threads'] > 0: - format += "%(threadName)s " - format += "%(message)s" - handler.setFormatter(logging.Formatter(format)) - - def remove_loghandler (self, handler): - """Remove log handler from root logger LOG_ROOT.""" - logging.getLogger(LOG_ROOT).removeHandler(handler) - - def reset_loglevel (self): - """Reset log level to display only warnings and errors.""" - self.set_loglevel(['all'], logging.WARN) - - def set_loglevel (self, loggers, level): - """Set logging levels for given loggers.""" - if not loggers: - return - if 'all' in loggers: - loggers = lognames.keys() - # disable threading if no thread debugging - if "thread" not in loggers and level == logging.DEBUG: - self['threads'] = 0 - for key in loggers: - logging.getLogger(lognames[key]).setLevel(level) - def logger_new (self, loggername, **kwargs): """Instantiate new logger and return it.""" args = self[loggername] diff --git a/linkcheck/configuration/confparse.py b/linkcheck/configuration/confparse.py index 70ad3f74..67751edd 100644 --- a/linkcheck/configuration/confparse.py +++ b/linkcheck/configuration/confparse.py @@ -18,7 +18,7 @@ import ConfigParser import os -from .. import LinkCheckerError, get_link_pat, LOG_CHECK, log, fileutil, plugins +from .. import LinkCheckerError, get_link_pat, LOG_CHECK, log, fileutil, plugins, logconf def read_multiline (value): @@ -113,7 +113,7 @@ class LCConfigParser (ConfigParser.RawConfigParser, object): if self.has_option(section, "debug"): val = self.get(section, "debug") parts = [f.strip().lower() for f in val.split(',')] - self.config.set_debug(parts) + logconf.set_debug(parts) self.read_boolean_option(section, "status") if self.has_option(section, "log"): val = self.get(section, "log").strip().lower() diff --git a/linkcheck/gui/__init__.py b/linkcheck/gui/__init__.py index 8bc55284..436bb96e 100644 --- a/linkcheck/gui/__init__.py +++ b/linkcheck/gui/__init__.py @@ -37,7 +37,7 @@ from .settings import Settings from .recentdocs import RecentDocumentModel from .projects import openproject, saveproject, loadproject, ProjectExt from .. import configuration, checker, director, get_link_pat, \ - strformat, fileutil, LinkCheckerError, i18n, httputil + strformat, mimeutil, LinkCheckerError, i18n, httputil, logconf from ..containers import enum from .. import url as urlutil @@ -111,6 +111,7 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow): self.label_busy.setText(u"") self.label_busy.setMovie(self.movie) # init the rest + self.init_logging() self.init_url(url) self.init_treeview() self.connect_widgets() @@ -121,6 +122,11 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow): self.init_drop() self.init_app(project) + def init_logging(self): + """Initialize logging.""" + self.handler = GuiLogHandler(self.debug.log_msg_signal) + logconf.init_log_config(handler=self.handler) + def init_url (self, url): """Initialize URL input.""" documents = self.settings.read_recent_documents() @@ -228,6 +234,8 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow): def init_config (self): """Create a configuration object.""" self.config = configuration.Configuration() + status = StatusLogger(self.log_status_signal) + self.config.set_status_logger(status) # dictionary holding overwritten values self.config_backup = {} # set standard GUI configuration values @@ -236,9 +244,6 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow): signal=self.log_url_signal, stats=self.log_stats_signal) self.config["status"] = True self.config["status_wait_seconds"] = 2 - self.handler = GuiLogHandler(self.debug.log_msg_signal) - status = StatusLogger(self.log_status_signal) - self.config.init_logging(status, handler=self.handler) def read_config (self, filename=None): """Read user and system configuration file.""" @@ -365,7 +370,7 @@ class LinkCheckerMain (QtGui.QMainWindow, Ui_MainWindow): self.settings.save_recent_documents(self.recent.get_documents()) self.settings.save_misc(dict(saveresultas=self.saveresultas)) self.settings.sync() - self.config.remove_loghandler(self.handler) + logconf.remove_loghandler(self.handler) if e is not None: e.accept() @@ -537,7 +542,7 @@ Version 2 or later. if not content_type: # read function for content type guessing read = lambda: data - content_type = fileutil.guess_mimetype(url, read=read) + content_type = mimeutil.guess_mimetype(url, read=read) self.editor.setContentType(content_type) self.editor.setText(data, line=line, col=col) self.editor.show() diff --git a/linkcheck/logconf.py b/linkcheck/logconf.py new file mode 100644 index 00000000..2f8f1069 --- /dev/null +++ b/linkcheck/logconf.py @@ -0,0 +1,101 @@ +# -*- coding: iso-8859-1 -*- +# Copyright (C) 2000-2014 Bastian Kleineidam +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +""" +Logging configuration +""" +import logging.config +import sys +from . import ansicolor + +# application log areas +LOG_ROOT = "linkcheck" +LOG_CMDLINE = "linkcheck.cmdline" +LOG_CHECK = "linkcheck.check" +LOG_CACHE = "linkcheck.cache" +LOG_GUI = "linkcheck.gui" +LOG_THREAD = "linkcheck.thread" +LOG_PLUGIN = "linkcheck.plugin" +lognames = { + "cmdline": LOG_CMDLINE, + "checking": LOG_CHECK, + "cache": LOG_CACHE, + "gui": LOG_GUI, + "thread": LOG_THREAD, + "plugin": LOG_PLUGIN, + "all": LOG_ROOT, +} + +lognamelist = ", ".join(repr(name) for name in lognames) + +# logging configuration +configdict = { + 'version': 1, + 'loggers': { + }, + 'root': { + 'level': 'DEBUG', + }, +} + +def init_log_config(handler=None): + """ + Set up the application logging (not to be confused with check + loggers). When debug is not None it is expected to be a list of + logger names for which debugging will be enabled. + + """ + for applog in lognames.values(): + # propagate except for root app logger 'linkcheck' + propagate = (applog != LOG_ROOT) + configdict['loggers'][applog] = dict(level='INFO', propagate=propagate) + + logging.config.dictConfig(configdict) + if handler is None: + handler = ansicolor.ColoredStreamHandler(strm=sys.stderr) + add_loghandler(handler) + + +def add_loghandler (handler): + """Add log handler to root logger LOG_ROOT and set formatting.""" + logging.getLogger(LOG_ROOT).addHandler(handler) + format = "%(levelname)s %(asctime)s %(threadName)s %(message)s" + handler.setFormatter(logging.Formatter(format)) + + +def remove_loghandler (handler): + """Remove log handler from root logger LOG_ROOT.""" + logging.getLogger(LOG_ROOT).removeHandler(handler) + + +def reset_loglevel(): + """Reset log level to display only warnings and errors.""" + set_loglevel(['all'], logging.WARN) + + +def set_debug(loggers): + """Set debugging log level.""" + set_loglevel(loggers, logging.DEBUG) + + +def set_loglevel(loggers, level): + """Set logging levels for given loggers.""" + if not loggers: + return + if 'all' in loggers: + loggers = lognames.keys() + for key in loggers: + logging.getLogger(lognames[key]).setLevel(level) diff --git a/linkchecker b/linkchecker index 833260a7..4b7b7bb2 100755 --- a/linkchecker +++ b/linkchecker @@ -28,12 +28,14 @@ import argparse import getpass # installs _() and _n() gettext functions into global namespace import linkcheck +from linkcheck import logconf, LOG_CMDLINE +logconf.init_log_config() # override argparse gettext method with the one from linkcheck.init_i18n() #argparse._ = _ # now import the rest of the linkchecker gang from linkcheck.cmdline import print_version, print_usage, aggregate_url, \ LCArgumentParser, print_plugins -from linkcheck import log, LOG_CMDLINE, i18n, strformat +from linkcheck import log, i18n, strformat import linkcheck.checker import linkcheck.configuration import linkcheck.fileutil @@ -212,6 +214,7 @@ argparser = LCArgumentParser( # build a config object for this check session config = linkcheck.configuration.Configuration() +config.set_status_logger(console.StatusLogger()) ################# general options ################## group = argparser.add_argument_group(_("General options")) @@ -253,7 +256,7 @@ The option can be given multiple times to debug with more than one logger. For accurate results, threading will be disabled during debug runs.""") % \ -{"lognamelist": linkcheck.lognamelist}) +{"lognamelist": logconf.lognamelist}) group.add_argument("-F", "--file-output", action="append", dest="fileoutput", metavar="TYPE[/ENCODING[/FILENAME]]", help=_( @@ -391,11 +394,11 @@ options = argparser.parse_args() # initialize logging if options.debug: - allowed_debugs = linkcheck.lognames.keys() + allowed_debugs = logconf.lognames.keys() for _name in options.debug: if _name not in allowed_debugs: print_usage(_("Invalid debug level %(level)r") % {'level': _name}) -config.init_logging(console.StatusLogger(), debug=options.debug) + # XXX set logging debug level log.debug(LOG_CMDLINE, _("Python %(version)s on %(platform)s") % \ {"version": sys.version, "platform": sys.platform}) # read configuration files