# -*- coding: iso-8859-1 -*- # Copyright (C) 2000-2014 Bastian Kleineidam # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. """ Main function module for link checking. """ # version checks import sys # Needs Python >= 2.7 because we use dictionary based logging config # Needs Python >= 2.7.2 which fixed http://bugs.python.org/issue11467 if not (hasattr(sys, 'version_info') or sys.version_info < (2, 7, 2, 'final', 0)): raise SystemExit("This program requires Python 2.7.2 or later.") import requests if requests.__version__ <= '2.2.0': raise SystemExit("This program requires Python requests 2.2.0 or later.") import os # add the custom linkcheck_dns directory to sys.path _dnspath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'linkcheck_dns')) if _dnspath not in sys.path: sys.path.insert(0, _dnspath) del _dnspath import re import signal import traceback from . import i18n import _LinkChecker_configdata as configdata def module_path (): """Return absolute directory of system executable.""" return os.path.dirname(os.path.abspath(sys.executable)) def get_install_data (): """Return absolute path of LinkChecker data installation directory.""" from .loader import is_frozen if is_frozen(): return module_path() return configdata.install_data # application log areas LOG_ROOT = "linkcheck" LOG_CMDLINE = "linkcheck.cmdline" LOG_CHECK = "linkcheck.check" LOG_CACHE = "linkcheck.cache" LOG_GUI = "linkcheck.gui" LOG_THREAD = "linkcheck.thread" LOG_PLUGIN = "linkcheck.plugin" lognames = { "cmdline": LOG_CMDLINE, "checking": LOG_CHECK, "cache": LOG_CACHE, "gui": LOG_GUI, "thread": LOG_THREAD, "plugin": LOG_PLUGIN, "all": LOG_ROOT, } # XXX debug httplib #import httplib #httplib.HTTPConnection.debuglevel = 1 lognamelist = ", ".join(repr(name) for name in lognames) # logging configuration configdict = { 'version': 1, 'loggers': { }, 'root': { 'level': 'DEBUG', }, } def init_log_config(): """Configure the application loggers.""" for applog in lognames.values(): # propagate except for root app logger 'linkcheck' propagate = (applog != LOG_ROOT) configdict['loggers'][applog] = dict(level='INFO', propagate=propagate) init_log_config() from . import log class LinkCheckerError (StandardError): """Exception to be raised on linkchecker-specific check errors.""" pass class LinkCheckerInterrupt (StandardError): """Used for testing.""" pass def get_link_pat (arg, strict=False): """Get a link pattern matcher for intern/extern links. Returns a compiled pattern and a negate and strict option. @param arg: pattern from config @type arg: string @param strict: if pattern is to be handled strict @type strict: bool @return: dictionary with keys 'pattern', 'negate' and 'strict' @rtype: dict @raises: re.error on invalid regular expressions """ log.debug(LOG_CHECK, "Link pattern %r strict=%s", arg, strict) if arg.startswith('!'): pattern = arg[1:] negate = True else: pattern = arg negate = False try: regex = re.compile(pattern) except re.error as msg: log.warn(LOG_CHECK, "invalid regular expression %r: %s" % (pattern, msg)) raise return { "pattern": regex, "negate": negate, "strict": strict, } def init_i18n (loc=None): """Initialize i18n with the configured locale dir. The environment variable LOCPATH can also specify a locale dir. @return: None """ if 'LOCPATH' in os.environ: locdir = os.environ['LOCPATH'] else: locdir = os.path.join(get_install_data(), 'share', 'locale') i18n.init(configdata.name.lower(), locdir, loc=loc) # install translated log level names import logging logging.addLevelName(logging.CRITICAL, _('CRITICAL')) logging.addLevelName(logging.ERROR, _('ERROR')) logging.addLevelName(logging.WARN, _('WARN')) logging.addLevelName(logging.WARNING, _('WARNING')) logging.addLevelName(logging.INFO, _('INFO')) logging.addLevelName(logging.DEBUG, _('DEBUG')) logging.addLevelName(logging.NOTSET, _('NOTSET')) # initialize i18n, puts _() and _n() function into global namespace init_i18n() def drop_privileges (): """Make sure to drop root privileges on POSIX systems.""" if os.name != 'posix': return if os.geteuid() == 0: log.warn(LOG_CHECK, _("Running as root user; " "dropping privileges by changing user to nobody.")) import pwd os.seteuid(pwd.getpwnam('nobody')[3]) def find_third_party_modules (): """Find third party modules and add them to the python path.""" parent = os.path.dirname(os.path.dirname(__file__)) third_party = os.path.join(parent, "third_party") if os.path.isdir(third_party): sys.path.append(os.path.join(third_party, "dnspython")) find_third_party_modules() if hasattr(signal, "SIGUSR1"): # install SIGUSR1 handler from .decorators import signal_handler @signal_handler(signal.SIGUSR1) def print_threadstacks(sig, frame): """Print stack traces of all running threads.""" log.warn(LOG_THREAD, "*** STACKTRACE START ***") for threadId, stack in sys._current_frames().items(): log.warn(LOG_THREAD, "# ThreadID: %s" % threadId) for filename, lineno, name, line in traceback.extract_stack(stack): log.warn(LOG_THREAD, 'File: "%s", line %d, in %s' % (filename, lineno, name)) line = line.strip() if line: log.warn(LOG_THREAD, " %s" % line) log.warn(LOG_THREAD, "*** STACKTRACE END ***")