linkchecker/linkcheck/checker/urlbase.py

# -*- coding: iso-8859-1 -*-
# Copyright (C) 2000-2010 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
Base URL handler.
"""
import sys
import os
import logging
import urlparse
import urllib2
import urllib
import time
import errno
import socket
import select
import tempfile

from . import absolute_url, StoringHandler, get_url_from
from .. import (log, LOG_CHECK, LOG_CACHE, httputil, httplib2 as httplib,
  strformat, LinkCheckerError, url as urlutil, trace, clamav, winutil, geoip)
from ..HtmlParser import htmlsax
from ..htmlutil import linkparse
from ..network import iputil
from .const import (WARN_URL_EFFECTIVE_URL, WARN_URL_UNICODE_DOMAIN,
    WARN_URL_ERROR_GETTING_CONTENT, WARN_URL_OBFUSCATED_IP,
    WARN_URL_ANCHOR_NOT_FOUND, WARN_URL_WARNREGEX_FOUND,
    WARN_URL_CONTENT_SIZE_TOO_LARGE, WARN_URL_CONTENT_SIZE_ZERO,
    WARN_URL_CONTENT_SIZE_UNEQUAL, ExcList, ExcSyntaxList, ExcNoCacheList)

# helper alias
unicode_safe = strformat.unicode_safe

def urljoin (parent, url, scheme):
    """
    If url is relative, join parent and url. Else leave url as-is.

    @return joined url
    """
    if url.startswith(scheme+":"):
        return url
    # work around a Python 2.6/3.1 bug cutting off characters when the URL
    # begins with semicolon
    if url.startswith(';'):
        url = "./%s" % url
    return urlparse.urljoin(parent, url)


def url_norm (url, encoding=None):
    """Wrapper for url.url_norm() to convert UnicodeError in
    LinkCheckerError."""
    try:
        return urlutil.url_norm(url, encoding=encoding)
    except UnicodeError:
        msg = _("URL has unparsable domain name: %(name)s") % \
            {"name": sys.exc_info()[1]}
        raise LinkCheckerError(msg)


class UrlBase (object):
    """An URL with additional information like validity etc."""

    # file types that can be parsed recursively
    ContentMimetypes = {
        "text/html": "html",
        "application/xhtml+xml": "html",
        "text/css": "css",
        "application/x-shockwave-flash": "swf",
        "application/msword": "word",
        "text/plain+linkchecker": "text",
        "text/plain+opera": "opera",
    }

    def __init__ (self, base_url, recursion_level, aggregate,
                  parent_url=None, base_ref=None, line=-1, column=-1,
                  name=u"", url_encoding=None):
        """
        Initialize check data, and store given variables.

        @param base_url: unquoted and possibly unnormed url
        @param recursion_level: on what check level lies the base url
        @param aggregate: aggregate instance
        @param parent_url: quoted and normed url of parent or None
        @param base_ref: quoted and normed url of <base href=""> or None
        @param line: line number of url in parent content
        @param column: column number of url in parent content
        @param name: name of url or empty
        @param url_encoding: encoding of URL or None
        """
        self.init(base_ref, base_url, parent_url, recursion_level,
                  aggregate, line, column, name, url_encoding)
        self.reset()
        self.check_syntax()

    def init (self, base_ref, base_url, parent_url, recursion_level,
              aggregate, line, column, name, url_encoding):
        """
        Initialize internal data.
        """
        self.base_ref = base_ref
        # note that self.base_url must not be modified
        self.base_url = base_url
        self.parent_url = parent_url
        self.recursion_level = recursion_level
        self.aggregate = aggregate
        self.line = line
        self.column = column
        self.name = name
        self.encoding = url_encoding
        if self.base_ref:
            assert not urlutil.url_needs_quoting(self.base_ref), \
                   "unquoted base reference URL %r" % self.base_ref
        if self.parent_url:
            assert not urlutil.url_needs_quoting(self.parent_url), \
                   "unquoted parent URL %r" % self.parent_url
        url = absolute_url(base_url, base_ref, parent_url)
        # assume file link if no scheme is found
        self.scheme = url.split(":", 1)[0] or "file"

    def reset (self):
        """
        Reset all variables to default values.
        """
        # self.url is constructed by self.build_url() out of base_url
        # and (base_ref or parent) as absolute and normed url.
        # This the real url we use when checking so it also referred to
        # as 'real url'
        self.url = None
        # a splitted version of url for convenience
        self.urlparts = None
        # the anchor part of url
        self.anchor = None
        # list of parsed anchors
        self.anchors = []
        # the result message string and flag
        self.result = u""
        self.has_result = False
        # cached or not
        self.cached = False
        # valid or not
        self.valid = True
        # list of warnings (without duplicates)
        self.warnings = []
        # list of infos
        self.info = []
        # content size
        self.size = -1
        # download time
        self.dltime = -1
        # download size
        self.dlsize = -1
        # check time
        self.checktime = 0
        # connection object
        self.url_connection = None
        # data of url content,  (data == None) means no data is available
        self.data = None
        # cache keys, are set by build_url() calling set_cache_keys()
        self.cache_url_key = None
        self.cache_content_key = None
        # extern flags (is_extern, is_strict), both enabled as default
        self.extern = (1, 1)
        # flag if the result should be cached
        self.caching = True
        # title is either the URL or parsed from content
        self.title = None
        # flag if content should be checked or not
        self.do_check_content = True

    def set_result (self, msg, valid=True, overwrite=False):
        """
        Set result string and validity.
        """
        if self.has_result and not overwrite:
            log.warn(LOG_CHECK,
              "Double result %r (previous %r) for %s", msg, self.result, self)
        else:
            self.has_result = True
        if not isinstance(msg, unicode):
            log.warn(LOG_CHECK, "Non-unicode result for %s: %r", self, msg)
        elif not msg:
            log.warn(LOG_CHECK, "Empty result for %s", self)
        self.result = msg
        self.valid = valid

    def get_title (self):
        """Return title of page the URL refers to.
        This is per default the filename or the URL."""
        if self.title is None:
            url = u""
            if self.base_url:
                url = self.base_url
            elif self.url:
                url = self.url
            self.title = url
            if "/" in url:
                title = url.rsplit("/", 1)[1]
                if title:
                    self.title = title
        return self.title

    def set_title_from_content (self):
        """Set title of page the URL refers to.from page content."""
        if self.valid:
            try:
                handler = linkparse.TitleFinder()
            except tuple(ExcList):
                return
            parser = htmlsax.parser(handler)
            handler.parser = parser
            # parse
            try:
                parser.feed(self.get_content())
                parser.flush()
            except linkparse.StopParse, msg:
                log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
            # break cyclic dependencies
            handler.parser = None
            parser.handler = None
            if handler.title:
                self.title = handler.title

    def is_parseable (self):
        """
        Return True iff content of this url is parseable.
        """
        return False

    def is_html (self):
        """
        Return True iff content of this url is HTML formatted.
        """
        return False

    def is_css (self):
        """Return True iff content of this url is CSS stylesheet."""
        return False

    def is_http (self):
        """
        Return True for http:// URLs.
        """
        return False

    def is_file (self):
        """
        Return True for file:// URLs.
        """
        return False

    def add_warning (self, s, tag=None):
        """
        Add a warning string.
        """
        item = (tag, s)
        if item not in self.warnings:
            self.warnings.append(item)

    def add_info (self, s):
        """
        Add an info string.
        """
        if s not in self.info:
            self.info.append(s)

    def copy_from_cache (self, cache_data):
        """
        Fill attributes from cache data.
        """
        self.result = cache_data["result"]
        self.has_result = True
        for tag, msg in cache_data["warnings"]:
            # do not copy anchor warnings, since the current anchor
            # might have changed
            if tag != WARN_URL_ANCHOR_NOT_FOUND:
                self.add_warning(msg, tag=tag)
        for info in cache_data["info"]:
            self.add_info(info)
        self.valid = cache_data["valid"]
        self.dltime = cache_data["dltime"]
        self.dlsize = cache_data["dlsize"]
        self.anchors = cache_data["anchors"]
        self.cached = True
        # recheck anchor
        if self.valid and self.anchor:
            self.check_anchor()

    def get_cache_data (self):
        """Return all data values that should be put in the cache."""
        return {"result": self.result,
                "warnings": self.warnings,
                "info": self.info,
                "valid": self.valid,
                "dltime": self.dltime,
                "dlsize": self.dlsize,
                "anchors": self.anchors,
               }

    def get_alias_cache_data (self):
        """Return all data values that should be put in the cache.
        Intended to be overridden by subclasses that handle aliases.
        """
        return self.get_cache_data()

    def set_cache_keys (self):
        """
        Set keys for URL checking and content recursion.
        """
        # remove anchor from content cache key since we assume
        # URLs with different anchors to have the same content
        self.cache_content_key = urlparse.urlunsplit(self.urlparts[:4]+[u''])
        assert isinstance(self.cache_content_key, unicode), self
        log.debug(LOG_CACHE, "Content cache key %r", self.cache_content_key)
        # construct cache key
        self.cache_url_key = self.cache_content_key
        assert isinstance(self.cache_url_key, unicode), self
        log.debug(LOG_CACHE, "URL cache key %r", self.cache_url_key)

    def check_syntax (self):
        """
        Called before self.check(), this function inspects the
        url syntax. Success enables further checking, failure
        immediately logs this url. Syntax checks must not
        use any network resources.
        """
        log.debug(LOG_CHECK, "checking syntax")
        if self.base_url is None:
            self.set_result(_("URL is missing"), valid=False)
            return
        if not (self.base_url or self.parent_url):
            self.set_result(_("URL is empty"), valid=False)
            return
        try:
            self.build_url()
            # check url warnings
            effectiveurl = urlparse.urlunsplit(self.urlparts)
            if self.url != effectiveurl:
                self.add_warning(_("Effective URL %(url)r.") %
                                 {"url": effectiveurl},
                                 tag=WARN_URL_EFFECTIVE_URL)
                self.url = effectiveurl
        except tuple(ExcSyntaxList), msg:
            self.set_result(unicode_safe(msg), valid=False)
            return
        self.set_cache_keys()

    def build_url (self):
        """
        Construct self.url and self.urlparts out of the given base
        url information self.base_url, self.parent_url and self.base_ref.
        """
        # norm base url - can raise UnicodeError from url.idna_encode()
        base_url, is_idn = url_norm(self.base_url, self.encoding)
        if is_idn:
            self.add_warning(_("""URL %(url)r has a unicode domain name which
                          is not yet widely supported. You should use
                          the URL %(idna_url)r instead.""") % \
                          {"url": self.base_url, "idna_url": base_url},
                          tag=WARN_URL_UNICODE_DOMAIN)
        # make url absolute
        if self.base_ref:
            # use base reference as parent url
            if ":" not in self.base_ref:
                # some websites have a relative base reference
                self.base_ref = urljoin(self.parent_url, self.base_ref,
                                        self.scheme)
            self.url = urljoin(self.base_ref, base_url, self.scheme)
        elif self.parent_url:
            # strip the parent url query and anchor
            urlparts = list(urlparse.urlsplit(self.parent_url))
            urlparts[3] = urlparts[4] = ""
            parent_url = urlparse.urlunsplit(urlparts)
            self.url = urljoin(parent_url, base_url, self.scheme)
        else:
            self.url = base_url
        # note: urljoin can unnorm the url path, so norm it again
        urlparts = list(urlparse.urlsplit(self.url))
        if urlparts[2]:
            urlparts[2] = urlutil.collapse_segments(urlparts[2])
        self.url = urlparse.urlunsplit(urlparts)
        # split into (modifiable) list
        self.urlparts = strformat.url_unicode_split(self.url)
        # and unsplit again
        self.url = urlparse.urlunsplit(self.urlparts)
        # check userinfo@host:port syntax
        self.userinfo, host = urllib.splituser(self.urlparts[1])
        # set host lowercase
        if self.userinfo:
            self.urlparts[1] = "%s@%s" % (self.userinfo, host.lower())
        else:
            self.urlparts[1] = host.lower()
        # safe anchor for later checking
        self.anchor = self.urlparts[4]
        self.host, self.port = urllib.splitport(host)
        if self.port is not None:
            if not urlutil.is_numeric_port(self.port):
                raise LinkCheckerError(_("URL has invalid port %(port)r") %
                    {"port": str(self.port)})
            self.port = int(self.port)
        self.check_obfuscated_ip()

    def check_obfuscated_ip (self):
        """Warn if host of this URL is obfuscated IP address."""
        # check if self.host can be an IP address
        if self.scheme not in ("ftp", "http", "mailto", "news", "nntp", "telnet"):
            return
        # check for obfuscated IP address
        if iputil.is_obfuscated_ip(self.host):
             ips = iputil.resolve_host(self.host)
             if ips:
                self.add_warning(
                   _("URL %(url)s has obfuscated IP address %(ip)s") % \
                   {"url": self.base_url, "ip": ips.pop()},
                          tag=WARN_URL_OBFUSCATED_IP)

    def check (self):
        """Main check function for checking this URL."""
        if self.aggregate.config["trace"]:
            trace.trace_on()
        try:
            self.local_check()
        except (socket.error, select.error):
            # on Unix, ctrl-c can raise
            # error: (4, 'Interrupted system call')
            etype, value = sys.exc_info()[:2]
            if etype == errno.EINTR:
                raise KeyboardInterrupt(value)
            else:
                raise
        finally:
            # close/release possible open connection
            self.close_connection()

    def add_country_info (self):
        """Try to ask GeoIP database for country info."""
        if self.host:
            country = geoip.get_country(self.host)
            if country is not None:
                self.add_info(_("URL is located in %(country)s.") %
                {"country": _(country)})

    def add_size_info (self):
        """Store size of URL content from meta info into self.size.
        Must be implemented in subclasses."""
        pass

    def local_check (self):
        """Local check function can be overridden in subclasses."""
        log.debug(LOG_CHECK, "Checking %s", self)
        # start time for check
        check_start = time.time()
        self.set_extern(self.url)
        if self.extern[0] and self.extern[1]:
            self.add_info(_("Outside of domain filter, checked only syntax."))
            return

        # check connection
        log.debug(LOG_CHECK, "checking connection")
        try:
            self.check_connection()
            self.add_size_info()
            self.add_country_info()
        except tuple(ExcList):
            value = self.handle_exception()
            # make nicer error msg for unknown hosts
            if isinstance(value, socket.error) and value.args[0] == -2:
                value = _('Hostname not found')
            # make nicer error msg for bad status line
            if isinstance(value, httplib.BadStatusLine):
                value = _('Bad HTTP response %(line)r') % {"line": str(value)}
            self.set_result(unicode_safe(value), valid=False)
        self.checktime = time.time() - check_start
        if self.do_check_content:
            # check content and recursion
            try:
                self.check_content()
                if self.allows_recursion():
                    self.parse_url()
                # check content size
                self.check_size()
            except tuple(ExcList):
                value = self.handle_exception()
                self.add_warning(_("could not get content: %(msg)r") %
                     {"msg": str(value)}, tag=WARN_URL_ERROR_GETTING_CONTENT)

    def close_connection (self):
        """
        Close an opened url connection.
        """
        if self.url_connection is None:
            # no connection is open
            return
        try:
            self.url_connection.close()
        except Exception:
            # ignore close errors
            pass
        self.url_connection = None

    def handle_exception (self):
        """
        An exception occurred. Log it and set the cache flag.
        """
        etype, value = sys.exc_info()[:2]
        log.debug(LOG_CHECK, "Error in %s: %s %s", self.url, etype, value, exception=True)
        # note: etype must be the exact class, not a subclass
        if (etype in ExcNoCacheList) or \
           (etype == socket.error and value.args[0]==errno.EBADF) or \
            not value:
            # EBADF occurs when operating on an already socket
            self.caching = False
        errmsg = etype.__name__
        if str(value):
            # use Exception class name
            errmsg += ": %s" % str(value)
        # limit length to 240
        return strformat.limit(errmsg, length=240)

    def check_connection (self):
        """
        The basic connection check uses urllib2.urlopen to initialize
        a connection object.
        """
        self.url_connection = urllib2.urlopen(self.url)

    def allows_recursion (self):
        """
        Return True iff we can recurse into the url's content.
        """
        log.debug(LOG_CHECK, "checking recursion of %r ...", self.url)
        # Test self.valid before self.is_parseable().
        if not self.valid:
            log.debug(LOG_CHECK, "... no, invalid.")
            return False
        if not self.is_parseable():
            log.debug(LOG_CHECK, "... no, not parseable.")
            return False
        if not self.can_get_content():
            log.debug(LOG_CHECK, "... no, cannot get content.")
            return False
        rec_level = self.aggregate.config["recursionlevel"]
        if rec_level >= 0 and self.recursion_level >= rec_level:
            log.debug(LOG_CHECK, "... no, maximum recursion level reached.")
            return False
        if self.extern[0]:
            log.debug(LOG_CHECK, "... no, extern.")
            return False
        if not self.content_allows_robots():
            log.debug(LOG_CHECK, "... no, robots.")
            return False
        log.debug(LOG_CHECK, "... yes, recursion.")
        return True

    def content_allows_robots (self):
        """
        Return False if the content of this URL forbids robots to
        search for recursive links.
        """
        if not self.is_html():
            return True
        if not (self.is_http() or self.is_file()):
            return True
        # construct parser object
        handler = linkparse.MetaRobotsFinder()
        parser = htmlsax.parser(handler)
        handler.parser = parser
        # parse
        try:
            parser.feed(self.get_content())
            parser.flush()
        except linkparse.StopParse, msg:
            log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
        # break cyclic dependencies
        handler.parser = None
        parser.handler = None
        return handler.follow

    def get_anchors (self):
        """Store list of anchors for this URL. Precondition: this URL is
        an HTML resource."""
        log.debug(LOG_CHECK, "Getting HTML anchors %s", self)
        handler = linkparse.LinkFinder(self.add_anchor,
                                   tags={'a': [u'name'], None: [u'id']})
        parser = htmlsax.parser(handler)
        handler.parser = parser
        # parse
        try:
            parser.feed(self.get_content())
            parser.flush()
        except linkparse.StopParse, msg:
            log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
        # break cyclic dependencies
        handler.parser = None
        parser.handler = None

    def add_anchor (self, url, line, column, name, base):
        """Add anchor URL."""
        self.anchors.append((url, line, column, name, base))

    def check_anchor (self):
        """If URL was valid and has an anchor, check it. A warning is
        logged if the anchor is not found.
        """
        if not self.aggregate.config["anchors"]:
            return
        log.debug(LOG_CHECK, "checking anchor %r", self.anchor)
        if any(x for x in self.anchors if x[0] == self.anchor):
            return
        anchors = u",".join(u"`%s'" % x[0] for x in self.anchors)
        args = {"name": self.anchor, "anchors": anchors}
        msg = u"%s %s" % (_("Anchor `%(name)s' not found.") % args,
                          _("Available anchors: %(anchors)s.") % args)
        self.add_warning(msg, tag=WARN_URL_ANCHOR_NOT_FOUND)

    def set_extern (self, url):
        """
        Match URL against extern and intern link patterns. If no pattern
        matches the URL is extern. Sets self.extern to a tuple (bool,
        bool) with content (is_extern, is_strict).

        @return: None
        """
        for entry in self.aggregate.config["externlinks"]:
            match = entry['pattern'].search(url)
            if (entry['negate'] and not match) or \
               (match and not entry['negate']):
                log.debug(LOG_CHECK, "Extern URL %r", url)
                self.extern = (1, entry['strict'])
                return
        for entry in self.aggregate.config["internlinks"]:
            match = entry['pattern'].search(url)
            if (entry['negate'] and not match) or \
               (match and not entry['negate']):
                log.debug(LOG_CHECK, "Intern URL %r", url)
                self.extern = (0, 0)
                return
        log.debug(LOG_CHECK, "Explicit extern URL %r", url)
        self.extern = (1, 0)
        return

    def can_get_content (self):
        """Indicate wether url get_content() can be called."""
        return True

    def get_content (self):
        """Precondition: url_connection is an opened URL."""
        if self.data is None:
            log.debug(LOG_CHECK, "Get content of %r", self.url)
            t = time.time()
            self.data, self.dlsize = self.read_content()
            self.dltime = time.time() - t
        return self.data

    def read_content (self):
        """Return data and data size for this URL.
        Can be overridden in subclasses."""
        data = self.url_connection.read()
        return data, len(data)

    def check_content (self):
        """Check content data for warnings, syntax errors, viruses etc."""
        if not (self.valid and self.can_get_content()):
            return
        if self.is_html():
            self.set_title_from_content()
            if self.aggregate.config["anchors"]:
                self.get_anchors()
        if self.anchor:
            self.check_anchor()
        self.check_warningregex()
        # is it an intern URL?
        if not self.extern[0]:
            # check HTML/CSS syntax
            if self.aggregate.config["checkhtml"] and self.is_html():
                self.check_html()
            if self.aggregate.config["checkcss"] and self.is_css():
                self.check_css()
            if self.aggregate.config["checkhtmlw3"] and self.is_html():
                self.check_html_w3()
            if self.aggregate.config["checkcssw3"] and self.is_css():
                self.check_css_w3()
            # check with clamav
            if self.aggregate.config["scanvirus"]:
                self.scan_virus()

    def check_warningregex (self):
        warningregex = self.aggregate.config["warningregex"]
        if warningregex:
            log.debug(LOG_CHECK, "checking content")
            try:
                match = warningregex.search(self.get_content())
                if match:
                    self.add_warning(_("Found %(match)r in link contents.") %
                       {"match": match.group()}, tag=WARN_URL_WARNREGEX_FOUND)
            except tuple(ExcList):
                value = self.handle_exception()
                self.set_result(unicode_safe(value), valid=False)

    def check_size (self):
        """Check content size if it is zero or larger than a given
        maximum size.
        """
        if self.dlsize == 0:
            self.add_warning(_("Content size is zero."),
                             tag=WARN_URL_CONTENT_SIZE_ZERO)
        else:
            maxbytes = self.aggregate.config["warnsizebytes"]
            if maxbytes is not None and self.dlsize >= maxbytes:
                self.add_warning(
                   _("Content size %(dlsize)s is larger than %(maxbytes)s.") %
                        {"dlsize": strformat.strsize(self.dlsize),
                         "maxbytes": strformat.strsize(maxbytes)},
                          tag=WARN_URL_CONTENT_SIZE_TOO_LARGE)
        if self.size != -1 and self.dlsize != -1 and self.dlsize != self.size:
                self.add_warning(_("Download size (%(dlsize)d Byte) "
                        "does not equal content size (%(size)d Byte).") %
                        {"dlsize": self.dlsize,
                         "size": self.size},
                          tag=WARN_URL_CONTENT_SIZE_UNEQUAL)

    def check_html (self):
        """Check HTML syntax of this page (which is supposed to be HTML)
        with the local HTML tidy module."""
        try:
            import tidy
        except ImportError:
            log.warn(LOG_CHECK, _("warning: tidy module is not available; " \
                 "download from http://utidylib.berlios.de/"))
            return
        options = dict(output_html=0, show_warnings=1, quiet=True,
            input_encoding='utf8', output_encoding='utf8', tidy_mark=0)
        try:
            doc = tidy.parseString(self.get_content(), **options)
            errors = filter_tidy_errors(doc.errors)
            if errors:
                for err in errors:
                    self.add_warning(u"HTMLTidy: %s" % err)
            else:
                self.add_info(u"HTMLTidy: %s" % _("valid HTML syntax"))
        except Exception:
            # catch _all_ exceptions since we dont want third party module
            # errors to propagate into this library
            err = str(sys.exc_info()[1])
            log.warn(LOG_CHECK,
                _("warning: tidy HTML parsing caused error: %(msg)s ") %
                {"msg": err})

    def check_css (self):
        """Check CSS syntax of this page (which is supposed to be CSS)
        with the local cssutils module."""
        try:
            import cssutils
        except ImportError:
            log.warn(LOG_CHECK,
                _("warning: cssutils module is not available; " \
                 "download from http://cthedot.de/cssutils/"))
            return
        try:
            csslog = logging.getLogger('cssutils')
            csslog.propagate = 0
            del csslog.handlers[:]
            handler = StoringHandler()
            csslog.addHandler(handler)
            csslog.setLevel(logging.WARN)
            cssparser = cssutils.CSSParser(log=csslog)
            cssparser.parseString(self.get_content(), href=self.url)
            if handler.storage:
                for record in handler.storage:
                    self.add_warning(u"cssutils: %s" % record.getMessage())
            else:
                self.add_info(u"cssutils: %s" % _("valid CSS syntax"))
        except Exception:
            # catch _all_ exceptions since we dont want third party module
            # errors to propagate into this library
            err = str(sys.exc_info()[1])
            log.warn(LOG_CHECK,
                _("warning: cssutils parsing caused error: %(msg)s") %
                {"msg": err})

    def check_html_w3 (self):
        """Check HTML syntax of this page (which is supposed to be HTML)
        with the online W3C HTML validator documented at
        http://validator.w3.org/docs/api.html
        """
        self.aggregate.check_w3_time()
        try:
            u = urllib2.urlopen('http://validator.w3.org/check',
                urllib.urlencode({
                    'fragment': self.get_content(),
                    'output': 'xml',
                }))
            if u.headers.get('x-w3c-validator-status', 'Invalid') == 'Valid':
                self.add_info(u"W3C Validator: %s" % _("valid HTML syntax"))
                return
            from xml.dom.minidom import parseString
            dom = parseString(u.read())
            elements = dom.getElementsByTagName('messages')[0].getElementsByTagName('msg')
            for msg in [e.firstChild.wholeText for e in elements]:
                self.add_warning(u"W3C HTML validation: %s" % msg)
        except Exception:
            # catch _all_ exceptions since we dont want third party module
            # errors to propagate into this library
            err = str(sys.exc_info()[1])
            log.warn(LOG_CHECK,
                _("warning: HTML W3C validation caused error: %(msg)s ") %
                {"msg": err})

    def check_css_w3 (self):
        """Check CSS syntax of this page (which is supposed to be CSS)
        with the online W3C CSS validator documented at
        http://jigsaw.w3.org/css-validator/manual.html#expert
        """
        self.aggregate.check_w3_time()
        try:
            host = 'jigsaw.w3.org'
            path = '/css-validator/validator'
            params = {
                'text': "div {}",
                'warning': '2',
                'output': 'soap12',
            }
            fields = params.items()
            content_type, body = httputil.encode_multipart_formdata(fields)
            h = httplib.HTTPConnection(host)
            h.putrequest('POST', path)
            h.putheader('Content-Type', content_type)
            h.putheader('Content-Length', str(len(body)))
            h.endheaders()
            h.send(body)
            r = h.getresponse(True)
            if r.getheader('X-W3C-Validator-Status', 'Invalid') == 'Valid':
                self.add_info(u"W3C Validator: %s" % _("valid CSS syntax"))
                return
            from xml.dom.minidom import parseString
            dom = parseString(r.read())
            elements = dom.getElementsByTagName('m:errors')[0].getElementsByTagName('m:error')
            for msg in [e.firstChild.wholeText for e in elements]:
                self.add_warning(u"W3C HTML validation: %s" % msg)
        except Exception:
            # catch _all_ exceptions since we dont want third party module
            # errors to propagate into this library
            err = str(sys.exc_info()[1])
            log.warn(LOG_CHECK,
                _("warning: CSS W3C validation caused error: %(msg)s ") %
                {"msg": err})

    def scan_virus (self):
        """Scan content for viruses."""
        infected, errors = clamav.scan(self.get_content())
        for msg in infected:
            self.add_warning(u"Virus scan infection: %s" % msg)
        for msg in errors:
            self.add_warning(u"Virus scan error: %s" % msg)

    def parse_url (self):
        """
        Parse url content and search for recursive links.
        Default parse type is html.
        """
        self.parse_html()

    def get_user_password (self):
        """Get tuple (user, password) from configured authentication.
        Both user and password can be None.
        """
        return self.aggregate.config.get_user_password(self.url)

    def parse_html (self):
        """Parse into HTML content and search for URLs to check.
        Found URLs are added to the URL queue.
        """
        log.debug(LOG_CHECK, "Parsing HTML %s", self)
        # construct parser object
        handler = linkparse.LinkFinder(self.add_url)
        parser = htmlsax.parser(handler)
        handler.parser = parser
        # parse
        try:
            parser.feed(self.get_content())
            parser.flush()
        except linkparse.StopParse, msg:
            log.debug(LOG_CHECK, "Stopped parsing: %s", msg)
        # break cyclic dependencies
        handler.parser = None
        parser.handler = None

    def add_url (self, url, line, column, name, base):
        """Queue URL data for checking."""
        base_ref = urlutil.url_norm(base)[0]
        url_data = get_url_from(url, self.recursion_level+1, self.aggregate,
            parent_url=self.url, base_ref=base_ref, line=line, column=column,
            name=name)
        self.aggregate.urlqueue.put(url_data)

    def parse_opera (self):
        """Parse an opera bookmark file."""
        log.debug(LOG_CHECK, "Parsing Opera bookmarks %s", self)
        name = None
        lineno = 0
        for line in self.get_content().splitlines():
            lineno += 1
            line = line.strip()
            if line.startswith("NAME="):
                name = line[5:]
            elif line.startswith("URL="):
                url = line[4:]
                if url and name is not None:
                    url_data = get_url_from(url, self.recursion_level+1,
                        self.aggregate, parent_url=self.url,
                        line=lineno, name=name)
                    self.aggregate.urlqueue.put(url_data)
            else:
                name = None

    def parse_text (self):
        """
        Parse a text file with on url per line; comment and blank
        lines are ignored.
        """
        log.debug(LOG_CHECK, "Parsing text %s", self)
        lineno = 0
        for line in self.get_content().splitlines():
            lineno += 1
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            url_data = get_url_from(line,
                              self.recursion_level+1, self.aggregate,
                              parent_url=self.url, line=lineno)
            self.aggregate.urlqueue.put(url_data)

    def parse_css (self):
        """
        Parse a CSS file for url() patterns.
        """
        log.debug(LOG_CHECK, "Parsing CSS %s", self)
        lineno = 0
        linkfinder = linkparse.css_url_re.finditer
        strip_comments = linkparse.strip_c_comments
        for line in strip_comments(self.get_content()).splitlines():
            lineno += 1
            for mo in linkfinder(line):
                column = mo.start("url")
                url = strformat.unquote(mo.group("url").strip())
                url_data = get_url_from(url,
                             self.recursion_level+1, self.aggregate,
                             parent_url=self.url, line=lineno, column=column)
                self.aggregate.urlqueue.put(url_data)

    def parse_swf (self):
        """Parse a SWF file for URLs."""
        linkfinder = linkparse.swf_url_re.finditer
        for mo in linkfinder(self.get_content()):
            url = mo.group()
            url_data = get_url_from(url,
                         self.recursion_level+1, self.aggregate,
                         parent_url=self.url)
            self.aggregate.urlqueue.put(url_data)

    def parse_word (self):
        """Parse a word file for hyperlinks."""
        if not winutil.has_word():
            return
        filename = self.get_temp_filename()
        # open word file and parse hyperlinks
        try:
            app = winutil.get_word_app()
            try:
                doc = winutil.open_wordfile(app, filename)
                try:
                    for link in doc.Hyperlinks:
                        url_data = get_url_from(link.Address,
                                 self.recursion_level+1, self.aggregate,
                                 parent_url=self.url, name=link.TextToDisplay)
                        self.aggregate.urlqueue.put(url_data)
                finally:
                    winutil.close_wordfile(doc)
            finally:
                winutil.close_word_app(app)
        except winutil.Error, msg:
            log.warn(LOG_CHECK, "Error parsing word file: %s", msg)

    def get_temp_filename (self):
        """Get temporary filename for content to parse."""
        # store content in temporary file
        fd, filename = tempfile.mkstemp(suffix='.doc', prefix='lc_')
        fp = os.fdopen(fd)
        fp.write(self.get_content())
        fp.close()

    def serialized (self):
        """
        Return serialized url check data as unicode string.
        """
        sep = unicode_safe(os.linesep)
        if self.base_url is not None:
            assert isinstance(self.base_url, unicode), self
        if self.parent_url is not None:
            assert isinstance(self.parent_url, unicode), self
        if self.base_ref is not None:
            assert isinstance(self.base_ref, unicode), self
        assert isinstance(self.name, unicode), self
        return sep.join([
            u"%s link" % self.scheme,
            u"base_url=%r" % self.base_url,
            u"parent_url=%r" % self.parent_url,
            u"base_ref=%r" % self.base_ref,
            u"recursion_level=%s" % self.recursion_level,
            u"url_connection=%s" % self.url_connection,
            u"line=%d" % self.line,
            u"column=%d" % self.column,
            u"name=%r" % self.name,
           ])

    def get_intern_pattern (self):
        """
        Get pattern for intern URL matching.

        @return non-empty regex pattern or None
        @rtype String or None
        """
        return None

    def __str__ (self):
        """
        Get URL info.

        @return: URL info, encoded with the output logger encoding
        @rtype: string
        """
        s = self.serialized()
        return self.aggregate.config['logger'].encode(s)

    def __repr__ (self):
        """
        Get URL info.

        @return: URL info
        @rtype: unicode
        """
        return u"<%s >" % self.serialized()

    def to_wire_dict (self):
        """Return a simplified transport object for logging.

        The transport object must contain these attributes:
        - url_data.valid: bool
          Indicates if URL is valid
        - url_data.cached: bool
          Indicates if URL data has been loaded from cache.
        - url_data.result: unicode
          Result string
        - url_data.warnings: list of unicode
          List of tagged warnings for this URL.
        - url_data.name: unicode string or None
          name of URL (eg. filename or link name)
        - url_data.parent_url: unicode or None
          Parent URL
        - url_data.base_ref: unicode or None
          HTML base reference URL of parent
        - url_data.url: unicode or None
          Fully qualified URL.
        - url_data.checktime: int
          Number of seconds needed to check this link, default: zero.
        - url_data.dltime: int
          Number of seconds needed to download URL content, default: -1
        - url_data.dlsize: int
          Size of downloaded URL content, default: -1
        - url_data.info: list of unicode
          Additional information about this URL.
        - url_data.line: int
          Line number of this URL at parent document, or -1
        - url_data.column: int
          Column number of this URL at parent document, or -1
        """
        return dict(valid=self.valid,
          extern=self.extern[0],
          cached=self.cached,
          result=self.result,
          warnings=[x[1] for x in self.warnings],
          name=self.name or u"",
          title=self.get_title(),
          parent_url=self.parent_url or u"",
          base_ref=self.base_ref or u"",
          base_url=self.base_url or u"",
          url=self.url or u"",
          checktime=self.checktime,
          dltime=self.dltime,
          dlsize=self.dlsize,
          info=self.info,
          line=self.line,
          column=self.column,
          cache_url_key=self.cache_url_key,
        )

    def to_wire (self):
        return CompactUrlData(self.to_wire_dict())


def filter_tidy_errors (errors):
    """Filter certain errors from HTML tidy run."""
    return [x for x in errors if not \
        (x.severity=='W' and x.message=='<table> lacks "summary" attribute')]


urlDataAttr = [
    'valid',
    'extern',
    'cached',
    'result',
    'warnings',
    'name',
    'title',
    'parent_url',
    'base_ref',
    'base_url',
    'url',
    'checktime',
    'dltime',
    'dlsize',
    'info',
    'line',
    'column',
    'cache_url_key',
]

class CompactUrlData (object):
    __slots__ = urlDataAttr

    def __init__(self, wired_url_data):
        '''Set all attributes according to the dictionnary wired_url_data'''
        for attr in urlDataAttr:
            setattr(self, attr, wired_url_data[attr])