mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-31 21:20:28 +00:00
3e56e96e14
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@1397 e7d03fd6-7b0d-0410-9947-9c21f3af8025
559 lines
20 KiB
Python
559 lines
20 KiB
Python
# -*- coding: iso-8859-1 -*-
|
|
"""Base URL handler"""
|
|
# Copyright (C) 2000-2004 Bastian Kleineidam
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import urlparse
|
|
import urllib2
|
|
import urllib
|
|
import time
|
|
import traceback
|
|
import socket
|
|
import select
|
|
import linkcheck
|
|
import linkcheck.linkparse
|
|
import bk.log
|
|
import bk.i18n
|
|
import bk.HtmlParser.htmlsax
|
|
|
|
|
|
ws_at_start_or_end = re.compile(r"(^\s+)|(\s+$)").search
|
|
|
|
# helper function for internal errors
|
|
def internal_error ():
|
|
print >>sys.stderr, bk.i18n._("""\n********** Oops, I did it again. *************
|
|
|
|
You have found an internal error in LinkChecker. Please write a bug report
|
|
at http://sourceforge.net/tracker/?func=add&group_id=1913&atid=101913
|
|
or send mail to %s and include the following information:
|
|
1) The URL or file you are testing
|
|
2) Your commandline arguments and/or configuration.
|
|
3) The system information below.
|
|
|
|
If you disclose some information because its too private to you thats ok.
|
|
I will try to help you nontheless (but you have to give me *something*
|
|
I can work with ;).
|
|
""") % linkcheck.Config.Email
|
|
etype, value = sys.exc_info()[:2]
|
|
print >>sys.stderr, etype, value
|
|
traceback.print_exc()
|
|
print_app_info()
|
|
print >>sys.stderr, bk.i18n._("\n******** LinkChecker internal error, bailing out ********")
|
|
sys.exit(1)
|
|
|
|
|
|
def print_app_info ():
|
|
print >>sys.stderr, bk.i18n._("System info:")
|
|
print >>sys.stderr, linkcheck.Config.App
|
|
print >>sys.stderr, "Python %s on %s" % (sys.version, sys.platform)
|
|
for key in ("LC_ALL", "LC_MESSAGES", "http_proxy", "ftp_proxy"):
|
|
value = os.getenv(key)
|
|
if value is not None:
|
|
print >>sys.stderr, key, "=", repr(value)
|
|
|
|
|
|
# regular expression for port numbers
|
|
is_valid_port = re.compile(r"\d+").match
|
|
|
|
|
|
class UrlData (object):
|
|
"Representing a URL with additional information like validity etc"
|
|
|
|
def __init__ (self,
|
|
urlName,
|
|
recursionLevel,
|
|
config,
|
|
parentName = None,
|
|
baseRef = None,
|
|
line = 0,
|
|
column = 0,
|
|
name = ""):
|
|
self.urlName = urlName
|
|
self.anchor = None
|
|
self.recursionLevel = recursionLevel
|
|
self.config = config
|
|
self.parentName = parentName
|
|
self.baseRef = baseRef
|
|
self.errorString = bk.i18n._("Error")
|
|
self.validString = bk.i18n._("Valid")
|
|
self.warningString = None
|
|
self.infoString = None
|
|
self.valid = True
|
|
self.url = None
|
|
self.urlparts = None
|
|
self.line = line
|
|
self.column = column
|
|
self.name = name
|
|
self.dltime = -1
|
|
self.dlsize = -1
|
|
self.checktime = 0
|
|
self.cached = False
|
|
self.urlConnection = None
|
|
self.extern = (1, 0)
|
|
self.data = None
|
|
self.has_content = False
|
|
url = linkcheck.checker.get_absolute_url(self.urlName, self.baseRef, self.parentName)
|
|
# assume file link if no scheme is found
|
|
self.scheme = url.split(":", 1)[0] or "file"
|
|
|
|
def setError (self, s):
|
|
self.valid = False
|
|
self.errorString = bk.i18n._("Error")+": "+s
|
|
|
|
def setValid (self, s):
|
|
self.valid = True
|
|
self.validString = bk.i18n._("Valid")+": "+s
|
|
|
|
def isParseable (self):
|
|
return False
|
|
|
|
def isHtml (self):
|
|
return False
|
|
|
|
def isHttp (self):
|
|
return False
|
|
|
|
def isFile (self):
|
|
return False
|
|
|
|
def setWarning (self, s):
|
|
if self.warningString:
|
|
self.warningString += "\n"+s
|
|
else:
|
|
self.warningString = s
|
|
|
|
def setInfo (self, s):
|
|
if self.infoString:
|
|
self.infoString += "\n"+s
|
|
else:
|
|
self.infoString = s
|
|
|
|
def copyFromCache (self, cacheData):
|
|
"""fill attributes from cache data"""
|
|
self.errorString = cacheData["errorString"]
|
|
self.validString = cacheData["validString"]
|
|
if self.warningString:
|
|
if cacheData["warningString"]:
|
|
self.warningString += "\n"+cacheData["warningString"]
|
|
else:
|
|
self.warningString = cacheData["warningString"]
|
|
self.infoString = cacheData["infoString"]
|
|
self.valid = cacheData["valid"]
|
|
self.dltime = cacheData["dltime"]
|
|
|
|
def getCacheData (self):
|
|
"""return all data values that should be put in the cache"""
|
|
return {"errorString": self.errorString,
|
|
"validString": self.validString,
|
|
"warningString": self.warningString,
|
|
"infoString": self.infoString,
|
|
"valid": self.valid,
|
|
"dltime": self.dltime,
|
|
}
|
|
|
|
def buildUrl (self):
|
|
if self.baseRef:
|
|
if ":" not in self.baseRef:
|
|
self.baseRef = urlparse.urljoin(self.parentName, self.baseRef)
|
|
self.url = urlparse.urljoin(self.baseRef, self.urlName)
|
|
elif self.parentName:
|
|
self.url = urlparse.urljoin(self.parentName, self.urlName)
|
|
else:
|
|
self.url = self.urlName
|
|
# unquote url
|
|
self.url = urllib.unquote(self.url)
|
|
# split into (modifiable) list
|
|
self.urlparts = list(urlparse.urlsplit(self.url))
|
|
# check userinfo@host:port syntax
|
|
self.userinfo, host = urllib.splituser(self.urlparts[1])
|
|
x, port = urllib.splitport(host)
|
|
if port is not None and not is_valid_port(port):
|
|
raise linkcheck.LinkCheckerError(bk.i18n._("URL has invalid port number %r")\
|
|
% str(port))
|
|
# set host lowercase and without userinfo
|
|
self.urlparts[1] = host.lower()
|
|
# safe anchor for later checking
|
|
self.anchor = self.urlparts[4]
|
|
|
|
def logMe (self):
|
|
bk.log.debug(linkcheck.LOG_CHECK, "logging url")
|
|
self.config.incrementLinknumber()
|
|
if self.config["verbose"] or not self.valid or \
|
|
(self.warningString and self.config["warnings"]):
|
|
self.config.log_newUrl(self)
|
|
|
|
def check (self):
|
|
try:
|
|
self._check()
|
|
except (socket.error, select.error):
|
|
# on Unix, ctrl-c can raise
|
|
# error: (4, 'Interrupted system call')
|
|
etype, value = sys.exc_info()[:2]
|
|
if etype!=4:
|
|
raise
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except:
|
|
internal_error()
|
|
|
|
def _check (self):
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Checking", self)
|
|
if self.recursionLevel and self.config['wait']:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "sleeping for", self.config['wait'], "seconds")
|
|
time.sleep(self.config['wait'])
|
|
t = time.time()
|
|
if not self.checkCache():
|
|
return
|
|
# apply filter
|
|
bk.log.debug(linkcheck.LOG_CHECK, "extern =", self.extern)
|
|
if self.extern[0] and (self.config["strict"] or self.extern[1]):
|
|
self.setWarning(
|
|
bk.i18n._("outside of domain filter, checked only syntax"))
|
|
self.logMe()
|
|
return
|
|
|
|
# check connection
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking connection")
|
|
try:
|
|
self.checkConnection()
|
|
if self.cached:
|
|
return
|
|
if self.config["anchors"]:
|
|
self.checkAnchors()
|
|
except tuple(linkcheck.checker.ExcList):
|
|
etype, evalue, etb = sys.exc_info()
|
|
bk.log.debug(linkcheck.LOG_CHECK, "exception", traceback.format_tb(etb))
|
|
# make nicer error msg for unknown hosts
|
|
if isinstance(evalue, socket.error) and evalue[0]==-2:
|
|
evalue = bk.i18n._('Hostname not found')
|
|
# make nicer error msg for bad status line
|
|
if isinstance(evalue, linkcheck.httplib2.BadStatusLine):
|
|
evalue = bk.i18n._('Bad HTTP response %r')%str(evalue)
|
|
self.setError(str(evalue))
|
|
|
|
# check content
|
|
warningregex = self.config["warningregex"]
|
|
if warningregex and self.valid:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking content")
|
|
try:
|
|
self.checkContent(warningregex)
|
|
except tuple(linkcheck.checker.ExcList):
|
|
value, tb = sys.exc_info()[1:]
|
|
bk.log.debug(linkcheck.LOG_CHECK, "exception", traceback.format_tb(tb))
|
|
self.setError(str(value))
|
|
|
|
self.checktime = time.time() - t
|
|
# check recursion
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking recursion")
|
|
try:
|
|
if self.allowsRecursion():
|
|
self.parseUrl()
|
|
# check content size
|
|
self.checkSize()
|
|
except tuple(linkcheck.checker.ExcList):
|
|
value, tb = sys.exc_info()[1:]
|
|
bk.log.debug(linkcheck.LOG_CHECK, "exception", traceback.format_tb(tb))
|
|
self.setError(bk.i18n._("could not parse content: %r")%str(value))
|
|
# close
|
|
self.closeConnection()
|
|
self.logMe()
|
|
bk.log.debug(linkcheck.LOG_CHECK, "caching")
|
|
self.putInCache()
|
|
|
|
def checkSyntax (self):
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking syntax")
|
|
if not self.urlName or self.urlName=="":
|
|
self.setError(bk.i18n._("URL is null or empty"))
|
|
self.logMe()
|
|
return False
|
|
if ws_at_start_or_end(self.urlName):
|
|
self.setError(bk.i18n._("URL has whitespace at beginning or end"))
|
|
self.logMe()
|
|
return False
|
|
try:
|
|
self.buildUrl()
|
|
self.extern = self._getExtern()
|
|
except linkcheck.LinkCheckerError, msg:
|
|
self.setError(str(msg))
|
|
self.logMe()
|
|
return False
|
|
return True
|
|
|
|
def checkCache (self):
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking cache")
|
|
for key in self.getCacheKeys():
|
|
if self.config.urlCache_has_key(key):
|
|
self.copyFromCache(self.config.urlCache_get(key))
|
|
self.cached = True
|
|
self.logMe()
|
|
return False
|
|
return True
|
|
|
|
def closeConnection (self):
|
|
# brute force closing
|
|
if self.urlConnection is not None:
|
|
try: self.urlConnection.close()
|
|
except: pass
|
|
# release variable for garbage collection
|
|
self.urlConnection = None
|
|
|
|
def putInCache (self):
|
|
if not self.cached:
|
|
data = self.getCacheData()
|
|
for key in self.getCacheKeys():
|
|
self.config.urlCache_set(key, data)
|
|
self.config.urlSeen_set(key)
|
|
self.cached = True
|
|
|
|
def getCacheKeys (self):
|
|
key = self.getCacheKey()
|
|
if key is None:
|
|
return []
|
|
return [key]
|
|
|
|
def isCached (self):
|
|
key = self.getCacheKey()
|
|
return self.cached or self.config.urlSeen_has_key(key)
|
|
|
|
def getCacheKey (self):
|
|
# note: the host is already lowercase
|
|
if self.urlparts:
|
|
if self.config["anchorcaching"]:
|
|
# do not ignore anchor
|
|
return urlparse.urlunsplit(self.urlparts)
|
|
else:
|
|
# removed anchor from cache key
|
|
return urlparse.urlunsplit(self.urlparts[:4]+[''])
|
|
return None
|
|
|
|
def checkConnection (self):
|
|
self.urlConnection = urllib2.urlopen(self.url)
|
|
|
|
def allowsRecursion (self):
|
|
# note: test self.valid before self.isParseable()
|
|
return self.valid and \
|
|
self.isParseable() and \
|
|
self.hasContent() and \
|
|
not self.isCached() and \
|
|
(self.config["recursionlevel"] < 0 or
|
|
self.recursionLevel < self.config["recursionlevel"]) and \
|
|
not self.extern[0] and self.contentAllowsRobots()
|
|
|
|
def contentAllowsRobots (self):
|
|
if not self.isHtml():
|
|
return True
|
|
if not (self.isHttp() or self.isFile()):
|
|
return True
|
|
h = linkcheck.linkparse.MetaRobotsFinder(self.getContent())
|
|
p = bk.HtmlParser.htmlsax.parser(h)
|
|
h.parser = p
|
|
p.feed(self.getContent())
|
|
p.flush()
|
|
h.parser = None
|
|
p.handler = None
|
|
return h.follow
|
|
|
|
|
|
def checkAnchors (self):
|
|
if not (self.valid and self.anchor and self.isHtml() and \
|
|
self.hasContent()):
|
|
# do not bother
|
|
return
|
|
bk.log.debug(linkcheck.LOG_CHECK, "checking anchor", self.anchor)
|
|
h = linkcheck.linkparse.LinkFinder(self.getContent(), tags={'a': ['name'], None: ['id']})
|
|
p = bk.HtmlParser.htmlsax.parser(h)
|
|
h.parser = p
|
|
p.feed(self.getContent())
|
|
p.flush()
|
|
h.parser = None
|
|
p.handler = None
|
|
for cur_anchor,line,column,name,base in h.urls:
|
|
if cur_anchor == self.anchor:
|
|
return
|
|
self.setWarning(bk.i18n._("anchor #%s not found") % self.anchor)
|
|
|
|
def _getExtern (self):
|
|
if not (self.config["externlinks"] or self.config["internlinks"]):
|
|
return (0, 0)
|
|
# deny and allow external checking
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Url", self.url)
|
|
if self.config["denyallow"]:
|
|
for entry in self.config["externlinks"]:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Extern entry", entry)
|
|
match = entry['pattern'].search(self.url)
|
|
if (entry['negate'] and not match) or \
|
|
(match and not entry['negate']):
|
|
return (1, entry['strict'])
|
|
for entry in self.config["internlinks"]:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Intern entry", entry)
|
|
match = entry['pattern'].search(self.url)
|
|
if (entry['negate'] and not match) or \
|
|
(match and not entry['negate']):
|
|
return (0, 0)
|
|
return (0, 0)
|
|
else:
|
|
for entry in self.config["internlinks"]:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Intern entry", entry)
|
|
match = entry['pattern'].search(self.url)
|
|
if (entry['negate'] and not match) or \
|
|
(match and not entry['negate']):
|
|
return (0, 0)
|
|
for entry in self.config["externlinks"]:
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Extern entry", entry)
|
|
match = entry['pattern'].search(self.url)
|
|
if (entry['negate'] and not match) or \
|
|
(match and not entry['negate']):
|
|
return (1, entry['strict'])
|
|
return (1,0)
|
|
|
|
def hasContent (self):
|
|
"""indicate wether url getContent() can be called"""
|
|
return True
|
|
|
|
def getContent (self):
|
|
"""Precondition: urlConnection is an opened URL."""
|
|
if not self.has_content:
|
|
self.has_content = True
|
|
t = time.time()
|
|
self.data = self.urlConnection.read()
|
|
self.dltime = time.time() - t
|
|
self.dlsize = len(self.data)
|
|
return self.data
|
|
|
|
def checkContent (self, warningregex):
|
|
"""if a warning expression was given, call this function to check it
|
|
against the content of this url"""
|
|
if not self.hasContent():
|
|
return
|
|
match = warningregex.search(self.getContent())
|
|
if match:
|
|
self.setWarning(bk.i18n._("Found %r in link contents")%match.group())
|
|
|
|
def checkSize (self):
|
|
"""if a maximum size was given, call this function to check it
|
|
against the content size of this url"""
|
|
maxbytes = self.config["warnsizebytes"]
|
|
if maxbytes is not None and self.dlsize >= maxbytes:
|
|
self.setWarning(bk.i18n._("Content size %s is larger than %s")%\
|
|
(linkcheck.StringUtil.strsize(self.dlsize),
|
|
linkcheck.StringUtil.strsize(maxbytes)))
|
|
|
|
def parseUrl (self):
|
|
# default parse type is html
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Parsing recursively into", self)
|
|
self.parse_html();
|
|
|
|
def getUserPassword (self):
|
|
for auth in self.config["authentication"]:
|
|
if auth['pattern'].match(self.url):
|
|
return auth['user'], auth['password']
|
|
return None,None
|
|
|
|
def parse_html (self):
|
|
# search for a possible base reference
|
|
h = linkcheck.linkparse.LinkFinder(self.getContent(), tags={'base': ['href']})
|
|
p = bk.HtmlParser.htmlsax.parser(h)
|
|
h.parser = p
|
|
p.feed(self.getContent())
|
|
p.flush()
|
|
h.parser = None
|
|
p.handler = None
|
|
baseRef = None
|
|
if len(h.urls)>=1:
|
|
baseRef = h.urls[0][0]
|
|
if len(h.urls)>1:
|
|
self.setWarning(bk.i18n._(
|
|
"more than one <base> tag found, using only the first one"))
|
|
h = linkcheck.linkparse.LinkFinder(self.getContent())
|
|
p = bk.HtmlParser.htmlsax.parser(h)
|
|
h.parser = p
|
|
p.feed(self.getContent())
|
|
p.flush()
|
|
h.parser = None
|
|
p.handler = None
|
|
for s in h.parse_info:
|
|
# the parser had warnings/errors
|
|
self.setWarning(s)
|
|
for url,line,column,name,codebase in h.urls:
|
|
if codebase:
|
|
base = codebase
|
|
else:
|
|
base = baseRef
|
|
bk.log.debug(linkcheck.LOG_CHECK, "Put url %r in queue"%url)
|
|
self.config.appendUrl(linkcheck.checker.getUrlDataFrom(url,
|
|
self.recursionLevel+1, self.config,
|
|
parentName=self.url, baseRef=base,
|
|
line=line, column=column, name=name))
|
|
|
|
def parse_opera (self):
|
|
# parse an opera bookmark file
|
|
name = ""
|
|
lineno = 0
|
|
lines = self.getContent().splitlines()
|
|
for line in lines:
|
|
lineno += 1
|
|
line = line.strip()
|
|
if line.startswith("NAME="):
|
|
name = line[5:]
|
|
elif line.startswith("URL="):
|
|
url = line[4:]
|
|
if url:
|
|
self.config.appendUrl(linkcheck.checker.getUrlDataFrom(url,
|
|
self.recursionLevel+1, self.config, self.url, None, lineno, name))
|
|
name = ""
|
|
|
|
def parse_text (self):
|
|
"""parse a text file with on url per line; comment and blank
|
|
lines are ignored
|
|
UNUSED and UNTESTED, just use linkchecker `cat file.txt`
|
|
"""
|
|
lineno = 0
|
|
for line in self.getContent().splitlines():
|
|
lineno += 1
|
|
line = line.strip()
|
|
if not line or line.startswith('#'): continue
|
|
self.config.appendUrl(linkcheck.checker.getUrlDataFrom(line, self.recursionLevel+1,
|
|
self.config, parentName=self.url, line=lineno))
|
|
|
|
def parse_css (self):
|
|
"""parse a CSS file for url() patterns"""
|
|
lineno = 0
|
|
for line in self.getContent().splitlines():
|
|
lineno += 1
|
|
for mo in linkcheck.linkparse.css_url_re.finditer(line):
|
|
column = mo.start("url")
|
|
self.config.appendUrl(linkcheck.checker.getUrlDataFrom(mo.group("url"),
|
|
self.recursionLevel+1, self.config,
|
|
parentName=self.url, line=lineno, column=column))
|
|
|
|
def __str__ (self):
|
|
return ("%s link\n"
|
|
"urlname=%s\n"
|
|
"parentName=%s\n"
|
|
"baseRef=%s\n"
|
|
"cached=%s\n"
|
|
"recursionLevel=%s\n"
|
|
"urlConnection=%s\n"
|
|
"line=%s\n"
|
|
"column=%s\n"
|
|
"name=%s" % \
|
|
(self.scheme, self.urlName, self.parentName, self.baseRef,
|
|
self.cached, self.recursionLevel, self.urlConnection, self.line,
|
|
self.column, self.name))
|
|
|