Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-19 13:51:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

more import fixes

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@1383 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2004-07-22 10:54:47 +00:00
parent 6c2c8f78b6
commit 32bc5cd292
15 changed files with 86 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
linkcheck/__init__.py
View file

@ -26,7 +26,6 @@ import bk.strtime
# logger areas
LOG_CMDLINE = "linkcheck.cmdline"
LOG_CHECK = "linkcheck.check"
LOG_DNS = "linkcheck.dns"
LOG_GUI = "linkcheck.gui"

7
linkcheck/checker/FileUrlData.py
View file

@ -19,10 +19,7 @@
import re
import os
import urlparse
import linkcheck.checker
# OSError is thrown on Windows when a file is not found
linkcheck.checker.ExcList.append(OSError)
import linkcheck.checker.UrlData
# if file extension was fruitless, look at the content
contents = {
@ -72,7 +69,7 @@ acap # application configuration access protocol
|nntp # news
)"""
class FileUrlData (linkcheck.UrlData.UrlData):
class FileUrlData (linkcheck.checker.UrlData.UrlData):
"Url link with file scheme"
def __init__ (self,

9
linkcheck/checker/FtpUrlData.py
View file

@ -18,16 +18,9 @@
import ftplib
import linkcheck
from linkcheck.debug import *
linkcheck.UrlData.ExcList.extend([
ftplib.error_reply,
ftplib.error_temp,
ftplib.error_perm,
ftplib.error_proto,
])
class FtpUrlData (linkcheck.ProxyUrlData.ProxyUrlData):
class FtpUrlData (linkcheck.checker.ProxyUrlData.ProxyUrlData):
"""
Url link with ftp scheme.
"""

2
linkcheck/checker/GopherUrlData.py
View file

@ -18,6 +18,6 @@
import linkcheck
class GopherUrlData (linkcheck.UrlData.UrlData):
class GopherUrlData (linkcheck.checker.UrlData.UrlData):
"Url link with gopher scheme"
pass

2
linkcheck/checker/HostCheckingUrlData.py
View file

@ -20,7 +20,7 @@ import socket
import urllib
import linkcheck
class HostCheckingUrlData (linkcheck.UrlData.UrlData):
class HostCheckingUrlData (linkcheck.checker.UrlData.UrlData):
"Url link for which we have to connect to a specific host"
def __init__ (self, urlName, recursionLevel, config, parentName=None,

5
linkcheck/checker/HttpUrlData.py
View file

@ -25,19 +25,16 @@ import gzip
import socket
import cStringIO as StringIO
import linkcheck
from linkcheck.debug import *
supportHttps = hasattr(linkcheck.httplib2, "HTTPSConnection") and \
hasattr(socket, "ssl")
linkcheck.UrlData.ExcList.extend([linkcheck.httplib2.error,])
_supported_encodings = ('gzip', 'x-gzip', 'deflate')
# Amazon blocks all HEAD requests
_isAmazonHost = re.compile(r'^www\.amazon\.(com|de|ca|fr|co\.(uk|jp))').search
class HttpUrlData (linkcheck.ProxyUrlData.ProxyUrlData):
class HttpUrlData (linkcheck.checker.ProxyUrlData.ProxyUrlData):
"Url link with http scheme"
def __init__ (self, urlName, recursionLevel, config, parentName=None,

5
linkcheck/checker/HttpsUrlData.py
View file

@ -17,14 +17,13 @@
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import linkcheck
from linkcheck.debug import *
class HttpsUrlData (linkcheck.HttpUrlData.HttpUrlData):
class HttpsUrlData (linkcheck.checker.HttpUrlData.HttpUrlData):
"""Url link with https scheme"""
def _check (self):
if linkcheck.HttpUrlData.supportHttps:
if linkcheck.checker.HttpUrlData.supportHttps:
super(HttpsUrlData, self)._check()
else:
self.setWarning(bk.i18n._("%s url ignored")%self.scheme.capitalize())

2
linkcheck/checker/IgnoredUrlData.py
View file

@ -55,7 +55,7 @@ acap # application configuration access protocol
ignored_schemes_re = re.compile(ignored_schemes, re.VERBOSE)
class IgnoredUrlData (linkcheck.UrlData.UrlData):
class IgnoredUrlData (linkcheck.checker.UrlData.UrlData):
"""Some schemes are defined in http://www.w3.org/Addressing/schemes"""
def _check (self):

35
linkcheck/checker/MailtoUrlData.py
View file

@ -20,29 +20,28 @@ import re
import sys
import cgi
import urllib
import smtplib
import rfc822
import linkcheck
from linkcheck.DNS import mxlookup
from rfc822 import AddressList
from HostCheckingUrlData import HostCheckingUrlData
from smtplib import SMTP
from debug import *
import bk.log
import bk.net.dns.lazy
# regular expression for RFC2368 compliant mailto: scanning
headers_re = re.compile(r"\?(.+)$")
class MailtoUrlData (HostCheckingUrlData):
class MailtoUrlData (linkcheck.checker.HostCheckingUrlData.HostCheckingUrlData):
"Url link with mailto scheme"
def buildUrl (self):
super(MailtoUrlData, self).buildUrl()
self.headers = {}
self.adresses = AddressList(self._cutout_adresses()).addresslist
self.adresses = rfc822.AddressList(self._cutout_adresses()).addresslist
for key in ("to", "cc", "bcc"):
if self.headers.has_key(key):
for val in self.headers[key]:
a = urllib.unquote(val)
self.adresses.extend(AddressList(a).addresslist)
linkcheck.Config.debug(BRING_IT_ON, "adresses: ", self.adresses)
self.adresses.extend(rfc822.AddressList(a).addresslist)
bk.log.debug(BRING_IT_ON, "adresses: ", self.adresses)
def _cutout_adresses (self):
@ -74,25 +73,25 @@ class MailtoUrlData (HostCheckingUrlData):
value = "unknown reason"
for name,mail in self.adresses:
linkcheck.Config.debug(BRING_IT_ON, "checking mail address", mail)
linkcheck.Config.debug(HURT_ME_PLENTY, "splitting address")
bk.log.debug(BRING_IT_ON, "checking mail address", mail)
bk.log.debug(HURT_ME_PLENTY, "splitting address")
user,host = self._split_adress(mail)
linkcheck.Config.debug(HURT_ME_PLENTY, "looking up MX mailhost")
mxrecords = mxlookup(host)
linkcheck.Config.debug(HURT_ME_PLENTY, "found mailhosts", mxrecords)
bk.log.debug(HURT_ME_PLENTY, "looking up MX mailhost")
mxrecords = bk.net.dns.lazy.mxlookup(host, config.dnsconfig)
bk.log.debug(HURT_ME_PLENTY, "found mailhosts", mxrecords)
if not len(mxrecords):
self.setWarning(bk.i18n._("No MX mail host for %s found")%host)
return
smtpconnect = 0
for mxrecord in mxrecords:
try:
linkcheck.Config.debug(BRING_IT_ON, "SMTP check for", mxrecord)
self.urlConnection = SMTP(mxrecord[1])
linkcheck.Config.debug(HURT_ME_PLENTY, "SMTP connected!")
bk.log.debug(BRING_IT_ON, "SMTP check for", mxrecord)
self.urlConnection = smtplib.SMTP(mxrecord[1])
bk.log.debug(HURT_ME_PLENTY, "SMTP connected!")
smtpconnect = 1
self.urlConnection.helo()
info = self.urlConnection.verify(user)
linkcheck.Config.debug(HURT_ME_PLENTY, "SMTP user info", info)
bk.log.debug(HURT_ME_PLENTY, "SMTP user info", info)
if info[0]==250:
self.setInfo(bk.i18n._("Verified adress: %s")%str(info[1]))
except:

24
linkcheck/checker/NntpUrlData.py
View file

@ -16,20 +16,18 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import re, time, sys, nntplib, urlparse, random
from linkcheck import linkcheck.LinkCheckerError, Config
from UrlData import ExcList, UrlData
from debug import *
import re
import time
import sys
import nntplib
import urlparse
import random
import linkcheck
import bk.log
random.seed()
ExcList.extend([nntplib.error_reply,
nntplib.error_temp,
nntplib.error_perm,
nntplib.error_proto,
EOFError,
])
class NntpUrlData (UrlData):
class NntpUrlData (linkcheck.checker.UrlData.UrlData):
"Url link with NNTP scheme"
def buildUrl (self):
@ -41,7 +39,7 @@ class NntpUrlData (UrlData):
else:
self.url = self.urlName
self.urlparts = urlparse.urlsplit(self.url)
Config.debug(BRING_IT_ON, self.urlparts)
bk.log.debug(BRING_IT_ON, self.urlparts)
def checkConnection (self):

2
linkcheck/checker/ProxyUrlData.py
View file

@ -17,7 +17,7 @@
from UrlData import UrlData
from urllib import splittype, splithost, splituser
class ProxyUrlData (UrlData):
class ProxyUrlData (linkcheck.checker.UrlData.UrlData):
"""urldata with ability for proxying and for urls with user:pass@host
setting"""

3
linkcheck/checker/TelnetUrlData.py
View file

@ -20,10 +20,9 @@ import telnetlib
import urlparse
import urllib
import linkcheck
from linkcheck.debug import *
class TelnetUrlData (linkcheck.HostCheckingUrlData.HostCheckingUrlData):
class TelnetUrlData (linkcheck.checker.HostCheckingUrlData.HostCheckingUrlData):
"Url link with telnet scheme"
def buildUrl (self):

15
linkcheck/checker/UrlData.py
View file

@ -26,7 +26,7 @@ import traceback
import socket
import select
import linkcheck
import linkcheck.DNS
import bk.log
ws_at_start_or_end = re.compile(r"(^\s+)|(\s+$)").search
@ -78,9 +78,6 @@ def get_absolute_url (urlName, baseRef, parentName):
return ""
if hasattr(socket, "sslerror"):
ExcList.append(socket.sslerror)
# regular expression for port numbers
is_valid_port = re.compile(r"\d+").match
@ -405,16 +402,16 @@ class UrlData (object):
if not (self.config["externlinks"] or self.config["internlinks"]):
return (0, 0)
# deny and allow external checking
linkcheck.Config.debug(HURT_ME_PLENTY, "Url", self.url)
bk.log.debug(HURT_ME_PLENTY, "Url", self.url)
if self.config["denyallow"]:
for entry in self.config["externlinks"]:
linkcheck.Config.debug(HURT_ME_PLENTY, "Extern entry", entry)
bk.log.debug(HURT_ME_PLENTY, "Extern entry", entry)
match = entry['pattern'].search(self.url)
if (entry['negate'] and not match) or \
(match and not entry['negate']):
return (1, entry['strict'])
for entry in self.config["internlinks"]:
linkcheck.Config.debug(HURT_ME_PLENTY, "Intern entry", entry)
bk.log.debug(HURT_ME_PLENTY, "Intern entry", entry)
match = entry['pattern'].search(self.url)
if (entry['negate'] and not match) or \
(match and not entry['negate']):
@ -422,13 +419,13 @@ class UrlData (object):
return (0, 0)
else:
for entry in self.config["internlinks"]:
linkcheck.Config.debug(HURT_ME_PLENTY, "Intern entry", entry)
bk.log.debug(HURT_ME_PLENTY, "Intern entry", entry)
match = entry['pattern'].search(self.url)
if (entry['negate'] and not match) or \
(match and not entry['negate']):
return (0, 0)
for entry in self.config["externlinks"]:
linkcheck.Config.debug(HURT_ME_PLENTY, "Extern entry", entry)
bk.log.debug(HURT_ME_PLENTY, "Extern entry", entry)
match = entry['pattern'].search(self.url)
if (entry['negate'] and not match) or \
(match and not entry['negate']):

37
linkcheck/checker/__init__.py
View file

@ -21,21 +21,40 @@ import socket
import select
import re
import urlparse
import linkcheck
import linkcheck.DNS
import nntplib
import ftplib
import linkcheck.httplib2
import bk.net.dns.Base
# we catch these exceptions, all other exceptions are internal
# or system errors
ExcList = [
IOError,
ValueError, # from httplib.py
linkcheck.LinkCheckerError,
linkcheck.DNS.Error,
socket.timeout,
socket.error,
select.error,
IOError,
OSError, # OSError is thrown on Windows when a file is not found
ValueError, # from httplib.py
linkcheck.LinkCheckerError,
bk.net.dns.Base.DNSError,
socket.timeout,
socket.error,
select.error,
# nttp errors (including EOFError)
nntplib.error_reply,
nntplib.error_temp,
nntplib.error_perm,
nntplib.error_proto,
EOFError,
# http error
linkcheck.httplib2.error,
# ftp errors
ftplib.error_reply,
ftplib.error_temp,
ftplib.error_perm,
ftplib.error_proto,
]
if hasattr(socket, "sslerror"):
ExcList.append(socket.sslerror)
# main check function

26
linkcheck/logger/ColoredLogger.py
View file

@ -16,7 +16,7 @@
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import bk.i18n
import linkcheck.AnsiColor
import bk.ansicolor
import linkcheck.logger.StandardLogger
@ -25,18 +25,18 @@ class ColoredLogger (linkcheck.logger.StandardLogger.StandardLogger):
def __init__ (self, **args):
super(ColoredLogger, self).__init__(**args)
self.colorparent = linkcheck.AnsiColor.esc_ansicolor(args['colorparent'])
self.colorurl = linkcheck.AnsiColor.esc_ansicolor(args['colorurl'])
self.colorname = linkcheck.AnsiColor.esc_ansicolor(args['colorname'])
self.colorreal = linkcheck.AnsiColor.esc_ansicolor(args['colorreal'])
self.colorbase = linkcheck.AnsiColor.esc_ansicolor(args['colorbase'])
self.colorvalid = linkcheck.AnsiColor.esc_ansicolor(args['colorvalid'])
self.colorinvalid = linkcheck.AnsiColor.esc_ansicolor(args['colorinvalid'])
self.colorinfo = linkcheck.AnsiColor.esc_ansicolor(args['colorinfo'])
self.colorwarning = linkcheck.AnsiColor.esc_ansicolor(args['colorwarning'])
self.colordltime = linkcheck.AnsiColor.esc_ansicolor(args['colordltime'])
self.colordlsize = linkcheck.AnsiColor.esc_ansicolor(args['colordlsize'])
self.colorreset = linkcheck.AnsiColor.esc_ansicolor(args['colorreset'])
self.colorparent = bk.ansicolor.esc_ansicolor(args['colorparent'])
self.colorurl = bk.ansicolor.esc_ansicolor(args['colorurl'])
self.colorname = bk.ansicolor.esc_ansicolor(args['colorname'])
self.colorreal = bk.ansicolor.esc_ansicolor(args['colorreal'])
self.colorbase = bk.ansicolor.esc_ansicolor(args['colorbase'])
self.colorvalid = bk.ansicolor.esc_ansicolor(args['colorvalid'])
self.colorinvalid = bk.ansicolor.esc_ansicolor(args['colorinvalid'])
self.colorinfo = bk.ansicolor.esc_ansicolor(args['colorinfo'])
self.colorwarning = bk.ansicolor.esc_ansicolor(args['colorwarning'])
self.colordltime = bk.ansicolor.esc_ansicolor(args['colordltime'])
self.colordlsize = bk.ansicolor.esc_ansicolor(args['colordlsize'])
self.colorreset = bk.ansicolor.esc_ansicolor(args['colorreset'])
self.currentPage = None
self.prefix = 0