mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-05-08 22:54:51 +00:00
docs
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@288 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin
parent
2859abd8b5
commit
fd1409b7ab
9 changed files with 42 additions and 293 deletions
|
|
@ -126,7 +126,6 @@ class DnsRequest:
|
|||
self.args=args
|
||||
|
||||
def socketInit(self,a,b):
|
||||
import socket
|
||||
self.s = socket.socket(a,b)
|
||||
|
||||
def processUDPReply(self):
|
||||
|
|
|
|||
28
DNS/Lib.py
28
DNS/Lib.py
|
|
@ -11,8 +11,6 @@
|
|||
# ------------------------------------------------------------------------
|
||||
|
||||
|
||||
import string
|
||||
|
||||
import DNS.Type
|
||||
import DNS.Class
|
||||
import DNS.Opcode
|
||||
|
|
@ -38,10 +36,10 @@ def unpack32bit(s):
|
|||
def addr2bin(addr):
|
||||
if type(addr) == type(0):
|
||||
return addr
|
||||
bytes = string.splitfields(addr, '.')
|
||||
bytes = addr.split('.')
|
||||
if len(bytes) != 4: raise ValueError, 'bad IP address'
|
||||
n = 0
|
||||
for byte in bytes: n = n<<8 | string.atoi(byte)
|
||||
for byte in bytes: n = n<<8 | int(byte)
|
||||
return n
|
||||
|
||||
def bin2addr(n):
|
||||
|
|
@ -77,21 +75,21 @@ class Packer:
|
|||
# Add a domain name to the buffer, possibly using pointers.
|
||||
# The case of the first occurrence of a name is preserved.
|
||||
# Redundant dots are ignored.
|
||||
list = []
|
||||
for label in string.splitfields(name, '.'):
|
||||
lst = []
|
||||
for label in name.split('.'):
|
||||
if label:
|
||||
if len(label) > 63:
|
||||
raise PackError, 'label too long'
|
||||
list.append(label)
|
||||
lst.append(label)
|
||||
keys = []
|
||||
for i in range(len(list)):
|
||||
key = string.joinfields(list[i:], '.').upper()
|
||||
for i in range(len(lst)):
|
||||
key = '.'.join(lst[i:]).upper()
|
||||
keys.append(key)
|
||||
if self.index.has_key(key):
|
||||
pointer = self.index[key]
|
||||
break
|
||||
else:
|
||||
i = len(list)
|
||||
i = len(lst)
|
||||
pointer = None
|
||||
# Do it into temporaries first so exceptions don't
|
||||
# mess up self.index and self.buf
|
||||
|
|
@ -99,18 +97,18 @@ class Packer:
|
|||
offset = len(self.buf)
|
||||
index = []
|
||||
for j in range(i):
|
||||
label = list[j]
|
||||
label = lst[j]
|
||||
n = len(label)
|
||||
if offset + len(buf) < 0x3FFF:
|
||||
index.append((keys[j], offset + len(buf)))
|
||||
else:
|
||||
print 'DNS.Lib.Packer.addname:',
|
||||
print 'warning: pointer too big'
|
||||
buf = buf + (chr(n) + label)
|
||||
buf += chr(n) + label
|
||||
if pointer:
|
||||
buf = buf + pack16bit(pointer | 0xC000)
|
||||
buf += pack16bit(pointer | 0xC000)
|
||||
else:
|
||||
buf = buf + '\0'
|
||||
buf += '\0'
|
||||
self.buf = self.buf + buf
|
||||
for key, value in index:
|
||||
self.index[key] = value
|
||||
|
|
@ -494,7 +492,7 @@ class DnsResult:
|
|||
h['opcode'],h['status'],h['id'])
|
||||
flags=filter(lambda x,h=h:h[x],('qr','aa','rd','ra','tc'))
|
||||
print ';; flags: %s; Ques: %d, Ans: %d, Auth: %d, Addit: %d'%(
|
||||
string.join(flags),h['qdcount'],h['ancount'],h['nscount'],
|
||||
' '.join(flags),h['qdcount'],h['ancount'],h['nscount'],
|
||||
h['arcount'])
|
||||
print ';; QUESTIONS:'
|
||||
for q in self.questions:
|
||||
|
|
|
|||
265
DNS/asyncore.py
265
DNS/asyncore.py
|
|
@ -1,265 +0,0 @@
|
|||
# $Id$
|
||||
# Author: Sam Rushing <rushing@nightmare.com>
|
||||
|
||||
# A simple unix version of the asynchronous socket support.
|
||||
# There are lots of problems with this still - I only wrote it to show
|
||||
# that it could be done, and for my own testing purposes.
|
||||
# [960206: servtest, asynfing, asynhttp, and pop3demo work, asyndns doesn't.]
|
||||
# [960321: servtest, asynfing, asynhttp, pop3demo, pop3_2 work]
|
||||
import select
|
||||
import socket
|
||||
import sys
|
||||
|
||||
# you need to generate ERRNO.py from Tools/scripts/h2py.py in the Python
|
||||
# distribution.
|
||||
|
||||
try:
|
||||
import ERRNO
|
||||
except ImportError:
|
||||
raise ImportError,'you need to generate ERRNO.py from Tools/scripts/h2py.py in the Python distribution'
|
||||
|
||||
# look what I can get away with... 8^)
|
||||
socket.socket_map = {}
|
||||
|
||||
ALL_EVENTS = []
|
||||
|
||||
DEFAULT_TIMEOUT = 30.0
|
||||
|
||||
loop_running = 0
|
||||
|
||||
stop_loop_exception = "stop running the select loop"
|
||||
|
||||
# we want to select for read only those sockets
|
||||
# to which we are already connected to, -OR- those
|
||||
# sockets we are accepting on.
|
||||
def readables (sock_fds):
|
||||
sm = socket.socket_map
|
||||
def readable_test (fd, sm=sm):
|
||||
sock = sm[fd]
|
||||
return sock.connected or sock.accepting
|
||||
return filter (readable_test, sock_fds)
|
||||
|
||||
# only those fd's we are 'write blocked' on, -OR-
|
||||
# those sockets we are waiting for a connection on.
|
||||
def writables (sock_fds):
|
||||
sm = socket.socket_map
|
||||
def writable_test (fd, sm=sm):
|
||||
sock = sm[fd]
|
||||
return sock.write_blocked or not sock.connected
|
||||
return filter (writable_test, sock_fds)
|
||||
|
||||
def loop(timeout=DEFAULT_TIMEOUT):
|
||||
loop_running = 1
|
||||
try:
|
||||
while 1:
|
||||
sock_fds = socket.socket_map.keys()
|
||||
|
||||
read_fds = readables (sock_fds)
|
||||
write_fds = writables (sock_fds)
|
||||
expt_fds = sock_fds[:]
|
||||
|
||||
(read_fds,
|
||||
write_fds,
|
||||
expt_fds) = select.select (read_fds,
|
||||
write_fds,
|
||||
expt_fds,
|
||||
timeout)
|
||||
print read_fds,write_fds,expt_fds
|
||||
try:
|
||||
for x in expt_fds:
|
||||
socket.socket_map[x].handle_expt_event()
|
||||
for x in read_fds:
|
||||
socket.socket_map[x].handle_read_event()
|
||||
for x in write_fds:
|
||||
socket.socket_map[x].handle_write_event()
|
||||
except KeyError:
|
||||
# handle_expt handle_read might remove as socket
|
||||
# from the map by calling self.close().
|
||||
pass
|
||||
except stop_loop_exception:
|
||||
print 'loop stopped'
|
||||
|
||||
class dispatcher:
|
||||
def __init__ (self, sock=None):
|
||||
self.debug = 0
|
||||
self.log_queue = []
|
||||
self.connected = 0
|
||||
self.accepting = 0
|
||||
self.write_blocked = 1
|
||||
if sock:
|
||||
self.socket = sock
|
||||
self.fileno = self.socket.fileno()
|
||||
# I think it should inherit this anyway
|
||||
self.socket.setblocking (0)
|
||||
self.connected = 1
|
||||
self.add_channel()
|
||||
|
||||
def add_channel (self, events=ALL_EVENTS):
|
||||
self.log ('adding channel %s' % self)
|
||||
socket.socket_map [self.fileno] = self
|
||||
|
||||
def del_channel (self):
|
||||
if socket.socket_map.has_key (self.fileno):
|
||||
del socket.socket_map [self.fileno]
|
||||
if not len(socket.socket_map.keys()):
|
||||
raise stop_loop_exception
|
||||
|
||||
def create_socket (self, family, type):
|
||||
self.socket = socket.socket (family, type)
|
||||
self.socket.setblocking(0)
|
||||
self.fileno = self.socket.fileno()
|
||||
self.add_channel()
|
||||
|
||||
def bind (self, *args):
|
||||
return apply (self.socket.bind, args)
|
||||
|
||||
def go (self):
|
||||
if not loop_running:
|
||||
loop()
|
||||
|
||||
def listen (self, num):
|
||||
self.accepting = 1
|
||||
self.socket.listen (num)
|
||||
|
||||
def accept (self):
|
||||
return self.socket.accept()
|
||||
|
||||
def connect (self, host, port):
|
||||
try:
|
||||
self.socket.connect (host, port)
|
||||
except socket.error, why:
|
||||
if type(why) == type(()) \
|
||||
and why[0] in (ERRNO.EINPROGRESS, ERRNO.EALREADY, ERRNO.EWOULDBLOCK):
|
||||
return
|
||||
else:
|
||||
raise socket.error, why
|
||||
self.connected = 1
|
||||
self.handle_connect()
|
||||
|
||||
def send (self, data):
|
||||
try:
|
||||
result = self.socket.send (data)
|
||||
if result != len(data):
|
||||
self.write_blocked = 1
|
||||
else:
|
||||
self.write_blocked = 0
|
||||
return result
|
||||
except socket.error, why:
|
||||
if type(why) == type(()) and why[0] == ERRNO.EWOULDBLOCK:
|
||||
self.write_blocked = 1
|
||||
return 0
|
||||
else:
|
||||
raise socket.error, why
|
||||
return 0
|
||||
|
||||
def recv (self, buffer_size):
|
||||
data = self.socket.recv (buffer_size)
|
||||
if not data:
|
||||
self.handle_close()
|
||||
return ''
|
||||
else:
|
||||
return data
|
||||
|
||||
def close (self):
|
||||
self.socket.close()
|
||||
self.del_channel()
|
||||
|
||||
def shutdown (self, how):
|
||||
self.socket.shutdown (how)
|
||||
|
||||
def log (self, message):
|
||||
#self.log_queue.append ('%s:%d %s' %
|
||||
# (self.__class__.__name__, self.fileno, message))
|
||||
print 'log:', message
|
||||
|
||||
def done (self):
|
||||
self.print_log()
|
||||
|
||||
def print_log (self):
|
||||
for x in self.log_queue:
|
||||
print x
|
||||
|
||||
def handle_read_event (self):
|
||||
# getting a read implies that we are connected
|
||||
if not self.connected:
|
||||
self.handle_connect()
|
||||
self.connected = 1
|
||||
self.handle_read()
|
||||
elif self.accepting:
|
||||
if not self.connected:
|
||||
self.connected = 1
|
||||
self.handle_accept()
|
||||
else:
|
||||
self.handle_read()
|
||||
|
||||
def more_to_send (self, yesno=1):
|
||||
self.write_blocked = yesno
|
||||
|
||||
def handle_write_event (self):
|
||||
# getting a read implies that we are connected
|
||||
if not self.connected:
|
||||
self.handle_connect()
|
||||
self.connected = 1
|
||||
self.write_blocked = 0
|
||||
self.handle_write()
|
||||
|
||||
def handle_expt_event (self):
|
||||
self.handle_error()
|
||||
|
||||
def handle_error (self, error=0):
|
||||
self.close()
|
||||
|
||||
def handle_read (self):
|
||||
self.log ('unhandled FD_READ')
|
||||
|
||||
def handle_write (self):
|
||||
self.log ('unhandled FD_WRITE')
|
||||
|
||||
def handle_connect (self):
|
||||
self.log ('unhandled FD_CONNECT')
|
||||
|
||||
def handle_oob (self):
|
||||
self.log ('unhandled FD_OOB')
|
||||
|
||||
def handle_accept (self):
|
||||
self.log ('unhandled FD_ACCEPT')
|
||||
|
||||
def handle_close (self):
|
||||
self.log ('unhandled FD_CLOSE')
|
||||
|
||||
def handle_disconnect (self, error):
|
||||
self.log ('unexpected disconnect, error:%d' % error)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# adds async send capability, useful for simple clients.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class dispatcher_with_send (dispatcher):
|
||||
def __init__ (self, sock=None):
|
||||
dispatcher.__init__ (self, sock)
|
||||
self.out_buffer = ''
|
||||
|
||||
def initiate_send (self):
|
||||
while self.out_buffer:
|
||||
num_sent = 0
|
||||
num_sent = dispatcher.send (self, self.out_buffer[:512])
|
||||
self.out_buffer = self.out_buffer[num_sent:]
|
||||
|
||||
def handle_write (self):
|
||||
self.initiate_send()
|
||||
|
||||
def send (self, data):
|
||||
if self.debug:
|
||||
self.log ('sending %s' % repr(data))
|
||||
self.out_buffer = data
|
||||
self.initiate_send()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# used a lot when debugging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def close_all ():
|
||||
for x in socket.socket_map.items():
|
||||
x[1].socket.close()
|
||||
socket.socket_map = {}
|
||||
|
||||
|
|
@ -10,12 +10,12 @@ def revlookup(name):
|
|||
# this will only return one of any records returned.
|
||||
return Base.DnsRequest(b, qtype = 'ptr').req().answers[0]['data']
|
||||
|
||||
def mxlookup(name):
|
||||
def mxlookup(name, protocol="udp"):
|
||||
"""
|
||||
convenience routine for doing an MX lookup of a name. returns a
|
||||
sorted list of (preference, mail exchanger) records
|
||||
"""
|
||||
a = Base.DnsRequest(name, qtype = 'mx').req().answers
|
||||
a = Base.DnsRequest(name, qtype='mx', protocol=protocol).req().answers
|
||||
l = map(lambda x:x['data'], a)
|
||||
l.sort()
|
||||
return l
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -79,3 +79,7 @@ onlinetest:
|
|||
.PHONY: locale
|
||||
locale:
|
||||
$(MAKE) -C po
|
||||
|
||||
.PHONY: timeouttest
|
||||
timeouttest:
|
||||
./$(PACKAGE) -DDD --timeout=2 mailto:calvin@cs.uni-sb.de
|
||||
|
|
|
|||
15
debian/changelog
vendored
15
debian/changelog
vendored
|
|
@ -1,12 +1,19 @@
|
|||
linkchecker (1.3.6) unstable; urgency=low
|
||||
|
||||
* fix typo in Copyright blurb
|
||||
* implement Proxy Authentication
|
||||
* Opera Bookmark files (opera.adr) support
|
||||
* support for socket timeouts
|
||||
* implemented Proxy Authentication (untested)
|
||||
* Opera Bookmark files (opera.adr) support, they get parsed
|
||||
automatically like HTML files
|
||||
* support for TCP socket timeouts with --timeout (untested)
|
||||
* replace string module functions with Python 2.x string methods
|
||||
* use TCP for DNS MX mailhost lookup as default, not UDP as it seems
|
||||
to hang forever (dont know why). This should fix bug #106393, but
|
||||
only time will tell.
|
||||
* updated DNS.init_dns_config with Windows-specific functions from
|
||||
pydns.sf.net (only tested under Win2000 with DNS server settings
|
||||
over DHCP)
|
||||
|
||||
-- Bastian Kleineidam <calvin@debian.org> Wed, 22 Aug 2001 15:52:18 +0200
|
||||
-- Bastian Kleineidam <calvin@debian.org> Wed, 22 Aug 2001 19:34:13 +0200
|
||||
|
||||
linkchecker (1.3.5) unstable; urgency=low
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ class FileUrlData(UrlData):
|
|||
def isHtml(self):
|
||||
if html_re.search(self.url) or opera_re.search(self.url):
|
||||
return 1
|
||||
# try to read content (attention: could be a directory)
|
||||
# try to read content (can fail, so catch error)
|
||||
try:
|
||||
return html_content_re.search(self.getContent()) or \
|
||||
opera_content_re.search(self.getContent())
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
||||
|
||||
import os,re,string,DNS,sys,Config,cgi,urllib,linkcheck
|
||||
import os, re, DNS, sys, Config, cgi, urllib, linkcheck
|
||||
from rfc822 import AddressList
|
||||
from HostCheckingUrlData import HostCheckingUrlData
|
||||
from smtplib import SMTP
|
||||
|
|
@ -33,7 +33,7 @@ DNS.init_dns_resolver()
|
|||
|
||||
class MailtoUrlData(HostCheckingUrlData):
|
||||
"Url link with mailto scheme"
|
||||
|
||||
|
||||
def buildUrl(self):
|
||||
HostCheckingUrlData.buildUrl(self)
|
||||
self.headers = {}
|
||||
|
|
@ -53,6 +53,7 @@ class MailtoUrlData(HostCheckingUrlData):
|
|||
return self.urlName[7:mo.start()]
|
||||
return self.urlName[7:]
|
||||
|
||||
|
||||
def checkConnection(self, config):
|
||||
"""Verify a list of email adresses. If one adress fails,
|
||||
the whole list will fail.
|
||||
|
|
@ -72,8 +73,11 @@ class MailtoUrlData(HostCheckingUrlData):
|
|||
value = "unknown reason"
|
||||
for name,mail in self.adresses:
|
||||
Config.debug(BRING_IT_ON, "checking mail address", mail)
|
||||
Config.debug(HURT_ME_PLENTY, "splitting address")
|
||||
user,host = self._split_adress(mail)
|
||||
mxrecords = DNS.mxlookup(host)
|
||||
Config.debug(HURT_ME_PLENTY, "looking up MX mailhost")
|
||||
mxrecords = DNS.mxlookup(host, protocol="tcp")
|
||||
Config.debug(HURT_ME_PLENTY, "found mailhosts", mxrecords)
|
||||
if not len(mxrecords):
|
||||
self.setError(_("No mail host for %s found")%host)
|
||||
return
|
||||
|
|
@ -82,9 +86,11 @@ class MailtoUrlData(HostCheckingUrlData):
|
|||
try:
|
||||
Config.debug(BRING_IT_ON, "SMTP check for", mxrecord)
|
||||
self.urlConnection = SMTP(mxrecord[1])
|
||||
Config.debug(HURT_ME_PLENTY, "SMTP connected!")
|
||||
smtpconnect = 1
|
||||
self.urlConnection.helo()
|
||||
info = self.urlConnection.verify(user)
|
||||
Config.debug(HURT_ME_PLENTY, "SMTP user info", info)
|
||||
if info[0]==250:
|
||||
self.setInfo(_("Verified adress: %s")%str(info[1]))
|
||||
except:
|
||||
|
|
@ -102,7 +108,7 @@ class MailtoUrlData(HostCheckingUrlData):
|
|||
|
||||
|
||||
def _split_adress(self, adress):
|
||||
split = string.split(adress, "@", 1)
|
||||
split = adress.split("@", 1)
|
||||
if len(split)==2:
|
||||
if not split[1]:
|
||||
return (split[0], "localhost")
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ import sys
|
|||
if sys.version[:5] < "2.0":
|
||||
raise SystemExit, "This program requires Python 2.0 or later."
|
||||
|
||||
from linkcheck import timeoutsocket
|
||||
import getopt, re, os, urlparse, linkcheck
|
||||
from linkcheck import _,StringUtil,timeoutsocket
|
||||
|
||||
|
||||
Usage = _("""USAGE\tlinkchecker [options] file-or-url...
|
||||
|
||||
OPTIONS
|
||||
|
|
|
|||
Loading…
Reference in a new issue