Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-02 20:04:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Python 2.0 required

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@182 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2000-10-29 15:55:56 +00:00
parent ce3eb27e1f
commit 2b605bf92b
11 changed files with 25 additions and 1285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
INSTALL
View file

@ -3,25 +3,12 @@
Requirements
------------
Python >= 1.5.2 from http://www.python.org/
Distutils >= 0.9.1 from http://www.python.org/sigs/distutils-sig/
Python 1.6 includes the Distutils 0.9.1,
Python 2.0 includes the Distutils 1.0
Optionally packages
-------------------
OpenSSL from http://www.openssl.org/
You will need Perl for Win32 (available from
http://www.activestate.com/ActivePerl) if you want to install OpenSSL
on Windows!
Python >= 2.0 from http://www.python.org/
Setup
-----
Run "python setup.py config" to configure.
Linux users should run "python setup.py config -lcrypto" to use the SSL
module.
Run "python setup.py install --prefix=<prefix>" to install.
Default prefix is /usr.
Run "python setup.py --help" for help.
@ -37,13 +24,6 @@ to check.
Type "linkchecker -h" for help.
Note
----
If you want to make your own distribution with "python setup.py sdist",
you will need Distutils >= 0.9.4. Older versions are hanging when
they try to parse the MANIFEST.in file.
(Fast)CGI web interface
-----------------------
The *cgi files are three CGI script which you can use to run LinkChecker

6
debian/changelog vendored
View file

@ -1,3 +1,9 @@
linkchecker (1.3.0) unstable; urgency=low
* require Python 2.0
-- Bastian Kleineidam <calvin@users.sourceforge.net> Sun, 29 Oct 2000 16:45:42 +0100
linkchecker (1.2.6) unstable; urgency=low
* made a FAQ

394
http11lib.py
View file

@ -1,394 +0,0 @@
#
# HTTP/1.1 client library
#
# Copyright (C) 1998-1999 Guido van Rossum. All Rights Reserved.
# Written by Greg Stein. Given to Guido. Licensed using the Python license.
#
# This module is maintained by Greg and is available at:
# http://www.lyra.org/greg/python/httplib.py
#
# Since this isn't in the Python distribution yet, we'll use the CVS ID
# for tracking:
# $Id$
#
# Modified by Bastian Kleineidam to squish a bug.
import socket,string,mimetools,httplib
error = __name__ + '.error'
HTTP_PORT = 80
class HTTPResponse(mimetools.Message):
def __init__(self, fp, version, errcode):
mimetools.Message.__init__(self, fp, 0)
if version == 'HTTP/1.0':
self.version = 10
elif version[:7] == 'HTTP/1.':
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
else:
raise error, 'unknown HTTP protocol'
# are we using the chunked-style of transfer encoding?
tr_enc = self.getheader('transfer-encoding')
if tr_enc:
if string.lower(tr_enc) != 'chunked':
raise error, 'unknown transfer-encoding'
self.chunked = 1
self.chunk_left = None
else:
self.chunked = 0
# will the connection close at the end of the response?
conn = self.getheader('connection')
if conn:
conn = string.lower(conn)
# a "Connection: close" will always close the connection. if we
# don't see that and this is not HTTP/1.1, then the connection will
# close unless we see a Keep-Alive header.
self.will_close = string.find(conn, 'close') != -1 or \
( self.version != 11 and \
not self.getheader('keep-alive') )
else:
# for HTTP/1.1, the connection will always remain open
# otherwise, it will remain open IFF we see a Keep-Alive header
self.will_close = self.version != 11 and \
not self.getheader('keep-alive')
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 states we ignore this if tr_enc is "chunked"
length = self.getheader('content-length')
if length and not self.chunked:
self.length = int(length)
else:
self.length = None
# does the body have a fixed length? (of zero)
if (errcode == 204 or # No Content
errcode == 304 or # Not Modified
100 <= errcode < 200): # 1xx codes
self.length = 0
# if the connection remains open, and we aren't using chunked, and
# a content-length was not provided, then assume that the connection
# WILL close.
if not self.will_close and \
not self.chunked and \
self.length is None:
self.will_close = 1
def close(self):
if self.fp:
self.fp.close()
self.fp = None
def isclosed(self):
# NOTE: it is possible that we will not ever call self.close(). This
# case occurs when will_close is TRUE, length is None, and we
# read up to the last byte, but NOT past it.
#
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
# called, meaning self.isclosed() is meaningful.
return self.fp is None
def read(self, amt=None):
if not self.fp:
return ''
if self.chunked:
chunk_left = self.chunk_left
value = ''
while 1:
if not chunk_left:
line = self.fp.readline()
i = string.find(line, ';')
if i >= 0:
line = line[:i] # strip chunk-extensions
chunk_left = string.atoi(line, 16)
if chunk_left == 0:
break
if not amt:
value = value + self.fp.read(chunk_left)
elif amt < chunk_left:
value = value + self.fp.read(amt)
self.chunk_left = chunk_left - amt
return value
elif amt == chunk_left:
value = value + self.fp.read(amt)
self.fp.read(2) # toss the CRLF at the end of the chunk
self.chunk_left = None
return value
else:
value = value + self.fp.read(chunk_left)
amt = amt - chunk_left
# we read the whole chunk, get another
self.fp.read(2) # toss the CRLF at the end of the chunk
chunk_left = None
# read and discard trailer up to the CRLF terminator
### note: we shouldn't have any trailers!
while 1:
line = self.fp.readline()
if line == '\r\n':
break
# we read everything; close the "file"
self.close()
return value
elif not amt:
# unbounded read
if self.will_close:
s = self.fp.read()
else:
s = self.fp.read(self.length)
self.close() # we read everything
return s
if self.length is not None:
if amt > self.length:
# clip the read to the "end of response"
amt = self.length
self.length = self.length - amt
s = self.fp.read(amt)
# close our "file" if we know we should
### I'm not sure about the len(s) < amt part; we should be safe because
### we shouldn't be using non-blocking sockets
if self.length == 0 or len(s) < amt:
self.close()
return s
class HTTPConnection:
_http_vsn = 11
_http_vsn_str = 'HTTP/1.1'
response_class = HTTPResponse
def __init__(self, host, port=None):
self.sock = None
self.response = None
self._set_hostport(host, port)
def _set_hostport(self, host, port):
if port is None:
i = string.find(host, ':')
if i >= 0:
port = int(host[i+1:])
host = host[:i]
else:
port = HTTP_PORT
self.host = host
self.port = port
def connect(self):
"""Connect to the host and port specified in __init__."""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, self.port))
def close(self):
"""Close the connection to the HTTP server."""
if self.sock:
self.sock.close() # close it manually... there may be other refs
self.sock = None
if self.response:
self.response.close()
self.response = None
def send(self, str):
"""Send `str' to the server."""
if not self.sock:
self.connect()
# send the data to the server. if we get a broken pipe, then close
# the socket. we want to reconnect when somebody tries to send again.
#
# NOTE: we DO propagate the error, though, because we cannot simply
# ignore the error... the caller will know if they can retry.
try:
self.sock.send(str)
except socket.error, v:
if v[0] == 32: # Broken pipe
self.close()
raise
def putrequest(self, method, url='/'):
"""Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g.
'/index.html'.
"""
if self.response:
if not self.response.isclosed():
### implies half-duplex!
raise error, 'prior response has not been fully handled'
self.response = None
if not url:
url = '/'
str = '%s %s %s\r\n' % (method, url, self._http_vsn_str)
try:
self.send(str)
except socket.error, v:
if v[0] != 32: # Broken pipe
raise
# try one more time (the socket was closed; this will reopen)
self.send(str)
#self.putheader('Host', self.host)
if self._http_vsn == 11:
# Issue some standard headers for better HTTP/1.1 compliance
# note: we are assuming that clients will not attempt to set these
# headers since *this* library must deal with the consequences.
# this also means that when the supporting libraries are
# updated to recognize other forms, then this code should be
# changed (removed or updated).
# we only want a Content-Encoding of "identity" since we don't
# support encodings such as x-gzip or x-deflate.
self.putheader('Accept-Encoding', 'identity')
# we can accept "chunked" Transfer-Encodings, but no others
# NOTE: no TE header implies *only* "chunked"
#self.putheader('TE', 'chunked')
# if TE is supplied in the header, then it must appear in a
# Connection header.
#self.putheader('Connection', 'TE')
else:
# For HTTP/1.0, the server will assume "not chunked"
pass
def putheader(self, header, value):
"""Send a request header line to the server.
For example: h.putheader('Accept', 'text/html')
"""
str = '%s: %s\r\n' % (header, value)
self.send(str)
def endheaders(self):
"""Indicate that the last header line has been sent to the server."""
self.send('\r\n')
def request(self, method, url='/', body=None, headers={}):
"""Send a complete request to the server."""
self.putrequest(method, url)
if body:
self.putheader('Content-Length', str(len(body)))
for hdr, value in headers.items():
self.putheader(hdr, value)
self.endheaders()
if body:
self.send(body)
def getreply(self):
"""Get a reply from the server.
Returns a tuple consisting of:
- server response code (e.g. '200' if all goes well)
- server response string corresponding to response code
- any RFC822 headers in the response from the server
"""
file = self.sock.makefile('rb')
line = file.readline()
try:
[ver, code, msg] = string.split(line, None, 2)
except ValueError:
try:
[ver, code] = string.split(line, None, 1)
msg = ""
except ValueError:
self.close()
return -1, line, file
if ver[:5] != 'HTTP/':
self.close()
return -1, line, file
errcode = int(code)
errmsg = string.strip(msg)
response = self.response_class(file, ver, errcode)
if response.will_close:
# this effectively passes the connection to the response
self.close()
else:
# remember this, so we can tell when it is complete
self.response = response
return errcode, errmsg, response
class HTTP(HTTPConnection):
"Compatibility class with httplib.py from 1.5."
_http_vsn = 10
_http_vsn_str = 'HTTP/1.0'
def __init__(self, host='', port=None):
"Provide a default host, since the superclass requires one."
# Note that we may pass an empty string as the host; this will throw
# an error when we attempt to connect. Presumably, the client code
# will call connect before then, with a proper host.
HTTPConnection.__init__(self, host, port)
self.debuglevel=0
def connect(self, host=None, port=None):
"Accept arguments to set the host/port, since the superclass doesn't."
if host:
self._set_hostport(host, port)
HTTPConnection.connect(self)
def set_debuglevel(self, debuglevel):
self.debuglevel=debuglevel
def getfile(self):
"Provide a getfile, since the superclass' use of HTTP/1.1 prevents it."
return self.file
def putheader(self, header, *values):
"The superclass allows only one value argument."
HTTPConnection.putheader(self, header, string.joinfields(values,'\r\n\t'))
def getreply(self):
"Compensate for an instance attribute shuffling."
errcode, errmsg, response = HTTPConnection.getreply(self)
if errcode == -1:
self.file = response # response is the "file" when errcode==-1
self.headers = None
return -1, errmsg, None
self.headers = response
self.file = response.fp
return errcode, errmsg, response
def _test():
h = HTTP('www.siemens.de')
h.putrequest("GET")
h.putheader("Host", 'www.siemens.de')
h.endheaders()
status,text,reply = h.getreply()
print status,text,reply
if __name__=='__main__':
_test()

170
httpslib.py
View file

@ -1,170 +0,0 @@
# @(#)httpslib.py 1.1 VMS-99/01/30 https support
import ssl,httplib,string,socket,mimetools
HTTP_PREF = 'HTTP/'
HTTPS_PORT = 443
class HTTPS(httplib.HTTP):
def connect (self, host, port = 0):
"""Connect to a host on a given port.
Note: This method is automatically invoked by __init__,
if a host is specified during instantiation.
"""
if not port:
i = string.find(host, ':')
if i >= 0:
host, port = host[:i], host[i+1:]
try: port = string.atoi(port)
except string.atoi_error:
raise socket.error, "nonnumeric port"
if not port: port = HTTPS_PORT
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.debuglevel > 0: print 'connect:', (host, port)
self.sock.connect(host, port)
self.ssl = ssl.ssl(self.sock.fileno())
def send (self, str):
if self.debuglevel > 0: print 'send:', `str`
self.ssl.write(str,len(str))
def makefile (self, mode='r', bufsize=-1):
return _fileobject(self.sock,self.ssl,mode,bufsize)
def getreply (self):
self.file = self.makefile('rb')
# self.sock = None
line = self.file.readline()
if self.debuglevel > 0: print 'reply:',`line`
try:
[ver,code,msg] = string.split(line,None,2)
except ValueError:
try:
[ver,code] = string.split(line,None,1)
msg = ""
except ValueError:
ver = ""
if ver[:len(HTTP_PREF)] != HTTP_PREF:
self.headers = None
return -1, line, self.headers
self.headers = mimetools.Message(self.file,0)
return string.atoi(code), string.strip(msg), self.headers
def close (self):
if self.file:
self.file.close()
self.file = self.sock = self.ssl = None
class _fileobject:
def __init__ (self, sock, ssl, mode, bufsize):
import string
self._sock = sock
self._ssl = ssl
self._mode = mode
if bufsize < 0:
bufsize = 512
self._rbufsize = max(1,bufsize)
self._wbufsize = bufsize
self._wbuf = self._rbuf = ""
def close (self):
try:
if self._sock:
self.flush()
finally:
self._sock = None
def __del__ (self):
self.close()
def flush (self):
if self._wbuf:
self._sock.write(self._wbuf,len(self._wbuf))
self._wbuf = ""
def fileno (self):
return self._sock.fileno()
def write (self, data):
self._wbuf = self._wbuf + data
if self._wbufsize == 1:
if '\n' in data:
self.flush()
else:
if len(self._wbuf) >= self._wbufsize:
self.flush()
def writelines (self, lst):
filter(self._sock.send,lst)
self.flush()
def read (self, n=-1):
if n >= 0:
while len(self._rbuf) < n:
new = self._ssl.read(self._rbufsize)
if not new: break
self._rbuf = self._rbuf + new
data,self._rbuf = self._rbuf[:n],self._rbuf[n:]
return data
while 1:
new = self._ssl.read(self._rbufsize)
if not new: break
self._rbuf = self._rbuf + new
data,self._rbuf = self._rbuf,""
return data
def readline (self):
data = ""
i = string.find(self._rbuf,'\n')
while i < 0:
new = self._ssl.read(self._rbufsize)
if not new: break
i = string.find(new,'\n')
if i >= 0: i = i + len(self._rbuf)
self._rbuf = self._rbuf + new
if i < 0: i = len(self._rbuf)
else: i = i+1
data,self._rbuf = self._rbuf[:i],self._rbuf[i:]
return data
def readlines (self):
l = []
while 1:
line = self.readline()
if not line: break
l.append(line)
return l
def _test():
import sys
import getopt
opts, args = getopt.getopt(sys.argv[1:], 'd')
dl = 0
for o, a in opts:
if o == '-d': dl = dl + 1
if args[0:]: host = args[0]
if args[1:]: selector = args[1]
h = HTTPS()
host = 'synergy.as.cmu.edu'
selector = '/~geek/'
# host = 'tls.cryptsoft.com'
# selector = '/'
h.set_debuglevel(dl)
h.connect(host)
h.putrequest('GET', selector)
h.endheaders()
errcode, errmsg, headers = h.getreply()
print 'errcode =', errcode
print 'errmsg =', errmsg
print "\tHEADERS:"
if headers:
for header in headers.headers: print string.strip(header)
print "\tTEXT:"
print h.getfile().read()
if __name__ == '__main__':
_test()

2
linkcheck/GopherUrlData.py
View file

@ -23,5 +23,3 @@ class GopherUrlData(UrlData):
def __str__(self):
return "Gopher link\n"+UrlData.__str__(self)

25
linkcheck/HttpUrlData.py
View file

@ -15,9 +15,8 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""
import http11lib,urlparse,sys,time,re
import httplib,urlparse,sys,time,re,robotparser
from UrlData import UrlData
from RobotsTxt import RobotsTxt
import Config,StringUtil
from linkcheck import _
@ -173,7 +172,7 @@ class HttpUrlData(UrlData):
return self.urlConnection.getreply()
def _getHTTPObject(self, host):
return http11lib.HTTP(host)
return httplib.HTTP(host)
def getContent(self):
if not self.data:
@ -193,18 +192,14 @@ class HttpUrlData(UrlData):
return self.mime.gettype()=="text/html"
def robotsTxtAllowsUrl(self, config):
try:
if config.robotsTxtCache_has_key(self.urlTuple[1]):
robotsTxt = config.robotsTxtCache_get(self.urlTuple[1])
else:
robotsTxt = RobotsTxt(self.urlTuple, Config.UserAgent)
Config.debug("DEBUG: "+str(robotsTxt)+"\n")
config.robotsTxtCache_set(self.urlTuple[1], robotsTxt)
except:
type, value = sys.exc_info()[:2]
Config.debug("Heieiei: "+str(value)+"\n")
return 1
return robotsTxt.allowance(Config.UserAgent, self.urlTuple[2])
if not config.robotsTxtCache_has_key(self.url):
roboturl="%s://%s/robots.txt" % self.urlTuple[0:2]
rp = robotparser.RobotFileParser()
rp.set_url(roboturl)
rp.read()
robotsTxt = rp.can_fetch(Config.UserAgent, self.url)
config.robotsTxtCache_set(self.urlTuple[0:2], robotsTxt)
return config.robotsTxtCache_get(self.url)
def __str__(self):

7
linkcheck/HttpsUrlData.py
View file

@ -18,9 +18,8 @@
from UrlData import UrlData
from HttpUrlData import HttpUrlData
from linkcheck import _
_supportHttps=1
try: import httpslib
except ImportError: _supportHttps=0
import httplib,socket
_supportHttps=hasattr(socket.socket, 'ssl')
class HttpsUrlData(HttpUrlData):
"""Url link with https scheme"""
@ -35,7 +34,7 @@ class HttpsUrlData(HttpUrlData):
parentName, baseRef, line)
def _getHTTPObject(self, host):
return httpslib.HTTPS(host)
return httplib.HTTPS(host)
def check(self, config):
if _supportHttps:

7
linkcheck/MailtoUrlData.py
View file

@ -18,7 +18,7 @@
import os,re,string,DNS,sys,Config,cgi,urllib,linkcheck
from rfc822 import AddressList
from HostCheckingUrlData import HostCheckingUrlData
from smtplib import SMTP
from smtplib import SMTP,SMTPException
from linkcheck import _
# regular expression for RFC2368 compliant mailto: scanning
@ -85,9 +85,8 @@ class MailtoUrlData(HostCheckingUrlData):
info = self.urlConnection.verify(user)
if info[0]==250:
self.setInfo("Verified adress: "+info[1])
except:
type, value = sys.exc_info()[:2]
#print type,value
except SMTPException:
pass
if smtpconnect: break
if not smtpconnect:

182
linkcheck/RobotsTxt.py
View file

@ -1,182 +0,0 @@
"""
Copyright (C) 2000 Bastian Kleineidam
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""
import re,urlparse,string,httplib,urllib,sys,StringUtil,Config
from linkcheck import _
class RobotsTxt:
def __init__(self, urltuple, useragent):
self.entries = []
self.disallowAll = 0
self.allowAll = 0
self.base = urltuple[0]+"://"+urltuple[1]+"/robots.txt"
try:
urlConnection = None
if urltuple[0]=="http":
urlConnection = httplib.HTTP(urltuple[1])
else:
import httpslib
urlConnection = httpslib.HTTPS(urltuple[1])
urlConnection.putrequest("GET", "/robots.txt")
urlConnection.putheader("User-agent", useragent)
urlConnection.endheaders()
status = urlConnection.getreply()[0]
if status==401 or status==403:
self.disallowAll = 1
else:
if status>=400:
self.allowAll = 1
if status<400:
self.parseUrl(urlConnection)
except:
type, value = sys.exc_info()[:2]
Config.debug("DEBUG: RobotsTxt error:"+str(value)+"\n")
self.allowAll = 1
def parseUrl(self, urlConnection):
data = urlConnection.getfile().readlines()
state = 0
linenumber = 0
entry = Entry()
for line in data:
line = string.lower(string.strip(line))
linenumber = linenumber + 1
if len(line)<=0:
if state==1:
raise ParseException, \
"robots.txt:"+`linenumber`+": no rules found"
elif state==2:
self.entries.append(entry)
entry = Entry()
state = 0
line = string.strip(StringUtil.stripFenceComments(line))
if len(line)<=0:
continue
if re.compile("^user-agent:.+").match(line):
if state==2:
raise ParseException, \
_("robots.txt:%d: user-agent in the middle of rules") % \
linenumber
entry.useragents.append(string.strip(line[11:]))
state = 1
elif re.compile("^disallow:.+").match(line):
if state==0:
raise ParseException, \
_("robots.txt:%d: disallow without user agents") % \
linenumber
line = string.strip(line[9:])
entry.rulelines.append(RuleLine(line, 0))
state = 2
elif re.compile("^allow:.+").match(line):
if state==0:
raise ParseException, \
_("robots.txt:%d: allow without user agents") % \
linenumber
line = string.strip(line[6:])
entry.rulelines.append(RuleLine(line, 1))
else:
# ignore extensions
pass
def allowance(self, useragent, path):
Config.debug("DEBUG: checking allowance\n")
if self.disallowAll:
return 0
if self.allowAll:
return 1
# search for given user agent matches
# the first match counts
useragent = string.lower(useragent)
for entry in self.entries:
if entry.appliesToAgent(useragent):
return entry.allowance(path)
# agent not found ==> access granted
Config.debug("DEBUG: no match, access granted\n")
return 1
def __str__(self):
ret = "RobotsTxt\n"+\
"Base: "+self.base+"\n"+\
"AllowAll: "+`self.allowAll`+"\n"+\
"DisallowAll: "+`self.disallowAll`+"\n"
for entry in self.entries:
ret = ret + str(entry) + "\n"
return ret
class RuleLine:
def __init__(self, path, allowance):
self.path = urllib.unquote(path)
self.allowance = allowance
def appliesTo(self, filename):
return self.path=="*" or re.compile(self.path).match(filename)
def __str__(self):
if self.allowance:
return "Allow: "+self.path
return "Disallow: "+self.path
class Entry:
def __init__(self):
self.useragents = []
self.rulelines = []
def __str__(self):
ret = ""
for agent in self.useragents:
ret = ret + "User-agent: "+agent+"\n"
for line in self.rulelines:
ret = ret + str(line) + "\n"
return ret
def appliesToAgent(self, agent):
"check if this entry applies to the specified agent"
for cur_agent in self.useragents:
if cur_agent=="*":
return 1
if re.compile("^"+cur_agent).match(agent):
return 1
return 0
def allowance(self, filename):
"""Preconditions:
- our agent applies to this entry
- file is URL decoded"""
for line in self.rulelines:
if line.appliesTo(filename):
return line.allowance
return 1

80
setup.py
View file

@ -43,62 +43,8 @@ class LCInstall(install):
class LCConfig(config):
user_options = config.user_options + [
('ssl-include-dirs=', None,
"directories to search for SSL header files"),
('ssl-library-dirs=', None,
"directories to search for SSL library files"),
]
def initialize_options (self):
config.initialize_options(self)
self.ssl_include_dirs = None
self.ssl_library_dirs = None
def finalize_options(self):
# we have some default include and library directories
self.basic_finalize_options()
if self.ssl_include_dirs is None:
self.ssl_include_dirs = ['/usr/include/openssl',
'/usr/local/include/openssl']
if self.ssl_library_dirs is None:
self.ssl_library_dirs = ['/usr/lib',
'/usr/local/lib']
def basic_finalize_options(self):
"""fix up types of option values"""
# this should be in config.finalize_options
# I submitted a patch
if self.include_dirs is None:
self.include_dirs = self.distribution.include_dirs or []
elif type(self.include_dirs) is StringType:
self.include_dirs = string.split(self.include_dirs, os.pathsep)
if self.libraries is None:
self.libraries = []
elif type(self.libraries) is StringType:
self.libraries = [self.libraries]
if self.library_dirs is None:
self.library_dirs = []
elif type(self.library_dirs) is StringType:
self.library_dirs = [self.library_dirs]
def run (self):
# try to compile a test program with SSL
config.run(self)
have_ssl = self.check_lib("ssl",
library_dirs = self.ssl_library_dirs,
include_dirs = self.ssl_include_dirs,
headers = ["ssl.h"])
# write the result in the configuration file
data = []
data.append("have_ssl = %d" % (have_ssl))
data.append("ssl_library_dirs = %s" % `self.ssl_library_dirs`)
data.append("ssl_include_dirs = %s" % `self.ssl_include_dirs`)
data.append("libraries = %s" % `['ssl', 'crypto']`)
data.append("install_data = %s" % `os.getcwd()`)
data = ["install_data = %s" % `os.getcwd()`]
self.distribution.create_conf_file(".", data)
@ -108,28 +54,6 @@ class LCDistribution(Distribution):
self.config_file = self.get_name()+"Conf.py"
def run_commands(self):
if "config" not in self.commands:
self.check_ssl()
Distribution.run_commands(self)
def check_ssl(self):
if not os.path.exists(self.config_file):
raise SystemExit, "please run 'python setup.py config'"
#self.announce("generating default configuration")
#self.run_command('config')
import LinkCheckerConf
if 'bdist_wininst' in self.commands and os.name!='nt':
self.announce("bdist_wininst command found on non-Windows "
"platform. Disabling SSL compilation")
elif LinkCheckerConf.have_ssl:
self.ext_modules = [Extension('ssl', ['ssl.c'],
include_dirs=LinkCheckerConf.ssl_include_dirs,
library_dirs=LinkCheckerConf.ssl_library_dirs,
libraries=LinkCheckerConf.libraries)]
def create_conf_file(self, directory, data=[]):
data.insert(0, "# this file is automatically created by setup.py")
filename = os.path.join(directory, self.config_file)
@ -145,7 +69,7 @@ class LCDistribution(Distribution):
setup (name = "LinkChecker",
version = "1.2.6",
version = "1.3.0",
description = "check links of HTML pages",
author = "Bastian Kleineidam",
author_email = "calvin@users.sourceforge.net",

415
ssl.c
View file

@ -1,415 +0,0 @@
/* @(#)ssl.c 1.1 VMS-99/01/30 python wrapper for SSLeay https
*/
#include "Python.h"
#if defined(WITH_THREAD) && !defined(HAVE_GETHOSTBYNAME_R) &&\
!defined(MS_WINDOWS)
#include "thread.h"
#endif
#include <sys/types.h>
#ifndef MS_WINDOWS
#include <sys/socket.h>
#else
#include <winsock.h>
#endif
#if defined(PYOS_OS2)
#define INCL_DOS
#define INCL_DOSERRORS
#define INCL_NOPMAPI
#include <os2.h>
#endif
#include "ssl.h"
#include "err.h"
/*
some hacks to choose between K&R or ANSI style function
definitions. For NT to build this as an extension module (ie, DLL)
it must be compiled by the C++ compiler, as it takes the address of
a static data item exported from the main Python DLL.
*/
#ifdef MS_WINDOWS
#define FORCE_ANSI_FUNC_DEFS
#endif
#if defined(PYOS_OS2)
#define FORCE_ANSI_FUNC_DEFS
#endif
#ifdef FORCE_ANSI_FUNC_DEFS
#define BUILD_FUNC_DEF_1( fnname, arg1type, arg1name ) \
fnname( arg1type arg1name )
#define BUILD_FUNC_DEF_2( fnname, arg1type, arg1name, arg2type, arg2name ) \
fnname( arg1type arg1name, arg2type arg2name )
#else /* !FORCE_ANSI_FN_DEFS */
#define BUILD_FUNC_DEF_1( fnname, arg1type, arg1name ) \
fnname( arg1name ) \
arg1type arg1name;
#define BUILD_FUNC_DEF_2( fnname, arg1type, arg1name, arg2type, arg2name ) \
fnname( arg1name, arg2name ) \
arg1type arg1name; \
arg2type arg2name;
#endif /* !FORCE_ANSI_FN_DEFS */
/* Global variable holding the exception type for errors detected
by this module (but not argument type or memory errors, etc.). */
static PyObject *PySslError;
typedef struct {
PyObject_HEAD
int sock_fd;
PyObject *x_attr; /* attributes dictionary */
SSL_CTX *ctx;
SSL *ssl;
X509 *server_cert;
BIO *sbio;
char server[256];
char issuer[256];
} PySslObject;
staticforward PyTypeObject SSL_Type;
#define PySslObject_Check(v) ((v)->ob_type == &SSL_Type)
/*
* raise an error according to errno, return NULL
*/
static PyObject *
PySsl_errno ()
{
#ifdef MS_WINDOWS
if (WSAGetLastError()) {
PyObject *v = Py_BuildValue("(is)",WSAGetLastError(),"winsock error");
if (v) {
PyErr_SetObject(PySslError,v);
Py_DECREF(v);
}
return NULL;
}
#endif
return PyErr_SetFromErrno(PySslError);
}
/*
* format SSl error string
*/
static int
BUILD_FUNC_DEF_2 (PySsl_err_str, unsigned long, e, char *, buf)
{
unsigned long l = ERR_GET_LIB(e);
unsigned long f = ERR_GET_FUNC(e);
unsigned long r = ERR_GET_REASON(e);
char* ls = (char*)ERR_lib_error_string(e);
char* fs = (char*)ERR_func_error_string(e);
char* rs = (char*)ERR_reason_error_string(e);
char* bp = buf + 2; /* skip two initial blanks */
(void)strcpy(buf," none:"); /* initialize buffer */
bp += (ls) ? sprintf(bp,"%s:",ls) :
((l) ? sprintf(bp,"lib %lu:",l) : 0);
bp += (fs) ? sprintf(bp,"%s ",fs) :
((f) ? sprintf(bp,"func %lu:",f) : 0);
bp += (rs) ? sprintf(bp,"%s:",rs) :
((r) ? sprintf(bp,"reason(%lu):",r) : 0);
*bp-- = 0; /* suppress last divider (:) */
return (bp - buf);
}
/*
* report SSL core errors
*/
static PySslObject *
PySsl_errors ()
{
#define PY_SSL_ERR_MAX 256
unsigned long e;
char buf[2 * PY_SSL_ERR_MAX];
char *bf = buf;
while (((bf - buf) < PY_SSL_ERR_MAX) && (e = ERR_get_error()))
bf += PySsl_err_str(e,bf);
{
PyObject *v = Py_BuildValue("(sss)", "ssl","core",buf+2);
if (v != NULL) {
PyErr_SetObject(PySslError,v);
Py_DECREF(v);
}
}
return (NULL);
}
/*
* report SSL application layer errors
*/
static PySslObject *
BUILD_FUNC_DEF_2 (PySsl_app_errors, SSL *, s, int, ret)
{
int err = SSL_get_error(s,ret);
char *str;
switch (err) {
case SSL_ERROR_SSL:
return (PySsl_errors());
case SSL_ERROR_SYSCALL:
return ((PySslObject *)PySsl_errno());
case SSL_ERROR_ZERO_RETURN:
str = "End of data";
break;
case SSL_ERROR_WANT_READ:
str = "Want read";
break;
case SSL_ERROR_WANT_WRITE:
str = "Want write";
break;
case SSL_ERROR_WANT_X509_LOOKUP:
str = "Want x509 lookup";
break;
case SSL_ERROR_WANT_CONNECT:
str = "Want connect";
break;
default:
str = "Unknown";
break;
}
{
PyObject *v = Py_BuildValue("(sis)", "ssl",err, str);
if (v != NULL) {
PyErr_SetObject(PySslError,v);
Py_DECREF(v);
}
}
return (NULL);
}
/* ssl.read(len) method */
static PyObject *
BUILD_FUNC_DEF_2 (PySslObj_read, PySslObject *, self, PyObject *, args)
{
int len, n;
PyObject *buf;
if (!PyArg_ParseTuple(args,"i",&len))
return (NULL);
if (!(buf = PyString_FromStringAndSize((char *)0,len)))
return (NULL);
Py_BEGIN_ALLOW_THREADS
n = SSL_read(self->ssl,PyString_AsString(buf),len);
Py_END_ALLOW_THREADS
switch (SSL_get_error(self->ssl,n)) {
case SSL_ERROR_NONE: /* good return value */
break;
case SSL_ERROR_ZERO_RETURN:
case SSL_ERROR_SYSCALL:
if (!n) /* fix SSL_ERROR_SYCSALL errno=0 case */
break;
/* fall thru here */
default:
Py_DECREF(buf);
(void)PySsl_app_errors(self->ssl,n);
return (NULL);
}
if ((n != len) && (_PyString_Resize(&buf,n) < 0))
return (NULL);
return (buf);
}
/* ssl.write(data,len) method */
static PyObject *
BUILD_FUNC_DEF_2 (PySslObj_write, PySslObject *, self, PyObject *, args)
{
char *buf;
int len, n;
if (!PyArg_ParseTuple(args, "si", &buf, &len))
return NULL;
/* Note: flags are ignored */
Py_BEGIN_ALLOW_THREADS
n = SSL_write(self->ssl,buf,len);
Py_END_ALLOW_THREADS
if (n < 0)
return (PySsl_errno());
return (PyInt_FromLong((long)n));
}
/* ssl.server() method */
static PyObject *
BUILD_FUNC_DEF_2 (PySslObj_server, PySslObject *, self, PyObject *, args)
{
if (!PyArg_NoArgs(args))
return (NULL);
return (PyString_FromString(self->server));
}
/* ssl.issuer() method */
static PyObject *
BUILD_FUNC_DEF_2 (PySslObj_issuer, PySslObject *, self, PyObject *, args)
{
if (!PyArg_NoArgs(args))
return (NULL);
return (PyString_FromString(self->issuer));
}
/* SSL object methods */
static PyMethodDef PySslObj_methods[] = {
{"read", (PyCFunction)PySslObj_read,1},
{"write", (PyCFunction)PySslObj_write,1},
{"server", (PyCFunction)PySslObj_server},
{"issuer", (PyCFunction)PySslObj_issuer},
{ NULL, NULL}
};
static void
BUILD_FUNC_DEF_1 (PySsl_dealloc, PySslObject *, self)
{
if (self->server_cert) /* possible not to have one? */
X509_free(self->server_cert);
SSL_CTX_free(self->ctx);
SSL_free(self->ssl);
Py_XDECREF(self->x_attr);
PyMem_DEL(self);
}
static PyObject *
BUILD_FUNC_DEF_2 (PySsl_getattr, PySslObject *, self, char *, name)
{
return (Py_FindMethod(PySslObj_methods,(PyObject *)self,name));
}
staticforward PyTypeObject SSL_Type = {
PyObject_HEAD_INIT(&PyType_Type)
0, /*ob_size*/
"SSL", /*tp_name*/
sizeof(PySslObject), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor)PySsl_dealloc, /*tp_dealloc*/
0, /*tp_print*/
(getattrfunc)PySsl_getattr, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
};
/*
* C function called for new object initialization
* Note: SSL protocol version 2, 3, or 2+3 set at compile time
*/
static PySslObject *
BUILD_FUNC_DEF_1 (newPySslObject, int, sock_fd)
{
PySslObject *self;
SSL_METHOD *meth;
int ret;
#if 0
meth=SSLv3_client_method();
meth=SSLv23_client_method();
#endif
meth=SSLv2_client_method();
if (!(self = PyObject_NEW(PySslObject,&SSL_Type))) /* create new object */
return (NULL);
(void)memset(self->server,0,sizeof(self->server));
(void)memset(self->issuer,0,sizeof(self->issuer));
self->x_attr = PyDict_New();
if (!(self->ctx = SSL_CTX_new(meth))) { /* set up context */
PyMem_DEL(self);
return (PySsl_errors());
}
#if 0 /* Note: set this for v23, Netscape server */
SSL_CTX_set_options(self->ctx,SSL_OP_ALL);
#endif
self->ssl = SSL_new(self->ctx); /* new ssl struct */
if (!(ret = SSL_set_fd(self->ssl,sock_fd))) { /* set the socket for SSL */
PyMem_DEL(self);
return (PySsl_app_errors(self->ssl,ret));
}
SSL_CTX_set_verify(self->ctx,SSL_VERIFY_NONE,NULL); /* set verify lvl */
SSL_set_connect_state(self->ssl);
if ((ret = SSL_connect(self->ssl)) < 0) { /* negotiate SSL connection */
PyMem_DEL(self);
return (PySsl_app_errors(self->ssl,ret));
}
self->ssl->debug = 1;
if ((self->server_cert = SSL_get_peer_certificate(self->ssl))) {
X509_NAME_oneline(X509_get_subject_name(self->server_cert),
self->server,sizeof(self->server));
X509_NAME_oneline(X509_get_issuer_name(self->server_cert),
self->issuer, sizeof(self->issuer));
}
self->x_attr = NULL;
self->sock_fd = sock_fd;
return (self);
}
/*
* Python function called for new object initialization
*/
static PyObject *
BUILD_FUNC_DEF_2 (PySsl_ssl_new, PyObject *, self, PyObject *, args)
{
int sock_fd;
if (!PyArg_ParseTuple(args, "i", &sock_fd))
return (NULL);
return ((PyObject *)newPySslObject(sock_fd));
}
/* List of functions exported by this module. */
static PyMethodDef PySsl_methods[] = {
{"ssl", (PyCFunction)PySsl_ssl_new, 1},
{NULL, NULL} /* sentinel */
};
/*
* Initialize this module, called when the first 'import ssl' is done
*/
void
initssl ()
{
PyObject *m, *d;
m = Py_InitModule("ssl", PySsl_methods);
d = PyModule_GetDict(m);
SSL_load_error_strings();
SSLeay_add_ssl_algorithms();
/* *** Python 1.5 ***
if (!(PySssl_Error = PyErr_NewException("ssl.error",NULL,NULL)))
return;
*/
if (!(PySslError = PyString_FromString("ssl.error")) ||
PyDict_SetItemString(d,"error",PySslError))
Py_FatalError("can't define ssl.error");
if (PyDict_SetItemString(d,"SSLType",(PyObject *)&SSL_Type))
return;
}