devel changes

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@226 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2001-01-22 23:02:54 +00:00
parent a2ceebf3eb
commit ed1ab5c3a9
35 changed files with 206 additions and 609 deletions

View file

@ -3,6 +3,7 @@ build-stamp
*-out.*
build
dist
foo
MANIFEST
VERSION
LinkCheckerConf.py

12
INSTALL
View file

@ -3,10 +3,7 @@
Requirements
------------
Python >= 1.5.2 from http://www.python.org/
Distutils >= 0.9.1 from http://www.python.org/sigs/distutils-sig/
Python 1.6 includes the Distutils 0.9.1,
Python 2.0 includes the Distutils 1.0.1
Python >= 2.0 from http://www.python.org/
Optionally packages
@ -41,13 +38,6 @@ to check.
Type "linkchecker -h" for help.
Note
----
If you want to make your own distribution with "python setup.py sdist",
you will need Distutils >= 0.9.4. Older versions are hanging when
they try to parse the MANIFEST.in file.
(Fast)CGI web interface
-----------------------
The *cgi files are three CGI scripts which you can use to run LinkChecker

View file

@ -15,4 +15,4 @@ recursive-include locale *.mo
recursive-include po *.po *.py Makefile
recursive-include lconline *
recursive-include tests *.py
exclude LinkCheckerConf.py
exclude linkcheckerConf.py

View file

@ -1,9 +1,10 @@
# This Makefile is only used by developers.
# You will need a Debian Linux system to use this Makefile because
# some targets produce Debian .deb packages
VERSION=$(shell python setup.py --version)
PYTHON=python2.0
VERSION=$(shell $(PYTHON) setup.py --version)
PACKAGE = linkchecker
NAME = $(shell python setup.py --name)
NAME = $(shell $(PYTHON) setup.py --name)
HOST=treasure.calvinsplayground.de
#LCOPTS=-ocolored -Ftext -Fhtml -Fgml -Fsql -Fcsv -Fxml -R -t0 -v -s
LCOPTS=-ocolored -Ftext -Fhtml -Fgml -Fsql -Fcsv -Fxml -R -t0 -v -s
@ -18,13 +19,13 @@ all:
@echo "Read the file INSTALL to see how to build and install"
clean:
-python setup.py clean --all # ignore errors of this command
-$(PYTHON) setup.py clean --all # ignore errors of this command
$(MAKE) -C po clean
find . -name '*.py[co]' | xargs rm -f
distclean: clean cleandeb
rm -rf dist build # just to be sure clean also the build dir
rm -f $(PACKAGE)-out.* VERSION LinkCheckerConf.py MANIFEST Packages.gz
rm -f $(PACKAGE)-out.* VERSION $(PACKAGE)Conf.py MANIFEST Packages.gz
cleandeb:
rm -rf debian/$(PACKAGE) debian/tmp
@ -36,9 +37,9 @@ dist: locale
# cleandeb because distutils choke on dangling symlinks
# (linkchecker.1 -> undocumented.1)
$(MAKE) cleandeb
python setup.py sdist --formats=gztar,zip bdist_rpm
$(PYTHON) setup.py sdist --formats=gztar,zip bdist_rpm
# extra run without SSL compilation
python setup.py bdist_wininst
$(PYTHON) setup.py bdist_wininst
mv -f ../$(DEBPACKAGE) dist
package:

2
README
View file

@ -50,14 +50,12 @@ So for example 1.1.5 is the fifth release of the 1.1 development package.
Included packages
-----------------
httplib from http://www.lyra.org/greg/python/
httpslib from http://home.att.net/~nvsoft1/ssl_wrapper.html
DNS see DNS/README
fcgi.py and sz_fcgi.py from http://saarland.sz-sb.de/~ajung/sz_fcgi/
fintl.py from http://sourceforge.net/snippet/detail.php?type=snippet&id=100059
Note that the following packages are modified by me:
httplib.py (renamed to http11lib.py and a bug fixed)
fcgi.py (implemented streamed output)
sz_fcgi.py (simplified the code)
DNS/Lib.py:566 fixed rdlength name error

3
TODO
View file

@ -1 +1,2 @@
Feature complete, only fixes.
Better link name parsing
Embed the Mozilla spidermonkey JavaScript engine for JS links

6
debian/changelog vendored
View file

@ -1,3 +1,9 @@
linkchecker (1.3.0) unstable; urgency=low
* require Python 2.0
-- Bastian Kleineidam <calvin@users.sourceforge.net> Sat, 13 Jan 2001 11:52:30 +0100
linkchecker (1.2.14) unstable; urgency=low
* fixed Python 2.0 bug: TypeError: object has read-only attributes

4
debian/control vendored
View file

@ -2,13 +2,13 @@ Source: linkchecker
Section: web
Priority: optional
Maintainer: Bastian Kleineidam <calvin@users.sourceforge.net>
Build-Depends: python-base (>= 1.5.2), python-base (<< 1.6), python-dev (>= 1.5.2), python-dev (<< 1.6), python-distutils (>= 0.9.4), debhelper
Build-Depends: python2-base (>= 2.0), python2-base (<= 2.0), python2-dev (>= 1.5.2), python2-dev (<= 2.0), debhelper
Build-Depends-Indep: gettext
Standards-Version: 3.2.1
Package: linkchecker
Architecture: any
Depends: python-base (>= 1.5.2), python-base (<< 1.6)
Depends: python2-base (>= 2.0), python-base (<= 2.0)
Suggests: libssl09|libssl095a, httpd
Description: check HTML documents for broken links
Features:

2
debian/postinst vendored
View file

@ -4,7 +4,7 @@
# used by Bastian Kleineidam for LinkChecker
DIRLIST="/usr/lib/python1.5/site-packages/linkcheck"
FILELIST="LinkCheckerConf.py"
FILELIST="linkcheckerConf.py"
SITEPACKAGES="/usr/lib/python1.5/site-packages"
COMMAND="'import sys,py_compile;py_compile.compile(sys.argv[1])'"

6
debian/rules vendored
View file

@ -16,7 +16,7 @@ export DH_OPTIONS
configure: configure-stamp
configure-stamp:
dh_testdir
python setup.py config -lcrypto
./setup.py config -lcrypto
touch configure-stamp
@ -24,7 +24,7 @@ build: configure-stamp build-stamp
build-stamp:
dh_testdir
rm -rf debian/$(PACKAGE)
python setup.py build
./setup.py build
touch build-stamp
clean:
@ -38,7 +38,7 @@ install: build
dh_clean -k
dh_installdirs
$(MAKE) locale
python setup.py install --root=`pwd`/debian/$(PACKAGE) --no-compile
./setup.py install --root=`pwd`/debian/$(PACKAGE) --no-compile
# remove man pages, we install them with dh_installmanpages
rm -rf debian/$(PACKAGE)/usr/man
# remove example files, we install them with dh_installexamples

4
lc.cgi
View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (C) 2000 Bastian Kleineidam
#!/usr/bin/env python2
# Copyright (C) 2000,2001 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (C) 2000 Bastian Kleineidam
#!/usr/bin/env python2
# Copyright (C) 2000,2001 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (C) 2000 Bastian Kleineidam
#!/usr/bin/env python2
# Copyright (C) 2000,2001 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

View file

@ -1,10 +1,8 @@
#
# CSV 0.17 8 June 1999 Copyright ©Laurence Tratt 1998 - 1999
# e-mail: tratt@dcs.kcl.ac.uk
# home-page: http://eh.org/~laurie/comp/python/csv/index.html
#
#
#
# CSV.py is copyright ©1998 - 1999 by Laurence Tratt
#
# All rights reserved
@ -22,28 +20,18 @@
# IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
import re, string, types, UserList
###################################################################################################
#
# CSV class
#
class CSV(UserList.UserList):
""" Manage a CSV (comma separated values) file
The data is held in a list.
Methods:
__init__()
load() load from file
@ -55,10 +43,8 @@ class CSV(UserList.UserList):
"""
def __init__(self, separator = ','):
""" Initialise CVS class instance.
""" Initialise CVS class instance.
Arguments:
separator : The field delimiter. Defaults to ','
@ -72,7 +58,6 @@ class CSV(UserList.UserList):
def load(self, file__data__name, fields__title__have, convert_numbers = 0, separator = None, comments = None):
""" Load up a CSV file
Arguments:
@ -94,7 +79,6 @@ class CSV(UserList.UserList):
def save(self, file__data__name, separator = None):
""" Save data to CSV file.
Arguments:
@ -434,4 +418,4 @@ class Entry(UserList.UserList):
def __str__(self):
return `self.data`
return `self.data`

View file

@ -23,24 +23,24 @@ This module stores
"""
import ConfigParser, sys, os, re, UserDict, string, time
import Logging, LinkCheckerConf
import Logging, linkcheckerConf
from os.path import expanduser,normpath,normcase,join,isfile
from types import StringType
from urllib import getproxies
from linkcheck import _
Version = LinkCheckerConf.version
AppName = LinkCheckerConf.name
Version = linkcheckerConf.version
AppName = linkcheckerConf.name
App = AppName+" "+Version
UserAgent = AppName+"/"+Version
Author = LinkCheckerConf.author
Author = linkcheckerConf.author
HtmlAuthor = string.replace(Author, ' ', '&nbsp;')
Copyright = "Copyright © 2000,2001 by "+Author
HtmlCopyright = "Copyright &copy; 2000,2001 by "+HtmlAuthor
AppInfo = App+" "+Copyright
HtmlAppInfo = App+", "+HtmlCopyright
Url = LinkCheckerConf.url
Email = LinkCheckerConf.author_email
Url = linkcheckerConf.url
Email = linkcheckerConf.author_email
Freeware = AppName+""" comes with ABSOLUTELY NO WARRANTY!
This is free software, and you are welcome to redistribute it
under certain conditions. Look at the file `LICENSE' whithin this
@ -263,7 +263,7 @@ class Configuration(UserDict.UserDict):
return apply(Loggers[name], (), namespace)
def incrementLinknumber_NoThreads(self):
self['linknumber'] = self['linknumber'] + 1
self['linknumber'] += 1
def log_newUrl_NoThreads(self, url):
if not self["quiet"]: self["log"].newUrl(url)
@ -284,7 +284,7 @@ class Configuration(UserDict.UserDict):
def incrementLinknumber_Threads(self):
try:
self.dataLock.acquire()
self['linknumber'] = self['linknumber'] + 1
self['linknumber'] += 1
finally:
self.dataLock.release()
@ -297,7 +297,7 @@ class Configuration(UserDict.UserDict):
self.reduceCount=0
self.threader.reduceThreads()
else:
self.reduceCount = self.reduceCount + 1
self.reduceCount += 1
return self.threader.finished() and self.urls.empty()
def finish_Threads(self):
@ -366,7 +366,7 @@ class Configuration(UserDict.UserDict):
def read(self, files = []):
if not files:
# system wide config settings
config_dir = join(LinkCheckerConf.install_data, 'linkchecker')
config_dir = join(linkcheckerConf.install_data, 'linkchecker')
files.append(norm(join(config_dir, "linkcheckerrc")))
# per user config settings
files.append(norm("~/.linkcheckerrc"))
@ -463,7 +463,7 @@ class Configuration(UserDict.UserDict):
if len(tuple)!=3: break
tuple[0] = re.compile(tuple[0])
self["authentication"].insert(0, tuple)
i = i + 1
i += 1
except ConfigParser.Error: pass
section = "filtering"
@ -474,7 +474,7 @@ class Configuration(UserDict.UserDict):
if len(tuple)!=2: break
self["externlinks"].append((re.compile(tuple[0]),
int(tuple[1])))
i = i + 1
i += 1
except ConfigParser.Error: pass
try: self["internlinks"].append(re.compile(cfgparser.get(section, "internlinks")))
except ConfigParser.Error: pass

View file

@ -16,7 +16,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""
import http11lib,urlparse,sys,time,re
import httplib,urlparse,sys,time,re
import Config,StringUtil,robotparser2
from UrlData import UrlData
from urllib import splittype, splithost
@ -101,7 +101,7 @@ class HttpUrlData(UrlData):
self.urlTuple = urlparse.urlparse(redirected)
status, statusText, self.mime = self._getHttpRequest()
Config.debug("DEBUG: Redirected\n"+str(self.mime))
tries = tries + 1
tries += 1
# authentication
if status==401:
@ -186,7 +186,7 @@ class HttpUrlData(UrlData):
return self.urlConnection.getreply()
def _getHTTPObject(self, host):
return http11lib.HTTP(host)
return httplib.HTTP(host)
def getContent(self):
if not self.data:

View file

@ -22,6 +22,9 @@ init(self)
Because we initialize the start time in init and __init__ gets not
called at the time the checking starts but when the logger object is
created.
Another reason is that we dont want might create several loggers
as a default and then switch to another configured output. So we
must not print anything out at __init__ time.
newUrl(self,urlData)
Called every time an url finished checking. All data we checked is in
@ -135,7 +138,7 @@ class StandardLogger:
StringUtil.blocktext(urlData.infoString, 65),
MaxIndent)+"\n")
if urlData.warningString:
self.warnings = self.warnings+1
self.warnings += 1
self.fd.write(_("Warning")+Spaces["Warning"]+
StringUtil.indent(
StringUtil.blocktext(urlData.warningString, 65),
@ -145,7 +148,7 @@ class StandardLogger:
if urlData.valid:
self.fd.write(urlData.validString+"\n")
else:
self.errors = self.errors+1
self.errors += 1
self.fd.write(urlData.errorString+"\n")
self.fd.flush()
@ -258,7 +261,7 @@ class HtmlLogger(StandardLogger):
StringUtil.htmlify(urlData.infoString)+
"</td></tr>\n")
if urlData.warningString:
self.warnings = self.warnings+1
self.warnings += 1
self.fd.write("<tr>"+self.tablewarning+_("Warning")+
"</td>"+self.tablewarning+
string.replace(urlData.warningString,"\n", "<br>")+
@ -267,7 +270,7 @@ class HtmlLogger(StandardLogger):
self.fd.write("<tr>"+self.tableok+_("Result")+"</td>"+
self.tableok+urlData.validString+"</td></tr>\n")
else:
self.errors = self.errors+1
self.errors += 1
self.fd.write("<tr>"+self.tableerror+_("Result")+
"</td>"+self.tableerror+
urlData.errorString+"</td></tr>\n")
@ -402,7 +405,7 @@ class ColoredLogger(StandardLogger):
self.fd.write(self.colorreset+"\n")
if urlData.warningString:
self.warnings = self.warnings+1
self.warnings += 1
if self.prefix:
self.fd.write("| ")
self.fd.write(_("Warning")+Spaces["Warning"]+self.colorwarning+
@ -415,7 +418,7 @@ class ColoredLogger(StandardLogger):
self.fd.write(self.colorvalid+urlData.validString+
self.colorreset+"\n")
else:
self.errors = self.errors+1
self.errors += 1
self.fd.write(self.colorinvalid+urlData.errorString+
self.colorreset+"\n")
self.fd.flush()
@ -453,7 +456,7 @@ class GMLLogger(StandardLogger):
if node.url and not self.nodes.has_key(node.url):
node.id = self.nodeid
self.nodes[node.url] = node
self.nodeid = self.nodeid + 1
self.nodeid += 1
self.fd.write(" node [\n")
self.fd.write(" id %d\n" % node.id)
self.fd.write(' label "%s"\n' % node.url)
@ -527,7 +530,7 @@ class XMLLogger(StandardLogger):
if node.url and not self.nodes.has_key(node.url):
node.id = self.nodeid
self.nodes[node.url] = node
self.nodeid = self.nodeid + 1
self.nodeid += 1
self.fd.write(' <node name="%d" ' % node.id)
self.fd.write(">\n")
self.fd.write(" <label>%s</label>\n" % quote(node.url))

View file

@ -52,7 +52,7 @@ def stripFenceComments(data):
for line in lines:
if not re.compile("\s*#.*").match(line):
if ret:
ret = ret + "\n" + line
ret += "\n" + line
else:
ret = line
return ret
@ -98,7 +98,7 @@ def indentWith(s, indent):
while i < len(s):
if s[i]=="\n" and (i+1) < len(s):
s = s[0:(i+1)] + indent + s[(i+1):]
i = i+1
i += 1
return s
@ -111,12 +111,12 @@ def blocktext(s, width):
ret = ""
while len(s):
if line:
line = line+"\n"+s.pop()
line += "\n"+s.pop()
else:
line = s.pop()
while len(line) > width:
i = getLastWordBoundary(line, width)
ret = ret + string.strip(line[0:i]) + "\n"
ret += string.strip(line[0:i]) + "\n"
line = string.strip(line[i:])
return ret + line
@ -163,8 +163,8 @@ def getLineNumber(str, index):
line=1
while i<index:
if str[i]=='\n':
line = line + 1
i = i+1
line += 1
i += 1
return line
def paginate(text, lines=22):
@ -173,7 +173,7 @@ def paginate(text, lines=22):
curline = 1
for line in textlines:
print line
curline = curline + 1
curline += 1
if curline >= lines and sys.stdin.isatty():
curline = 1
print "press return to continue..."

View file

@ -16,16 +16,17 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
"""
import sys,re,string,urlparse,urllib,time
import sys,re,string,urlparse,urllib,time,DNS
import Config,StringUtil,linkcheck
from linkcheck import _
debug = linkcheck.Config.debug
ExcList = [
IOError,
ValueError, # from http11lib.py
ValueError, # from httplib.py
linkcheck.error,
EOFError, # from ftplib.py
DNS.Error,
]
try:
import socket
@ -127,13 +128,13 @@ class UrlData:
def setWarning(self, s):
if self.warningString:
self.warningString = self.warningString+"\n" + s
self.warningString += "\n" + s
else:
self.warningString = s
def setInfo(self, s):
if self.infoString:
self.infoString = self.infoString+"\n"+s
self.infoString += "\n"+s
else:
self.infoString = s

View file

@ -25,12 +25,12 @@ class error(Exception):
# i18n suppport
LANG="EN" # default language (used for HTML output)
import LinkCheckerConf
import linkcheckerConf
try:
import fintl,os,string
gettext = fintl.gettext
domain = 'linkcheck'
localedir = os.path.join(LinkCheckerConf.install_data, 'locale')
localedir = os.path.join(linkcheckerConf.install_data, 'locale')
fintl.bindtextdomain(domain, localedir)
fintl.textdomain(domain)
languages = []

View file

@ -105,7 +105,7 @@ class record:
self.content = ""
while len(self.content) < contentLength:
data = sock.recv(contentLength - len(self.content))
self.content = self.content + data
self.content += data
if paddingLength != 0:
padding = sock.recv(paddingLength)
@ -141,12 +141,12 @@ class record:
elif self.recType==FCGI_GET_VALUES or self.recType==FCGI_PARAMS:
content = ""
for i in self.values.keys():
content = content + writePair(i, self.values[i])
content += writePair(i, self.values[i])
elif self.recType==FCGI_END_REQUEST:
v = self.appStatus
content = chr((v>>24)&255) + chr((v>>16)&255) + chr((v>>8)&255) + chr(v&255)
content = content + chr(self.protocolStatus) + 3*'\000'
content = chr((v>>24)&255) + chr((v>>16)&255) + chr((v>>8)&255) +\
chr(v&255) + chr(self.protocolStatus) + 3*'\000'
cLen = len(content)
eLen = (cLen + 7) & (0xFFFF - 7) # align to an 8-byte boundary
@ -167,13 +167,17 @@ class record:
#---------------------------------------------------------------------------
def readPair(s, pos):
nameLen=ord(s[pos]) ; pos=pos+1
nameLen=ord(s[pos])
pos += 1
if nameLen & 128:
b=map(ord, s[pos:pos+3]) ; pos=pos+3
b=map(ord, s[pos:pos+3])
pos += 3
nameLen=((nameLen&127)<<24) + (b[0]<<16) + (b[1]<<8) + b[2]
valueLen=ord(s[pos]) ; pos=pos+1
valueLen=ord(s[pos])
pos += 1
if valueLen & 128:
b=map(ord, s[pos:pos+3]) ; pos=pos+3
b=map(ord, s[pos:pos+3])
pos += 3
valueLen=((valueLen&127)<<24) + (b[0]<<16) + (b[1]<<8) + b[2]
return ( s[pos:pos+nameLen], s[pos+nameLen:pos+nameLen+valueLen],
pos+nameLen+valueLen )
@ -182,13 +186,15 @@ def readPair(s, pos):
def writePair(name, value):
l=len(name)
if l<128: s=chr(l)
if l<128:
s = chr(l)
else:
s=chr(128|(l>>24)&255) + chr((l>>16)&255) + chr((l>>8)&255) + chr(l&255)
s = chr(128|(l>>24)&255) + chr((l>>16)&255) + chr((l>>8)&255) + chr(l&255)
l=len(value)
if l<128: s=s+chr(l)
if l<128:
s += chr(l)
else:
s=s+chr(128|(l>>24)&255) + chr((l>>16)&255) + chr((l>>8)&255) + chr(l&255)
s += chr(128|(l>>24)&255) + chr((l>>16)&255) + chr((l>>8)&255) + chr(l&255)
return s + name + value
#---------------------------------------------------------------------------
@ -342,22 +348,22 @@ class FCGI:
elif r.recType == FCGI_PARAMS:
if r.content == "":
remaining=remaining-1
remaining -= 1
else:
for i in r.values.keys():
self.env[i] = r.values[i]
elif r.recType == FCGI_STDIN:
if r.content == "":
remaining=remaining-1
remaining = remaining-1
else:
stdin=stdin+r.content
stdin += r.content
elif r.recType==FCGI_DATA:
if r.content == "":
remaining=remaining-1
remaining -= 1
else:
data=data+r.content
data += r.content
# end of while remaining:
self.stdin = sys.stdin = StringIO(stdin)
@ -425,7 +431,7 @@ def _test():
try:
while isFCGI():
req = FCGI()
counter=counter+1
counter += 1
try:
fs = req.getFieldStorage()

View file

@ -143,8 +143,8 @@ except ImportError:
else:
raise error, ".mo file '%s' is corrupt" % mo_filename
# advance to the next entry in seek tables:
master_index= master_index + 8
transl_index= transl_index + 8
master_index += 8
transl_index += 8
def gettext(self, message):
"""return the translation of a given message"""

View file

@ -1,394 +0,0 @@
#
# HTTP/1.1 client library
#
# Copyright (C) 1998-1999 Guido van Rossum. All Rights Reserved.
# Written by Greg Stein. Given to Guido. Licensed using the Python license.
#
# This module is maintained by Greg and is available at:
# http://www.lyra.org/greg/python/httplib.py
#
# Since this isn't in the Python distribution yet, we'll use the CVS ID
# for tracking:
# $Id$
#
# Modified by Bastian Kleineidam to squish a bug.
import socket,string,mimetools,httplib
error = __name__ + '.error'
HTTP_PORT = 80
class HTTPResponse(mimetools.Message):
def __init__(self, fp, version, errcode):
mimetools.Message.__init__(self, fp, 0)
if version == 'HTTP/1.0':
self.version = 10
elif version[:7] == 'HTTP/1.':
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
else:
raise error, 'unknown HTTP protocol'
# are we using the chunked-style of transfer encoding?
tr_enc = self.getheader('transfer-encoding')
if tr_enc:
if string.lower(tr_enc) != 'chunked':
raise error, 'unknown transfer-encoding'
self.chunked = 1
self.chunk_left = None
else:
self.chunked = 0
# will the connection close at the end of the response?
conn = self.getheader('connection')
if conn:
conn = string.lower(conn)
# a "Connection: close" will always close the connection. if we
# don't see that and this is not HTTP/1.1, then the connection will
# close unless we see a Keep-Alive header.
self.will_close = string.find(conn, 'close') != -1 or \
( self.version != 11 and \
not self.getheader('keep-alive') )
else:
# for HTTP/1.1, the connection will always remain open
# otherwise, it will remain open IFF we see a Keep-Alive header
self.will_close = self.version != 11 and \
not self.getheader('keep-alive')
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 states we ignore this if tr_enc is "chunked"
length = self.getheader('content-length')
if length and not self.chunked:
self.length = int(length)
else:
self.length = None
# does the body have a fixed length? (of zero)
if (errcode == 204 or # No Content
errcode == 304 or # Not Modified
100 <= errcode < 200): # 1xx codes
self.length = 0
# if the connection remains open, and we aren't using chunked, and
# a content-length was not provided, then assume that the connection
# WILL close.
if not self.will_close and \
not self.chunked and \
self.length is None:
self.will_close = 1
def close(self):
if self.fp:
self.fp.close()
self.fp = None
def isclosed(self):
# NOTE: it is possible that we will not ever call self.close(). This
# case occurs when will_close is TRUE, length is None, and we
# read up to the last byte, but NOT past it.
#
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
# called, meaning self.isclosed() is meaningful.
return self.fp is None
def read(self, amt=None):
if not self.fp:
return ''
if self.chunked:
chunk_left = self.chunk_left
value = ''
while 1:
if not chunk_left:
line = self.fp.readline()
i = string.find(line, ';')
if i >= 0:
line = line[:i] # strip chunk-extensions
chunk_left = string.atoi(line, 16)
if chunk_left == 0:
break
if not amt:
value = value + self.fp.read(chunk_left)
elif amt < chunk_left:
value = value + self.fp.read(amt)
self.chunk_left = chunk_left - amt
return value
elif amt == chunk_left:
value = value + self.fp.read(amt)
self.fp.read(2) # toss the CRLF at the end of the chunk
self.chunk_left = None
return value
else:
value = value + self.fp.read(chunk_left)
amt = amt - chunk_left
# we read the whole chunk, get another
self.fp.read(2) # toss the CRLF at the end of the chunk
chunk_left = None
# read and discard trailer up to the CRLF terminator
### note: we shouldn't have any trailers!
while 1:
line = self.fp.readline()
if line == '\r\n':
break
# we read everything; close the "file"
self.close()
return value
elif not amt:
# unbounded read
if self.will_close:
s = self.fp.read()
else:
s = self.fp.read(self.length)
self.close() # we read everything
return s
if self.length is not None:
if amt > self.length:
# clip the read to the "end of response"
amt = self.length
self.length = self.length - amt
s = self.fp.read(amt)
# close our "file" if we know we should
### I'm not sure about the len(s) < amt part; we should be safe because
### we shouldn't be using non-blocking sockets
if self.length == 0 or len(s) < amt:
self.close()
return s
class HTTPConnection:
_http_vsn = 11
_http_vsn_str = 'HTTP/1.1'
response_class = HTTPResponse
def __init__(self, host, port=None):
self.sock = None
self.response = None
self._set_hostport(host, port)
def _set_hostport(self, host, port):
if port is None:
i = string.find(host, ':')
if i >= 0:
port = int(host[i+1:])
host = host[:i]
else:
port = HTTP_PORT
self.host = host
self.port = port
def connect(self):
"""Connect to the host and port specified in __init__."""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.host, self.port))
def close(self):
"""Close the connection to the HTTP server."""
if self.sock:
self.sock.close() # close it manually... there may be other refs
self.sock = None
if self.response:
self.response.close()
self.response = None
def send(self, str):
"""Send `str' to the server."""
if not self.sock:
self.connect()
# send the data to the server. if we get a broken pipe, then close
# the socket. we want to reconnect when somebody tries to send again.
#
# NOTE: we DO propagate the error, though, because we cannot simply
# ignore the error... the caller will know if they can retry.
try:
self.sock.send(str)
except socket.error, v:
if v[0] == 32: # Broken pipe
self.close()
raise
def putrequest(self, method, url='/'):
"""Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g.
'/index.html'.
"""
if self.response:
if not self.response.isclosed():
### implies half-duplex!
raise error, 'prior response has not been fully handled'
self.response = None
if not url:
url = '/'
str = '%s %s %s\r\n' % (method, url, self._http_vsn_str)
try:
self.send(str)
except socket.error, v:
if v[0] != 32: # Broken pipe
raise
# try one more time (the socket was closed; this will reopen)
self.send(str)
#self.putheader('Host', self.host)
if self._http_vsn == 11:
# Issue some standard headers for better HTTP/1.1 compliance
# note: we are assuming that clients will not attempt to set these
# headers since *this* library must deal with the consequences.
# this also means that when the supporting libraries are
# updated to recognize other forms, then this code should be
# changed (removed or updated).
# we only want a Content-Encoding of "identity" since we don't
# support encodings such as x-gzip or x-deflate.
self.putheader('Accept-Encoding', 'identity')
# we can accept "chunked" Transfer-Encodings, but no others
# NOTE: no TE header implies *only* "chunked"
#self.putheader('TE', 'chunked')
# if TE is supplied in the header, then it must appear in a
# Connection header.
#self.putheader('Connection', 'TE')
else:
# For HTTP/1.0, the server will assume "not chunked"
pass
def putheader(self, header, value):
"""Send a request header line to the server.
For example: h.putheader('Accept', 'text/html')
"""
str = '%s: %s\r\n' % (header, value)
self.send(str)
def endheaders(self):
"""Indicate that the last header line has been sent to the server."""
self.send('\r\n')
def request(self, method, url='/', body=None, headers={}):
"""Send a complete request to the server."""
self.putrequest(method, url)
if body:
self.putheader('Content-Length', str(len(body)))
for hdr, value in headers.items():
self.putheader(hdr, value)
self.endheaders()
if body:
self.send(body)
def getreply(self):
"""Get a reply from the server.
Returns a tuple consisting of:
- server response code (e.g. '200' if all goes well)
- server response string corresponding to response code
- any RFC822 headers in the response from the server
"""
file = self.sock.makefile('rb')
line = file.readline()
try:
[ver, code, msg] = string.split(line, None, 2)
except ValueError:
try:
[ver, code] = string.split(line, None, 1)
msg = ""
except ValueError:
self.close()
return -1, line, file
if ver[:5] != 'HTTP/':
self.close()
return -1, line, file
errcode = int(code)
errmsg = string.strip(msg)
response = self.response_class(file, ver, errcode)
if response.will_close:
# this effectively passes the connection to the response
self.close()
else:
# remember this, so we can tell when it is complete
self.response = response
return errcode, errmsg, response
class HTTP(HTTPConnection):
"Compatibility class with httplib.py from 1.5."
_http_vsn = 10
_http_vsn_str = 'HTTP/1.0'
def __init__(self, host='', port=None):
"Provide a default host, since the superclass requires one."
# Note that we may pass an empty string as the host; this will throw
# an error when we attempt to connect. Presumably, the client code
# will call connect before then, with a proper host.
HTTPConnection.__init__(self, host, port)
self.debuglevel=0
def connect(self, host=None, port=None):
"Accept arguments to set the host/port, since the superclass doesn't."
if host:
self._set_hostport(host, port)
HTTPConnection.connect(self)
def set_debuglevel(self, debuglevel):
self.debuglevel=debuglevel
def getfile(self):
"Provide a getfile, since the superclass' use of HTTP/1.1 prevents it."
return self.file
def putheader(self, header, *values):
"The superclass allows only one value argument."
HTTPConnection.putheader(self, header, string.joinfields(values,'\r\n\t'))
def getreply(self):
"Compensate for an instance attribute shuffling."
errcode, errmsg, response = HTTPConnection.getreply(self)
if errcode == -1:
self.file = response # response is the "file" when errcode==-1
self.headers = None
return -1, errmsg, None
self.headers = response
self.file = response.fp
return errcode, errmsg, response
def _test():
h = HTTP('www.siemens.de')
h.putrequest("GET")
h.putheader("Host", 'www.siemens.de')
h.endheaders()
status,text,reply = h.getreply()
print status,text,reply
if __name__=='__main__':
_test()

View file

@ -90,7 +90,7 @@ class _fileobject:
return self._sock.fileno()
def write (self, data):
self._wbuf = self._wbuf + data
self._wbuf += data
if self._wbufsize == 1:
if '\n' in data:
self.flush()
@ -107,13 +107,13 @@ class _fileobject:
while len(self._rbuf) < n:
new = self._ssl.read(self._rbufsize)
if not new: break
self._rbuf = self._rbuf + new
self._rbuf += new
data,self._rbuf = self._rbuf[:n],self._rbuf[n:]
return data
while 1:
new = self._ssl.read(self._rbufsize)
if not new: break
self._rbuf = self._rbuf + new
self._rbuf += new
data,self._rbuf = self._rbuf,""
return data
@ -124,10 +124,13 @@ class _fileobject:
new = self._ssl.read(self._rbufsize)
if not new: break
i = string.find(new,'\n')
if i >= 0: i = i + len(self._rbuf)
self._rbuf = self._rbuf + new
if i < 0: i = len(self._rbuf)
else: i = i+1
if i >= 0:
i += len(self._rbuf)
self._rbuf += new
if i < 0:
i = len(self._rbuf)
else:
i += 1
data,self._rbuf = self._rbuf[:i],self._rbuf[i:]
return data
@ -145,14 +148,15 @@ def _test():
opts, args = getopt.getopt(sys.argv[1:], 'd')
dl = 0
for o, a in opts:
if o == '-d': dl = dl + 1
if args[0:]: host = args[0]
if args[1:]: selector = args[1]
if o == '-d':
dl += 1
if args[0:]:
host = args[0]
if args[1:]:
selector = args[1]
h = HTTPS()
host = 'synergy.as.cmu.edu'
selector = '/~geek/'
# host = 'tls.cryptsoft.com'
# selector = '/'
h.set_debuglevel(dl)
h.connect(host)
h.putrequest('GET', selector)
@ -162,9 +166,10 @@ def _test():
print 'errmsg =', errmsg
print "\tHEADERS:"
if headers:
for header in headers.headers: print string.strip(header)
for header in headers.headers:
print string.strip(header)
print "\tTEXT:"
print h.getfile().read()
if __name__ == '__main__':
_test()
_test()

View file

@ -1,6 +1,6 @@
""" common CGI functions used by the CGI scripts
Copyright (C) 2000 Bastian Kleineidam
Copyright (C) 2000,2001 Bastian Kleineidam
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

View file

@ -228,7 +228,7 @@ class TimeoutSocket:
raise Timeout("Send timed out")
sentlen = sock.send(data, flags)
data = data[sentlen:]
totallen = totallen + sentlen
totallen += sentlen
return totallen
# end send
@ -282,7 +282,7 @@ class TimeoutFile:
buf = self.recv(bufsize)
if not buf:
break
data = data + buf
data += buf
if datalen > size > 0:
self._sock._inqueue = data[size:]
data = data[:size]
@ -305,10 +305,10 @@ class TimeoutFile:
buf = self.recv(bufsize)
if not buf:
break
data = data + buf
data += buf
if idx >= 0:
idx = idx + 1
idx += 1
self._sock._inqueue = data[idx:]
data = data[:idx]
elif size > 0 and datalen > size:

View file

@ -142,7 +142,7 @@ def hotzenplotz(w,y,x,l):
for li in l:
w.move(y,x)
w.addstr(li)
y = y+1
y += 1
def wischi(w, ls):
my,mx = w.getmaxyx()
@ -152,8 +152,8 @@ def wischi(w, ls):
up = 1
w.erase()
while i<11:
i = i+1
j = j + (up and 1 or -1)
i += 1
j += (up and 1 or -1)
if j==-1: up = 1
elif j==1: up = 0
hotzenplotz(w,my/3,mx/2+j,ls[j+1])

View file

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (C) 2000 Bastian Kleineidam
#!/usr/bin/env python2
# Copyright (C) 2000,2001 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -17,8 +17,8 @@
# imports and checks
import sys
if sys.version[:5] < "1.5.2":
raise SystemExit, "This program requires Python 1.5.2 or later."
if sys.version[:5] < "2.0":
raise SystemExit, "This program requires Python 2.0 or later."
import getopt,re,string,os,urlparse
# 90 seconds timeout for all connections
#import timeoutsocket

View file

@ -40,19 +40,19 @@
# HTML logger
[html]
#filename=linkchecker-out.html
#colorbackground=#ffffff
# colors for the various parts
#colorbackground="#ffffff"
#colorurl=blue
#colorborder=
#colorlink=
#tablewarning=
#tableok=
#tableok=
#rowend=
#myfont=
#tableerror=
# ANSI color logger
[colored]
#filename=linkchecker-out.ansi
# colors for the various parts
#colorparent=
#colorurl=
#colorname=

View file

@ -1,8 +1,9 @@
# we use the scripts in Tools/i18n of the Python 2.0 distribution
# we copied the scripts in Tools/i18n of the Python 2.0 distribution
I18NTOOLS=.
GETTEXT=python $(I18NTOOLS)/pygettext.py
#MSGFMT=python $(I18NTOOLS)/msgfmt.py
MSGFMT=msgfmt
PYTHON=python2.0
GETTEXT=$(PYTHON) $(I18NTOOLS)/pygettext.py
MSGFMT=$(PYTHON) $(I18NTOOLS)/msgfmt.py
#MSGFMT=msgfmt
MSGMERGE=msgmerge
SOURCES=\
../linkcheck/Config.py \
@ -36,8 +37,10 @@ all:
$(GETTEXT) --default-domain=$(PACKAGE) --no-location $(SOURCES)
touch .time.pot
# for GNU msgfmt use -o flag:
#$(MSGFMT) $< -o $@ && mkdir -p ../locale/$*/LC_MESSAGES && mv $@ ../locale/$*/LC_MESSAGES/$(PACKAGE).mo
%.mo: %.po
$(MSGFMT) $< -o $@ && mkdir -p ../locale/$*/LC_MESSAGES && mv $@ ../locale/$*/LC_MESSAGES/$(PACKAGE).mo
$(MSGFMT) $< && mkdir -p ../locale/$*/LC_MESSAGES && mv $@ ../locale/$*/LC_MESSAGES/$(PACKAGE).mo
clean:
rm -f .time.pot

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2
# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
@ -21,21 +21,25 @@ Options:
"""
import sys, getopt, struct, array, string
import sys
import getopt
import struct
import array
__version__ = "1.0"
MESSAGES = {}
def usage(code, msg=''):
sys.stderr.write(__doc__)
print >> sys.stderr, __doc__
if msg:
sys.stderr.write(msg)
print >> sys.stderr, msg
sys.exit(code)
def add(id, str, fuzzy):
"Add a non-fuzzy translation to the dictionary."
global MESSAGES
@ -43,7 +47,7 @@ def add(id, str, fuzzy):
MESSAGES[id] = str
def generate():
"Return the generated output."
global MESSAGES
@ -56,8 +60,8 @@ def generate():
# For each string, we need size and file offset. Each string is NUL
# terminated; the NUL does not count into the size.
offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
ids = ids + id + '\0'
strs = strs + MESSAGES[id] + '\0'
ids += id + '\0'
strs += MESSAGES[id] + '\0'
output = ''
# The header is 7 32-bit unsigned integers. We don't use hash tables, so
# the keys start right after the index tables.
@ -70,8 +74,8 @@ def generate():
# The string table first has the list of keys, then the list of values.
# Each entry has first the size of the string, then the file offset.
for o1, l1, o2, l2 in offsets:
koffsets = koffsets + [l1, o1+keystart]
voffsets = voffsets + [l2, o2+valuestart]
koffsets += [l1, o1+keystart]
voffsets += [l2, o2+valuestart]
offsets = koffsets + voffsets
output = struct.pack("iiiiiii",
0x950412de, # Magic
@ -80,19 +84,19 @@ def generate():
7*4, # start of key index
7*4+len(keys)*8, # start of value index
0, 0) # size and offset of hash table
output = output + array.array("i", offsets).tostring()
output = output + ids
output = output + strs
output += array.array("i", offsets).tostring()
output += ids
output += strs
return output
def make(filename):
ID = 1
STR = 2
# Compute .mo name from .po name
if filename[-3:] == '.po':
if filename.endswith('.po'):
infile = filename
outfile = filename[:-2] + 'mo'
else:
@ -101,7 +105,7 @@ def make(filename):
try:
lines = open(infile).readlines()
except IOError, msg:
sys.stderr.write(msg)
print >> sys.stderr, msg
sys.exit(1)
section = None
@ -110,42 +114,43 @@ def make(filename):
# Parse the catalog
lno = 0
for l in lines:
lno = lno + 1
lno += 1
# If we get a comment line after a msgstr, this is a new entry
if l[0] == '#' and section == STR:
add(msgid, msgstr, fuzzy)
section = None
fuzzy = 0
# Record a fuzzy mark
if l[:2] == '#,' and string.find(l, 'fuzzy') != -1:
if l[:2] == '#,' and l.find('fuzzy'):
fuzzy = 1
# Skip comments
if l[0] == '#':
continue
# Now we are in a msgid section, output previous section
if l[:5] == 'msgid':
if l.startswith('msgid'):
if section == STR:
add(msgid, msgstr, fuzzy)
section = ID
l = l[5:]
msgid = msgstr = ''
# Now we are in a msgstr section
elif l[:6] == 'msgstr':
elif l.startswith('msgstr'):
section = STR
l = l[6:]
# Skip empty lines
l = string.strip(l)
l = l.strip()
if not l:
continue
# XXX: Does this always follow Python escape semantics?
l = eval(l)
if section == ID:
msgid = msgid + l
msgid += l
elif section == STR:
msgstr = msgstr + l
msgstr += l
else:
sys.stderr.write('Syntax error on %s:%d\n'
'before: %s\n' % (infile, lno, l))
print >> sys.stderr, 'Syntax error on %s:%d' % (infile, lno), \
'before:'
print >> sys.stderr, l
sys.exit(1)
# Add last entry
if section == STR:
@ -158,10 +163,10 @@ def make(filename):
try:
open(outfile,"wb").write(output)
except IOError,msg:
sys.stderr.write(msg)
print >> sys.stderr, msg
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'hV', ['help','version'])
@ -173,12 +178,12 @@ def main():
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-V', '--version'):
sys.stderr.write("msgfmt.py %s" % __version__)
print >> sys.stderr, "msgfmt.py", __version__
sys.exit(0)
# do it
if not args:
sys.stderr.write('No input file given\n')
sys.stderr.write("Try `msgfmt --help' for more information.\n")
print >> sys.stderr, 'No input file given'
print >> sys.stderr, "Try `msgfmt --help' for more information."
return
for filename in args:

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#! /usr/bin/env python2
# Originally written by Barry Warsaw <bwarsaw@python.org>
#
# minimally patched to make it even more xgettext compatible
@ -134,14 +134,21 @@ If `inputfile' is -, standard input is read.
""")
import os, sys, time, getopt, tokenize, string
import os
import sys
import time
import getopt
import tokenize
__version__ = '1.1'
default_keywords = ['_']
DEFAULTKEYWORDS = string.join(default_keywords, ', ')
DEFAULTKEYWORDS = ', '.join(default_keywords)
EMPTYSTRING = ''
# The normal pot-file header. msgmerge and EMACS' po-mode work better if
# it's there.
pot_header = _('''\
@ -162,7 +169,7 @@ msgstr ""
''')
def usage(code, msg=''):
print __doc__ % globals()
if msg:
@ -170,7 +177,7 @@ def usage(code, msg=''):
sys.exit(code)
escapes = []
def make_escapes(pass_iso8859):
@ -199,7 +206,7 @@ def escape(s):
s = list(s)
for i in range(len(s)):
s[i] = escapes[ord(s[i])]
return string.join(s, EMPTYSTRING)
return EMPTYSTRING.join(s)
def safe_eval(s):
@ -210,7 +217,7 @@ def safe_eval(s):
def normalize(s):
# This converts the various Python string types into a format that is
# appropriate for .po files, namely much closer to C style.
lines = string.split(s, '\n')
lines = s.split('\n')
if len(lines) == 1:
s = '"' + escape(s) + '"'
else:
@ -220,11 +227,11 @@ def normalize(s):
for i in range(len(lines)):
lines[i] = escape(lines[i])
lineterm = '\\n"\n"'
s = '""\n"' + string.join(lines, lineterm) + '"'
s = '""\n"' + lineterm.join(lines) + '"'
return s
class TokenEater:
def __init__(self, options):
self.__options = options
@ -256,7 +263,7 @@ class TokenEater:
# of messages seen. Reset state for the next batch. If there
# were no strings inside _(), then just ignore this entry.
if self.__data:
msg = string.join(self.__data, EMPTYSTRING)
msg = EMPTYSTRING.join(self.__data)
if not msg in self.__options.toexclude:
entry = (self.__curfile, self.__lineno)
linenos = self.__messages.get(msg)
@ -309,7 +316,7 @@ class TokenEater:
finally:
sys.stdout = sys.__stdout__
def main():
global default_keywords
try:
@ -439,8 +446,8 @@ def main():
if closep:
fp.close()
if __name__ == '__main__':
main()
# some more test strings
#_(u'a unicode string')
_(u'a unicode string')

View file

@ -1,3 +1,3 @@
#!/bin/sh
python setup.py config -lcrypto
env CFLAGS="-Wall" python setup.py build
python2 setup.py config -lcrypto
python2 setup.py build

View file

@ -12,3 +12,4 @@ doc_files = INSTALL
provides = linkchecker
group = Web/Utilities
build_script = rpm_build_script
python = python2

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2
# Copyright (C) 2000 Bastian Kleineidam
#
@ -115,7 +115,7 @@ class MyConfig(config):
def finalize_options(self):
# we have some default include and library directories
# suitable for each platform
self.basic_finalize_options()
config.finalize_options(self)
if self.ssl_include_dirs is None:
if os.name=='posix':
self.ssl_include_dirs = ['/usr/include/openssl',
@ -131,27 +131,6 @@ class MyConfig(config):
self.ssl_library_dirs = []
def basic_finalize_options(self):
"""fix up types of option values"""
# this should be in config.finalize_options
# I submitted a patch
# ok, its in 1.0.1, but I still leave this here for compatibility
if self.include_dirs is None:
self.include_dirs = self.distribution.include_dirs or []
elif type(self.include_dirs) is StringType:
self.include_dirs = string.split(self.include_dirs, os.pathsep)
if self.libraries is None:
self.libraries = []
elif type(self.libraries) is StringType:
self.libraries = [self.libraries]
if self.library_dirs is None:
self.library_dirs = []
elif type(self.library_dirs) is StringType:
self.library_dirs = string.split(self.library_dirs, os.pathsep)
def run (self):
# try to compile a test program with SSL
config.run(self)
@ -189,7 +168,7 @@ class MyDistribution(Distribution):
import LinkCheckerConf
if 'bdist_wininst' in self.commands and os.name!='nt':
self.announce("bdist_wininst command found on non-Windows "
"platform. Disabling SSL compilation")
"platform. Disabling SSL compilation")
elif LinkCheckerConf.have_ssl:
self.ext_modules = [Extension('ssl', ['ssl.c'],
include_dirs=LinkCheckerConf.ssl_include_dirs,
@ -214,8 +193,8 @@ class MyDistribution(Distribution):
myname = "Bastian Kleineidam"
myemail = "calvin@users.sourceforge.net"
setup (name = "LinkChecker",
version = "1.2.14",
setup (name = "linkchecker",
version = "1.3.0",
description = "check HTML documents for broken links",
author = myname,
author_email = myemail,