remove DNS Tests

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@244 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2001-04-03 18:59:38 +00:00
parent 8f4260458a
commit 62cbdf15d9
24 changed files with 170 additions and 190 deletions

View file

@ -6,7 +6,7 @@ include lc.cgi lc.fcgi lc.sz_fcgi
include Makefile
include create.sql
include debian/rules debian/changelog debian/copyright debian/control
include debian/dirs debian/docs debian/links debian/postinst
include debian/docs debian/links debian/postinst
include debian/prerm
include DNS/README
include test/viewprof.py test/profiletest.py test/*.html test/robots.txt
@ -15,4 +15,4 @@ recursive-include locale *.mo
recursive-include po *.po *.py Makefile
recursive-include lconline *
recursive-include tests *.py
exclude linkcheckerConf.py
exclude _linkchecker_configdata.py

View file

@ -7,7 +7,6 @@ NAME=$(shell ./setup.py --name)
HOST=treasure.calvinsplayground.de
#LCOPTS=-ocolored -Ftext -Fhtml -Fgml -Fsql -Fcsv -Fxml -R -t0 -v -s
LCOPTS=-ocolored -Ftext -Fhtml -Fgml -Fsql -Fcsv -Fxml -R -t0 -v -s
DEBPACKAGE = $(PACKAGE)_$(VERSION)_all.deb $(PACKAGE)-ssl_$(VERSION)_i386.deb
OFFLINETESTS = test_base test_misc test_file test_frames
ONLINETESTS = test_mail test_http test_https test_news test_ftp
@ -24,7 +23,7 @@ clean:
distclean: clean cleandeb
rm -rf dist build # just to be sure clean also the build dir
rm -f $(PACKAGE)-out.* VERSION $(PACKAGE)Conf.py MANIFEST Packages.gz
rm -f $(PACKAGE)-out.* VERSION _$(PACKAGE)_configdata.py MANIFEST Packages.gz
cleandeb:
rm -rf debian/$(PACKAGE) debian/$(PACKAGE)-ssl debian/tmp
@ -43,8 +42,11 @@ deb:
fakeroot debian/rules binary
fakeroot dpkg-buildpackage -sgpg -pgpg -k959C340F
package:
cd dist && dpkg-scanpackages . ../override.txt | gzip --best > Packages.gz
packages:
-cd .. && dpkg-scanpackages . | gzip --best > Packages.gz
sources:
-cd .. && dpkg-scansources . | gzip --best > Sources.gz
files: locale
env http_proxy="" ./$(PACKAGE) $(LCOPTS) -i$(HOST) http://$(HOST)/~calvin/
@ -52,7 +54,7 @@ files: locale
VERSION:
echo $(VERSION) > VERSION
upload: distclean dist package files VERSION
upload: distclean dist files VERSION
scp debian/changelog shell1.sourceforge.net:/home/groups/$(PACKAGE)/htdocs/changes.txt
scp README shell1.sourceforge.net:/home/groups/$(PACKAGE)/htdocs/readme.txt
scp linkchecker-out.* shell1.sourceforge.net:/home/groups/$(PACKAGE)/htdocs

23
debian/changelog vendored
View file

@ -6,7 +6,28 @@ linkchecker (1.3.0) unstable; urgency=low
Jamie Heilman <jamie@audible.transient.net>)
* config file option to control which fields the loggers should print
-- Bastian Kleineidam <calvin@users.sourceforge.net> Thu, 22 Feb 2001 01:35:50 +0100
-- Bastian Kleineidam <calvin@debian.org> Thu, 22 Feb 2001 01:35:50 +0100
linkchecker (1.2.15) unstable; urgency=low
* remove ssl modules for main linkchecker package; make a separate
package linkchecker-ssl for non-US
* adjust build-depends and depends
* fix anchor checking
* debian standards version 3.5.2.0
* require python-distutils >= 1.0.1, removed old compatibility code
* added FAQ entry for broken links
* change email address to calvin@debian.org
* add DNS.Error to caught exceptions
* add all html link tags (ripped from HTML::Tagset.pm)
* updated robotparser2.py to newest version
* catch errors when calling UrlData.checkContent
* added linkcheckerrc to the conffiles
* disable threading in the lc.cgi script so the machine wont be
hogged (closes: Bug#86788)
* fix https support: add -lssl when compiling ssl module
-- Bastian Kleineidam <calvin@debian.org> Sun, 4 Mar 2001 20:55:02 +0100
linkchecker (1.2.14) unstable; urgency=low

11
debian/control vendored
View file

@ -2,14 +2,14 @@ Source: linkchecker
Section: web
Priority: optional
Maintainer: Bastian Kleineidam <calvin@debian.org>
Build-Depends: python2-base, python2-dev, debhelper (>= 3.0.0), libssl096-dev
Build-Depends: python2-base, python2-dev, debhelper (>= 3.0.0)
Build-Depends-Indep: gettext
Standards-Version: 3.5.2
Package: linkchecker
Architecture: all
Depends: python2-base
Suggests: linkchecker-ssl
Suggests: linkchecker-ssl (>= ${Source-Version})
Description: check HTML documents for broken links
Features:
o recursive checking
@ -25,10 +25,3 @@ Description: check HTML documents for broken links
o i18n support
o command line interface
o (Fast)CGI web interface (requires HTTP server)
Package: linkchecker-ssl
Architecture: any
Depends: linkchecker (>= 1.3.0)
Description: HTTPS support for LinkChecker
Includes the Python modules 'ssl' and 'httpslib' to support https://
links.

6
debian/copyright vendored
View file

@ -1,10 +1,10 @@
This is LinkChecker, written and maintained by Bastian Kleineidam
<calvin@users.sourceforge.net> on Sat, 8 Jan 2000 11:00:35 +0100.
This is linkchecker, written and maintained by Bastian Kleineidam
<calvin@debian.org> on Sat, 8 Jan 2000 11:00:35 +0100.
The original source can always be found at:
http://linkchecker.sourceforge.net/
Copyright (C) 2000 Bastian Kleineidam
Copyright (C) 2000,2001 Bastian Kleineidam
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

View file

@ -1,2 +1,3 @@
usr/lib/python2.0/site-packages/linkcheck/httpslib.py
usr/lib/python2.0/site-packages/ssl.so
usr/lib/python2.0/site-packages/linkcheckssl/httpslib.py
usr/lib/python2.0/site-packages/linkcheckssl/ssl.so
usr/lib/python2.0/site-packages/linkcheckssl/__init__.py

View file

@ -3,21 +3,21 @@
# Written 1998 by Gregor Hoffleit <flight@debian.org>.
# used by Bastian Kleineidam for LinkChecker
DIRLIST="/usr/lib/python1.5/site-packages/linkcheck"
FILELIST="linkcheckerConf.py"
SITEPACKAGES="/usr/lib/python1.5/site-packages"
DIRLIST="/usr/lib/python2.0/site-packages/linkcheck"
FILELIST="_linkchecker_configdata.py"
SITEPACKAGES="/usr/lib/python2.0/site-packages"
COMMAND="'import sys,py_compile;py_compile.compile(sys.argv[1])'"
case "$1" in
configure|abort-upgrade|abort-remove|abort-deconfigure)
for i in $DIRLIST; do
python -O /usr/lib/python1.5/compileall.py -q $i
python /usr/lib/python1.5/compileall.py -q $i
python2 -O /usr/lib/python2.0/compileall.py -q $i
python2 /usr/lib/python2.0/compileall.py -q $i
done
# use /bin/sh -c, otherwise I get a SyntaxError from Python
for i in $FILELIST; do
/bin/sh -c "python -O -c $COMMAND $SITEPACKAGES/$i"
/bin/sh -c "python -c $COMMAND $SITEPACKAGES/$i"
/bin/sh -c "python2 -O -c $COMMAND $SITEPACKAGES/$i"
/bin/sh -c "python2 -c $COMMAND $SITEPACKAGES/$i"
done
;;
*)

View file

@ -9,7 +9,7 @@ dpkg --listfiles $PACKAGE |
awk '$0~/\.py$/ {print $0"c\n" $0"o"}' |
xargs rm -f >&2
rmdir /usr/lib/python1.5/site-packages/linkcheck 2>/dev/null || true
rmdir /usr/lib/python2.0/site-packages/linkcheck 2>/dev/null || true
#DEBHELPER#

View file

@ -188,7 +188,8 @@ class HttpUrlData(UrlData):
return httplib.HTTP(host)
def getContent(self):
if not self.data:
if not self.has_content:
self.has_content = 1
self.closeConnection()
t = time.time()
status, statusText, self.mime = self._getHttpRequest("GET")
@ -198,12 +199,14 @@ class HttpUrlData(UrlData):
self._init_html_comments()
Config.debug("DEBUG: comment spans %s\n" % self.html_comments)
return self.data
def isHtml(self):
if not (self.valid and self.mime):
return 0
return self.mime.gettype()[:9]=="text/html"
def robotsTxtAllowsUrl(self, config):
roboturl="%s://%s/robots.txt" % self.urlTuple[0:2]
if not config.robotsTxtCache_has_key(roboturl):

View file

@ -17,11 +17,9 @@
import string,re,sys,htmlentitydefs
HtmlTable = []
UnHtmlTable = []
for ent,ch in htmlentitydefs.entitydefs.items():
HtmlTable.append((ch, "&"+ent+";"))
UnHtmlTable.append(("&"+ent+";", ch))
entities = htmlentitydefs.entitydefs.items()
HtmlTable = map(lambda x: (x[1], "&"+x[0]+";"), entities)
UnHtmlTable = map(lambda x: ("&"+x[0]+";", x[1]), entities)
# order matters!
HtmlTable.sort()
UnHtmlTable.sort()
@ -153,9 +151,11 @@ def htmlify(str):
"Escape special HTML chars and strings"
return applyTable(HtmlTable, str)
def unhtmlify(str):
return applyTable(UnHtmlTable, str)
def getLineNumber(str, index):
"return the line number of str[index]"
i=0

View file

@ -52,6 +52,7 @@ _linkMatcher = r"""
> # close tag
"""
# ripped mainly from HTML::Tagset.pm
LinkTags = (
("a", ["href"]),
@ -103,6 +104,11 @@ BasePattern = {
'attr': 'href',
}
CommentPattern = re.compile("<!--.*?--\s*>", re.DOTALL)
# Workaround for Python 2.0 re module bug
CommentPatternBegin = re.compile("<!--")
CommentPatternEnd = re.compile("--\s*>")
class UrlData:
"Representing a URL with additional information like validity etc"
@ -133,6 +139,7 @@ class UrlData:
self.extern = 1
self.data = None
self.html_comments = []
self.has_content = 0
def setError(self, s):
@ -327,7 +334,8 @@ class UrlData:
def getContent(self):
"""Precondition: urlConnection is an opened URL."""
if not self.data:
if not self.has_content:
self.has_content = 1
t = time.time()
self.data = self.urlConnection.read()
self.downloadtime = time.time() - t
@ -339,13 +347,16 @@ class UrlData:
def _init_html_comments(self):
# if we find an URL inside HTML comments we ignore it
# so build a list of intervalls which are HTML comments
pattern = re.compile("<!--.*?--\s*>", re.DOTALL)
index = 0
while 1:
match = pattern.search(self.data, index)
match = CommentPatternBegin.search(self.getContent(), index)
if not match: break
index = match.end()
self.html_comments.append(match.span())
start = match.start()
index = match.end() + 1
end = CommentPatternEnd.search(self.getContent(), index)
if not match: break
index = match.end() + 1
self.html_comments.append(start, match.end())
def _isInComment(self, index):
for low,high in self.html_comments:

View file

@ -42,8 +42,8 @@ except ImportError:
# set _ as an alias for gettext
_ = gettext
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(20)
#import timeoutsocket
#timeoutsocket.setDefaultSocketTimeout(20)
import Config,UrlData,sys,lc_cgi

1
linkcheckssl/__init__.py Normal file
View file

@ -0,0 +1 @@
"""ssl wrapper module"""

View file

@ -31,12 +31,18 @@ msgstr "gefunden"
msgid "outside of domain filter, checked only syntax"
msgstr "außerhalb des Domain Filters; prüfe lediglich Syntax"
msgid "warning: no files or urls given"
msgstr "Warnung: keine Dateien oder URLs angegeben"
msgid "Result"
msgstr "Ergebnis"
msgid "Warning"
msgstr "Warnung"
msgid "Illegal argument '%s' for option "
msgstr "Ungültiges Argument '%s' für Option "
#, fuzzy
msgid ""
"EXAMPLES\n"
@ -225,6 +231,9 @@ msgstr ""
msgid "%.3f seconds\n"
msgstr "%.3f Sekunden\n"
msgid "realurl"
msgstr ""
msgid ""
"<html><head><title>LinkChecker Online Error</title></head><body "
"text=\"#192c83\" bgcolor=\"#fff7e5\" link=\"#191c83\" "
@ -261,22 +270,29 @@ msgstr "Fehler"
msgid "1 warning, "
msgstr "1 Warnung, "
msgid "Illegal argument '%s' for option "
msgstr "Ungültiges Argument '%s' für Option "
#, fuzzy
msgid "info"
msgstr "Info"
#, fuzzy
msgid "base"
msgstr "Basis"
msgid "Valid"
msgstr "Gültig"
msgid "hours"
msgstr "Stunden"
#, fuzzy
msgid "HTTPS url ignored"
msgstr "Javascript url ignoriert"
msgid "Start checking at %s\n"
msgstr "Beginne Prüfen am %s\n"
msgid "D/L Time"
msgstr "D/L Zeit"
msgid "Base"
msgstr "Basis"
msgid "seconds"
msgstr "Sekunden"
@ -308,8 +324,16 @@ msgid "HTTP 301 (moved permanent) encountered: you should update this link"
msgstr ""
"HTTP 301 (moved permanent) gefunden: Sie sollten diesen Link aktualisieren"
msgid "Base"
msgstr "Basis"
#, fuzzy
msgid "checktime"
msgstr "Prüfzeit"
msgid "dltime"
msgstr ""
#, fuzzy
msgid "HTTPS url ignored"
msgstr "Javascript url ignoriert"
msgid "Check Time"
msgstr "Prüfzeit"
@ -320,6 +344,9 @@ msgstr "Beende Pr
msgid "Group %s has %s articles, range %s to %s"
msgstr "Gruppe %s hat %s Artikel, von %s bis %s"
msgid "Javascript url ignored"
msgstr "Javascript url ignoriert"
msgid "No user or password found"
msgstr "Kein Benutzername oder Passwort gefunden"
@ -332,8 +359,8 @@ msgstr "Keine Antwort vom FTP Server"
msgid "found mail host %s"
msgstr "Mail host %s gefunden"
msgid "Javascript url ignored"
msgstr "Javascript url ignoriert"
msgid "parenturl"
msgstr ""
msgid "could not split the mail adress"
msgstr "konnte Mail Adresse nicht splitten"
@ -395,11 +422,13 @@ msgstr "Fehlendes '/' am Ende der URL"
msgid "# Format of the entries:\n"
msgstr "# Format der Einträge:\n"
msgid "warning: no files or urls given"
msgstr "Warnung: keine Dateien oder URLs angegeben"
#, fuzzy
msgid "result"
msgstr "Ergebnis"
msgid "Valid"
msgstr "Gültig"
#, fuzzy
msgid "warning"
msgstr "Warnung"
msgid " (cached)\n"
msgstr " (aus dem Cache)\n"

View file

@ -31,12 +31,18 @@ msgstr "trouv
msgid "outside of domain filter, checked only syntax"
msgstr "outside of domain filter, analyse de la syntaxe seulement"
msgid "warning: no files or urls given"
msgstr "attention: aucun fichier ou url donnée"
msgid "Result"
msgstr "Résultat"
msgid "Warning"
msgstr "Avertissement"
msgid "Illegal argument '%s' for option "
msgstr "Argumen illégal '%s' pour l'option "
#, fuzzy
msgid ""
"EXAMPLES\n"
@ -229,6 +235,9 @@ msgstr ""
msgid "%.3f seconds\n"
msgstr "%.3f secondes\n"
msgid "realurl"
msgstr ""
msgid ""
"<html><head><title>LinkChecker Online Error</title></head><body "
"text=\"#192c83\" bgcolor=\"#fff7e5\" link=\"#191c83\" "
@ -257,22 +266,28 @@ msgstr "Erreur"
msgid "1 warning, "
msgstr "1 avertissement, "
msgid "Illegal argument '%s' for option "
msgstr "Argumen illégal '%s' pour l'option "
msgid "info"
msgstr ""
#, fuzzy
msgid "base"
msgstr "Base"
msgid "Valid"
msgstr "Valide"
msgid "hours"
msgstr "heures"
#, fuzzy
msgid "HTTPS url ignored"
msgstr "Url Javascript ignorée"
msgid "Start checking at %s\n"
msgstr "Démarrage du controle à %s\n"
msgid "D/L Time"
msgstr "Durée D/L"
msgid "Base"
msgstr "Base"
msgid "seconds"
msgstr "secondes"
@ -303,8 +318,16 @@ msgstr "Ex
msgid "HTTP 301 (moved permanent) encountered: you should update this link"
msgstr "HTTP 301 (déplacé) rencontré: vous devez mettre à jour ce lien"
msgid "Base"
msgstr "Base"
#, fuzzy
msgid "checktime"
msgstr "Durée d'Analyse"
msgid "dltime"
msgstr ""
#, fuzzy
msgid "HTTPS url ignored"
msgstr "Url Javascript ignorée"
msgid "Check Time"
msgstr "Durée d'Analyse"
@ -315,6 +338,9 @@ msgstr "Arr
msgid "Group %s has %s articles, range %s to %s"
msgstr "Le groupe %s a %s articles, de %s à %s"
msgid "Javascript url ignored"
msgstr "Url Javascript ignorée"
msgid "No user or password found"
msgstr "Aucun utilisateur ou mot de passe trouvé"
@ -327,8 +353,8 @@ msgstr "N'obtient pas de r
msgid "found mail host %s"
msgstr "trouvé un hôte de messagerie %s"
msgid "Javascript url ignored"
msgstr "Url Javascript ignorée"
msgid "parenturl"
msgstr ""
msgid "could not split the mail adress"
msgstr "impossible de partager l'adresse e-mail"
@ -391,11 +417,13 @@ msgstr "Il manque '/'
msgid "# Format of the entries:\n"
msgstr "# Format des entrées:\n"
msgid "warning: no files or urls given"
msgstr "attention: aucun fichier ou url donnée"
#, fuzzy
msgid "result"
msgstr "Résultat"
msgid "Valid"
msgstr "Valide"
#, fuzzy
msgid "warning"
msgstr "Avertissement"
msgid " (cached)\n"
msgstr " (caché)\n"

View file

@ -152,7 +152,7 @@ class MyConfig(config):
class MyDistribution(Distribution):
def __init__(self, attrs=None):
Distribution.__init__(self, attrs=attrs)
self.config_file = self.get_name()+"Conf.py"
self.config_file = "_"+self.get_name()+"_configdata.py"
def run_commands(self):
@ -166,15 +166,16 @@ class MyDistribution(Distribution):
raise SystemExit, "please run 'python setup.py config'"
#self.announce("generating default configuration")
#self.run_command('config')
import linkcheckerConf
import _linkchecker_configdata
if 'bdist_wininst' in self.commands and os.name!='nt':
self.announce("bdist_wininst command found on non-Windows "
"platform. Disabling SSL compilation")
elif linkcheckerConf.have_ssl:
self.ext_modules = [Extension('ssl', ['ssl.c'],
include_dirs=linkcheckerConf.ssl_include_dirs,
library_dirs=linkcheckerConf.ssl_library_dirs,
libraries=linkcheckerConf.libraries)]
elif _linkchecker_configdata.have_ssl:
self.ext_modules = [Extension('linkcheckssl',
['linkcheckssl/ssl.c'],
include_dirs=_linkchecker_configdata.ssl_include_dirs,
library_dirs=_linkchecker_configdata.ssl_library_dirs,
libraries=_linkchecker_configdata.libraries)]
def create_conf_file(self, directory, data=[]):
@ -224,7 +225,7 @@ o a (Fast)CGI web interface (requires HTTP server)
'install': MyInstall,
'install_data': MyInstallData,
},
packages = ['','DNS','linkcheck'],
packages = ['','DNS','linkcheck','linkcheckssl'],
scripts = ['linkchecker'],
data_files = [('share/locale/de/LC_MESSAGES',
['locale/de/LC_MESSAGES/linkcheck.mo']),

View file

@ -13,6 +13,7 @@ Just some HTTP links
>invalid anchor</a>
<a href="HtTP://WWW.hEIsE.DE">should be cached</a>
<a href="HTTP://WWW.HEISE.DE">should be cached</a>
<a href="http://www.heise.de/?quoted=&uuml;">html entities</a>
<!-- <a href=http://nocheckin> no check because of comment -->
<a href=illegalquote1">no beginning quote</a>
<a href="illegalquote2>no ending quote</a>

View file

@ -1,22 +0,0 @@
#!/usr/bin/env python
import DNS
# automatically load nameserver(s) from /etc/resolv.conf
# (works on unix - on others, YMMV)
DNS.ParseResolvConf()
# lets do an all-in-one request
# set up the request object
r = DNS.DnsRequest(name='munnari.oz.au',qtype='A')
# do the request
a=r.req()
# and do a pretty-printed output
a.show()
# now lets setup a reusable request object
r = DNS.DnsRequest(qtype='ANY')
res = r.req("a.root-servers.nex",qtype='ANY')
res.show()
res = r.req("proxy.connect.com.au")
res.show()

View file

@ -1,17 +0,0 @@
#!/usr/bin/env python
import DNS
# automatically load nameserver(s) from /etc/resolv.conf
# (works on unix - on others, YMMV)
DNS.ParseResolvConf()
r=DNS.Request(qtype='mx')
res = r.req('connect.com.au')
res.show()
r=DNS.Request(qtype='soa')
res = r.req('connect.com.au')
res.show()
print DNS.revlookup('192.189.54.17')

View file

@ -1,13 +0,0 @@
#!/usr/bin/env python
import DNS
# automatically load nameserver(s) from /etc/resolv.conf
# (works on unix - on others, YMMV)
DNS.ParseResolvConf()
# web server reliability, the NT way. *snigger*
res = r.req('www.microsoft.com',qtype='A')
# res.answers is a list of dictionaries of answers
print len(res.answers),'different A records'
# each of these has an entry for 'data', which is the result.
print map(lambda x:x['data'], res.answers)

View file

@ -1,7 +0,0 @@
#!/usr/bin/env python
import DNS
DNS.ParseResolvConf()
print DNS.mxlookup("connect.com.au")

View file

@ -1,52 +0,0 @@
#!/usr/bin/env python
import DNS
DNS.ParseResolvConf()
def Error(mesg):
import sys
print sys.argv[0],"ERROR:"
print mesg
sys.exit(1)
def main():
import sys
if len(sys.argv) != 2:
Error("usage: %s somedomain.com"%sys.argv[0])
domain = sys.argv[1]
nslist = GetNS(domain)
print "According to the primary, the following are nameservers for this domain"
for ns in nslist:
print " ",ns
CheckNS(ns,domain)
def GetNS(domain):
import DNS
r = DNS.Request(domain,qtype='SOA').req()
if r.header['status'] != 'NOERROR':
Error("received status of %s when attempting to look up SOA for domain"%
(r.header['status']))
primary,email,serial,refresh,retry,expire,minimum = r.answers[0]['data']
print "Primary nameserver for domain %s is: %s"%(domain,primary)
r = DNS.Request(domain,qtype='NS',server=primary,aa=1).req()
if r.header['status'] != 'NOERROR':
Error("received status of %s when attempting to query %s for NSs"%
(r.header['status']))
if r.header['aa'] != 1:
Error("primary NS %s doesn't believe that it's authoritative!"% primary)
nslist = map(lambda x:x['data'], r.answers)
return nslist
def CheckNS(nameserver,domain):
r = DNS.Request(domain,qtype='SOA',server=nameserver,aa=1).req()
if r.header['status'] != 'NOERROR':
Error("received status of %s when attempting to query %s for NS"%
(r.header['status']))
if r.header['aa'] != 1:
Error("NS %s doesn't believe that it's authoritative!"% nameserver)
primary,email,serial,refresh,retry,expire,minimum = r.answers[0]['data']
print " NS has serial",serial[1]
if __name__ == "__main__":
main()