diff --git a/MANIFEST.in b/MANIFEST.in index c94bd2e9..67ac83e3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,5 +12,5 @@ include debian/dirs include debian/docs include DNS/README include linkcheck/__init__.py.tmpl -recursive-include test *.html *.py +recursive-include test *.html *.py *.tmpl recursive-include locale *.mo *.po diff --git a/debian/changelog b/debian/changelog index 1983e7d2..9ff28dff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -20,8 +20,11 @@ linkchecker (1.2.3) unstable; urgency=low * Only catch some exceptions in main check loop so the KeyboardInterrupt exception propagates through * Disable threading on non-POSIX systems + * Renice the main thread loop by sleep()ing 1/3 of a second in each + loop + * New function config.reset() - -- Bastian Kleineidam Sun, 11 Jun 2000 21:28:48 +0200 + -- Bastian Kleineidam Mon, 12 Jun 2000 14:50:44 +0200 linkchecker (1.2.2) unstable; urgency=low diff --git a/linkcheck/Config.py b/linkcheck/Config.py index 7ebac890..4d7bcb63 100644 --- a/linkcheck/Config.py +++ b/linkcheck/Config.py @@ -21,7 +21,7 @@ This module stores * Other configuration options """ -import ConfigParser,sys,os,re,UserDict,string +import ConfigParser,sys,os,re,UserDict,string,time from os.path import expanduser,normpath,normcase,join,isfile from types import StringType import Logging @@ -86,6 +86,10 @@ class Configuration(UserDict.UserDict): def __init__(self): """Initialize the default options""" UserDict.UserDict.__init__(self) + self.reset() + + def reset(self): + """Reset to default values""" self.data["verbose"] = 0 self.data["warnings"] = 0 self.data["anchors"] = 0 @@ -288,7 +292,7 @@ class Configuration(UserDict.UserDict): value = sys.exc_info()[1] self.debug("NNTP: "+value+"\n") if re.compile("^505").search(str(value)): - import whrandom,time + import whrandom time.sleep(whrandom.randint(30,60)) else: raise @@ -297,6 +301,7 @@ class Configuration(UserDict.UserDict): return not self.urls.empty() def finished_Threads(self): + time.sleep(0.3) self.threader.reduceThreads() return not self.hasMoreUrls() and self.threader.finished() diff --git a/test/profiletest.py.tmpl b/test/profiletest.py.tmpl index 41d48681..40f5ff41 100755 --- a/test/profiletest.py.tmpl +++ b/test/profiletest.py.tmpl @@ -7,12 +7,23 @@ $syspath import linkcheck url='http://www.yahoo.de/' config = linkcheck.Config.Configuration() -config['recursionlevel'] = 5 + +config['recursionlevel'] = 3 config['anchors'] = 1 -config['internlinks'].append(re.compile(r"^(ftp|https?)://www\.yahoo\.de/")) +config['internlinks'].append(re.compile(r"^(ftp|https?)://.*yahoo.*")) # avoid checking of local files (security!) config["externlinks"].append((re.compile("^file:"), 1)) config.appendUrl(linkcheck.UrlData.GetUrlDataFrom(url, 0)) -profile.run("linkcheck.checkUrls(config)", "test.prof") -p = pstats.Stats("test.prof") -p.strip_dirs().sort_stats("time").print_stats(20) +profile.run("linkcheck.checkUrls(config)", "threads.prof") + +config.reset() + +config.disableThreading() +config['recursionlevel'] = 3 +config['anchors'] = 1 +config['internlinks'].append(re.compile(r"^(ftp|https?)://.*yahoo.*")) +# avoid checking of local files (security!) +config["externlinks"].append((re.compile("^file:"), 1)) +config.appendUrl(linkcheck.UrlData.GetUrlDataFrom(url, 0)) +profile.run("linkcheck.checkUrls(config)", "nothreads.prof") + diff --git a/test/viewprof.py b/test/viewprof.py new file mode 100644 index 00000000..1f38d283 --- /dev/null +++ b/test/viewprof.py @@ -0,0 +1,4 @@ +import pstats,glob + +for file in glob.glob('*.prof'): + pstats.Stats(file).strip_dirs().sort_stats("time").print_stats(20)