profiling

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@111 e7d03fd6-7b0d-0410-9947-9c21f3af8025

MANIFEST.in

@@ -12,5 +12,5 @@ include debian/dirs
 include debian/docs
 include DNS/README
 include linkcheck/__init__.py.tmpl
-recursive-include test *.html *.py
+recursive-include test *.html *.py *.tmpl
 recursive-include locale *.mo *.po

debian/changelog

@@ -20,8 +20,11 @@ linkchecker (1.2.3) unstable; urgency=low
   * Only catch some exceptions in main check loop so the KeyboardInterrupt
     exception propagates through
   * Disable threading on non-POSIX systems
+  * Renice the main thread loop by sleep()ing 1/3 of a second in each
+    loop
+  * New function config.reset()
 
- -- Bastian Kleineidam <calvin@users.sourceforge.net>  Sun, 11 Jun 2000 21:28:48 +0200
+ -- Bastian Kleineidam <calvin@users.sourceforge.net>  Mon, 12 Jun 2000 14:50:44 +0200
 
 linkchecker (1.2.2) unstable; urgency=low
 

linkcheck/Config.py

@@ -21,7 +21,7 @@ This module stores
 * Other configuration options
 """
 
-import ConfigParser,sys,os,re,UserDict,string
+import ConfigParser,sys,os,re,UserDict,string,time
 from os.path import expanduser,normpath,normcase,join,isfile
 from types import StringType
 import Logging
@@ -86,6 +86,10 @@ class Configuration(UserDict.UserDict):
     def __init__(self):
         """Initialize the default options"""
         UserDict.UserDict.__init__(self)
+        self.reset()
+
+    def reset(self):
+        """Reset to default values"""
         self.data["verbose"] = 0
         self.data["warnings"] = 0
         self.data["anchors"] = 0
@@ -288,7 +292,7 @@ class Configuration(UserDict.UserDict):
                 value = sys.exc_info()[1]
                 self.debug("NNTP: "+value+"\n")
                 if re.compile("^505").search(str(value)):
-                    import whrandom,time
+                    import whrandom
                     time.sleep(whrandom.randint(30,60))
                 else:
                     raise
@@ -297,6 +301,7 @@ class Configuration(UserDict.UserDict):
         return not self.urls.empty()
 
     def finished_Threads(self):
+        time.sleep(0.3)
         self.threader.reduceThreads()
         return not self.hasMoreUrls() and self.threader.finished()
 

test/profiletest.py.tmpl

@@ -7,12 +7,23 @@ $syspath
 import linkcheck
 url='http://www.yahoo.de/'
 config = linkcheck.Config.Configuration()
-config['recursionlevel'] = 5
+
+config['recursionlevel'] = 3
 config['anchors'] = 1
-config['internlinks'].append(re.compile(r"^(ftp|https?)://www\.yahoo\.de/"))
+config['internlinks'].append(re.compile(r"^(ftp|https?)://.*yahoo.*"))
 # avoid checking of local files (security!)
 config["externlinks"].append((re.compile("^file:"), 1))
 config.appendUrl(linkcheck.UrlData.GetUrlDataFrom(url, 0))
-profile.run("linkcheck.checkUrls(config)", "test.prof")
-p = pstats.Stats("test.prof")
-p.strip_dirs().sort_stats("time").print_stats(20)
+profile.run("linkcheck.checkUrls(config)", "threads.prof")
+
+config.reset()
+
+config.disableThreading()
+config['recursionlevel'] = 3
+config['anchors'] = 1
+config['internlinks'].append(re.compile(r"^(ftp|https?)://.*yahoo.*"))
+# avoid checking of local files (security!)
+config["externlinks"].append((re.compile("^file:"), 1))
+config.appendUrl(linkcheck.UrlData.GetUrlDataFrom(url, 0))
+profile.run("linkcheck.checkUrls(config)", "nothreads.prof")
+

test/viewprof.py

@@ -0,0 +1,4 @@
+import pstats,glob
+
+for file in glob.glob('*.prof'):
+    pstats.Stats(file).strip_dirs().sort_stats("time").print_stats(20)