fix broken HttpUrlData method request

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@287 e7d03fd6-7b0d-0410-9947-9c21f3af8025

DNS/Base.py

@@ -18,12 +18,21 @@ defaults['server']=[]
 
 
 def init_dns_resolver():
+    """init the DNS resolver. This function can be called more than once"""
     global defaults
+    # reset to old values
+    defaults['server'] = []
+    defaults['search_domains'] = []
+    # platform specific config
     import os
     if os.name=="posix":
         init_dns_resolver_posix()
     elif os.name=="nt":
         init_dns_resolver_nt()
+    else:
+        # other platforms not supported (what about Mac?)
+        pass
+    # default values
     if not defaults['search_domains']:
         defaults['search_domains'].append('')
     if not defaults['server']:
@@ -31,6 +40,7 @@ def init_dns_resolver():
 
 def init_dns_resolver_posix():
     "parses the /etc/resolv.conf file and sets defaults for name servers"
+    # XXX this needs to be dynamic?
     global defaults
     for line in open('/etc/resolv.conf', 'r').readlines():
         line = line.strip()
@@ -50,21 +60,42 @@ def init_dns_resolver_nt():
     """Windows network config read from registry"""
     import winreg
     global defaults
+    key = None
     try:
         key = winreg.key_handle(winreg.HKEY_LOCAL_MACHINE,
-                 r"SYSTEM\CurrentControlSet\Services\Tcpip\Parameters")
-    except WindowsError:
-        # key not found :(
-        return
-    if key.get("EnableDhcp"):
-        nameserver = key.get("DhcpNameServer")
-    else:
-        nameserver = key.get("NameServer")
-    if nameserver:
-        defaults['server'].append(nameserver)
-    searchlist = key.get("SearchList", [])
-    for domain in searchlist:
-        defaults['search_domains'].append('.'+domain.lower())
+               r"SYSTEM\CurrentControlSet\Services\Tcpip\Parameters")
+    except EnvironmentError:
+        try: # for Windows ME
+            key = winreg.key_handle(winreg.HKEY_LOCAL_MACHINE,
+                    r"SYSTEM\CurrentControlSet\Services\VxD\MSTCP")
+        except EnvironmentError:
+            pass
+    if key:
+        nameserver = key["NameServer"][0] or ""
+        for server in nameserver.split(","):
+            defaults['nameserver'].append(server)
+    # XXX search for "EnableDhcp", "DhcpNameServer", "SearchList"
+
+    try: # for win2000
+        key = winreg.handle_key(winreg.HKEY_LOCAL_MACHINE,
+  r"SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\DNSRegisteredAdapters")
+        for subkey in key.subkeys():
+            count, counttype = subkey['DNSServerAddressCount']
+            values, valuestype = subkey['DNSServerAddresses']
+            for server in winreg.binipdisplay(values):
+                defaults['nameserver'].append(server)
+    except EnvironmentError:
+        pass
+
+    try: # for whistler
+        key = winreg.handle_key(winreg.HKEY_LOCAL_MACHINE,
+           r"SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces")
+        for subkey in key.subkeys():
+            nameserver = subkey['NameServer'][0] or ""
+            for server in winreg.stringdisplay(nameserver):
+	        defaults['nameserver'].append(server)
+    except EnvironmentError:
+        pass
 
 
 class DnsRequest:

DNS/winreg.py

@@ -23,6 +23,16 @@ class key_handle:
         except IndexError:
             return default
 
+    def subkeys(self):
+        i = 0
+        keys = []
+        while 1:
+            try:
+                keys.append(OpenKey(self._key, EnumKey(self._key, i)))
+            except EnvironmentError:
+                break
+        return keys
+
     def __len__(self):
         return QueryInfoKey(self._key)[0]
 
@@ -33,6 +43,30 @@ class key_handle:
         pass
 
 
+# helper functions from pydns at sourceforge
+# (c) 2001 Copyright by Wolfgang Strobl ws@mystrobl.de,
+#          License analog to the current Python license
+
+def binipdisplay(s):
+    "convert a binary array of ip adresses to a python list"
+    if len(s)%4!= 0:
+        raise EnvironmentError # well ...
+    ol=[]
+    for i in range(len(s)/4):
+        s1=s[:4]
+        s=s[4:]
+        ip=[]
+        for j in s1:
+            ip.append(str(ord(j)))
+        ol.append('.'.join(ip))
+    return ol
+
+def stringdisplay(s):
+    'convert "d.d.d.d,d.d.d.d" to ["d.d.d.d","d.d.d.d"]'
+    return s.split(",")
+
+#################################################################
+
 def test():
     pass
 

linkcheck/Config.py

@@ -60,7 +60,7 @@ DebugLevel = 0
 # note: debugging with more than 1 thread can be painful
 def debug(level, *args):
     if DebugLevel > level:
-        sys.stderr.write("DEBUG: ")
+        sys.stderr.write("DEBUG(%d):"%level)
         for arg in args:
             sys.stderr.write(" %s"%str(arg))
         sys.stderr.write("\n")

linkcheck/FileUrlData.py

@@ -64,9 +64,15 @@ class FileUrlData(UrlData):
 
 
     def isHtml(self):
-        return html_re.search(self.url) or opera_re.search(self.url) or \
-               html_content_re.search(self.getContent()) or \
-               opera_content_re.search(self.getContent())
+        if html_re.search(self.url) or opera_re.search(self.url):
+            return 1
+        # try to read content (attention: could be a directory)
+        try:
+            return html_content_re.search(self.getContent()) or \
+                   opera_content_re.search(self.getContent())
+        except IOError:
+            pass
+        return None
 
 
     def parseUrl(self, config):
@@ -77,15 +83,15 @@ class FileUrlData(UrlData):
         # parse an opera bookmark file
         name = ""
         lineno = 0
-        # XXX use iterators for this?
-        for line in StringUtil.lines(self.getContent()):
+        for line in self.getContent().split("\n"):
             lineno += 1
             line = line.strip()
-            if line.startwith("NAME="):
+            if line.startswith("NAME="):
                 name = line[5:]
             elif line.startswith("URL="):
                 url = line[4:]
                 if url:
+                    from UrlData import GetUrlDataFrom
                     config.appendUrl(GetUrlDataFrom(url,
                         self.recursionLevel+1, self.url, None, lineno, name))
                 name = ""

linkcheck/HttpUrlData.py

@@ -82,7 +82,7 @@ class HttpUrlData(UrlData):
             return
 
         # first try
-        status, statusText, self.mime = self._getHttpRequest(self.urlTuple[1])
+        status, statusText, self.mime = self._getHttpRequest()
         Config.debug(BRING_IT_ON, status, statusText, self.mime)
         has301status = 0
         while 1:
@@ -137,7 +137,7 @@ class HttpUrlData(UrlData):
             #   content-type
             elif status in [405,501,500]:
                 # HEAD method not allowed ==> try get
-                self.setWarning(_("Server does not support HEAD request, falling back to GET"))
+                self.setWarning(_("Server does not support HEAD request (got %d status), falling back to GET")%status)
                 status, statusText, self.mime = self._getHttpRequest("GET")
             elif status>=400 and self.mime:
                 server = self.mime.getheader("Server")

linkchecker

@@ -20,8 +20,9 @@
 import sys
 if sys.version[:5] < "2.0":
     raise SystemExit, "This program requires Python 2.0 or later."
+
 import getopt, re, os, urlparse, linkcheck
-from linkcheck import _,StringUtil
+from linkcheck import _,StringUtil,timeoutsocket
 
 
 Usage = _("""USAGE\tlinkchecker [options] file-or-url...
@@ -81,8 +82,8 @@ For single-letter option arguments the space is not a necessity. So
         Generate no more than num threads. Default number of threads is 5.
         To disable threading specify a non-positive number.
 --timeout=secs
-        Set the timeout for connection attempts in seconds. Default timeout
-        is system dependant.
+        Set the timeout for connection attempts in seconds. the default
+        timeout is system dependant.
 -u name, --user=name
         Try username name for HTML and FTP authorization.
         Default is 'anonymous'. See also -p.
@@ -274,7 +275,6 @@ for opt,arg in options:
                 config.disableThreading()
 
     elif opt=="--timeout":
-        import timeoutsocket
         timeoutsocket.setDefaultSocketTimeout(int(arg))
 
     elif opt=="-u" or opt=="--user":

linkchecker.1

@@ -98,6 +98,10 @@ Check only the syntax of extern links, do not try to connect to them.
 Generate no more than \fInum\fP threads. Default number of threads is 5.
 To disable threading specify a non-positive number.
 .TP
+\fB--timeout=\fIsecs\fP
+Set the timeout for connection attempts in seconds. The default timeout
+is system dependant.
+.TP
 \fB-u \fIname\fP, \fB--user=\fIname\fP
 Try username \fIname\fP for HTML and FTP authorization.
 Default is \fIanonymous\fP. See also \fB-p\fP.