diff --git a/ChangeLog b/ChangeLog index cfd52f0b..d93f57c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,11 @@ Type: bugfix Changed: linkcheck/checker/httpurl.py + * Workaround a urllib2.py bug raising ValueError on some failed + HTTP authorisations. + Type: bugfix + Changed: linkcheck/robotparser2.py + 3.1 "Suspicious" (released 18.7.2005) * Updated documentation for the HTML parser. diff --git a/linkcheck/robotparser2.py b/linkcheck/robotparser2.py index 4dc83b67..a3251723 100755 --- a/linkcheck/robotparser2.py +++ b/linkcheck/robotparser2.py @@ -207,6 +207,12 @@ class RobotFileParser (object): self.allow_all = True debug("robots.txt allow all") return + except ValueError: + # XXX bug workaround: + # urllib2.AbstractDigestAuthHandler raises ValueError on + # failed authorisation + self.disallow_all = True + debug("robotst.txt disallow all") lines = [] line = f.readline() while line: