From 29921eb3efbe355de3fde1e060a95dbfa01dc3cb Mon Sep 17 00:00:00 2001 From: calvin Date: Thu, 4 Jan 2001 23:26:53 +0000 Subject: [PATCH] robotparser2 git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@217 e7d03fd6-7b0d-0410-9947-9c21f3af8025 --- linkcheck/robotparser2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/linkcheck/robotparser2.py b/linkcheck/robotparser2.py index 6f6ab6ee..5094e361 100755 --- a/linkcheck/robotparser2.py +++ b/linkcheck/robotparser2.py @@ -50,13 +50,12 @@ class RobotFileParser: import httplib tries = 0 while tries<5: - _debug(self.host) - _debug(self.path) + _debug(self.host+self.path) connection = httplib.HTTP(self.host) connection.putrequest("GET", self.path) connection.endheaders() status, text, mime = connection.getreply() - if status in [301,302]: + if status in [301,302] and mime: tries = tries + 1 newurl = mime.get("Location", mime.get("Uri", "")) newurl = urlparse.urljoin(self.url, newurl)