Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-24 14:13:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

robotparser2

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@217 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2001-01-04 23:26:53 +00:00
parent 3254f921ee
commit 29921eb3ef
1 changed files with 2 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
linkcheck/robotparser2.py
View file

@ -50,13 +50,12 @@ class RobotFileParser:
import httplib import httplib
tries = 0 tries = 0
while tries<5: while tries<5:
_debug(self.host) _debug(self.host+self.path)
_debug(self.path)
connection = httplib.HTTP(self.host) connection = httplib.HTTP(self.host)
connection.putrequest("GET", self.path) connection.putrequest("GET", self.path)
connection.endheaders() connection.endheaders()
status, text, mime = connection.getreply() status, text, mime = connection.getreply()
if status in [301,302]: if status in [301,302] and mime:
tries = tries + 1 tries = tries + 1
newurl = mime.get("Location", mime.get("Uri", "")) newurl = mime.get("Location", mime.get("Uri", ""))
newurl = urlparse.urljoin(self.url, newurl) newurl = urlparse.urljoin(self.url, newurl)