Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-04 21:04:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

getRobotsTxtUrl function

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@938 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2003-07-04 15:06:35 +00:00
parent 5e69d743d0
commit 1d9a192f0e
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
linkcheck/HttpUrlData.py
View file

@ -314,8 +314,12 @@ class HttpUrlData (ProxyUrlData):
return 1
def getRobotsTxtUrl (self):
return self.urlparts[0]+"://"+self.urlparts[1]+"/robots.txt"
def robotsTxtAllowsUrl (self):
roboturl = self.urlparts[0]+"://"+self.urlparts[1]+"/robots.txt"
roboturl = self.getRobotsTxtUrl()
debug(HURT_ME_PLENTY, "robots.txt url", roboturl)
debug(HURT_ME_PLENTY, "url", self.url)
if not self.config.robotsTxtCache_has_key(roboturl):