Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-15 01:53:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add callback after robots.txt parse

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3225 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2006-05-16 23:19:18 +00:00
parent 2aed0f3bc5
commit fb319b3785
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
linkcheck/cache/robots_txt.py vendored
View file

@ -37,7 +37,7 @@ class RobotsTxt (object):
self.cache = {}
@synchronized(_lock)
def allows_url (self, roboturl, url, user, password):
def allows_url (self, roboturl, url, user, password, callback=None):
"""
Ask robots.txt allowance.
"""
@ -46,6 +46,8 @@ class RobotsTxt (object):
user=user, password=password)
rp.set_url(roboturl)
rp.read()
if callback is not None:
callback(rp)
self.cache[roboturl] = rp
else:
rp = self.cache[roboturl]