Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-19 03:51:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove lock from check_url in case of disabled threading

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@2617 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2005-05-19 18:30:06 +00:00
parent 3407ab362d
commit d6e56ceba2
2 changed files with 21 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
linkcheck/checker/__init__.py
View file

@ -134,8 +134,21 @@ def _check_urls (consumer):
start_time = time.time()
status_time = start_time
while not consumer.finished():
if not consumer.check_url():
url_data = consumer.incoming_get_url()
if url_data is None:
# wait for incoming queue to fill
time.sleep(0.1)
elif url_data.cached:
# was cached -> can be logged
consumer.log_url(url_data)
else:
# go check this url
if url_data.parent_url and not url_is_absolute(url_data.base_url):
name = url_data.parent_url
else:
name = u""
name += url_data.base_url
consumer.check_url(url_data, name)
if consumer.config('status'):
curtime = time.time()
if (curtime - status_time) > 5:

28
linkcheck/checker/consumer.py
View file

@ -95,29 +95,15 @@ class Consumer (object):
# can be logged
self._log_url(url_data)
@synchronized(_lock)
def check_url (self):
def check_url (self, url_data, name):
"""
Start new thread checking the given url.
Check given URL data, spawning a new thread with given name.
This eventually calls either Consumer.checked() or
Consumer.interrupted().
This method is not thread safe (hence it should only be called
from a single thread).
"""
url_data = self._cache.incoming_get_url()
if url_data is None:
# active connections are downloading/parsing
pass
elif url_data.cached:
# was cached -> can be logged
self._log_url(url_data)
else:
# go check this url
# this calls either self.checked() or self.interrupted()
if url_data.parent_url and \
not linkcheck.url.url_is_absolute(url_data.base_url):
name = url_data.parent_url
else:
name = u""
name += url_data.base_url
self._threader.start_thread(url_data.check, (), name=name)
return url_data and not url_data.cached
self._threader.start_thread(url_data.check, (), name=name)
@synchronized(_lock)
def checked (self, url_data):