mirror of
https://github.com/Hopiu/linkchecker.git
synced 2026-03-24 18:00:24 +00:00
3eac1be9ab
Use Python 2.5 features and get rid of old compat code. Also some code cleanups have been made. git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3737 e7d03fd6-7b0d-0410-9947-9c21f3af8025
78 lines
2.7 KiB
Python
78 lines
2.7 KiB
Python
# -*- coding: iso-8859-1 -*-
|
|
# Copyright (C) 2006-2008 Bastian Kleineidam
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
|
"""
|
|
Aggregate needed object instances for checker threads.
|
|
"""
|
|
from .. import log, LOG_CHECK
|
|
import linkcheck.director
|
|
import logger
|
|
import status
|
|
import checker
|
|
import cleanup
|
|
|
|
|
|
class Aggregate (object):
|
|
"""Store thread-safe data collections for checker threads."""
|
|
|
|
def __init__ (self, config, urlqueue, connections, cookies, robots_txt):
|
|
self.config = config
|
|
self.urlqueue = urlqueue
|
|
self.connections = connections
|
|
self.cookies = cookies
|
|
self.robots_txt = robots_txt
|
|
self.logger = logger.Logger(config)
|
|
self.threads = []
|
|
|
|
def start_threads (self):
|
|
"""Spawn threads for URL checking and status printing."""
|
|
if self.config["status"]:
|
|
t = status.Status(self.urlqueue)
|
|
t.start()
|
|
self.threads.append(t)
|
|
t = cleanup.Cleanup(self.connections)
|
|
t.start()
|
|
self.threads.append(t)
|
|
num = self.config["threads"]
|
|
if num >= 1:
|
|
for dummy in xrange(num):
|
|
t = checker.Checker(self.urlqueue, self.logger)
|
|
t.start()
|
|
self.threads.append(t)
|
|
else:
|
|
checker.check_url(self.urlqueue, self.logger)
|
|
|
|
def abort (self):
|
|
"""Empty the URL queue."""
|
|
self.urlqueue.do_shutdown()
|
|
try:
|
|
self.urlqueue.join(timeout=self.config["timeout"])
|
|
except linkcheck.cache.urlqueue.Timeout:
|
|
log.warn(LOG_CHECK, "Abort timed out")
|
|
|
|
def remove_stopped_threads (self):
|
|
"Remove the stopped threads from the internal thread list."""
|
|
self.threads = [t for t in self.threads if t.isAlive()]
|
|
|
|
def finish (self):
|
|
"""Wait for checker threads to finish."""
|
|
assert self.urlqueue.empty()
|
|
for t in self.threads:
|
|
t.stop()
|
|
t.join(2)
|
|
if t.isAlive():
|
|
log.warn(LOG_CHECK, "Thread %s still active", t)
|
|
self.connections.clear()
|