Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-26 17:14:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use timeout value from configuration.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-10-10 10:53:52 +02:00
parent 871508ef5d
commit f484a6776d
8 changed files with 37 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
linkcheck/checker/ftpurl.py
View file

@ -25,8 +25,6 @@ from .. import log, LOG_CHECK, LinkCheckerError, fileutil
from . import proxysupport, httpurl, internpaturl, get_index_html, pooledconnection
from .const import WARN_FTP_MISSING_SLASH
DEFAULT_TIMEOUT_SECS = 300
class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport, pooledconnection.PooledConnection):
"""
@ -74,7 +72,7 @@ class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport, pooledco
"""Log into ftp server and check the welcome message."""
def create_connection(scheme, host, port):
"""Create a new ftp connection."""
connection = ftplib.FTP()
connection = ftplib.FTP(timeout=self.aggregate.config["timeout"])
if log.is_debug(LOG_CHECK):
connection.set_debuglevel(1)
return connection

26
linkcheck/checker/httpheaders.py
View file

@ -18,7 +18,7 @@
Helper functions dealing with HTTP headers.
"""
DEFAULT_TIMEOUT_SECS = 300
DEFAULT_KEEPALIVE = 300
MAX_HEADER_BYTES = 8*1024
@ -54,25 +54,25 @@ def http_persistent (response):
return has_header_value(headers, "Connection", "Keep-Alive")
def http_timeout (response):
def http_keepalive (headers):
"""
Get HTTP timeout value, either from the Keep-Alive header or a
Get HTTP keepalive value, either from the Keep-Alive header or a
default value.
@param response: response instance
@type response: httplib.HTTPResponse
@return: timeout
@param headers: HTTP headers
@type headers: dict
@return: keepalive in seconds
@rtype: int
"""
timeout = response.getheader("Keep-Alive")
if timeout is not None:
keepalive = headers.get("Keep-Alive")
if keepalive is not None:
try:
timeout = int(timeout[8:].strip())
except ValueError:
timeout = DEFAULT_TIMEOUT_SECS
keepalive = int(keepalive[8:].strip())
except (ValueError, OverflowError):
keepalive = DEFAULT_KEEPALIVE
else:
timeout = DEFAULT_TIMEOUT_SECS
return timeout
keepalive = DEFAULT_KEEPALIVE
return keepalive
def get_content_type (headers):

12
linkcheck/checker/httpurl.py
View file

@ -528,7 +528,6 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport, pooledc
self.add_connection_headers()
buffering = True
response = self.url_connection.getresponse(buffering)
self.timeout = headers.http_timeout(response)
self.headers = response.msg
self.content_type = None
self.persistent = not response.will_close
@ -650,13 +649,13 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport, pooledc
self.close_connection()
def create_connection(scheme, host, port):
"""Create a new http or https connection."""
kwargs = dict(port=port, strict=True, timeout=self.aggregate.config["timeout"])
if scheme == "http":
strict = True
h = httplib.HTTPConnection(host, port, strict)
h = httplib.HTTPConnection(host, **kwargs)
elif scheme == "https" and supportHttps:
devel_dir = os.path.join(configuration.configdata.install_data, "config")
ca_certs = configuration.get_share_file(devel_dir, 'ca-certificates.crt')
h = httplib.HTTPSConnection(host, port, ca_certs=ca_certs)
kwargs["ca_certs"] = configuration.get_share_file(devel_dir, 'ca-certificates.crt')
h = httplib.HTTPSConnection(host, **kwargs)
else:
msg = _("Unsupported HTTP url scheme `%(scheme)s'") % {"scheme": scheme}
raise LinkCheckerError(msg)
@ -830,8 +829,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport, pooledc
# add to cached connections
scheme, host, port = self.get_netloc()
if self.persistent and self.url_connection.is_idle():
# XXX use self.repsonse
expiration = time.time() + self.timeout
expiration = time.time() + headers.http_keepalive(self.headers)
else:
expiration = None
self.aggregate.connections.release(scheme, host, port, self.url_connection, expiration=expiration)

4
linkcheck/checker/mailtourl.py
View file

@ -32,8 +32,6 @@ from ..network import iputil
from .const import WARN_MAIL_NO_MX_HOST, \
WARN_MAIL_UNVERIFIED_ADDRESS, WARN_MAIL_NO_CONNECTION
DEFAULT_TIMEOUT_SECS = 300
def getaddresses (addr):
"""Return list of email addresses from given field value."""
@ -308,7 +306,7 @@ class MailtoUrl (urlbase.UrlBase):
try:
log.debug(LOG_CHECK,
"SMTP check for %r (preference %d)", host, preference)
self.url_connection = smtplib.SMTP(timeout=DEFAULT_TIMEOUT_SECS)
self.url_connection = smtplib.SMTP(timeout=self.aggregate.config["timeout"])
if log.is_debug(LOG_CHECK):
self.url_connection.set_debuglevel(1)
self.url_connection.connect(host)

2
linkcheck/checker/telneturl.py
View file

@ -54,7 +54,7 @@ class TelnetUrl (urlbase.UrlBase):
Open a telnet connection and try to login. Expected login
label is "login: ", expected password label is "Password: ".
"""
self.url_connection = telnetlib.Telnet()
self.url_connection = telnetlib.Telnet(timeout=self.aggregate.config["timeout"])
if log.is_debug(LOG_CHECK):
self.url_connection.set_debuglevel(1)
self.url_connection.open(self.host, self.port)

3
linkcheck/configuration/__init__.py
View file

@ -25,6 +25,7 @@ import logging.config
import urllib
import urlparse
import shutil
import socket
import _LinkChecker_configdata as configdata
from .. import (log, LOG_CHECK, LOG_ROOT, ansicolor, lognames, clamav,
get_config_dir, fileutil, configdict)
@ -354,6 +355,8 @@ class Configuration (dict):
if self['loginurl']:
self.sanitize_loginurl()
self.sanitize_proxies()
# set default socket timeout
socket.setdefaulttimeout(self['timeout'])
def sanitize_anchors (self):
"""Make anchor configuration consistent."""

23
linkcheck/configuration/confparse.py
View file

@ -69,13 +69,16 @@ class LCConfigParser (ConfigParser.RawConfigParser, object):
if self.has_option(section, option):
self.config[option] = self.getboolean(section, option)
def read_int_option (self, section, option, key=None, allownegative=False):
def read_int_option (self, section, option, key=None, min=None, max=None):
"""Read an integer option."""
if self.has_option(section, option):
num = self.getint(section, option)
if not allownegative and num < 0:
if min is not None and num < min:
raise LinkCheckerError(
_("invalid negative value for %s: %d\n") % (option, num))
_("invalid value for %s: %d must not be less than %d") % (option, num, min))
if max is not None and num < max:
raise LinkCheckerError(
_("invalid value for %s: %d must not be greater than %d") % (option, num, max))
if key is None:
key = option
self.config[key] = num
@ -128,19 +131,19 @@ class LCConfigParser (ConfigParser.RawConfigParser, object):
def read_checking_config (self):
"""Read configuration options in section "checking"."""
section = "checking"
self.read_int_option(section, "threads", allownegative=True)
self.read_int_option(section, "threads", min=-1)
self.config['threads'] = max(0, self.config['threads'])
self.read_int_option(section, "timeout")
self.read_int_option(section, "timeout", min=1)
self.read_boolean_option(section, "anchors")
self.read_int_option(section, "recursionlevel", allownegative=True)
self.read_int_option(section, "recursionlevel", min=-1)
if self.has_option(section, "warningregex"):
val = self.get(section, "warningregex")
if val:
self.config["warningregex"] = re.compile(val)
self.read_int_option(section, "warnsizebytes")
self.read_int_option(section, "warnsizebytes", min=1)
self.read_string_option(section, "nntpserver")
self.read_string_option(section, "useragent")
self.read_int_option(section, "pause", key="wait")
self.read_int_option(section, "pause", key="wait", min=0)
self.read_check_options(section)
def read_check_options (self, section):
@ -155,8 +158,8 @@ class LCConfigParser (ConfigParser.RawConfigParser, object):
self.getboolean(section, "cookies")
self.read_string_option(section, "cookiefile")
self.read_string_option(section, "localwebroot")
self.read_int_option(section, "warnsslcertdaysvalid")
self.read_int_option(section, "maxrunseconds")
self.read_int_option(section, "warnsslcertdaysvalid", min=1)
self.read_int_option(section, "maxrunseconds", min=0)
def read_authentication_config (self):
"""Read configuration options in section "authentication"."""

2
linkchecker
View file

@ -25,7 +25,6 @@ import codecs
import re
import os
import pprint
import socket
import optparse
import getpass
# installs _() and _n() gettext functions into global namespace
@ -598,7 +597,6 @@ if options.timeout is not None:
else:
print_usage(_("Illegal argument %(arg)r for option %(option)s") % \
{"arg": options.timeout, "option": "'--timeout'"})
socket.setdefaulttimeout(config["timeout"])
if options.version is not None:
print_version()
if options.verbose is not None: