Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-15 10:03:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use the same request arguments for redirects.

Browse source
This commit is contained in:
Bastian Kleineidam 2014-04-23 22:03:44 +02:00
parent 981079c041
commit 6bae3e0f49
1 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
linkcheck/checker/httpurl.py
View file

@ -163,15 +163,8 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
"""Send request and store response in self.url_connection."""
# throttle the number of requests to each host
self.aggregate.wait_for_host(self.urlparts[1])
kwargs = dict(
stream=True,
timeout=self.aggregate.config["timeout"],
allow_redirects=False,
)
if self.scheme == u"https" and self.aggregate.config["sslverify"]:
kwargs['verify'] = self.aggregate.config["sslverify"]
else:
kwargs['verify'] = False
kwargs = self.get_request_kwargs()
kwargs["allow_redirects"] = False
self._send_request(request, **kwargs)
def _send_request(self, request, **kwargs):
@ -230,20 +223,29 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
return ('location' in self.headers and
self.url_connection.status_code in REDIRECT_STATI)
def get_request_kwargs(self):
"""Construct keyword parameters for Session.request() and
Session.resolve_redirects()."""
kwargs = dict(stream=True, timeout=self.aggregate.config["timeout"])
if self.scheme == u"https" and self.aggregate.config["sslverify"]:
kwargs['verify'] = self.aggregate.config["sslverify"]
else:
kwargs['verify'] = False
return kwargs
def follow_redirections(self, request):
"""Follow all redirections of http response."""
log.debug(LOG_CHECK, "follow all redirections")
if self.is_redirect():
# run plugins for old connection
self.aggregate.plugin_manager.run_connection_plugins(self)
kwargs = dict(
stream=True,
)
kwargs = self.get_request_kwargs()
response = None
for response in self.session.resolve_redirects(self.url_connection, request, **kwargs):
newurl = response.url
log.debug(LOG_CHECK, "Redirected to %r", newurl)
self.aliases.append(newurl)
# XXX on redirect errors this is not printed
self.add_info(_("Redirected to `%(url)s'.") % {'url': newurl})
self.urlparts = strformat.url_unicode_split(newurl)
self.build_url_parts()