Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-11 08:03:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

more response closing, and cleanups

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3443 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2006-09-15 19:51:02 +00:00
parent 6fe2db6755
commit d6676ab0a0
1 changed files with 9 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
linkcheck/checker/httpurl.py
View file

@ -205,7 +205,10 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
@return: response or None if url is already handled
@rtype: HttpResponse or None
"""
response = None
while True:
if response is not None:
response.close()
try:
response = self._get_http_response()
except linkcheck.httplib2.BadStatusLine:
@ -250,6 +253,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
if tries == -1:
assert None == linkcheck.log.debug(linkcheck.LOG_CHECK,
"already handled")
response.close()
return None
if tries >= self.max_redirects:
if self.method == "HEAD":
@ -406,7 +410,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
if response.status == 204:
# no content
self.add_warning(unicode_safe(response.reason),
tag="http-empty-content")
tag="http-empty-content")
# store cookies for valid links
if self.aggregate.config['storecookies']:
for c in self.cookies:
@ -484,11 +488,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
name = c.client_header_name()
value = c.client_header_value()
self.url_connection.putheader(name, value)
try:
self.url_connection.endheaders()
except:
print "XXX", self.url_connection
raise
self.url_connection.endheaders()
response = self.url_connection.getresponse()
self.persistent = not response.will_close
self.timeout = headers.http_timeout(response)
@ -522,8 +522,8 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
elif scheme == "https" and supportHttps:
h = linkcheck.httplib2.HTTPSConnection(host)
else:
raise linkcheck.LinkCheckerError(
_("Unsupported HTTP url scheme %r") % scheme)
msg = _("Unsupported HTTP url scheme %r") % scheme
raise linkcheck.LinkCheckerError(msg)
if linkcheck.log.is_debug(linkcheck.LOG_CHECK):
h.set_debuglevel(1)
h.connect()
@ -560,6 +560,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
f = StringIO.StringIO(self.data)
self.data = f.read()
self.downloadtime = time.time() - t
response.close()
return self.data
def is_html (self):