From a68329329fa19c6f3b14a00a225a3bd1cfe8a740 Mon Sep 17 00:00:00 2001 From: Bastian Kleineidam Date: Sun, 3 Oct 2010 12:11:25 +0200 Subject: [PATCH] Fix get_content() function. --- linkcheck/url.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/linkcheck/url.py b/linkcheck/url.py index 9983aff3..f5febbc2 100644 --- a/linkcheck/url.py +++ b/linkcheck/url.py @@ -546,16 +546,19 @@ def get_content (url, user=None, password=None, proxy=None): from . import configuration headers = { 'User-Agent': configuration.UserAgent, - 'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5', + # makes problems with some sites + #'Accept-Encoding': 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5', } req = urllib2.Request(url, None, headers) try: f = get_opener(user=user, password=password, proxy=proxy) + res = None try: - f.open(req) - return (f.info(), f.read()) + res = f.open(req) + return (res.info(), res.read()) finally: - f.close() + if res is not None: + res.close() except (urllib2.HTTPError, socket.timeout, urllib2.URLError, socket.gaierror, socket.error, IOError, httplib.HTTPException, ValueError):