Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-03-23 09:20:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix get_content() function.

Browse source
This commit is contained in:
Bastian Kleineidam 2010-10-03 12:11:25 +02:00
parent 59b54730de
commit a68329329f
1 changed files with 7 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
linkcheck/url.py
View file

@ -546,16 +546,19 @@ def get_content (url, user=None, password=None, proxy=None):
from . import configuration
headers = {
'User-Agent': configuration.UserAgent,
'Accept-Encoding' : 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5',
# makes problems with some sites
#'Accept-Encoding': 'gzip;q=1.0, deflate;q=0.9, identity;q=0.5',
}
req = urllib2.Request(url, None, headers)
try:
f = get_opener(user=user, password=password, proxy=proxy)
res = None
try:
f.open(req)
return (f.info(), f.read())
res = f.open(req)
return (res.info(), res.read())
finally:
f.close()
if res is not None:
res.close()
except (urllib2.HTTPError, socket.timeout, urllib2.URLError,
socket.gaierror, socket.error, IOError, httplib.HTTPException,
ValueError):