Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-18 11:31:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #572 from cjmayo/latin1

Browse source 
Ignore an encoding of ISO-8859-1 returned by Requests
This commit is contained in:
Chris Mayo 2021-11-29 19:55:16 +00:00 committed by GitHub
commit 606472e910
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
linkcheck/checker/httpurl.py
View file

@ -177,7 +177,14 @@ class HttpUrl(internpaturl.InternPatternUrl, proxysupport.ProxySupport):
log.debug(LOG_CHECK, "Request headers %s", request.headers)
self.url_connection = self.session.send(request, **kwargs)
self.headers = self.url_connection.headers
self.encoding = self.url_connection.encoding
log.debug(LOG_CHECK, "Response headers %s", self.headers)
if self.url_connection.encoding == "ISO-8859-1":
# Can't trust ISO-8859-1 because it is Requests' fallback for text
# content-types. We fall back to it in UrlBase.get_content() if
# Beautiful Soup doesn't return an encoding.
self.encoding = None
else:
self.encoding = self.url_connection.encoding
log.debug(LOG_CHECK, "Response encoding %s", self.encoding)
self._add_ssl_info()