Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-05 07:20:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix non-ascii HTTP header debugging.

Browse source
This commit is contained in:
Bastian Kleineidam 2012-03-09 11:54:18 +01:00
parent 187a94312b
commit 5e13a78f66
4 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
doc/changelog.txt
View file

@ -6,6 +6,9 @@ Fixes:
Closes: SF bug #3495407
- checking: Fix non-ascii HTTP header handling.
Closes: SF bug #3495621
- checking: Fix non-ascii HTTP header debugging.
Closes: SF bug #3488675
7.5 "Kukushka" (released 13.02.2012)

8
linkcheck/checker/httpurl.py
View file

@ -37,6 +37,9 @@ from .const import WARN_HTTP_ROBOTS_DENIED, \
WARN_HTTP_DECOMPRESS_ERROR, WARN_HTTP_UNSUPPORTED_ENCODING, \
WARN_HTTP_AUTH_UNKNOWN
# assumed HTTP header encoding
HEADER_ENCODING = "iso-8859-1"
# helper alias
unicode_safe = strformat.unicode_safe
@ -197,7 +200,8 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
response.reason = unicode_safe(response.reason)
log.debug(LOG_CHECK,
"Response: %s %s", response.status, response.reason)
log.debug(LOG_CHECK, "Headers: %s", self.headers)
uheaders = unicode_safe(self.headers, encoding=HEADER_ENCODING)
log.debug(LOG_CHECK, "Headers: %s", uheaders)
# proxy enforcement (overrides standard proxy)
if response.status == 305 and self.headers:
oldproxy = (self.proxy, self.proxyauth)
@ -453,7 +457,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
value = self.headers.get(name)
if value is None:
return default
return value.decode("iso-8859-1", "replace")
return unicode_safe(value, encoding=HEADER_ENCODING)
def get_alias_cache_data (self):
"""

2
linkcheck/i18n.py
View file

@ -1,5 +1,5 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2000-2011 Bastian Kleineidam
# Copyright (C) 2000-2012 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

6
linkcheck/strformat.py
View file

@ -1,5 +1,5 @@
# -*- coding: iso-8859-1 -*-
# Copyright (C) 2000-2011 Bastian Kleineidam
# Copyright (C) 2000-2012 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -34,7 +34,7 @@ import pydoc
from . import i18n
def unicode_safe (s, encoding=i18n.default_encoding):
def unicode_safe (s, encoding=i18n.default_encoding, errors='replace'):
"""Get unicode string without raising encoding errors. Unknown
characters of the given encoding will be ignored.
@ -48,7 +48,7 @@ def unicode_safe (s, encoding=i18n.default_encoding):
if isinstance(s, unicode):
# s is already unicode, nothing to do
return s
return unicode(str(s), encoding, "ignore")
return unicode(str(s), encoding, errors)
def ascii_safe (s):