Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-03-16 22:10:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove duplicate code in http checker.

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3820 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2008-06-16 19:52:09 +00:00
parent 7f756ca40c
commit b30fb3b09c
2 changed files with 10 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
linkcheck/checker/const.py
View file

@ -144,6 +144,11 @@ PARSE_MIMETYPES = (
"application/x-shockwave-flash",
)
HTML_MIMETYPES = (
"text/html",
"application/xhtml+xml",
)
# if file extension lookup was unsuccessful, look at the content
PARSE_CONTENTS = {
"html": re.compile(r'^(?i)<(!DOCTYPE html|html|head|title)'),

17
linkcheck/checker/httpurl.py
View file

@ -36,7 +36,7 @@ from .const import WARN_HTTP_ROBOTS_DENIED, WARN_HTTP_NO_ANCHOR_SUPPORT, \
WARN_HTTP_WRONG_REDIRECT, WARN_HTTP_MOVED_PERMANENT, \
WARN_HTTP_EMPTY_CONTENT, WARN_HTTP_COOKIE_STORE_ERROR, \
WARN_HTTP_DECOMPRESS_ERROR, WARN_HTTP_UNSUPPORTED_ENCODING, \
PARSE_MIMETYPES
PARSE_MIMETYPES, HTML_MIMETYPES
# helper alias
unicode_safe = strformat.unicode_safe
@ -606,7 +606,7 @@ Use URL %(newurl)s instead for checking.""") % {
"""
if not (self.valid and self.headers):
return False
if headers.get_content_type(self.headers) not in ("text/html", "application/xhtml+xml"):
if headers.get_content_type(self.headers) not in HTML_MIMETYPES:
return False
return self.encoding_supported()
@ -638,23 +638,16 @@ Use URL %(newurl)s instead for checking.""") % {
return False
if headers.get_content_type(self.headers) not in PARSE_MIMETYPES:
return False
encoding = headers.get_content_encoding(self.headers)
if encoding and encoding not in _supported_encodings and \
encoding != 'identity':
self.add_warning(_('Unsupported content encoding %(encoding)r.') %
{"encoding": encoding},
tag=WARN_HTTP_UNSUPPORTED_ENCODING)
return False
return True
return self.encoding_supported()
def parse_url (self):
"""
Parse file contents for new links to check.
"""
ctype = headers.get_content_type(self.headers)
if ctype in ("text/html", "application/xhtml+xml"):
if self.is_html():
self.parse_html()
elif ctype == "text/css":
elif self.is_css():
self.parse_css()
elif ctype == "application/x-shockwave-flash":
self.parse_swf()