Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-05-03 12:24:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #325 from linkchecker/type-error-in-robot-parser

Browse source 
Fix TypeError: string arg required in content_allows_robots()
This commit is contained in:
Marius Gedminas 2019-10-22 18:07:31 +03:00 committed by GitHub
commit c6de64978c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 11 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
linkcheck/checker/httpurl.py
View file

@ -88,7 +88,7 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
parser.encoding = self.charset
# parse
try:
parser.feed(self.get_content())
parser.feed(self.get_raw_content())
parser.flush()
except linkparse.StopParse as msg:
log.debug(LOG_CHECK, "Stopped parsing: %s", msg)

5
tests/checker/data/http_utf8.html Normal file
View file

@ -0,0 +1,5 @@
<!DOCTYPE html>
<html lang="en">
<meta charset="utf-8">
<p>Some text — but with an em-dash.</p>
</html>

4
tests/checker/data/http_utf8.html.result Normal file
View file

@ -0,0 +1,4 @@
url http://localhost:%(port)d/%(datadir)s/http_utf8.html
cache key http://localhost:%(port)d/%(datadir)s/http_utf8.html
real url http://localhost:%(port)d/%(datadir)s/http_utf8.html
valid

1
tests/checker/test_http.py
View file

@ -39,6 +39,7 @@ class TestHttp (HttpServerTest):
self.file_test("http_slash.html", confargs=confargs)
self.file_test("http.xhtml", confargs=confargs)
self.file_test("http_file.html", confargs=confargs)
self.file_test("http_utf8.html", confargs=confargs)
def test_status(self):
for status in sorted(self.handler.responses.keys()):