Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-22 07:04:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #591 from cjmayo/robot

Browse source 
Assume robots.txt is UTF-8
This commit is contained in:
Chris Mayo 2021-12-15 19:31:00 +00:00 committed by GitHub
commit 5f3b007934
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
linkcheck/robotparser2.py
View file

@ -92,8 +92,9 @@ class RobotFileParser:
try:
response = self.session.get(self.url, **kwargs)
response.raise_for_status()
log.debug(LOG_CHECK, "Robots response headers: %s", response.headers)
content_type = response.headers.get('content-type')
self.encoding = response.encoding
self.encoding = response.encoding = "utf-8"
if content_type and content_type.lower().startswith('text/plain'):
self.parse(response.iter_lines(decode_unicode=True))
else: