Hopiu/linkchecker
1
0
Fork 0
mirror of https://github.com/Hopiu/linkchecker.git synced 2026-04-01 21:50:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Close robotparser URL connections; simplify line parsing.

Browse source 
git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3853 e7d03fd6-7b0d-0410-9947-9c21f3af8025
This commit is contained in:
calvin 2008-07-28 16:56:24 +00:00
parent 0cade7b308
commit bc48ce8a96
1 changed files with 2 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
linkcheck/robotparser2.py
View file

@ -172,14 +172,10 @@ class RobotFileParser (object):
f = self.get_opener().open(req)
ct = f.info().get("Content-Type")
if ct and ct.lower().startswith("text/plain"):
lines = []
line = f.readline()
while line:
lines.append(line.strip())
line = f.readline()
self.parse(lines)
self.parse([line.strip() for line in f])
else:
self.allow_all = True
f.close()
def _add_entry (self, entry):
"""Add a parsed entry to entry list.