From bc48ce8a965ee7bd2d61e4a8a81b2313a003bfad Mon Sep 17 00:00:00 2001
From: calvin <calvin@e7d03fd6-7b0d-0410-9947-9c21f3af8025>
Date: Mon, 28 Jul 2008 16:56:24 +0000
Subject: [PATCH] Close robotparser URL connections; simplify line parsing.

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3853 e7d03fd6-7b0d-0410-9947-9c21f3af8025
---
 linkcheck/robotparser2.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/linkcheck/robotparser2.py b/linkcheck/robotparser2.py
index 306ce2f0..353e9e90 100755
--- a/linkcheck/robotparser2.py
+++ b/linkcheck/robotparser2.py
@@ -172,14 +172,10 @@ class RobotFileParser (object):
         f = self.get_opener().open(req)
         ct = f.info().get("Content-Type")
         if ct and ct.lower().startswith("text/plain"):
-            lines = []
-            line = f.readline()
-            while line:
-                lines.append(line.strip())
-                line = f.readline()
-            self.parse(lines)
+            self.parse([line.strip() for line in f])
         else:
             self.allow_all = True
+        f.close()
 
     def _add_entry (self, entry):
         """Add a parsed entry to entry list.