From 0681b1889f6716d0dfe26709828e0ceb666a7340 Mon Sep 17 00:00:00 2001
From: calvin <calvin@e7d03fd6-7b0d-0410-9947-9c21f3af8025>
Date: Wed, 11 Jan 2006 22:03:58 +0000
Subject: [PATCH] limit amount of read chunks, and cleanups

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3017 e7d03fd6-7b0d-0410-9947-9c21f3af8025
---
 linkcheck/httplib2.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/linkcheck/httplib2.py b/linkcheck/httplib2.py
index ede42698..9a46bfcc 100644
--- a/linkcheck/httplib2.py
+++ b/linkcheck/httplib2.py
@@ -151,6 +151,9 @@ HTTP_VERSION_NOT_SUPPORTED = 505
 INSUFFICIENT_STORAGE = 507
 NOT_EXTENDED = 510
 
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -493,7 +496,7 @@ class HTTPResponse:
                     line = line[:i] # strip chunk-extensions
                 try:
                     chunk_left = int(line, 16)
-                except ValueError, msg:
+                except ValueError:
                     raise IncompleteRead("Invalid chunk length at %r" % line)
                 if chunk_left == 0:
                     break
@@ -542,14 +545,14 @@ class HTTPResponse:
         reading. If the bytes are truly not available (due to EOF), then the
         IncompleteRead exception can be used to detect the problem.
         """
-        s = ''
+        s = []
         while amt > 0:
-            chunk = self.fp.read(amt)
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
             if not chunk:
                 raise IncompleteRead(s)
-            s += chunk
+            s.append(chunk)
             amt -= len(chunk)
-        return s
+        return ''.join(s)
 
     def getheader(self, name, default=None):
         if self.msg is None: