Limit download file size to 5MB.

doc/changelog.txt

@@ -9,6 +9,8 @@ Fixes:
 Changes:
 - checking: Automatically allow redirections from URLs given by the
   user.
+- checking: Limit download file size to 5MB.
+  SF bug #3297970
 - gui: While checking, show new URLs added in the URL list view by
   scrolling down.
 - gui: Display release date in about dialog.

doc/todo.txt

@@ -1,5 +1,3 @@
-- [6.9] Test download of large binary files.
-  SF bug #3297970
 - [7.0] Put status info into GUI main window, do not make a modal window
   for that.
   SF bug #3297252

linkcheck/checker/__init__.py

@@ -24,6 +24,8 @@ import logging
 import urllib
 from .. import strformat, url as urlutil
 
+MAX_FILESIZE = 1024*1024*10 # 10MB
+
 
 def absolute_url (base_url, base_ref, parent_url):
     """

linkcheck/checker/fileurl.py

@@ -25,7 +25,7 @@ import urllib
 import urllib2
 
 from . import urlbase, get_index_html, get_url_from
-from .. import log, LOG_CHECK, fileutil
+from .. import log, LOG_CHECK, fileutil, LinkCheckerError
 from ..bookmarks import firefox
 from .const import WARN_FILE_MISSING_SLASH, WARN_FILE_SYSTEM_PATH
 
@@ -174,6 +174,8 @@ class FileUrl (urlbase.UrlBase):
     def read_content (self):
         """Return file content, or in case of directories a dummy HTML file
         with links to the files."""
+        if self.size > self.MaxFilesizeBytes:
+            raise LinkCheckerError(_("File size too large"))
         if self.is_directory():
             data = get_index_html(get_files(self.get_os_filename()))
             if isinstance(data, unicode):

linkcheck/checker/ftpurl.py

@@ -27,7 +27,6 @@ from . import proxysupport, httpurl, internpaturl, get_index_html
 from .const import WARN_FTP_MISSING_SLASH
 
 DEFAULT_TIMEOUT_SECS = 300
-MAX_FTP_FILESIZE = 1024*1024*10 # 10MB
 
 
 class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
@@ -228,7 +227,7 @@ class FtpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
             def stor_data (s):
                 """Helper method storing given data"""
                 # limit the download size
-                if (buf.tell() + len(s)) > MAX_FTP_FILESIZE:
+                if (buf.tell() + len(s)) > self.MaxFilesizeBytes:
                     raise LinkCheckerError(_("FTP file size too large"))
                 buf.write(s)
             self.url_connection.retrbinary(ftpcmd, stor_data)

linkcheck/checker/httpurl.py

@@ -631,6 +631,8 @@ class HttpUrl (internpaturl.InternPatternUrl, proxysupport.ProxySupport):
         # than a former HEAD request.
         self.add_size_info()
         if self._data is None:
+            if self.size > self.MaxFilesizeBytes:
+                raise LinkCheckerError(_("File size too large"))
             self._read_content(response)
         data, size = self._data, self._size
         self._data = self._size = None

linkcheck/checker/urlbase.py

@@ -90,6 +90,9 @@ class UrlBase (object):
         "text/plain+chromium": "chromium",
     }
 
+    # Set maximum file size for downloaded files in bytes.
+    MaxFilesizeBytes = 1024*1024*5
+
     def __init__ (self, base_url, recursion_level, aggregate,
                   parent_url=None, base_ref=None, line=-1, column=-1,
                   name=u"", url_encoding=None):
@@ -686,6 +689,8 @@ class UrlBase (object):
     def read_content (self):
         """Return data and data size for this URL.
         Can be overridden in subclasses."""
+        if self.size > self.MaxFilesizeBytes:
+            raise LinkCheckerError(_("File size too large"))
         data = self.url_connection.read()
         return data, len(data)