Refs #97 - Splitted StringIteratorIO into TextIteratorIO and BytesIteratorIO. StringIteratorIO remains for backward compatibility.

demo/demoproject/virtual/views.py

@@ -4,7 +4,7 @@ from django.core.files.base import ContentFile
 
 from django_downloadview import VirtualDownloadView
 from django_downloadview import VirtualFile
-from django_downloadview import StringIteratorIO
+from django_downloadview import TextIteratorIO
 
 
 class TextDownloadView(VirtualDownloadView):
@@ -15,7 +15,7 @@ class TextDownloadView(VirtualDownloadView):
 
 class StringIODownloadView(VirtualDownloadView):
     def get_file(self):
-        """Return wrapper on ``StringIO`` object."""
+        """Return wrapper on ``six.StringIO`` object."""
         file_obj = StringIO(u"Hello world!\n")
         return VirtualFile(file_obj, name='hello-world.txt')
 
@@ -29,5 +29,5 @@ def generate_hello():
 class GeneratedDownloadView(VirtualDownloadView):
     def get_file(self):
         """Return wrapper on ``StringIteratorIO`` object."""
-        file_obj = StringIteratorIO(generate_hello())
+        file_obj = TextIteratorIO(generate_hello())
         return VirtualFile(file_obj, name='hello-world.txt')

django_downloadview/api.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Declaration of API shortcuts."""
-from django_downloadview.io import StringIteratorIO  # NoQA
+from django_downloadview.io import (BytesIteratorIO,  # NoQA
+                                    TextIteratorIO)
 from django_downloadview.files import (StorageFile,  # NoQA
                                        VirtualFile,
                                        HTTPFile)
@@ -20,3 +21,7 @@ from django_downloadview.shortcuts import sendfile  # NoQA
 from django_downloadview.test import (assert_download_response,  # NoQA
                                       setup_view,
                                       temporary_media_root)
+
+
+# Backward compatibility.
+StringIteratorIO = TextIteratorIO

django_downloadview/files.py

@@ -9,7 +9,7 @@ from django.utils.encoding import force_bytes
 
 import requests
 
-from django_downloadview.io import StringIteratorIO
+from django_downloadview.io import BytesIteratorIO
 
 
 class StorageFile(File):
@@ -244,8 +244,8 @@ class HTTPFile(File):
         try:
             return self._file
         except AttributeError:
-            content = self.request.iter_content()
-            self._file = StringIteratorIO(content)
+            content = self.request.iter_content(decode_unicode=False)
+            self._file = BytesIteratorIO(content)
             return self._file
 
     @property

django_downloadview/io.py

@@ -3,9 +3,11 @@
 from __future__ import absolute_import
 import io
 
+from django.utils.encoding import force_text, force_bytes
 
-class StringIteratorIO(io.TextIOBase):
-    """A dynamically generated StringIO-like object.
+
+class TextIteratorIO(io.TextIOBase):
+    """A dynamically generated TextIO-like object.
 
     Original code by Matt Joiner <anacrolix@gmail.com> from:
 
@@ -14,8 +16,11 @@ class StringIteratorIO(io.TextIOBase):
 
     """
     def __init__(self, iterator):
+        #: Iterator/generator for content.
         self._iter = iterator
-        self._left = ''
+
+        #: Internal buffer.
+        self._left = u''
 
     def readable(self):
         return True
@@ -26,11 +31,15 @@ class StringIteratorIO(io.TextIOBase):
                 self._left = next(self._iter)
             except StopIteration:
                 break
+            else:
+                # Make sure we handle text.
+                self._left = force_text(self._left)
         ret = self._left[:n]
         self._left = self._left[len(ret):]
         return ret
 
     def read(self, n=None):
+        """Return content up to ``n`` length."""
         l = []
         if n is None or n < 0:
             while True:
@@ -45,21 +54,89 @@ class StringIteratorIO(io.TextIOBase):
                     break
                 n -= len(m)
                 l.append(m)
-        return ''.join(l)
+        return u''.join(l)
 
     def readline(self):
         l = []
         while True:
-            i = self._left.find('\n')
+            i = self._left.find(u'\n')
             if i == -1:
                 l.append(self._left)
                 try:
                     self._left = next(self._iter)
                 except StopIteration:
-                    self._left = ''
+                    self._left = u''
                     break
             else:
                 l.append(self._left[:i + 1])
                 self._left = self._left[i + 1:]
                 break
-        return ''.join(l)
+        return u''.join(l)
+
+
+class BytesIteratorIO(io.BytesIO):
+    """A dynamically generated BytesIO-like object.
+
+    Original code by Matt Joiner <anacrolix@gmail.com> from:
+
+    * http://stackoverflow.com/questions/12593576/
+    * https://gist.github.com/anacrolix/3788413
+
+    """
+    def __init__(self, iterator):
+        #: Iterator/generator for content.
+        self._iter = iterator
+
+        #: Internal buffer.
+        self._left = b''
+
+    def readable(self):
+        return True
+
+    def _read1(self, n=None):
+        while not self._left:
+            try:
+                self._left = next(self._iter)
+            except StopIteration:
+                break
+            else:
+                # Make sure we handle text.
+                self._left = force_bytes(self._left)
+        ret = self._left[:n]
+        self._left = self._left[len(ret):]
+        return ret
+
+    def read(self, n=None):
+        """Return content up to ``n`` length."""
+        l = []
+        if n is None or n < 0:
+            while True:
+                m = self._read1()
+                if not m:
+                    break
+                l.append(m)
+        else:
+            while n > 0:
+                m = self._read1(n)
+                if not m:
+                    break
+                n -= len(m)
+                l.append(m)
+        return b''.join(l)
+
+    def readline(self):
+        l = []
+        while True:
+            i = self._left.find(b'\n')
+            if i == -1:
+                l.append(self._left)
+                try:
+                    self._left = next(self._iter)
+                except StopIteration:
+                    self._left = b''
+                    break
+            else:
+                l.append(self._left[:i + 1])
+                self._left = self._left[i + 1:]
+                break
+        return b''.join(l)

django_downloadview/test.py

@@ -139,9 +139,9 @@ class DownloadResponseValidator(object):
         test_case.assertTrue(response['Content-Type'].startswith(value))
 
     def assert_content(self, test_case, response, value):
-        test_case.assertEqual(
-            ''.join([s.decode('utf-8') for s in response.streaming_content]),
-            value)
+        from django.utils.encoding import force_bytes
+        parts = [force_bytes(s) for s in response.streaming_content]
+        test_case.assertEqual(b''.join(parts), force_bytes(value))
 
     def assert_attachment(self, test_case, response, value):
         if value:

docs/files.txt

@@ -48,6 +48,27 @@ django-downloadview builtins
   This is a convenient wrapper to use in :doc:`/views/virtual` subclasses.
 
 
+**********************
+Low-level IO utilities
+**********************
+
+`django-downloadview` provides two classes to implement file-like objects
+whose content is dynamically generated:
+
+* :class:`~django_downloadview.io.TextIteratorIO` for generated text;
+* :class:`~django_downloadview.io.BytesIteratorIO` for generated bytes.
+
+These classes may be handy to serve dynamically generated files. See
+:doc:`/views/virtual` for details.
+
+.. tip::
+
+   **Text or bytes?** (formerly "unicode or str?") As `django-downloadview`
+   is meant to serve files, as opposed to read or parse files, what matters
+   is file contents is preserved. `django-downloadview` tends to handle files
+   in binary mode and as bytes.
+
+
 *************
 API reference
 *************
@@ -81,6 +102,26 @@ VirtualFile
    :member-order: bysource
 
 
+BytesIteratorIO
+===============
+
+.. autoclass:: ~django_downloadview.io.BytesIteratorIO
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :member-order: bysource
+
+
+TextIteratorIO
+==============
+
+.. autoclass:: ~django_downloadview.io.TextIteratorIO
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :member-order: bysource
+
+
 .. rubric:: Notes & references
 
 .. target-notes::

docs/views/virtual.txt

@@ -14,8 +14,8 @@ it returns a suitable file wrapper...
 .. note::
 
    Current implementation does not support reverse-proxy optimizations,
-   because there is no place reverse-proxy can load files from after Django
-   exited.
+   because content is actually generated within Django, not stored in some
+   third-party place.
 
 
 ************
@@ -68,7 +68,7 @@ Let's consider you have a generator function (``yield``) or an iterator object
 
 Stream generated content using :class:`VirtualDownloadView`,
 :class:`~django_downloadview.files.VirtualFile` and
-:class:`~django_downloadview.file.StringIteratorIO`:
+:class:`~django_downloadview.io.BytesIteratorIO`:
 
 .. literalinclude:: /../demo/demoproject/virtual/views.py
    :language: python

tests/io.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+"""Tests around :mod:`django_downloadview.io`."""
+import unittest
+
+from django_downloadview import TextIteratorIO, BytesIteratorIO
+
+
+HELLO_TEXT = u'Hello world!\né\n'
+HELLO_BYTES = b'Hello world!\n\xc3\xa9\n'
+
+
+def generate_hello_text():
+    """Generate u'Hello world!\n'."""
+    yield u'Hello '
+    yield u'world!'
+    yield u'\n'
+    yield u'é'
+    yield u'\n'
+
+
+def generate_hello_bytes():
+    """Generate b'Hello world!\n'."""
+    yield b'Hello '
+    yield b'world!'
+    yield b'\n'
+    yield b'\xc3\xa9'
+    yield b'\n'
+
+
+class TextIteratorIOTestCase(unittest.TestCase):
+    """Tests around :class:`~django_downloadview.io.TextIteratorIO`."""
+    def test_read_text(self):
+        """TextIteratorIO obviously accepts text generator."""
+        file_obj = TextIteratorIO(generate_hello_text())
+        self.assertEqual(file_obj.read(), HELLO_TEXT)
+
+    def test_read_bytes(self):
+        """TextIteratorIO converts bytes as text."""
+        file_obj = TextIteratorIO(generate_hello_bytes())
+        self.assertEqual(file_obj.read(), HELLO_TEXT)
+
+
+class BytesIteratorIOTestCase(unittest.TestCase):
+    """Tests around :class:`~django_downloadview.io.BytesIteratorIO`."""
+    def test_read_bytes(self):
+        """BytesIteratorIO obviously accepts bytes generator."""
+        file_obj = BytesIteratorIO(generate_hello_bytes())
+        self.assertEqual(file_obj.read(), HELLO_BYTES)
+
+    def test_read_text(self):
+        """BytesIteratorIO converts text as bytes."""
+        file_obj = BytesIteratorIO(generate_hello_text())
+        self.assertEqual(file_obj.read(), HELLO_BYTES)