From d122c68455a0eb6b9bbccc704092006927e38e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Bryon?= Date: Fri, 12 Jun 2015 16:47:46 +0200 Subject: [PATCH] Refs #97 - Splitted StringIteratorIO into TextIteratorIO and BytesIteratorIO. StringIteratorIO remains for backward compatibility. --- demo/demoproject/virtual/views.py | 6 +- django_downloadview/api.py | 7 ++- django_downloadview/files.py | 6 +- django_downloadview/io.py | 91 ++++++++++++++++++++++++++++--- django_downloadview/test.py | 6 +- docs/files.txt | 41 ++++++++++++++ docs/views/virtual.txt | 6 +- tests/io.py | 53 ++++++++++++++++++ 8 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 tests/io.py diff --git a/demo/demoproject/virtual/views.py b/demo/demoproject/virtual/views.py index 805651f..3dc8ed2 100644 --- a/demo/demoproject/virtual/views.py +++ b/demo/demoproject/virtual/views.py @@ -4,7 +4,7 @@ from django.core.files.base import ContentFile from django_downloadview import VirtualDownloadView from django_downloadview import VirtualFile -from django_downloadview import StringIteratorIO +from django_downloadview import TextIteratorIO class TextDownloadView(VirtualDownloadView): @@ -15,7 +15,7 @@ class TextDownloadView(VirtualDownloadView): class StringIODownloadView(VirtualDownloadView): def get_file(self): - """Return wrapper on ``StringIO`` object.""" + """Return wrapper on ``six.StringIO`` object.""" file_obj = StringIO(u"Hello world!\n") return VirtualFile(file_obj, name='hello-world.txt') @@ -29,5 +29,5 @@ def generate_hello(): class GeneratedDownloadView(VirtualDownloadView): def get_file(self): """Return wrapper on ``StringIteratorIO`` object.""" - file_obj = StringIteratorIO(generate_hello()) + file_obj = TextIteratorIO(generate_hello()) return VirtualFile(file_obj, name='hello-world.txt') diff --git a/django_downloadview/api.py b/django_downloadview/api.py index c768ad5..f0726b7 100644 --- a/django_downloadview/api.py +++ b/django_downloadview/api.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Declaration of API shortcuts.""" -from django_downloadview.io import StringIteratorIO # NoQA +from django_downloadview.io import (BytesIteratorIO, # NoQA + TextIteratorIO) from django_downloadview.files import (StorageFile, # NoQA VirtualFile, HTTPFile) @@ -20,3 +21,7 @@ from django_downloadview.shortcuts import sendfile # NoQA from django_downloadview.test import (assert_download_response, # NoQA setup_view, temporary_media_root) + + +# Backward compatibility. +StringIteratorIO = TextIteratorIO diff --git a/django_downloadview/files.py b/django_downloadview/files.py index ea2efbf..a83f31b 100644 --- a/django_downloadview/files.py +++ b/django_downloadview/files.py @@ -9,7 +9,7 @@ from django.utils.encoding import force_bytes import requests -from django_downloadview.io import StringIteratorIO +from django_downloadview.io import BytesIteratorIO class StorageFile(File): @@ -244,8 +244,8 @@ class HTTPFile(File): try: return self._file except AttributeError: - content = self.request.iter_content() - self._file = StringIteratorIO(content) + content = self.request.iter_content(decode_unicode=False) + self._file = BytesIteratorIO(content) return self._file @property diff --git a/django_downloadview/io.py b/django_downloadview/io.py index 9c3056e..1935413 100644 --- a/django_downloadview/io.py +++ b/django_downloadview/io.py @@ -3,9 +3,11 @@ from __future__ import absolute_import import io +from django.utils.encoding import force_text, force_bytes -class StringIteratorIO(io.TextIOBase): - """A dynamically generated StringIO-like object. + +class TextIteratorIO(io.TextIOBase): + """A dynamically generated TextIO-like object. Original code by Matt Joiner from: @@ -14,8 +16,11 @@ class StringIteratorIO(io.TextIOBase): """ def __init__(self, iterator): + #: Iterator/generator for content. self._iter = iterator - self._left = '' + + #: Internal buffer. + self._left = u'' def readable(self): return True @@ -26,11 +31,15 @@ class StringIteratorIO(io.TextIOBase): self._left = next(self._iter) except StopIteration: break + else: + # Make sure we handle text. + self._left = force_text(self._left) ret = self._left[:n] self._left = self._left[len(ret):] return ret def read(self, n=None): + """Return content up to ``n`` length.""" l = [] if n is None or n < 0: while True: @@ -45,21 +54,89 @@ class StringIteratorIO(io.TextIOBase): break n -= len(m) l.append(m) - return ''.join(l) + return u''.join(l) def readline(self): l = [] while True: - i = self._left.find('\n') + i = self._left.find(u'\n') if i == -1: l.append(self._left) try: self._left = next(self._iter) except StopIteration: - self._left = '' + self._left = u'' break else: l.append(self._left[:i + 1]) self._left = self._left[i + 1:] break - return ''.join(l) + return u''.join(l) + + +class BytesIteratorIO(io.BytesIO): + """A dynamically generated BytesIO-like object. + + Original code by Matt Joiner from: + + * http://stackoverflow.com/questions/12593576/ + * https://gist.github.com/anacrolix/3788413 + + """ + def __init__(self, iterator): + #: Iterator/generator for content. + self._iter = iterator + + #: Internal buffer. + self._left = b'' + + def readable(self): + return True + + def _read1(self, n=None): + while not self._left: + try: + self._left = next(self._iter) + except StopIteration: + break + else: + # Make sure we handle text. + self._left = force_bytes(self._left) + ret = self._left[:n] + self._left = self._left[len(ret):] + return ret + + def read(self, n=None): + """Return content up to ``n`` length.""" + l = [] + if n is None or n < 0: + while True: + m = self._read1() + if not m: + break + l.append(m) + else: + while n > 0: + m = self._read1(n) + if not m: + break + n -= len(m) + l.append(m) + return b''.join(l) + + def readline(self): + l = [] + while True: + i = self._left.find(b'\n') + if i == -1: + l.append(self._left) + try: + self._left = next(self._iter) + except StopIteration: + self._left = b'' + break + else: + l.append(self._left[:i + 1]) + self._left = self._left[i + 1:] + break + return b''.join(l) diff --git a/django_downloadview/test.py b/django_downloadview/test.py index ef0ba00..d9d001a 100644 --- a/django_downloadview/test.py +++ b/django_downloadview/test.py @@ -139,9 +139,9 @@ class DownloadResponseValidator(object): test_case.assertTrue(response['Content-Type'].startswith(value)) def assert_content(self, test_case, response, value): - test_case.assertEqual( - ''.join([s.decode('utf-8') for s in response.streaming_content]), - value) + from django.utils.encoding import force_bytes + parts = [force_bytes(s) for s in response.streaming_content] + test_case.assertEqual(b''.join(parts), force_bytes(value)) def assert_attachment(self, test_case, response, value): if value: diff --git a/docs/files.txt b/docs/files.txt index 7511040..8cfc548 100644 --- a/docs/files.txt +++ b/docs/files.txt @@ -48,6 +48,27 @@ django-downloadview builtins This is a convenient wrapper to use in :doc:`/views/virtual` subclasses. +********************** +Low-level IO utilities +********************** + +`django-downloadview` provides two classes to implement file-like objects +whose content is dynamically generated: + +* :class:`~django_downloadview.io.TextIteratorIO` for generated text; +* :class:`~django_downloadview.io.BytesIteratorIO` for generated bytes. + +These classes may be handy to serve dynamically generated files. See +:doc:`/views/virtual` for details. + +.. tip:: + + **Text or bytes?** (formerly "unicode or str?") As `django-downloadview` + is meant to serve files, as opposed to read or parse files, what matters + is file contents is preserved. `django-downloadview` tends to handle files + in binary mode and as bytes. + + ************* API reference ************* @@ -81,6 +102,26 @@ VirtualFile :member-order: bysource +BytesIteratorIO +=============== + +.. autoclass:: ~django_downloadview.io.BytesIteratorIO + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + + +TextIteratorIO +============== + +.. autoclass:: ~django_downloadview.io.TextIteratorIO + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + + .. rubric:: Notes & references .. target-notes:: diff --git a/docs/views/virtual.txt b/docs/views/virtual.txt index d7ea396..e08d4d1 100644 --- a/docs/views/virtual.txt +++ b/docs/views/virtual.txt @@ -14,8 +14,8 @@ it returns a suitable file wrapper... .. note:: Current implementation does not support reverse-proxy optimizations, - because there is no place reverse-proxy can load files from after Django - exited. + because content is actually generated within Django, not stored in some + third-party place. ************ @@ -68,7 +68,7 @@ Let's consider you have a generator function (``yield``) or an iterator object Stream generated content using :class:`VirtualDownloadView`, :class:`~django_downloadview.files.VirtualFile` and -:class:`~django_downloadview.file.StringIteratorIO`: +:class:`~django_downloadview.io.BytesIteratorIO`: .. literalinclude:: /../demo/demoproject/virtual/views.py :language: python diff --git a/tests/io.py b/tests/io.py new file mode 100644 index 0000000..d90816f --- /dev/null +++ b/tests/io.py @@ -0,0 +1,53 @@ +# coding=utf-8 +"""Tests around :mod:`django_downloadview.io`.""" +import unittest + +from django_downloadview import TextIteratorIO, BytesIteratorIO + + +HELLO_TEXT = u'Hello world!\né\n' +HELLO_BYTES = b'Hello world!\n\xc3\xa9\n' + + +def generate_hello_text(): + """Generate u'Hello world!\n'.""" + yield u'Hello ' + yield u'world!' + yield u'\n' + yield u'é' + yield u'\n' + + +def generate_hello_bytes(): + """Generate b'Hello world!\n'.""" + yield b'Hello ' + yield b'world!' + yield b'\n' + yield b'\xc3\xa9' + yield b'\n' + + +class TextIteratorIOTestCase(unittest.TestCase): + """Tests around :class:`~django_downloadview.io.TextIteratorIO`.""" + def test_read_text(self): + """TextIteratorIO obviously accepts text generator.""" + file_obj = TextIteratorIO(generate_hello_text()) + self.assertEqual(file_obj.read(), HELLO_TEXT) + + def test_read_bytes(self): + """TextIteratorIO converts bytes as text.""" + file_obj = TextIteratorIO(generate_hello_bytes()) + self.assertEqual(file_obj.read(), HELLO_TEXT) + + +class BytesIteratorIOTestCase(unittest.TestCase): + """Tests around :class:`~django_downloadview.io.BytesIteratorIO`.""" + def test_read_bytes(self): + """BytesIteratorIO obviously accepts bytes generator.""" + file_obj = BytesIteratorIO(generate_hello_bytes()) + self.assertEqual(file_obj.read(), HELLO_BYTES) + + def test_read_text(self): + """BytesIteratorIO converts text as bytes.""" + file_obj = BytesIteratorIO(generate_hello_text()) + self.assertEqual(file_obj.read(), HELLO_BYTES)