Merge pull request #71 from benoitbryon/36-apache-x-sendfile

Closes #36 - Introduced support of Apache X-Sendfile

CHANGELOG

@@ -8,7 +8,9 @@ future releases, check `milestones`_ and :doc:`/about/vision`.
 1.5 (unreleased)
 ----------------
 
-- Nothing changed yet.
+X-Sendfile support.
+
+- Feature #36 - Introduced support of Apache's mod_xsendfile.
 
 
 1.4 (2013-11-24)

demo/demoproject/apache/__init__.py

@@ -0,0 +1 @@
+"""Apache optimizations."""

demo/demoproject/apache/models.py

@@ -0,0 +1 @@
+"""Required to make a Django application."""

demo/demoproject/apache/tests.py

@@ -0,0 +1,43 @@
+import os
+
+from django.core.files.base import ContentFile
+from django.core.urlresolvers import reverse
+import django.test
+
+from django_downloadview.apache import assert_x_sendfile
+
+from demoproject.apache.views import storage, storage_dir
+
+
+def setup_file():
+    if not os.path.exists(storage_dir):
+        os.makedirs(storage_dir)
+    storage.save('hello-world.txt', ContentFile(u'Hello world!\n'))
+
+
+class OptimizedByMiddlewareTestCase(django.test.TestCase):
+    def test_response(self):
+        """'apache:optimized_by_middleware' returns X-Sendfile response."""
+        setup_file()
+        url = reverse('apache:optimized_by_middleware')
+        response = self.client.get(url)
+        assert_x_sendfile(
+            self,
+            response,
+            content_type="text/plain; charset=utf-8",
+            basename="hello-world.txt",
+            file_path="/apache-optimized-by-middleware/hello-world.txt")
+
+
+class OptimizedByDecoratorTestCase(django.test.TestCase):
+    def test_response(self):
+        """'apache:optimized_by_decorator' returns X-Sendfile response."""
+        setup_file()
+        url = reverse('apache:optimized_by_decorator')
+        response = self.client.get(url)
+        assert_x_sendfile(
+            self,
+            response,
+            content_type="text/plain; charset=utf-8",
+            basename="hello-world.txt",
+            file_path="/apache-optimized-by-decorator/hello-world.txt")

demo/demoproject/apache/urls.py

@@ -0,0 +1,13 @@
+"""URL mapping."""
+from django.conf.urls import patterns, url
+
+
+urlpatterns = patterns(
+    'demoproject.apache.views',
+    url(r'^optimized-by-middleware/$',
+        'optimized_by_middleware',
+        name='optimized_by_middleware'),
+    url(r'^optimized-by-decorator/$',
+        'optimized_by_decorator',
+        name='optimized_by_decorator'),
+)

demo/demoproject/apache/views.py

@@ -0,0 +1,22 @@
+import os
+
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+
+from django_downloadview import StorageDownloadView
+from django_downloadview.apache import x_sendfile
+
+
+storage_dir = os.path.join(settings.MEDIA_ROOT, 'apache')
+storage = FileSystemStorage(location=storage_dir,
+                            base_url=''.join([settings.MEDIA_URL, 'apache/']))
+
+
+optimized_by_middleware = StorageDownloadView.as_view(storage=storage,
+                                                      path='hello-world.txt')
+
+
+optimized_by_decorator = x_sendfile(
+    StorageDownloadView.as_view(storage=storage, path='hello-world.txt'),
+    source_url=storage.base_url,
+    destination_dir='/apache-optimized-by-decorator/')

demo/demoproject/settings.py

@@ -52,6 +52,7 @@ INSTALLED_APPS = (
     'demoproject.http',  # Demo around HTTPDownloadView
     'demoproject.virtual',  # Demo around VirtualDownloadView
     'demoproject.nginx',  # Sample optimizations for Nginx X-Accel.
+    'demoproject.apache',  # Sample optimizations for Apache X-Sendfile.
     # For test purposes. The demo project is part of django-downloadview
     # test suite.
     'django_nose',
@@ -71,9 +72,23 @@ MIDDLEWARE_CLASSES = [
 
 # Specific configuration for django_downloadview.SmartDownloadMiddleware.
 DOWNLOADVIEW_BACKEND = 'django_downloadview.nginx.XAccelRedirectMiddleware'
+"""Could also be:
+DOWNLOADVIEW_BACKEND = 'django_downloadview.apache.XSendfileMiddleware'
+"""
 DOWNLOADVIEW_RULES = [
-    {'source_url': '/media/nginx/',
-     'destination_url': '/nginx-optimized-by-middleware/'},
+    {
+        'source_url': '/media/nginx/',
+        'destination_url': '/nginx-optimized-by-middleware/',
+    },
+    {
+        'source_url': '/media/apache/',
+        'destination_dir': '/apache-optimized-by-middleware/',
+        # Bypass global default backend with additional argument "backend".
+        # Notice that in general use case, ``DOWNLOADVIEW_BACKEND`` should be
+        # enough. Here, the django_downloadview demo project needs to
+        # demonstrate usage of several backends.
+        'backend': 'django_downloadview.apache.XSendfileMiddleware',
+    },
 ]
 
 

demo/demoproject/urls.py

@@ -31,6 +31,10 @@ urlpatterns = patterns(
     url(r'^nginx/', include('demoproject.nginx.urls',
                             app_name='nginx',
                             namespace='nginx')),
+    # Apache optimizations.
+    url(r'^apache/', include('demoproject.apache.urls',
+                             app_name='apache',
+                             namespace='apache')),
     # An informative homepage.
-    url(r'', home, name='home')
+    url(r'$', home, name='home')
 )

django_downloadview/apache/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+"""Optimizations for Apache.
+
+See also `documentation of mod_xsendfile for Apache
+<https://tn123.org/mod_xsendfile/>`_ and :doc:`narrative documentation about
+Apache optimizations </optimizations/apache>`.
+
+"""
+# API shortcuts.
+from django_downloadview.apache.decorators import x_sendfile  # NoQA
+from django_downloadview.apache.response import XSendfileResponse  # NoQA
+from django_downloadview.apache.tests import assert_x_sendfile  # NoQA
+from django_downloadview.apache.middlewares import XSendfileMiddleware  # NoQA

django_downloadview/apache/decorators.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""Decorators to apply Apache X-Sendfile on a specific view."""
+from django_downloadview.decorators import DownloadDecorator
+from django_downloadview.apache.middlewares import XSendfileMiddleware
+
+
+def x_sendfile(view_func, *args, **kwargs):
+    """Apply
+    :class:`~django_downloadview.apache.middlewares.XSendfileMiddleware` to
+    ``view_func``.
+
+    Proxies (``*args``, ``**kwargs``) to middleware constructor.
+
+    """
+    decorator = DownloadDecorator(XSendfileMiddleware)
+    return decorator(view_func, *args, **kwargs)

django_downloadview/apache/middlewares.py

@@ -0,0 +1,30 @@
+from django_downloadview.apache.response import XSendfileResponse
+from django_downloadview.middlewares import (ProxiedDownloadMiddleware,
+                                             NoRedirectionMatch)
+
+
+class XSendfileMiddleware(ProxiedDownloadMiddleware):
+    """Configurable middleware, for use in decorators or in global middlewares.
+
+    Standard Django middlewares are configured globally via settings. Instances
+    of this class are to be configured individually. It makes it possible to
+    use this class as the factory in
+    :py:class:`django_downloadview.decorators.DownloadDecorator`.
+
+    """
+    def __init__(self, source_dir=None, source_url=None, destination_dir=None):
+        """Constructor."""
+        super(XSendfileMiddleware, self).__init__(source_dir,
+                                                  source_url,
+                                                  destination_dir)
+
+    def process_download_response(self, request, response):
+        """Replace DownloadResponse instances by XSendfileResponse ones."""
+        try:
+            redirect_url = self.get_redirect_url(response)
+        except NoRedirectionMatch:
+            return response
+        return XSendfileResponse(file_path=redirect_url,
+                                 content_type=response['Content-Type'],
+                                 basename=response.basename,
+                                 attachment=response.attachment)

django_downloadview/apache/response.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""Apache's specific responses."""
+import os.path
+
+from django_downloadview.response import (ProxiedDownloadResponse,
+                                          content_disposition)
+
+
+class XSendfileResponse(ProxiedDownloadResponse):
+    "Delegates serving file to Apache via X-Sendfile header."
+    def __init__(self, file_path, content_type, basename=None,
+                 attachment=True):
+        """Return a HttpResponse with headers for Apache X-Sendfile."""
+        super(XSendfileResponse, self).__init__(content_type=content_type)
+        if attachment:
+            self.basename = basename or os.path.basename(file_path)
+            self['Content-Disposition'] = content_disposition(self.basename)
+        self['X-Sendfile'] = file_path

django_downloadview/apache/tests.py

@@ -0,0 +1,61 @@
+from django_downloadview.apache.response import XSendfileResponse
+
+
+class XSendfileValidator(object):
+    """Utility class to validate XSendfileResponse instances.
+
+    See also :py:func:`assert_x_sendfile` shortcut function.
+
+    """
+    def __call__(self, test_case, response, **assertions):
+        """Assert that ``response`` is a valid X-Sendfile response.
+
+        Optional ``assertions`` dictionary can be used to check additional
+        items:
+
+        * ``basename``: the basename of the file in the response.
+
+        * ``content_type``: the value of "Content-Type" header.
+
+        * ``file_path``: the value of "X-Sendfile" header.
+
+        """
+        self.assert_x_sendfile_response(test_case, response)
+        for key, value in assertions.iteritems():
+            assert_func = getattr(self, 'assert_%s' % key)
+            assert_func(test_case, response, value)
+
+    def assert_x_sendfile_response(self, test_case, response):
+        test_case.assertTrue(isinstance(response, XSendfileResponse))
+
+    def assert_basename(self, test_case, response, value):
+        test_case.assertEqual(response.basename, value)
+
+    def assert_content_type(self, test_case, response, value):
+        test_case.assertEqual(response['Content-Type'], value)
+
+    def assert_file_path(self, test_case, response, value):
+        test_case.assertEqual(response['X-Sendfile'], value)
+
+    def assert_attachment(self, test_case, response, value):
+        header = 'Content-Disposition'
+        if value:
+            test_case.assertTrue(response[header].startswith('attachment'))
+        else:
+            test_case.assertFalse(header in response)
+
+
+def assert_x_sendfile(test_case, response, **assertions):
+    """Make ``test_case`` assert that ``response`` is a XSendfileResponse.
+
+    Optional ``assertions`` dictionary can be used to check additional items:
+
+    * ``basename``: the basename of the file in the response.
+
+    * ``content_type``: the value of "Content-Type" header.
+
+    * ``file_path``: the value of "X-Sendfile" header.
+
+    """
+    validator = XSendfileValidator()
+    return validator(test_case, response, **assertions)

django_downloadview/middlewares.py

@@ -5,6 +5,7 @@ Download middlewares capture :py:class:`django_downloadview.DownloadResponse`
 responses and may replace them with optimized download responses.
 
 """
+import copy
 import collections
 import os
 
@@ -134,7 +135,7 @@ class SmartDownloadMiddleware(BaseDownloadMiddleware):
         """Populate :attr:`dispatcher` using :attr:`factory` and
         ``settings.DOWNLOADVIEW_RULES``."""
         try:
-            options_list = settings.DOWNLOADVIEW_RULES
+            options_list = copy.deepcopy(settings.DOWNLOADVIEW_RULES)
         except AttributeError:
             raise ImproperlyConfigured('SmartDownloadMiddleware requires '
                                        'settings.DOWNLOADVIEW_RULES')
@@ -145,7 +146,12 @@ class SmartDownloadMiddleware(BaseDownloadMiddleware):
                 kwargs = options
             else:
                 args = options
-            middleware_instance = self.backend_factory(*args, **kwargs)
+            if 'backend' in kwargs:  # Specific backend for this rule.
+                factory = import_member(kwargs['backend'])
+                del kwargs['backend']
+            else:  # Fallback to global backend.
+                factory = self.backend_factory
+            middleware_instance = factory(*args, **kwargs)
             self.dispatcher.middlewares.append((key, middleware_instance))
 
     def process_download_response(self, request, response):

django_downloadview/nginx/response.py

@@ -4,7 +4,8 @@ from datetime import timedelta
 
 from django.utils.timezone import now
 
-from django_downloadview.response import ProxiedDownloadResponse
+from django_downloadview.response import (ProxiedDownloadResponse,
+                                          content_disposition)
 from django_downloadview.utils import content_type_to_charset, url_basename
 
 
@@ -17,8 +18,7 @@ class XAccelRedirectResponse(ProxiedDownloadResponse):
         if attachment:
             self.basename = basename or url_basename(redirect_url,
                                                      content_type)
-            self['Content-Disposition'] = 'attachment; filename={name}'.format(
-                name=self.basename)
+            self['Content-Disposition'] = content_disposition(self.basename)
         self['X-Accel-Redirect'] = redirect_url
         self['X-Accel-Charset'] = content_type_to_charset(content_type)
         if with_buffering is not None:

django_downloadview/response.py

@@ -56,6 +56,18 @@ def encode_basename_utf8(value):
     return urllib.quote(force_str(value))
 
 
+def content_disposition(filename):
+    """Return value of ``Content-Disposition`` header."""
+    ascii_filename = encode_basename_ascii(filename)
+    utf8_filename = encode_basename_utf8(filename)
+    if ascii_filename == utf8_filename:  # ASCII only.
+        return "attachment; filename={ascii}".format(ascii=ascii_filename)
+    else:
+        return "attachment; filename={ascii}; filename*=UTF-8''{utf8}" \
+               .format(ascii=ascii_filename,
+                       utf8=utf8_filename)
+
+
 class DownloadResponse(StreamingHttpResponse):
     """File download response (Django serves file, client downloads it).
 
@@ -151,10 +163,7 @@ class DownloadResponse(StreamingHttpResponse):
                 pass  # Generated files.
             if self.attachment:
                 basename = self.get_basename()
-                headers['Content-Disposition'] = \
-                    "attachment; filename={ascii}; filename*=UTF-8''{utf8}" \
-                    .format(ascii=encode_basename_ascii(basename),
-                            utf8=encode_basename_utf8(basename))
+                headers['Content-Disposition'] = content_disposition(basename)
             self._default_headers = headers
             return self._default_headers
 

django_downloadview/test.py

@@ -112,6 +112,8 @@ class DownloadResponseValidator(object):
         """Implies ``attachement is True``."""
         ascii_name = encode_basename_ascii(value)
         utf8_name = encode_basename_utf8(value)
+        check_utf8 = False
+        check_ascii = False
         if ascii_name == utf8_name:  # Only ASCII characters.
             check_ascii = True
             if "filename*=" in response['Content-Disposition']:

docs/optimizations/apache.txt

@@ -0,0 +1,131 @@
+######
+Apache
+######
+
+If you serve Django behind Apache, then you can delegate the file streaming
+to Apache and get increased performance:
+
+* lower resources used by Python/Django workers ;
+* faster download.
+
+See `Apache mod_xsendfile documentation`_ for details.
+
+
+*****************
+Known limitations
+*****************
+
+* Apache needs access to the resource by path on local filesystem.
+* Thus only files that live on local filesystem can be streamed by Apache.
+
+
+************
+Given a view
+************
+
+Let's consider the following view:
+
+.. literalinclude:: /../demo/demoproject/apache/views.py
+   :language: python
+   :lines: 1-6, 8-16
+
+What is important here is that the files will have an ``url`` property
+implemented by storage. Let's setup an optimization rule based on that URL.
+
+.. note::
+
+   It is generally easier to setup rules based on URL rather than based on
+   name in filesystem. This is because path is generally relative to storage,
+   whereas URL usually contains some storage identifier, i.e. it is easier to
+   target a specific location by URL rather than by filesystem name.
+
+
+***************************
+Setup XSendfile middlewares
+***************************
+
+Make sure ``django_downloadview.SmartDownloadMiddleware`` is in
+``MIDDLEWARE_CLASSES`` of your `Django` settings.
+
+Example:
+
+.. literalinclude:: /../demo/demoproject/settings.py
+   :language: python
+   :lines: 63-70
+
+Then set ``django_downloadview.apache.XSendfileMiddleware`` as
+``DOWNLOADVIEW_BACKEND``:
+
+.. literalinclude:: /../demo/demoproject/settings.py
+   :language: python
+   :lines: 76
+
+Then register as many ``DOWNLOADVIEW_RULES`` as you wish:
+
+.. literalinclude:: /../demo/demoproject/settings.py
+   :language: python
+   :lines: 78, 79-82, 92
+
+Each item in ``DOWNLOADVIEW_RULES`` is a dictionary of keyword arguments passed
+to the middleware factory. In the example above, we capture responses by
+``source_url`` and convert them to internal redirects to ``destination_dir``.
+
+.. autoclass:: django_downloadview.apache.middlewares.XSendfileMiddleware
+   :members:
+   :inherited-members:
+   :undoc-members:
+   :show-inheritance:
+   :member-order: bysource
+
+
+****************************************
+Per-view setup with x_sendfile decorator
+****************************************
+
+Middlewares should be enough for most use cases, but you may want per-view
+configuration. For `Apache`, there is ``x_sendfile``:
+
+.. autofunction:: django_downloadview.apache.decorators.x_sendfile
+
+As an example:
+
+.. literalinclude:: /../demo/demoproject/apache/views.py
+   :language: python
+   :lines: 1-7, 17-
+
+
+*************************************
+Test responses with assert_x_sendfile
+*************************************
+
+Use :func:`~django_downloadview.apache.decorators.assert_x_sendfile`
+function as a shortcut in your tests.
+
+.. literalinclude:: /../demo/demoproject/apache/tests.py
+   :language: python
+
+.. autofunction:: django_downloadview.apache.tests.assert_x_sendfile
+
+The tests above assert the `Django` part is OK. Now let's configure `Apache`.
+
+
+************
+Setup Apache
+************
+
+See `Apache mod_xsendfile documentation`_ for details.
+
+
+*********************************************
+Assert everything goes fine with healthchecks
+*********************************************
+
+:doc:`Healthchecks </healthchecks>` are the best way to check the complete
+setup.
+
+
+.. rubric:: References
+
+.. target-notes::
+
+.. _`Apache mod_xsendfile documentation`: https://tn123.org/mod_xsendfile/ 

docs/optimizations/index.txt

@@ -11,17 +11,37 @@ proxy:
 As a result, you get increased performance: reverse proxies are more efficient
 than Django at serving static files.
 
-The setup depends on the reverse proxy:
 
-.. toctree::
-   :titlesonly:
+***********************
+Supported features grid
+***********************
 
-   nginx
+Supported features depend on backend. Given the file you want to stream, the
+backend may or may not be able to handle it:
 
-.. note::
++-----------------------+-------------------------+-------------------------+
+| View / File           | :doc:`nginx`            | :doc:`apache`           |
++=======================+=========================+=========================+
+| :doc:`/views/path`    | Yes, local filesystem.  | Yes, local filesystem.  |
++-----------------------+-------------------------+-------------------------+
+| :doc:`/views/storage` | Yes, local and remote.  | Yes, local filesystem.  |
++-----------------------+-------------------------+-------------------------+
+| :doc:`/views/object`  | Yes, local and remote.  | Yes, local filesystem.  |
++-----------------------+-------------------------+-------------------------+
+| :doc:`/views/http`    | Yes.                    | No.                     |
++-----------------------+-------------------------+-------------------------+
+| :doc:`/views/virtual` | No.                     | No.                     |
++-----------------------+-------------------------+-------------------------+
 
-   Currently, only `nginx's X-Accel`_ is supported, but `contributions are
-   welcome`_!
+As an example, :doc:`Nginx X-Accel </optimizations/nginx>` handles URL for
+internal redirects, so it can manage
+:class:`~django_downloadview.files.HTTPFile`; whereas :doc:`Apache X-Sendfile
+</optimizations/apache>` handles absolute path, so it can only deal with files
+on local filesystem.
+
+There are currently no optimizations to stream in-memory files, since they only
+live on Django side, i.e. they do not persist after Django returned a response.
+Note: there is `a feature request about "local cache" for streamed files`_.
 
 
 *****************
@@ -36,16 +56,36 @@ able to capture :class:`~django_downloadview.response.DownloadResponse`
 instances and convert them to
 :class:`~django_downloadview.response.ProxiedDownloadResponse`.
 
+The :class:`~django_downloadview.response.ProxiedDownloadResponse` is specific
+to the reverse-proxy (backend): it tells the reverse proxy to stream some
+resource.
+
 .. note::
 
    The feature is inspired by :mod:`Django's TemplateResponse
    <django.template.response>`
 
 
+***********************
+Available optimizations
+***********************
+
+Here are optimizations builtin `django_downloadview`:
+
+.. toctree::
+   :titlesonly:
+
+   nginx
+   apache
+
+.. note:: If you need support for additional optimizations, `tell us`_!
+
+
 .. rubric:: Notes & references
 
 .. target-notes::
 
-.. _`nginx's X-Accel`: http://wiki.nginx.org/X-accel
-.. _`contributions are welcome`:
+.. _`tell us`:
    https://github.com/benoitbryon/django-downloadview/issues?labels=optimizations
+.. _`a feature request about "local cache" for streamed files`:
+   https://github.com/benoitbryon/django-downloadview/issues/70

docs/optimizations/nginx.txt

@@ -11,6 +11,15 @@ to Nginx and get increased performance:
 See `Nginx X-accel documentation`_ for details.
 
 
+*****************
+Known limitations
+*****************
+
+* Nginx needs access to the resource by URL (proxy) or path (location).
+* Thus :class:`~django_downloadview.files.VirtualFile` and any generated files
+  cannot be streamed by Nginx.
+
+
 ************
 Given a view
 ************
@@ -43,20 +52,20 @@ Example:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 62-69
+   :lines: 63-70
 
 Then set ``django_downloadview.nginx.XAccelRedirectMiddleware`` as
 ``DOWNLOADVIEW_BACKEND``:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 73
+   :lines: 74
 
 Then register as many ``DOWNLOADVIEW_RULES`` as you wish:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 74-77
+   :lines: 78, 79-82, 92
 
 Each item in ``DOWNLOADVIEW_RULES`` is a dictionary of keyword arguments passed
 to the middleware factory. In the example above, we capture responses by

docs/settings.txt

@@ -26,7 +26,7 @@ Example:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 62-69
+   :lines: 63-70
 
 
 ********************
@@ -43,7 +43,7 @@ Example:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 73
+   :lines: 74
 
 See :doc:`/optimizations/index` for a list of available backends (middlewares).
 
@@ -69,7 +69,7 @@ Here is an example containing one rule using keyword arguments:
 
 .. literalinclude:: /../demo/demoproject/settings.py
    :language: python
-   :lines: 74-77
+   :lines: 78, 79-82, 92
 
 See :doc:`/optimizations/index` for details about builtin backends
 (middlewares) and their options.