Merge pull request #6 from benoitbryon/1-nginx

Closes #1. Introducing Nginx optimisations. Should be enough for a start.

demo/demoproject/download/tests.py

@@ -10,6 +10,8 @@ from django.core.urlresolvers import reverse_lazy as reverse
 from django.test import TestCase
 from django.test.utils import override_settings
 
+from django_downloadview.nginx import XAccelRedirectResponse
+
 from demoproject.download.models import Document
 
 
@@ -98,3 +100,24 @@ class ObjectDownloadViewTestCase(DownloadTestCase):
                           'attachment; filename=hello-world.txt')
         self.assertEqual(open(self.files['hello-world.txt']).read(),
                          response.content)
+
+
+class XAccelRedirectDecoratorTestCase(DownloadTestCase):
+    @temporary_media_root()
+    def test_response(self):
+        document = Document.objects.create(
+            slug='hello-world',
+            file=File(open(self.files['hello-world.txt'])),
+        )
+        download_url = reverse('download_document_nginx',
+                               kwargs={'slug': 'hello-world'})
+        response = self.client.get(download_url)
+        self.assertEquals(response.status_code, 200)
+        self.assertTrue(isinstance(response, XAccelRedirectResponse))
+        self.assertEquals(response['Content-Type'],
+                          'text/plain; charset=utf-8')
+        self.assertFalse('ContentEncoding' in response)
+        self.assertEquals(response['Content-Disposition'],
+                          'attachment; filename=hello-world.txt')
+        self.assertEquals(response['X-Accel-Redirect'],
+                          '/download-optimized/document/hello-world.txt') 

demo/demoproject/download/urls.py

@@ -7,4 +7,6 @@ urlpatterns = patterns('demoproject.download.views',
         name='download_hello_world'),
     url(r'^document/(?P<slug>[a-zA-Z0-9_-]+)/$', 'download_document',
         name='download_document'),
+    url(r'^document-nginx/(?P<slug>[a-zA-Z0-9_-]+)/$',
+        'download_document_nginx', name='download_document_nginx'),
 )

demo/demoproject/download/views.py

@@ -1,6 +1,7 @@
 from os.path import abspath, dirname, join
 
 from django_downloadview import DownloadView, ObjectDownloadView
+from django_downloadview.nginx import x_accel_redirect
 
 from demoproject.download.models import Document
 
@@ -14,3 +15,7 @@ download_hello_world = DownloadView.as_view(filename=hello_world_file,
                                             storage=None)
 
 download_document = ObjectDownloadView.as_view(model=Document)
+
+download_document_nginx = x_accel_redirect(download_document,
+                                           media_root='/var/www/files',
+                                           media_url='/download-optimized')

django_downloadview/middlewares.py

@@ -16,6 +16,7 @@ class BaseDownloadMiddleware(object):
         """Call ``process_download_response()`` if ``response`` is download."""
         if self.is_download_response(response):
             return self.process_download_response(request, response)
+        return response
 
     def process_download_response(self, request, response):
         """Handle file download response."""

django_downloadview/nginx.py

@@ -0,0 +1,127 @@
+"""Let Nginx serve files for increased performance.
+
+See `Nginx X-accel documentation <http://wiki.nginx.org/X-accel>`_.
+
+"""
+from datetime import datetime, timedelta
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.http import HttpResponse
+
+from django_downloadview.decorators import DownloadDecorator
+from django_downloadview.middlewares import BaseDownloadMiddleware
+from django_downloadview.utils import content_type_to_charset
+
+
+#: Default value for X-Accel-Buffering header.
+DEFAULT_WITH_BUFFERING = None
+if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING'):
+    setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING',
+            DEFAULT_WITH_BUFFERING)
+
+
+#: Default value for X-Accel-Limit-Rate header.
+DEFAULT_LIMIT_RATE = None
+if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE'):
+    setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE', DEFAULT_LIMIT_RATE)
+
+
+#: Default value for X-Accel-Limit-Rate header.
+DEFAULT_EXPIRES = None
+if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES'):
+    setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES', DEFAULT_EXPIRES)
+
+
+class XAccelRedirectResponse(HttpResponse):
+    """Http response that delegate serving file to Nginx."""
+    def __init__(self, redirect_url, content_type, basename=None, expires=None,
+                 with_buffering=None, limit_rate=None):
+        """Return a HttpResponse with headers for Nginx X-Accel-Redirect."""
+        super(XAccelRedirectResponse, self).__init__(content_type=content_type)
+        basename = basename or redirect_url.split('/')[-1]
+        self['Content-Disposition'] = 'attachment; filename=%s' % basename
+        self['X-Accel-Redirect'] = redirect_url
+        self['X-Accel-Charset'] = content_type_to_charset(content_type)
+        if with_buffering is not None:
+            self['X-Accel-Buffering'] = with_buffering and 'yes' or 'no'
+        if expires:
+            expire_seconds = timedelta(expires - datetime.now()).seconds
+            self['X-Accel-Expires'] = expire_seconds
+        elif expires is not None:  # We explicitely want it off.
+            self['X-Accel-Expires'] = 'off'
+        if limit_rate is not None:
+            self['X-Accel-Limit-Rate'] = limit_rate and '%d' % limit_rate \
+                                                    or 'off'
+
+
+class BaseXAccelRedirectMiddleware(BaseDownloadMiddleware):
+    """Looks like a middleware, but configurable."""
+    def __init__(self, media_root, media_url, expires=None,
+                 with_buffering=None, limit_rate=None):
+        """Constructor."""
+        self.media_root = media_root
+        self.media_url = media_url
+        self.expires = expires
+        self.with_buffering = with_buffering
+        self.limit_rate = limit_rate
+
+    def get_redirect_url(self, response):
+        """Return redirect URL for file wrapped into response."""
+        absolute_filename = response.filename
+        relative_filename = absolute_filename[len(self.media_root):]
+        return '/'.join((self.media_url.rstrip('/'),
+                         relative_filename.strip('/')))
+
+    def process_download_response(self, request, response):
+        """Replace DownloadResponse instances by NginxDownloadResponse ones."""
+        redirect_url = self.get_redirect_url(response)
+        if self.expires:
+            expires = self.expires
+        else:
+            try:
+                expires = response.expires
+            except AttributeError:
+                expires = None
+        return XAccelRedirectResponse(redirect_url=redirect_url,
+                                      content_type=response['Content-Type'],
+                                      basename=response.basename,
+                                      expires=expires,
+                                      with_buffering=self.with_buffering,
+                                      limit_rate=self.limit_rate)
+
+
+class XAccelRedirectMiddleware(BaseXAccelRedirectMiddleware):
+    """Apply X-Accel-Redirect globally.
+
+    XAccelRedirectResponseHandler with django settings.
+
+    """
+    def __init__(self):
+        """Use Django settings as configuration."""
+        try:
+            media_root = settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT
+        except AttributeError:
+            raise ImproperlyConfigured(
+                'settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT is required by '
+                '%s middleware' % self.__class__.name)
+        try:
+            media_url = settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL
+        except AttributeError:
+            raise ImproperlyConfigured(
+                'settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL is required by '
+                '%s middleware' % self.__class__.name)
+        super(XAccelRedirectMiddleware, self).__init__(
+            media_root,
+            media_url,
+            expires=settings.NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES,
+            with_buffering=settings.NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING,
+            limit_rate=settings.NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE)
+
+
+#: Apply BaseXAccelRedirectMiddleware to ``view_func`` response.
+#:
+#: Proxies additional arguments (``*args``, ``**kwargs``) to
+#: :py:meth:`django_downloadview.nginx.BaseXAccelRedirectMiddleware.__init__`:
+#: ``expires``, ``with_buffering``, and ``limit_rate``.
+x_accel_redirect = DownloadDecorator(BaseXAccelRedirectMiddleware)

django_downloadview/utils.py

@@ -0,0 +1,18 @@
+"""Utility functions."""
+import re
+
+
+charset_pattern = re.compile(r'charset=(?P<charset>.+)$', re.I | re.U)
+
+
+def content_type_to_charset(content_type):
+    """Return charset part of content-type header.
+
+    >>> from django_downloadview.utils import content_type_to_charset
+    >>> content_type_to_charset('text/html; charset=utf-8')
+    'utf-8'
+
+    """
+    match = re.search(charset_pattern, content_type)
+    if match:
+        return match.group('charset')

docs/api/django_downloadview.txt

@@ -25,6 +25,14 @@ django_downloadview Package
     :undoc-members:
     :show-inheritance:
 
+:mod:`nginx` Module
+-------------------
+
+.. automodule:: django_downloadview.nginx
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 :mod:`response` Module
 ----------------------
 
@@ -33,6 +41,14 @@ django_downloadview Package
     :undoc-members:
     :show-inheritance:
 
+:mod:`utils` Module
+-------------------
+
+.. automodule:: django_downloadview.utils
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 :mod:`views` Module
 -------------------
 

docs/index.txt

@@ -13,6 +13,7 @@ Contents
 .. toctree::
    :maxdepth: 2
 
+   nginx
    api/modules
 
 

docs/nginx.txt

@@ -0,0 +1,243 @@
+###################
+Nginx optimisations
+###################
+
+If you serve Django behind Nginx, then you can delegate the file download
+service to Nginx and get increased performance:
+
+* lower resources used by Python/Django workers ;
+* faster download.
+
+See `Nginx X-accel documentation`_ for details.
+
+
+****************************
+Configure some download view
+****************************
+
+As an example, let's consider the following download view:
+
+* mapped on ``/document/<object-slug>/download``
+* returns DownloadResponse corresponding to Document's model FileField
+* Document storage root is :file:`/var/www/files/`
+* FileField's ``upload_to`` is "document".
+
+Files live in ``/var/www/files/document/`` folder.
+
+As is, Django is to serve the files, i.e. load chunks into memory and stream
+them.
+
+Nginx is much more efficient for the actual streaming.
+
+
+***************
+Configure Nginx
+***************
+
+See `Nginx X-accel documentation`_ for details.
+
+In this documentation, let's suppose we have something like this:
+
+.. code-block:: nginx
+
+   # Will serve /var/www/files/myfile.tar.gz
+   # When passed URI /protected_files/myfile.tar.gz
+   location /optimized-download {
+       internal;
+       alias /var/www/files;
+   }
+
+.. note::
+
+   ``/optimized-download`` is not available for the client, i.e. users
+   won't be able to download files via ``/optimized-download/<filename>``.
+
+.. warning::
+
+   Make sure Nginx can read the files to download! Check permissions.
+
+
+************************************************
+Global delegation, with XAccelRedirectMiddleware
+************************************************
+
+If you want to delegate all file downloads to Nginx, then use
+:py:class:`django_downloadview.nginx.XAccelRedirectMiddleware`.
+
+Register it in your settings:
+
+.. code-block:: python
+
+   MIDDLEWARE_CLASSES = (
+       # ...
+       'django_downloadview.nginx.XAccelRedirectMiddleware',
+       # ...
+   )
+
+Optionally customize configuration (default is "use Nginx's defaults").
+
+.. code-block:: python
+
+   NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT = MEDIA_ROOT
+   NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL = '/optimized-download'
+   NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES = False  # Force no expiration.
+   NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING = False  # Force buffering off.
+   NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE = False  # Force limit rate off.
+
+
+*************************************************
+Local delegation, with x_accel_redirect decorator
+*************************************************
+
+If you want to delegate file downloads to Nginx on a per-view basis, then use
+:py:func:`django_downloadview.nginx.x_accel_redirect` decorator.
+
+In some urls.py:
+
+.. code-block:: python
+
+   # ... import Document and django.core.urls
+
+   from django_downloadview import ObjectDownloadView
+   from django_downloadview.nginx import x_accel_redirect
+
+
+   download = x_accel_redirect(ObjectDownloadView.as_view(model=Document),
+                               media_root=settings.MEDIA_ROOT,
+                               media_url='/optimized-download')
+
+   # ... URL patterns using ``download`` 
+
+
+********************
+Sample configuration
+********************
+
+In this sample configuration...
+
+* we register files in some "myapp.models.Document" model
+* store files in :file:`/var/www/private/` folder
+* publish files at ``/download/<pk>/`` URL
+* restrict access to authenticated users with the ``login_required`` decorator
+* delegate file download to Nginx, via ``/private/`` internal URL.
+
+Nginx
+=====
+
+:file:`/etc/nginx/sites-available/default`:
+
+.. code-block:: nginx
+
+   charset utf-8;
+
+   # Django-powered service.
+   upstream frontend {
+       server 127.0.0.1:8000 fail_timeout=0;
+   }
+
+   server {
+       listen 80 default;
+
+       # File-download proxy.
+       # See http://wiki.nginx.org/X-accel
+       # and https://github.com/benoitbryon/django-downloadview
+       location /private/ {
+           internal;
+           # Location to files on disk.
+           # See Django's settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT
+           alias /var/www/private/;
+       }
+
+       # Proxy to Django-powered frontend.
+       location / {
+           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+           proxy_set_header Host $http_host;
+           proxy_redirect off;
+           proxy_pass http://frontend;
+       }
+   }
+
+Django settings
+===============
+
+:file:`settings.py`:
+
+.. code-block:: python
+
+   MYAPP_STORAGE_LOCATION = '/var/www/private/'
+   NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT = MYAPP_STORAGE_LOCATION
+   MIDDLEWARE_CLASSES = (
+       # ...
+       'django_downloadview.nginx.XAccelRedirectMiddleware',
+       # ...
+   ) 
+   INSTALLED_APPS = (
+       # ...
+       'myapp',
+       # ...
+   )
+
+In some model
+=============
+
+:file:`myapp/models.py`:
+
+.. code-block:: python
+
+   from django.conf import settings
+   from django.db import models
+   from django.core.files.storage import FileSystemStorage
+
+
+   storage = FileSystemStorage(location=settings.MYAPP_STORAGE_LOCATION)
+
+
+   class Document(models.Model):
+       file = models.ImageField(storage=storage)
+
+URL patterns
+============
+
+:file:`myapp/urls.py`:
+
+.. code-block:: python
+
+   from django.conf.urls import url, url_patterns
+   from django.contrib.auth.decorators import login_required
+
+   from django_downloadview import ObjectDownloadView
+
+   from myapp.models import Document
+
+
+   download = login_required(ObjectDownloadView.as_view(model=Document))
+
+   url_patterns = ('',
+       url('^download/(?P<pk>[0-9]+/$', download, name='download'),
+   )
+
+
+*************
+Common issues
+*************
+
+``Unknown charset "utf-8" to override``
+=======================================
+
+Add ``charset utf-8;`` in your nginx configuration file.
+
+``open() "path/to/something" failed (2: No such file or directory)``
+====================================================================
+
+Check your ``settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT`` in Django
+configuration VS ``alias`` in nginx configuration: in a standard configuration,
+they should be equal.
+
+
+**********
+References
+**********
+
+.. target-notes::
+
+.. _`Nginx X-accel documentation`: http://wiki.nginx.org/X-accel