diff --git a/demo/demoproject/download/tests.py b/demo/demoproject/download/tests.py index b006131..6e426f5 100644 --- a/demo/demoproject/download/tests.py +++ b/demo/demoproject/download/tests.py @@ -10,6 +10,8 @@ from django.core.urlresolvers import reverse_lazy as reverse from django.test import TestCase from django.test.utils import override_settings +from django_downloadview.nginx import XAccelRedirectResponse + from demoproject.download.models import Document @@ -98,3 +100,24 @@ class ObjectDownloadViewTestCase(DownloadTestCase): 'attachment; filename=hello-world.txt') self.assertEqual(open(self.files['hello-world.txt']).read(), response.content) + + +class XAccelRedirectDecoratorTestCase(DownloadTestCase): + @temporary_media_root() + def test_response(self): + document = Document.objects.create( + slug='hello-world', + file=File(open(self.files['hello-world.txt'])), + ) + download_url = reverse('download_document_nginx', + kwargs={'slug': 'hello-world'}) + response = self.client.get(download_url) + self.assertEquals(response.status_code, 200) + self.assertTrue(isinstance(response, XAccelRedirectResponse)) + self.assertEquals(response['Content-Type'], + 'text/plain; charset=utf-8') + self.assertFalse('ContentEncoding' in response) + self.assertEquals(response['Content-Disposition'], + 'attachment; filename=hello-world.txt') + self.assertEquals(response['X-Accel-Redirect'], + '/download-optimized/document/hello-world.txt') diff --git a/demo/demoproject/download/urls.py b/demo/demoproject/download/urls.py index 477aa40..32da47d 100644 --- a/demo/demoproject/download/urls.py +++ b/demo/demoproject/download/urls.py @@ -7,4 +7,6 @@ urlpatterns = patterns('demoproject.download.views', name='download_hello_world'), url(r'^document/(?P[a-zA-Z0-9_-]+)/$', 'download_document', name='download_document'), + url(r'^document-nginx/(?P[a-zA-Z0-9_-]+)/$', + 'download_document_nginx', name='download_document_nginx'), ) diff --git a/demo/demoproject/download/views.py b/demo/demoproject/download/views.py index 6ba9854..6c46508 100644 --- a/demo/demoproject/download/views.py +++ b/demo/demoproject/download/views.py @@ -1,6 +1,7 @@ from os.path import abspath, dirname, join from django_downloadview import DownloadView, ObjectDownloadView +from django_downloadview.nginx import x_accel_redirect from demoproject.download.models import Document @@ -14,3 +15,7 @@ download_hello_world = DownloadView.as_view(filename=hello_world_file, storage=None) download_document = ObjectDownloadView.as_view(model=Document) + +download_document_nginx = x_accel_redirect(download_document, + media_root='/var/www/files', + media_url='/download-optimized') diff --git a/django_downloadview/middlewares.py b/django_downloadview/middlewares.py index d0c2cbb..31b983f 100644 --- a/django_downloadview/middlewares.py +++ b/django_downloadview/middlewares.py @@ -16,6 +16,7 @@ class BaseDownloadMiddleware(object): """Call ``process_download_response()`` if ``response`` is download.""" if self.is_download_response(response): return self.process_download_response(request, response) + return response def process_download_response(self, request, response): """Handle file download response.""" diff --git a/django_downloadview/nginx.py b/django_downloadview/nginx.py new file mode 100644 index 0000000..8f69011 --- /dev/null +++ b/django_downloadview/nginx.py @@ -0,0 +1,127 @@ +"""Let Nginx serve files for increased performance. + +See `Nginx X-accel documentation `_. + +""" +from datetime import datetime, timedelta + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.http import HttpResponse + +from django_downloadview.decorators import DownloadDecorator +from django_downloadview.middlewares import BaseDownloadMiddleware +from django_downloadview.utils import content_type_to_charset + + +#: Default value for X-Accel-Buffering header. +DEFAULT_WITH_BUFFERING = None +if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING'): + setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING', + DEFAULT_WITH_BUFFERING) + + +#: Default value for X-Accel-Limit-Rate header. +DEFAULT_LIMIT_RATE = None +if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE'): + setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE', DEFAULT_LIMIT_RATE) + + +#: Default value for X-Accel-Limit-Rate header. +DEFAULT_EXPIRES = None +if not hasattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES'): + setattr(settings, 'NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES', DEFAULT_EXPIRES) + + +class XAccelRedirectResponse(HttpResponse): + """Http response that delegate serving file to Nginx.""" + def __init__(self, redirect_url, content_type, basename=None, expires=None, + with_buffering=None, limit_rate=None): + """Return a HttpResponse with headers for Nginx X-Accel-Redirect.""" + super(XAccelRedirectResponse, self).__init__(content_type=content_type) + basename = basename or redirect_url.split('/')[-1] + self['Content-Disposition'] = 'attachment; filename=%s' % basename + self['X-Accel-Redirect'] = redirect_url + self['X-Accel-Charset'] = content_type_to_charset(content_type) + if with_buffering is not None: + self['X-Accel-Buffering'] = with_buffering and 'yes' or 'no' + if expires: + expire_seconds = timedelta(expires - datetime.now()).seconds + self['X-Accel-Expires'] = expire_seconds + elif expires is not None: # We explicitely want it off. + self['X-Accel-Expires'] = 'off' + if limit_rate is not None: + self['X-Accel-Limit-Rate'] = limit_rate and '%d' % limit_rate \ + or 'off' + + +class BaseXAccelRedirectMiddleware(BaseDownloadMiddleware): + """Looks like a middleware, but configurable.""" + def __init__(self, media_root, media_url, expires=None, + with_buffering=None, limit_rate=None): + """Constructor.""" + self.media_root = media_root + self.media_url = media_url + self.expires = expires + self.with_buffering = with_buffering + self.limit_rate = limit_rate + + def get_redirect_url(self, response): + """Return redirect URL for file wrapped into response.""" + absolute_filename = response.filename + relative_filename = absolute_filename[len(self.media_root):] + return '/'.join((self.media_url.rstrip('/'), + relative_filename.strip('/'))) + + def process_download_response(self, request, response): + """Replace DownloadResponse instances by NginxDownloadResponse ones.""" + redirect_url = self.get_redirect_url(response) + if self.expires: + expires = self.expires + else: + try: + expires = response.expires + except AttributeError: + expires = None + return XAccelRedirectResponse(redirect_url=redirect_url, + content_type=response['Content-Type'], + basename=response.basename, + expires=expires, + with_buffering=self.with_buffering, + limit_rate=self.limit_rate) + + +class XAccelRedirectMiddleware(BaseXAccelRedirectMiddleware): + """Apply X-Accel-Redirect globally. + + XAccelRedirectResponseHandler with django settings. + + """ + def __init__(self): + """Use Django settings as configuration.""" + try: + media_root = settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT + except AttributeError: + raise ImproperlyConfigured( + 'settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT is required by ' + '%s middleware' % self.__class__.name) + try: + media_url = settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL + except AttributeError: + raise ImproperlyConfigured( + 'settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL is required by ' + '%s middleware' % self.__class__.name) + super(XAccelRedirectMiddleware, self).__init__( + media_root, + media_url, + expires=settings.NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES, + with_buffering=settings.NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING, + limit_rate=settings.NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE) + + +#: Apply BaseXAccelRedirectMiddleware to ``view_func`` response. +#: +#: Proxies additional arguments (``*args``, ``**kwargs``) to +#: :py:meth:`django_downloadview.nginx.BaseXAccelRedirectMiddleware.__init__`: +#: ``expires``, ``with_buffering``, and ``limit_rate``. +x_accel_redirect = DownloadDecorator(BaseXAccelRedirectMiddleware) diff --git a/django_downloadview/utils.py b/django_downloadview/utils.py new file mode 100644 index 0000000..daddb05 --- /dev/null +++ b/django_downloadview/utils.py @@ -0,0 +1,18 @@ +"""Utility functions.""" +import re + + +charset_pattern = re.compile(r'charset=(?P.+)$', re.I | re.U) + + +def content_type_to_charset(content_type): + """Return charset part of content-type header. + + >>> from django_downloadview.utils import content_type_to_charset + >>> content_type_to_charset('text/html; charset=utf-8') + 'utf-8' + + """ + match = re.search(charset_pattern, content_type) + if match: + return match.group('charset') diff --git a/docs/api/django_downloadview.txt b/docs/api/django_downloadview.txt index 4b70ab3..c790b25 100644 --- a/docs/api/django_downloadview.txt +++ b/docs/api/django_downloadview.txt @@ -25,6 +25,14 @@ django_downloadview Package :undoc-members: :show-inheritance: +:mod:`nginx` Module +------------------- + +.. automodule:: django_downloadview.nginx + :members: + :undoc-members: + :show-inheritance: + :mod:`response` Module ---------------------- @@ -33,6 +41,14 @@ django_downloadview Package :undoc-members: :show-inheritance: +:mod:`utils` Module +------------------- + +.. automodule:: django_downloadview.utils + :members: + :undoc-members: + :show-inheritance: + :mod:`views` Module ------------------- diff --git a/docs/index.txt b/docs/index.txt index 235d7e8..9fa5743 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -13,6 +13,7 @@ Contents .. toctree:: :maxdepth: 2 + nginx api/modules diff --git a/docs/nginx.txt b/docs/nginx.txt new file mode 100644 index 0000000..6020f60 --- /dev/null +++ b/docs/nginx.txt @@ -0,0 +1,243 @@ +################### +Nginx optimisations +################### + +If you serve Django behind Nginx, then you can delegate the file download +service to Nginx and get increased performance: + +* lower resources used by Python/Django workers ; +* faster download. + +See `Nginx X-accel documentation`_ for details. + + +**************************** +Configure some download view +**************************** + +As an example, let's consider the following download view: + +* mapped on ``/document//download`` +* returns DownloadResponse corresponding to Document's model FileField +* Document storage root is :file:`/var/www/files/` +* FileField's ``upload_to`` is "document". + +Files live in ``/var/www/files/document/`` folder. + +As is, Django is to serve the files, i.e. load chunks into memory and stream +them. + +Nginx is much more efficient for the actual streaming. + + +*************** +Configure Nginx +*************** + +See `Nginx X-accel documentation`_ for details. + +In this documentation, let's suppose we have something like this: + +.. code-block:: nginx + + # Will serve /var/www/files/myfile.tar.gz + # When passed URI /protected_files/myfile.tar.gz + location /optimized-download { + internal; + alias /var/www/files; + } + +.. note:: + + ``/optimized-download`` is not available for the client, i.e. users + won't be able to download files via ``/optimized-download/``. + +.. warning:: + + Make sure Nginx can read the files to download! Check permissions. + + +************************************************ +Global delegation, with XAccelRedirectMiddleware +************************************************ + +If you want to delegate all file downloads to Nginx, then use +:py:class:`django_downloadview.nginx.XAccelRedirectMiddleware`. + +Register it in your settings: + +.. code-block:: python + + MIDDLEWARE_CLASSES = ( + # ... + 'django_downloadview.nginx.XAccelRedirectMiddleware', + # ... + ) + +Optionally customize configuration (default is "use Nginx's defaults"). + +.. code-block:: python + + NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT = MEDIA_ROOT + NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_URL = '/optimized-download' + NGINX_DOWNLOAD_MIDDLEWARE_EXPIRES = False # Force no expiration. + NGINX_DOWNLOAD_MIDDLEWARE_WITH_BUFFERING = False # Force buffering off. + NGINX_DOWNLOAD_MIDDLEWARE_LIMIT_RATE = False # Force limit rate off. + + +************************************************* +Local delegation, with x_accel_redirect decorator +************************************************* + +If you want to delegate file downloads to Nginx on a per-view basis, then use +:py:func:`django_downloadview.nginx.x_accel_redirect` decorator. + +In some urls.py: + +.. code-block:: python + + # ... import Document and django.core.urls + + from django_downloadview import ObjectDownloadView + from django_downloadview.nginx import x_accel_redirect + + + download = x_accel_redirect(ObjectDownloadView.as_view(model=Document), + media_root=settings.MEDIA_ROOT, + media_url='/optimized-download') + + # ... URL patterns using ``download`` + + +******************** +Sample configuration +******************** + +In this sample configuration... + +* we register files in some "myapp.models.Document" model +* store files in :file:`/var/www/private/` folder +* publish files at ``/download//`` URL +* restrict access to authenticated users with the ``login_required`` decorator +* delegate file download to Nginx, via ``/private/`` internal URL. + +Nginx +===== + +:file:`/etc/nginx/sites-available/default`: + +.. code-block:: nginx + + charset utf-8; + + # Django-powered service. + upstream frontend { + server 127.0.0.1:8000 fail_timeout=0; + } + + server { + listen 80 default; + + # File-download proxy. + # See http://wiki.nginx.org/X-accel + # and https://github.com/benoitbryon/django-downloadview + location /private/ { + internal; + # Location to files on disk. + # See Django's settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT + alias /var/www/private/; + } + + # Proxy to Django-powered frontend. + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://frontend; + } + } + +Django settings +=============== + +:file:`settings.py`: + +.. code-block:: python + + MYAPP_STORAGE_LOCATION = '/var/www/private/' + NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT = MYAPP_STORAGE_LOCATION + MIDDLEWARE_CLASSES = ( + # ... + 'django_downloadview.nginx.XAccelRedirectMiddleware', + # ... + ) + INSTALLED_APPS = ( + # ... + 'myapp', + # ... + ) + +In some model +============= + +:file:`myapp/models.py`: + +.. code-block:: python + + from django.conf import settings + from django.db import models + from django.core.files.storage import FileSystemStorage + + + storage = FileSystemStorage(location=settings.MYAPP_STORAGE_LOCATION) + + + class Document(models.Model): + file = models.ImageField(storage=storage) + +URL patterns +============ + +:file:`myapp/urls.py`: + +.. code-block:: python + + from django.conf.urls import url, url_patterns + from django.contrib.auth.decorators import login_required + + from django_downloadview import ObjectDownloadView + + from myapp.models import Document + + + download = login_required(ObjectDownloadView.as_view(model=Document)) + + url_patterns = ('', + url('^download/(?P[0-9]+/$', download, name='download'), + ) + + +************* +Common issues +************* + +``Unknown charset "utf-8" to override`` +======================================= + +Add ``charset utf-8;`` in your nginx configuration file. + +``open() "path/to/something" failed (2: No such file or directory)`` +==================================================================== + +Check your ``settings.NGINX_DOWNLOAD_MIDDLEWARE_MEDIA_ROOT`` in Django +configuration VS ``alias`` in nginx configuration: in a standard configuration, +they should be equal. + + +********** +References +********** + +.. target-notes:: + +.. _`Nginx X-accel documentation`: http://wiki.nginx.org/X-accel