diff --git a/CHANGELOG b/CHANGELOG index cd6be7b..fea7872 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,7 +8,9 @@ future releases, check `milestones`_ and :doc:`/about/vision`. 1.5 (unreleased) ---------------- -- Nothing changed yet. +X-Sendfile support. + +- Feature #36 - Introduced support of Apache's mod_xsendfile. 1.4 (2013-11-24) diff --git a/demo/demoproject/apache/__init__.py b/demo/demoproject/apache/__init__.py new file mode 100644 index 0000000..d8b1270 --- /dev/null +++ b/demo/demoproject/apache/__init__.py @@ -0,0 +1 @@ +"""Apache optimizations.""" diff --git a/demo/demoproject/apache/models.py b/demo/demoproject/apache/models.py new file mode 100644 index 0000000..35f7cd9 --- /dev/null +++ b/demo/demoproject/apache/models.py @@ -0,0 +1 @@ +"""Required to make a Django application.""" diff --git a/demo/demoproject/apache/tests.py b/demo/demoproject/apache/tests.py new file mode 100644 index 0000000..c24d7a9 --- /dev/null +++ b/demo/demoproject/apache/tests.py @@ -0,0 +1,43 @@ +import os + +from django.core.files.base import ContentFile +from django.core.urlresolvers import reverse +import django.test + +from django_downloadview.apache import assert_x_sendfile + +from demoproject.apache.views import storage, storage_dir + + +def setup_file(): + if not os.path.exists(storage_dir): + os.makedirs(storage_dir) + storage.save('hello-world.txt', ContentFile(u'Hello world!\n')) + + +class OptimizedByMiddlewareTestCase(django.test.TestCase): + def test_response(self): + """'apache:optimized_by_middleware' returns X-Sendfile response.""" + setup_file() + url = reverse('apache:optimized_by_middleware') + response = self.client.get(url) + assert_x_sendfile( + self, + response, + content_type="text/plain; charset=utf-8", + basename="hello-world.txt", + file_path="/apache-optimized-by-middleware/hello-world.txt") + + +class OptimizedByDecoratorTestCase(django.test.TestCase): + def test_response(self): + """'apache:optimized_by_decorator' returns X-Sendfile response.""" + setup_file() + url = reverse('apache:optimized_by_decorator') + response = self.client.get(url) + assert_x_sendfile( + self, + response, + content_type="text/plain; charset=utf-8", + basename="hello-world.txt", + file_path="/apache-optimized-by-decorator/hello-world.txt") diff --git a/demo/demoproject/apache/urls.py b/demo/demoproject/apache/urls.py new file mode 100644 index 0000000..c89140e --- /dev/null +++ b/demo/demoproject/apache/urls.py @@ -0,0 +1,13 @@ +"""URL mapping.""" +from django.conf.urls import patterns, url + + +urlpatterns = patterns( + 'demoproject.apache.views', + url(r'^optimized-by-middleware/$', + 'optimized_by_middleware', + name='optimized_by_middleware'), + url(r'^optimized-by-decorator/$', + 'optimized_by_decorator', + name='optimized_by_decorator'), +) diff --git a/demo/demoproject/apache/views.py b/demo/demoproject/apache/views.py new file mode 100644 index 0000000..cc4342e --- /dev/null +++ b/demo/demoproject/apache/views.py @@ -0,0 +1,22 @@ +import os + +from django.conf import settings +from django.core.files.storage import FileSystemStorage + +from django_downloadview import StorageDownloadView +from django_downloadview.apache import x_sendfile + + +storage_dir = os.path.join(settings.MEDIA_ROOT, 'apache') +storage = FileSystemStorage(location=storage_dir, + base_url=''.join([settings.MEDIA_URL, 'apache/'])) + + +optimized_by_middleware = StorageDownloadView.as_view(storage=storage, + path='hello-world.txt') + + +optimized_by_decorator = x_sendfile( + StorageDownloadView.as_view(storage=storage, path='hello-world.txt'), + source_url=storage.base_url, + destination_dir='/apache-optimized-by-decorator/') diff --git a/demo/demoproject/settings.py b/demo/demoproject/settings.py index 9782567..4066284 100755 --- a/demo/demoproject/settings.py +++ b/demo/demoproject/settings.py @@ -52,6 +52,7 @@ INSTALLED_APPS = ( 'demoproject.http', # Demo around HTTPDownloadView 'demoproject.virtual', # Demo around VirtualDownloadView 'demoproject.nginx', # Sample optimizations for Nginx X-Accel. + 'demoproject.apache', # Sample optimizations for Apache X-Sendfile. # For test purposes. The demo project is part of django-downloadview # test suite. 'django_nose', @@ -71,9 +72,23 @@ MIDDLEWARE_CLASSES = [ # Specific configuration for django_downloadview.SmartDownloadMiddleware. DOWNLOADVIEW_BACKEND = 'django_downloadview.nginx.XAccelRedirectMiddleware' +"""Could also be: +DOWNLOADVIEW_BACKEND = 'django_downloadview.apache.XSendfileMiddleware' +""" DOWNLOADVIEW_RULES = [ - {'source_url': '/media/nginx/', - 'destination_url': '/nginx-optimized-by-middleware/'}, + { + 'source_url': '/media/nginx/', + 'destination_url': '/nginx-optimized-by-middleware/', + }, + { + 'source_url': '/media/apache/', + 'destination_dir': '/apache-optimized-by-middleware/', + # Bypass global default backend with additional argument "backend". + # Notice that in general use case, ``DOWNLOADVIEW_BACKEND`` should be + # enough. Here, the django_downloadview demo project needs to + # demonstrate usage of several backends. + 'backend': 'django_downloadview.apache.XSendfileMiddleware', + }, ] diff --git a/demo/demoproject/urls.py b/demo/demoproject/urls.py index 80ffcab..154b9ff 100755 --- a/demo/demoproject/urls.py +++ b/demo/demoproject/urls.py @@ -31,6 +31,10 @@ urlpatterns = patterns( url(r'^nginx/', include('demoproject.nginx.urls', app_name='nginx', namespace='nginx')), + # Apache optimizations. + url(r'^apache/', include('demoproject.apache.urls', + app_name='apache', + namespace='apache')), # An informative homepage. - url(r'', home, name='home') + url(r'$', home, name='home') ) diff --git a/django_downloadview/apache/__init__.py b/django_downloadview/apache/__init__.py new file mode 100644 index 0000000..ec284eb --- /dev/null +++ b/django_downloadview/apache/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""Optimizations for Apache. + +See also `documentation of mod_xsendfile for Apache +`_ and :doc:`narrative documentation about +Apache optimizations `. + +""" +# API shortcuts. +from django_downloadview.apache.decorators import x_sendfile # NoQA +from django_downloadview.apache.response import XSendfileResponse # NoQA +from django_downloadview.apache.tests import assert_x_sendfile # NoQA +from django_downloadview.apache.middlewares import XSendfileMiddleware # NoQA diff --git a/django_downloadview/apache/decorators.py b/django_downloadview/apache/decorators.py new file mode 100644 index 0000000..7af7e3d --- /dev/null +++ b/django_downloadview/apache/decorators.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +"""Decorators to apply Apache X-Sendfile on a specific view.""" +from django_downloadview.decorators import DownloadDecorator +from django_downloadview.apache.middlewares import XSendfileMiddleware + + +def x_sendfile(view_func, *args, **kwargs): + """Apply + :class:`~django_downloadview.apache.middlewares.XSendfileMiddleware` to + ``view_func``. + + Proxies (``*args``, ``**kwargs``) to middleware constructor. + + """ + decorator = DownloadDecorator(XSendfileMiddleware) + return decorator(view_func, *args, **kwargs) diff --git a/django_downloadview/apache/middlewares.py b/django_downloadview/apache/middlewares.py new file mode 100644 index 0000000..ee854b3 --- /dev/null +++ b/django_downloadview/apache/middlewares.py @@ -0,0 +1,30 @@ +from django_downloadview.apache.response import XSendfileResponse +from django_downloadview.middlewares import (ProxiedDownloadMiddleware, + NoRedirectionMatch) + + +class XSendfileMiddleware(ProxiedDownloadMiddleware): + """Configurable middleware, for use in decorators or in global middlewares. + + Standard Django middlewares are configured globally via settings. Instances + of this class are to be configured individually. It makes it possible to + use this class as the factory in + :py:class:`django_downloadview.decorators.DownloadDecorator`. + + """ + def __init__(self, source_dir=None, source_url=None, destination_dir=None): + """Constructor.""" + super(XSendfileMiddleware, self).__init__(source_dir, + source_url, + destination_dir) + + def process_download_response(self, request, response): + """Replace DownloadResponse instances by XSendfileResponse ones.""" + try: + redirect_url = self.get_redirect_url(response) + except NoRedirectionMatch: + return response + return XSendfileResponse(file_path=redirect_url, + content_type=response['Content-Type'], + basename=response.basename, + attachment=response.attachment) diff --git a/django_downloadview/apache/response.py b/django_downloadview/apache/response.py new file mode 100644 index 0000000..9a37d8e --- /dev/null +++ b/django_downloadview/apache/response.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +"""Apache's specific responses.""" +import os.path + +from django_downloadview.response import (ProxiedDownloadResponse, + content_disposition) + + +class XSendfileResponse(ProxiedDownloadResponse): + "Delegates serving file to Apache via X-Sendfile header." + def __init__(self, file_path, content_type, basename=None, + attachment=True): + """Return a HttpResponse with headers for Apache X-Sendfile.""" + super(XSendfileResponse, self).__init__(content_type=content_type) + if attachment: + self.basename = basename or os.path.basename(file_path) + self['Content-Disposition'] = content_disposition(self.basename) + self['X-Sendfile'] = file_path diff --git a/django_downloadview/apache/tests.py b/django_downloadview/apache/tests.py new file mode 100644 index 0000000..b0ac5db --- /dev/null +++ b/django_downloadview/apache/tests.py @@ -0,0 +1,61 @@ +from django_downloadview.apache.response import XSendfileResponse + + +class XSendfileValidator(object): + """Utility class to validate XSendfileResponse instances. + + See also :py:func:`assert_x_sendfile` shortcut function. + + """ + def __call__(self, test_case, response, **assertions): + """Assert that ``response`` is a valid X-Sendfile response. + + Optional ``assertions`` dictionary can be used to check additional + items: + + * ``basename``: the basename of the file in the response. + + * ``content_type``: the value of "Content-Type" header. + + * ``file_path``: the value of "X-Sendfile" header. + + """ + self.assert_x_sendfile_response(test_case, response) + for key, value in assertions.iteritems(): + assert_func = getattr(self, 'assert_%s' % key) + assert_func(test_case, response, value) + + def assert_x_sendfile_response(self, test_case, response): + test_case.assertTrue(isinstance(response, XSendfileResponse)) + + def assert_basename(self, test_case, response, value): + test_case.assertEqual(response.basename, value) + + def assert_content_type(self, test_case, response, value): + test_case.assertEqual(response['Content-Type'], value) + + def assert_file_path(self, test_case, response, value): + test_case.assertEqual(response['X-Sendfile'], value) + + def assert_attachment(self, test_case, response, value): + header = 'Content-Disposition' + if value: + test_case.assertTrue(response[header].startswith('attachment')) + else: + test_case.assertFalse(header in response) + + +def assert_x_sendfile(test_case, response, **assertions): + """Make ``test_case`` assert that ``response`` is a XSendfileResponse. + + Optional ``assertions`` dictionary can be used to check additional items: + + * ``basename``: the basename of the file in the response. + + * ``content_type``: the value of "Content-Type" header. + + * ``file_path``: the value of "X-Sendfile" header. + + """ + validator = XSendfileValidator() + return validator(test_case, response, **assertions) diff --git a/django_downloadview/middlewares.py b/django_downloadview/middlewares.py index b02a24a..bcabce5 100644 --- a/django_downloadview/middlewares.py +++ b/django_downloadview/middlewares.py @@ -5,6 +5,7 @@ Download middlewares capture :py:class:`django_downloadview.DownloadResponse` responses and may replace them with optimized download responses. """ +import copy import collections import os @@ -134,7 +135,7 @@ class SmartDownloadMiddleware(BaseDownloadMiddleware): """Populate :attr:`dispatcher` using :attr:`factory` and ``settings.DOWNLOADVIEW_RULES``.""" try: - options_list = settings.DOWNLOADVIEW_RULES + options_list = copy.deepcopy(settings.DOWNLOADVIEW_RULES) except AttributeError: raise ImproperlyConfigured('SmartDownloadMiddleware requires ' 'settings.DOWNLOADVIEW_RULES') @@ -145,7 +146,12 @@ class SmartDownloadMiddleware(BaseDownloadMiddleware): kwargs = options else: args = options - middleware_instance = self.backend_factory(*args, **kwargs) + if 'backend' in kwargs: # Specific backend for this rule. + factory = import_member(kwargs['backend']) + del kwargs['backend'] + else: # Fallback to global backend. + factory = self.backend_factory + middleware_instance = factory(*args, **kwargs) self.dispatcher.middlewares.append((key, middleware_instance)) def process_download_response(self, request, response): diff --git a/django_downloadview/nginx/response.py b/django_downloadview/nginx/response.py index 7fdadae..727db2c 100644 --- a/django_downloadview/nginx/response.py +++ b/django_downloadview/nginx/response.py @@ -4,7 +4,8 @@ from datetime import timedelta from django.utils.timezone import now -from django_downloadview.response import ProxiedDownloadResponse +from django_downloadview.response import (ProxiedDownloadResponse, + content_disposition) from django_downloadview.utils import content_type_to_charset, url_basename @@ -17,8 +18,7 @@ class XAccelRedirectResponse(ProxiedDownloadResponse): if attachment: self.basename = basename or url_basename(redirect_url, content_type) - self['Content-Disposition'] = 'attachment; filename={name}'.format( - name=self.basename) + self['Content-Disposition'] = content_disposition(self.basename) self['X-Accel-Redirect'] = redirect_url self['X-Accel-Charset'] = content_type_to_charset(content_type) if with_buffering is not None: diff --git a/django_downloadview/response.py b/django_downloadview/response.py index 6a88952..3135f35 100644 --- a/django_downloadview/response.py +++ b/django_downloadview/response.py @@ -56,6 +56,18 @@ def encode_basename_utf8(value): return urllib.quote(force_str(value)) +def content_disposition(filename): + """Return value of ``Content-Disposition`` header.""" + ascii_filename = encode_basename_ascii(filename) + utf8_filename = encode_basename_utf8(filename) + if ascii_filename == utf8_filename: # ASCII only. + return "attachment; filename={ascii}".format(ascii=ascii_filename) + else: + return "attachment; filename={ascii}; filename*=UTF-8''{utf8}" \ + .format(ascii=ascii_filename, + utf8=utf8_filename) + + class DownloadResponse(StreamingHttpResponse): """File download response (Django serves file, client downloads it). @@ -151,10 +163,7 @@ class DownloadResponse(StreamingHttpResponse): pass # Generated files. if self.attachment: basename = self.get_basename() - headers['Content-Disposition'] = \ - "attachment; filename={ascii}; filename*=UTF-8''{utf8}" \ - .format(ascii=encode_basename_ascii(basename), - utf8=encode_basename_utf8(basename)) + headers['Content-Disposition'] = content_disposition(basename) self._default_headers = headers return self._default_headers diff --git a/django_downloadview/test.py b/django_downloadview/test.py index 50622f9..ac72999 100644 --- a/django_downloadview/test.py +++ b/django_downloadview/test.py @@ -112,6 +112,8 @@ class DownloadResponseValidator(object): """Implies ``attachement is True``.""" ascii_name = encode_basename_ascii(value) utf8_name = encode_basename_utf8(value) + check_utf8 = False + check_ascii = False if ascii_name == utf8_name: # Only ASCII characters. check_ascii = True if "filename*=" in response['Content-Disposition']: diff --git a/docs/optimizations/apache.txt b/docs/optimizations/apache.txt new file mode 100644 index 0000000..77676bd --- /dev/null +++ b/docs/optimizations/apache.txt @@ -0,0 +1,131 @@ +###### +Apache +###### + +If you serve Django behind Apache, then you can delegate the file streaming +to Apache and get increased performance: + +* lower resources used by Python/Django workers ; +* faster download. + +See `Apache mod_xsendfile documentation`_ for details. + + +***************** +Known limitations +***************** + +* Apache needs access to the resource by path on local filesystem. +* Thus only files that live on local filesystem can be streamed by Apache. + + +************ +Given a view +************ + +Let's consider the following view: + +.. literalinclude:: /../demo/demoproject/apache/views.py + :language: python + :lines: 1-6, 8-16 + +What is important here is that the files will have an ``url`` property +implemented by storage. Let's setup an optimization rule based on that URL. + +.. note:: + + It is generally easier to setup rules based on URL rather than based on + name in filesystem. This is because path is generally relative to storage, + whereas URL usually contains some storage identifier, i.e. it is easier to + target a specific location by URL rather than by filesystem name. + + +*************************** +Setup XSendfile middlewares +*************************** + +Make sure ``django_downloadview.SmartDownloadMiddleware`` is in +``MIDDLEWARE_CLASSES`` of your `Django` settings. + +Example: + +.. literalinclude:: /../demo/demoproject/settings.py + :language: python + :lines: 63-70 + +Then set ``django_downloadview.apache.XSendfileMiddleware`` as +``DOWNLOADVIEW_BACKEND``: + +.. literalinclude:: /../demo/demoproject/settings.py + :language: python + :lines: 76 + +Then register as many ``DOWNLOADVIEW_RULES`` as you wish: + +.. literalinclude:: /../demo/demoproject/settings.py + :language: python + :lines: 78, 79-82, 92 + +Each item in ``DOWNLOADVIEW_RULES`` is a dictionary of keyword arguments passed +to the middleware factory. In the example above, we capture responses by +``source_url`` and convert them to internal redirects to ``destination_dir``. + +.. autoclass:: django_downloadview.apache.middlewares.XSendfileMiddleware + :members: + :inherited-members: + :undoc-members: + :show-inheritance: + :member-order: bysource + + +**************************************** +Per-view setup with x_sendfile decorator +**************************************** + +Middlewares should be enough for most use cases, but you may want per-view +configuration. For `Apache`, there is ``x_sendfile``: + +.. autofunction:: django_downloadview.apache.decorators.x_sendfile + +As an example: + +.. literalinclude:: /../demo/demoproject/apache/views.py + :language: python + :lines: 1-7, 17- + + +************************************* +Test responses with assert_x_sendfile +************************************* + +Use :func:`~django_downloadview.apache.decorators.assert_x_sendfile` +function as a shortcut in your tests. + +.. literalinclude:: /../demo/demoproject/apache/tests.py + :language: python + +.. autofunction:: django_downloadview.apache.tests.assert_x_sendfile + +The tests above assert the `Django` part is OK. Now let's configure `Apache`. + + +************ +Setup Apache +************ + +See `Apache mod_xsendfile documentation`_ for details. + + +********************************************* +Assert everything goes fine with healthchecks +********************************************* + +:doc:`Healthchecks ` are the best way to check the complete +setup. + + +.. rubric:: References + +.. target-notes:: + +.. _`Apache mod_xsendfile documentation`: https://tn123.org/mod_xsendfile/ diff --git a/docs/optimizations/index.txt b/docs/optimizations/index.txt index 2ebbda5..29e52fb 100644 --- a/docs/optimizations/index.txt +++ b/docs/optimizations/index.txt @@ -11,17 +11,37 @@ proxy: As a result, you get increased performance: reverse proxies are more efficient than Django at serving static files. -The setup depends on the reverse proxy: -.. toctree:: - :titlesonly: +*********************** +Supported features grid +*********************** - nginx +Supported features depend on backend. Given the file you want to stream, the +backend may or may not be able to handle it: -.. note:: ++-----------------------+-------------------------+-------------------------+ +| View / File | :doc:`nginx` | :doc:`apache` | ++=======================+=========================+=========================+ +| :doc:`/views/path` | Yes, local filesystem. | Yes, local filesystem. | ++-----------------------+-------------------------+-------------------------+ +| :doc:`/views/storage` | Yes, local and remote. | Yes, local filesystem. | ++-----------------------+-------------------------+-------------------------+ +| :doc:`/views/object` | Yes, local and remote. | Yes, local filesystem. | ++-----------------------+-------------------------+-------------------------+ +| :doc:`/views/http` | Yes. | No. | ++-----------------------+-------------------------+-------------------------+ +| :doc:`/views/virtual` | No. | No. | ++-----------------------+-------------------------+-------------------------+ - Currently, only `nginx's X-Accel`_ is supported, but `contributions are - welcome`_! +As an example, :doc:`Nginx X-Accel ` handles URL for +internal redirects, so it can manage +:class:`~django_downloadview.files.HTTPFile`; whereas :doc:`Apache X-Sendfile +` handles absolute path, so it can only deal with files +on local filesystem. + +There are currently no optimizations to stream in-memory files, since they only +live on Django side, i.e. they do not persist after Django returned a response. +Note: there is `a feature request about "local cache" for streamed files`_. ***************** @@ -36,16 +56,36 @@ able to capture :class:`~django_downloadview.response.DownloadResponse` instances and convert them to :class:`~django_downloadview.response.ProxiedDownloadResponse`. +The :class:`~django_downloadview.response.ProxiedDownloadResponse` is specific +to the reverse-proxy (backend): it tells the reverse proxy to stream some +resource. + .. note:: The feature is inspired by :mod:`Django's TemplateResponse ` +*********************** +Available optimizations +*********************** + +Here are optimizations builtin `django_downloadview`: + +.. toctree:: + :titlesonly: + + nginx + apache + +.. note:: If you need support for additional optimizations, `tell us`_! + + .. rubric:: Notes & references .. target-notes:: -.. _`nginx's X-Accel`: http://wiki.nginx.org/X-accel -.. _`contributions are welcome`: +.. _`tell us`: https://github.com/benoitbryon/django-downloadview/issues?labels=optimizations +.. _`a feature request about "local cache" for streamed files`: + https://github.com/benoitbryon/django-downloadview/issues/70 diff --git a/docs/optimizations/nginx.txt b/docs/optimizations/nginx.txt index d4bc621..a704093 100644 --- a/docs/optimizations/nginx.txt +++ b/docs/optimizations/nginx.txt @@ -11,6 +11,15 @@ to Nginx and get increased performance: See `Nginx X-accel documentation`_ for details. +***************** +Known limitations +***************** + +* Nginx needs access to the resource by URL (proxy) or path (location). +* Thus :class:`~django_downloadview.files.VirtualFile` and any generated files + cannot be streamed by Nginx. + + ************ Given a view ************ @@ -43,20 +52,20 @@ Example: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 62-69 + :lines: 63-70 Then set ``django_downloadview.nginx.XAccelRedirectMiddleware`` as ``DOWNLOADVIEW_BACKEND``: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 73 + :lines: 74 Then register as many ``DOWNLOADVIEW_RULES`` as you wish: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 74-77 + :lines: 78, 79-82, 92 Each item in ``DOWNLOADVIEW_RULES`` is a dictionary of keyword arguments passed to the middleware factory. In the example above, we capture responses by diff --git a/docs/settings.txt b/docs/settings.txt index b5b218c..2c06c21 100644 --- a/docs/settings.txt +++ b/docs/settings.txt @@ -26,7 +26,7 @@ Example: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 62-69 + :lines: 63-70 ******************** @@ -43,7 +43,7 @@ Example: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 73 + :lines: 74 See :doc:`/optimizations/index` for a list of available backends (middlewares). @@ -69,7 +69,7 @@ Here is an example containing one rule using keyword arguments: .. literalinclude:: /../demo/demoproject/settings.py :language: python - :lines: 74-77 + :lines: 78, 79-82, 92 See :doc:`/optimizations/index` for details about builtin backends (middlewares) and their options.