From e33ff5d34d1ceba8a6e50e2355d4dd6850e9382c Mon Sep 17 00:00:00 2001 From: jordij Date: Fri, 10 Apr 2015 15:44:41 +1200 Subject: [PATCH 1/7] documents served using django-sendfile --- setup.py | 1 + tox.ini | 1 + wagtail/tests/settings.py | 3 +++ wagtail/wagtaildocs/tests.py | 3 +++ wagtail/wagtaildocs/views/serve.py | 17 ++--------------- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index a44d564e3..d0c29d9be 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ install_requires = [ "django-modelcluster>=0.6", "django-taggit>=0.13.0", "django-treebeard==3.0", + "django-sendfile==0.3.6", "Pillow>=2.6.1", "beautifulsoup4>=4.3.2", "html5lib==0.999", diff --git a/tox.ini b/tox.ini index d0d107896..53993ef7d 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ deps = django-modelcluster>=0.6 django-taggit==0.13.0 django-treebeard==3.0 + django-sendfile==0.3.6 Pillow>=2.3.0 beautifulsoup4>=4.3.2 html5lib==0.999 diff --git a/wagtail/tests/settings.py b/wagtail/tests/settings.py index 6bbe14e56..70268d82c 100644 --- a/wagtail/tests/settings.py +++ b/wagtail/tests/settings.py @@ -83,6 +83,7 @@ INSTALLED_APPS = ( 'taggit', 'compressor', + 'sendfile', 'wagtail.wagtailcore', 'wagtail.wagtailadmin', @@ -157,5 +158,7 @@ try: except ImportError: pass +# Sendfile dev backend, do NOT use in production https://github.com/johnsensible/django-sendfile#django-sendfile +SENDFILE_BACKEND = 'sendfile.backends.development' WAGTAIL_SITE_NAME = "Test Site" diff --git a/wagtail/wagtaildocs/tests.py b/wagtail/wagtaildocs/tests.py index ecde7fa09..68b186537 100644 --- a/wagtail/wagtaildocs/tests.py +++ b/wagtail/wagtaildocs/tests.py @@ -556,6 +556,9 @@ class TestServeView(TestCase): def test_content_length_header(self): self.assertEqual(self.get()['Content-Length'], '25') + def test_content_type_header(self): + self.assertEqual(self.get()['Content-Type'], 'application/msword') + def test_is_streaming_response(self): self.assertTrue(self.get().streaming) diff --git a/wagtail/wagtaildocs/views/serve.py b/wagtail/wagtaildocs/views/serve.py index 4bd8d4556..915511671 100644 --- a/wagtail/wagtaildocs/views/serve.py +++ b/wagtail/wagtaildocs/views/serve.py @@ -1,27 +1,14 @@ from django.shortcuts import get_object_or_404 -from wsgiref.util import FileWrapper -from django.http import StreamingHttpResponse, BadHeaderError -from unidecode import unidecode +from sendfile import sendfile from wagtail.wagtaildocs.models import Document, document_served def serve(request, document_id, document_filename): doc = get_object_or_404(Document, id=document_id) - wrapper = FileWrapper(doc.file) - response = StreamingHttpResponse(wrapper, content_type='application/octet-stream') - - try: - response['Content-Disposition'] = 'attachment; filename=%s' % doc.filename - except BadHeaderError: - # Unicode filenames can fail on Django <1.8, Python 2 due to - # https://code.djangoproject.com/ticket/20889 - try with an ASCIIfied version of the name - response['Content-Disposition'] = 'attachment; filename=%s' % unidecode(doc.filename) - - response['Content-Length'] = doc.file.size # Send document_served signal document_served.send(sender=Document, instance=doc, request=request) - return response + return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename) From b1062a712ec14d0d97e1ef22ce04b1fc764cc6e4 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Mon, 20 Apr 2015 16:43:05 +0100 Subject: [PATCH 2/7] Added tests for a few sendfile backends So we can test sendfile against our test matrix --- wagtail/wagtaildocs/tests.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/wagtail/wagtaildocs/tests.py b/wagtail/wagtaildocs/tests.py index 68b186537..4bb081568 100644 --- a/wagtail/wagtaildocs/tests.py +++ b/wagtail/wagtaildocs/tests.py @@ -4,6 +4,7 @@ from six import b import unittest import mock from bs4 import BeautifulSoup +import os.path from django.test import TestCase from django.contrib.auth import get_user_model @@ -11,6 +12,7 @@ from django.contrib.auth.models import Group, Permission from django.core.urlresolvers import reverse from django.core.files.base import ContentFile from django.test.utils import override_settings +from django.conf import settings from wagtail.tests.utils import WagtailTestUtils from wagtail.wagtailcore.models import Page @@ -584,6 +586,34 @@ class TestServeView(TestCase): response = self.client.get(reverse('wagtaildocs_serve', args=(self.document.id, 'incorrectfilename'))) self.assertEqual(response.status_code, 404) + def clear_sendfile_cache(self): + from sendfile import _get_sendfile + _get_sendfile.clear() + + @override_settings(SENDFILE_BACKEND='sendfile.backends.xsendfile') + def test_sendfile_xsendfile_backend(self): + self.clear_sendfile_cache() + response = self.get() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['X-Sendfile'], os.path.join(settings.MEDIA_ROOT, self.document.file.name)) + + @override_settings(SENDFILE_BACKEND='sendfile.backends.mod_wsgi', SENDFILE_ROOT=settings.MEDIA_ROOT, SENDFILE_URL=settings.MEDIA_URL[:-1]) + def test_sendfile_mod_wsgi_backend(self): + self.clear_sendfile_cache() + response = self.get() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Location'], os.path.join(settings.MEDIA_URL, self.document.file.name)) + + @override_settings(SENDFILE_BACKEND='sendfile.backends.nginx', SENDFILE_ROOT=settings.MEDIA_ROOT, SENDFILE_URL=settings.MEDIA_URL[:-1]) + def test_sendfile_nginx_backend(self): + self.clear_sendfile_cache() + response = self.get() + + self.assertEqual(response.status_code, 200) + self.assertEqual(response['X-Accel-Redirect'], os.path.join(settings.MEDIA_URL, self.document.file.name)) + class TestServeWithUnicodeFilename(TestCase): def setUp(self): From 6e0a448f111d6091dc608969ec20e5e8a1539664 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 23 Apr 2015 11:57:42 +0100 Subject: [PATCH 3/7] Import sendfile into wagtail utils --- wagtail/utils/sendfile.py | 92 ++++++++++++++++++++++++++++++ wagtail/wagtaildocs/tests.py | 2 +- wagtail/wagtaildocs/views/serve.py | 2 +- 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 wagtail/utils/sendfile.py diff --git a/wagtail/utils/sendfile.py b/wagtail/utils/sendfile.py new file mode 100644 index 000000000..0306e9819 --- /dev/null +++ b/wagtail/utils/sendfile.py @@ -0,0 +1,92 @@ +VERSION = (0, 3, 6) +__version__ = '.'.join(map(str, VERSION)) + +import os.path +from mimetypes import guess_type + + +def _lazy_load(fn): + _cached = [] + def _decorated(): + if not _cached: + _cached.append(fn()) + return _cached[0] + def clear(): + while _cached: + _cached.pop() + _decorated.clear = clear + return _decorated + + +@_lazy_load +def _get_sendfile(): + from django.utils.importlib import import_module + from django.conf import settings + from django.core.exceptions import ImproperlyConfigured + + backend = getattr(settings, 'SENDFILE_BACKEND', None) + if not backend: + raise ImproperlyConfigured('You must specify a value for SENDFILE_BACKEND') + module = import_module(backend) + return module.sendfile + + + +def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None): + ''' + create a response to send file using backend configured in SENDFILE_BACKEND + + If attachment is True the content-disposition header will be set. + This will typically prompt the user to download the file, rather + than view it. The content-disposition filename depends on the + value of attachment_filename: + + None (default): Same as filename + False: No content-disposition filename + String: Value used as filename + + If no mimetype or encoding are specified, then they will be guessed via the + filename (using the standard python mimetypes module) + ''' + _sendfile = _get_sendfile() + + if not os.path.exists(filename): + from django.http import Http404 + raise Http404('"%s" does not exist' % filename) + + guessed_mimetype, guessed_encoding = guess_type(filename) + if mimetype is None: + if guessed_mimetype: + mimetype = guessed_mimetype + else: + mimetype = 'application/octet-stream' + + response = _sendfile(request, filename, mimetype=mimetype) + if attachment: + if attachment_filename is None: + attachment_filename = os.path.basename(filename) + parts = ['attachment'] + if attachment_filename: + from unidecode import unidecode + try: + from django.utils.encoding import force_text + except ImportError: + # Django 1.3 + from django.utils.encoding import force_unicode as force_text + attachment_filename = force_text(attachment_filename) + ascii_filename = unidecode(attachment_filename) + parts.append('filename="%s"' % ascii_filename) + if ascii_filename != attachment_filename: + from django.utils.http import urlquote + quoted_filename = urlquote(attachment_filename) + parts.append('filename*=UTF-8\'\'%s' % quoted_filename) + response['Content-Disposition'] = '; '.join(parts) + + response['Content-length'] = os.path.getsize(filename) + response['Content-Type'] = mimetype + if not encoding: + encoding = guessed_encoding + if encoding: + response['Content-Encoding'] = encoding + + return response diff --git a/wagtail/wagtaildocs/tests.py b/wagtail/wagtaildocs/tests.py index 4bb081568..d9fca748f 100644 --- a/wagtail/wagtaildocs/tests.py +++ b/wagtail/wagtaildocs/tests.py @@ -587,7 +587,7 @@ class TestServeView(TestCase): self.assertEqual(response.status_code, 404) def clear_sendfile_cache(self): - from sendfile import _get_sendfile + from wagtail.utils.sendfile import _get_sendfile _get_sendfile.clear() @override_settings(SENDFILE_BACKEND='sendfile.backends.xsendfile') diff --git a/wagtail/wagtaildocs/views/serve.py b/wagtail/wagtaildocs/views/serve.py index 915511671..406920486 100644 --- a/wagtail/wagtaildocs/views/serve.py +++ b/wagtail/wagtaildocs/views/serve.py @@ -1,6 +1,6 @@ from django.shortcuts import get_object_or_404 -from sendfile import sendfile +from wagtail.utils.sendfile import sendfile from wagtail.wagtaildocs.models import Document, document_served From b3348534cc4fbd0bfa0d0463a14a540463abf373 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 23 Apr 2015 12:12:16 +0100 Subject: [PATCH 4/7] Fallback to simple sendfile backend If SENDFILE_BACKEND is not set --- wagtail/tests/settings.py | 2 -- wagtail/utils/sendfile.py | 8 ++++++-- wagtail/wagtaildocs/views/serve.py | 9 ++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/wagtail/tests/settings.py b/wagtail/tests/settings.py index 70268d82c..c5e06592a 100644 --- a/wagtail/tests/settings.py +++ b/wagtail/tests/settings.py @@ -158,7 +158,5 @@ try: except ImportError: pass -# Sendfile dev backend, do NOT use in production https://github.com/johnsensible/django-sendfile#django-sendfile -SENDFILE_BACKEND = 'sendfile.backends.development' WAGTAIL_SITE_NAME = "Test Site" diff --git a/wagtail/utils/sendfile.py b/wagtail/utils/sendfile.py index 0306e9819..f0fae979c 100644 --- a/wagtail/utils/sendfile.py +++ b/wagtail/utils/sendfile.py @@ -1,3 +1,7 @@ +# Copied from django-sendfile 0.3.6 and tweaked to allow a backend to be passed +# to sendfile() +# See: https://github.com/johnsensible/django-sendfile/pull/33 + VERSION = (0, 3, 6) __version__ = '.'.join(map(str, VERSION)) @@ -32,7 +36,7 @@ def _get_sendfile(): -def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None): +def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None, backend=None): ''' create a response to send file using backend configured in SENDFILE_BACKEND @@ -48,7 +52,7 @@ def sendfile(request, filename, attachment=False, attachment_filename=None, mime If no mimetype or encoding are specified, then they will be guessed via the filename (using the standard python mimetypes module) ''' - _sendfile = _get_sendfile() + _sendfile = backend or _get_sendfile() if not os.path.exists(filename): from django.http import Http404 diff --git a/wagtail/wagtaildocs/views/serve.py b/wagtail/wagtaildocs/views/serve.py index 406920486..42cf3c9a8 100644 --- a/wagtail/wagtaildocs/views/serve.py +++ b/wagtail/wagtaildocs/views/serve.py @@ -1,4 +1,5 @@ from django.shortcuts import get_object_or_404 +from django.conf import settings from wagtail.utils.sendfile import sendfile @@ -11,4 +12,10 @@ def serve(request, document_id, document_filename): # Send document_served signal document_served.send(sender=Document, instance=doc, request=request) - return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename) + if hasattr(settings, 'SENDFILE_BACKEND'): + return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename) + else: + # Fallback to simple backend if user hasn't specified SENDFILE_BACKEND (will crash by default) + from sendfile.backends.simple import sendfile as simple_sendfile_backend + + return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename, backend=simple_sendfile_backend) From f70eb09d072afad91bc131d7599ffc718852499f Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 23 Apr 2015 12:33:29 +0100 Subject: [PATCH 5/7] Added streaming sendfile backend The simple sendfile backend doesn't stream responses. This commit adds a backend that does. --- wagtail/utils/sendfile_streaming_backend.py | 63 +++++++++++++++++++++ wagtail/wagtaildocs/views/serve.py | 7 +-- 2 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 wagtail/utils/sendfile_streaming_backend.py diff --git a/wagtail/utils/sendfile_streaming_backend.py b/wagtail/utils/sendfile_streaming_backend.py new file mode 100644 index 000000000..a39521c14 --- /dev/null +++ b/wagtail/utils/sendfile_streaming_backend.py @@ -0,0 +1,63 @@ +# Sendfile "streaming" backend +# This is based on sendfiles builtin "simple" backend but uses a StreamingHttpResponse + +import os +import stat +import re +try: + from email.utils import parsedate_tz, mktime_tz +except ImportError: + from email.Utils import parsedate_tz, mktime_tz +from wsgiref.util import FileWrapper + +from django.core.files.base import File +from django.http import StreamingHttpResponse, HttpResponseNotModified +from django.utils.http import http_date + + +def sendfile(request, filename, **kwargs): + # Respect the If-Modified-Since header. + statobj = os.stat(filename) + + if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), + statobj[stat.ST_MTIME], statobj[stat.ST_SIZE]): + return HttpResponseNotModified() + + response = StreamingHttpResponse(FileWrapper(open(filename, 'rb'))) + + response["Last-Modified"] = http_date(statobj[stat.ST_MTIME]) + return response + + +def was_modified_since(header=None, mtime=0, size=0): + """ + Was something modified since the user last downloaded it? + + header + This is the value of the If-Modified-Since header. If this is None, + I'll just return True. + + mtime + This is the modification time of the item we're talking about. + + size + This is the size of the item we're talking about. + """ + try: + if header is None: + raise ValueError + matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, + re.IGNORECASE) + header_date = parsedate_tz(matches.group(1)) + if header_date is None: + raise ValueError + header_mtime = mktime_tz(header_date) + header_len = matches.group(3) + if header_len and int(header_len) != size: + raise ValueError + if mtime > header_mtime: + raise ValueError + except (AttributeError, ValueError, OverflowError): + return True + return False + diff --git a/wagtail/wagtaildocs/views/serve.py b/wagtail/wagtaildocs/views/serve.py index 42cf3c9a8..62bec3669 100644 --- a/wagtail/wagtaildocs/views/serve.py +++ b/wagtail/wagtaildocs/views/serve.py @@ -2,6 +2,7 @@ from django.shortcuts import get_object_or_404 from django.conf import settings from wagtail.utils.sendfile import sendfile +from wagtail.utils import sendfile_streaming_backend from wagtail.wagtaildocs.models import Document, document_served @@ -15,7 +16,5 @@ def serve(request, document_id, document_filename): if hasattr(settings, 'SENDFILE_BACKEND'): return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename) else: - # Fallback to simple backend if user hasn't specified SENDFILE_BACKEND (will crash by default) - from sendfile.backends.simple import sendfile as simple_sendfile_backend - - return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename, backend=simple_sendfile_backend) + # Fallback to streaming backend if user hasn't specified SENDFILE_BACKEND + return sendfile(request, doc.file.path, attachment=True, attachment_filename=doc.filename, backend=sendfile_streaming_backend.sendfile) From 5db78ec012c3cc533bb32e5218cd4a3e60b8400d Mon Sep 17 00:00:00 2001 From: jordij Date: Wed, 29 Apr 2015 09:55:25 +1200 Subject: [PATCH 6/7] PEP-8 fixes --- wagtail/utils/sendfile.py | 10 +++++----- wagtail/utils/sendfile_streaming_backend.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/wagtail/utils/sendfile.py b/wagtail/utils/sendfile.py index f0fae979c..a808065b9 100644 --- a/wagtail/utils/sendfile.py +++ b/wagtail/utils/sendfile.py @@ -1,20 +1,21 @@ # Copied from django-sendfile 0.3.6 and tweaked to allow a backend to be passed # to sendfile() # See: https://github.com/johnsensible/django-sendfile/pull/33 +import os.path +from mimetypes import guess_type VERSION = (0, 3, 6) __version__ = '.'.join(map(str, VERSION)) -import os.path -from mimetypes import guess_type - def _lazy_load(fn): _cached = [] + def _decorated(): if not _cached: _cached.append(fn()) return _cached[0] + def clear(): while _cached: _cached.pop() @@ -35,7 +36,6 @@ def _get_sendfile(): return module.sendfile - def sendfile(request, filename, attachment=False, attachment_filename=None, mimetype=None, encoding=None, backend=None): ''' create a response to send file using backend configured in SENDFILE_BACKEND @@ -64,7 +64,7 @@ def sendfile(request, filename, attachment=False, attachment_filename=None, mime mimetype = guessed_mimetype else: mimetype = 'application/octet-stream' - + response = _sendfile(request, filename, mimetype=mimetype) if attachment: if attachment_filename is None: diff --git a/wagtail/utils/sendfile_streaming_backend.py b/wagtail/utils/sendfile_streaming_backend.py index a39521c14..c42f06eb0 100644 --- a/wagtail/utils/sendfile_streaming_backend.py +++ b/wagtail/utils/sendfile_streaming_backend.py @@ -10,7 +10,6 @@ except ImportError: from email.Utils import parsedate_tz, mktime_tz from wsgiref.util import FileWrapper -from django.core.files.base import File from django.http import StreamingHttpResponse, HttpResponseNotModified from django.utils.http import http_date @@ -60,4 +59,3 @@ def was_modified_since(header=None, mtime=0, size=0): except (AttributeError, ValueError, OverflowError): return True return False - From 8673d07d8bac5bb81e44f388381c69b90b1bc9d6 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 7 May 2015 09:35:31 +0100 Subject: [PATCH 7/7] Updated sendfile to 0.3.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0c29d9be..7fd2b15cb 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ "django-modelcluster>=0.6", "django-taggit>=0.13.0", "django-treebeard==3.0", - "django-sendfile==0.3.6", + "django-sendfile==0.3.7", "Pillow>=2.6.1", "beautifulsoup4>=4.3.2", "html5lib==0.999",