From 260c6f5a10beb20265d0a303b43c071b9000de91 Mon Sep 17 00:00:00 2001 From: Markus Kaiserswerth Date: Sun, 27 Oct 2013 13:27:21 +0100 Subject: [PATCH 1/5] Added RQ-based async cache file backend --- imagekit/cachefiles/backends.py | 93 +++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/imagekit/cachefiles/backends.py b/imagekit/cachefiles/backends.py index b127430..71b5bab 100644 --- a/imagekit/cachefiles/backends.py +++ b/imagekit/cachefiles/backends.py @@ -17,7 +17,7 @@ def get_default_cachefile_backend(): """ from django.conf import settings return get_singleton(settings.IMAGEKIT_DEFAULT_CACHEFILE_BACKEND, - 'file backend') + 'file backend') class InvalidFileBackendError(ImproperlyConfigured): @@ -116,32 +116,81 @@ def _generate_file(backend, file, force=False): backend.generate_now(file, force=force) -try: - import celery -except ImportError: - pass -else: - _generate_file = celery.task(ignore_result=True)(_generate_file) - - -class Async(Simple): +class BaseAsync(Simple): """ - A backend that uses Celery to generate the images. + Base class for cache file backends that generate files asynchronously. """ - - def __init__(self, *args, **kwargs): - try: - import celery - except ImportError: - raise ImproperlyConfigured('You must install celery to use' - ' imagekit.cachefiles.backend.Async.') - super(Async, self).__init__(*args, **kwargs) - def generate(self, file, force=False): # Schedule the file for generation, unless we know for sure we don't # need to. If an already-generated file sneaks through, that's okay; # ``generate_now`` will catch it. We just want to make sure we don't # schedule anything we know is unnecessary--but we also don't want to # force a costly existence check. - if self.get_state(file, check_if_unknown=False) not in (CacheFileState.GENERATING, CacheFileState.EXISTS): - _generate_file.delay(self, file, force=force) + state = self.get_state(file, check_if_unknown=False) + if state not in (CacheFileState.GENERATING, CacheFileState.EXISTS): + self.schedule_generation(file, force=force) + + def schedule_generation(self, file, force=False): + # overwrite this to have the file generated in the background, + # e. g. in a worker queue. + raise NotImplementedError + + +try: + import celery +except ImportError: + pass + + +class CeleryAsync(BaseAsync): + """ + A backend that uses Celery to generate the images. + """ + def __init__(self, *args, **kwargs): + try: + import celery + except ImportError: + raise ImproperlyConfigured('You must install celery to use' + ' imagekit.cachefiles.backends.CeleryAsync.') + super(CeleryAsync, self).__init__(*args, **kwargs) + + def get_task(self): + if not hasattr(self, '_task'): + self._task = celery.task(ignore_result=True)(_generate_file) + return self._task + + def schedule_generation(self, file, force=False): + self.get_task().delay(self, file, force=force) + + +Async = CeleryAsync # backwards compatibility + + +try: + import django_rq +except ImportError: + pass + + +class RQAsync(BaseAsync): + """ + A backend that uses RQ to generate the images. + """ + queue_name = 'default' + + def __init__(self, *args, **kwargs): + try: + import django_rq + except ImportError: + raise ImproperlyConfigured('You must install django_rq to use' + ' imagekit.cachefiles.backends.RQAsync.') + super(RQAsync, self).__init__(*args, **kwargs) + + def get_queue(self): + # not caching property to avoid "can't pickle instancemethod objects", + # see https://github.com/nvie/rq/issues/189 + return django_rq.get_queue(self.queue_name) + + def schedule_generation(self, file, force=False): + self.get_queue().enqueue(_generate_file, args=(self, file), + kwargs=dict(force=force), result_ttl=0) From af3316278d739bbc78c6558bed2d563aba953df5 Mon Sep 17 00:00:00 2001 From: Markus Kaiserswerth Date: Mon, 4 Nov 2013 13:12:02 +0100 Subject: [PATCH 2/5] Cache file backends: dropped the "Async" in class names --- imagekit/cachefiles/backends.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/imagekit/cachefiles/backends.py b/imagekit/cachefiles/backends.py index 71b5bab..c7b997e 100644 --- a/imagekit/cachefiles/backends.py +++ b/imagekit/cachefiles/backends.py @@ -142,7 +142,7 @@ except ImportError: pass -class CeleryAsync(BaseAsync): +class Celery(BaseAsync): """ A backend that uses Celery to generate the images. """ @@ -151,8 +151,8 @@ class CeleryAsync(BaseAsync): import celery except ImportError: raise ImproperlyConfigured('You must install celery to use' - ' imagekit.cachefiles.backends.CeleryAsync.') - super(CeleryAsync, self).__init__(*args, **kwargs) + ' imagekit.cachefiles.backends.Celery.') + super(Celery, self).__init__(*args, **kwargs) def get_task(self): if not hasattr(self, '_task'): @@ -163,7 +163,7 @@ class CeleryAsync(BaseAsync): self.get_task().delay(self, file, force=force) -Async = CeleryAsync # backwards compatibility +Async = Celery # backwards compatibility try: @@ -172,7 +172,7 @@ except ImportError: pass -class RQAsync(BaseAsync): +class RQ(BaseAsync): """ A backend that uses RQ to generate the images. """ @@ -183,8 +183,8 @@ class RQAsync(BaseAsync): import django_rq except ImportError: raise ImproperlyConfigured('You must install django_rq to use' - ' imagekit.cachefiles.backends.RQAsync.') - super(RQAsync, self).__init__(*args, **kwargs) + ' imagekit.cachefiles.backends.RQ.') + super(RQ, self).__init__(*args, **kwargs) def get_queue(self): # not caching property to avoid "can't pickle instancemethod objects", From 43afb7c33d85a1369a916d5a6ae243d5562068e4 Mon Sep 17 00:00:00 2001 From: Matthew Dapena-Tretter Date: Thu, 28 Nov 2013 00:39:57 -0500 Subject: [PATCH 3/5] Fix celery backend --- imagekit/cachefiles/backends.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/imagekit/cachefiles/backends.py b/imagekit/cachefiles/backends.py index c7b997e..a781dc7 100644 --- a/imagekit/cachefiles/backends.py +++ b/imagekit/cachefiles/backends.py @@ -137,9 +137,11 @@ class BaseAsync(Simple): try: - import celery + from celery import task except ImportError: pass +else: + _celery_task = task(ignore_result=True)(_generate_file) class Celery(BaseAsync): @@ -154,13 +156,8 @@ class Celery(BaseAsync): ' imagekit.cachefiles.backends.Celery.') super(Celery, self).__init__(*args, **kwargs) - def get_task(self): - if not hasattr(self, '_task'): - self._task = celery.task(ignore_result=True)(_generate_file) - return self._task - def schedule_generation(self, file, force=False): - self.get_task().delay(self, file, force=force) + _celery_task.delay(self, file, force=force) Async = Celery # backwards compatibility From 68cfcce3f1c72033e939a5f305ab6f354cb4060c Mon Sep 17 00:00:00 2001 From: Matthew Dapena-Tretter Date: Sat, 14 Dec 2013 10:50:58 -0500 Subject: [PATCH 4/5] Correct reference to generateimage tag Closes GH-261 --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bdfed79..28c42dd 100644 --- a/README.rst +++ b/README.rst @@ -277,8 +277,8 @@ with the id "imagekit:thumbnail" which, by default, is Second, we're passing two positional arguments (the dimensions and the source image) as opposed to the keyword arguments we used with the generateimage tag. -Like with the generatethumbnail tag, you can also specify additional HTML -attributes for the thumbnail tag, or use it as an assignment tag: +Like with the generateimage tag, you can also specify additional HTML attributes +for the thumbnail tag, or use it as an assignment tag: .. code-block:: html From 452a9c1b3177d6acab61072fd719f7b855d3a879 Mon Sep 17 00:00:00 2001 From: Bryan Veloso Date: Fri, 20 Dec 2013 09:18:57 -0800 Subject: [PATCH 5/5] Tagging 3.1. --- imagekit/pkgmeta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imagekit/pkgmeta.py b/imagekit/pkgmeta.py index c530bc2..caeb043 100644 --- a/imagekit/pkgmeta.py +++ b/imagekit/pkgmeta.py @@ -1,5 +1,5 @@ __title__ = 'django-imagekit' -__author__ = 'Justin Driscoll, Bryan Veloso, Greg Newman, Chris Drackett, Matthew Tretter, Eric Eldredge' -__version__ = '3.0.4' +__author__ = 'Matthew Tretter, Eric Eldredge, Bryan Veloso, Greg Newman, Chris Drackett, Justin Driscoll' +__version__ = '3.1' __license__ = 'BSD' __all__ = ['__title__', '__author__', '__version__', '__license__']