From 42c79d7bb24f34b8627b74fe6b6c9aff2c6a9547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 12:27:53 +0100 Subject: [PATCH 1/8] declare celery backends in a separate module --- imagekit/imagecache/__init__.py | 31 ++++++++++++++++ .../{imagecache.py => imagecache/base.py} | 34 ------------------ imagekit/imagecache/celery.py | 35 +++++++++++++++++++ imagekit/settings.py | 2 +- 4 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 imagekit/imagecache/__init__.py rename imagekit/{imagecache.py => imagecache/base.py} (53%) create mode 100644 imagekit/imagecache/celery.py diff --git a/imagekit/imagecache/__init__.py b/imagekit/imagecache/__init__.py new file mode 100644 index 0000000..05414cc --- /dev/null +++ b/imagekit/imagecache/__init__.py @@ -0,0 +1,31 @@ +from django.core.exceptions import ImproperlyConfigured +from django.utils.importlib import import_module + +_default_image_cache_backend = None + + +def get_default_image_cache_backend(): + """ + Get the default image cache backend. Uses the same method as + django.core.file.storage.get_storage_class + + """ + global _default_image_cache_backend + if not _default_image_cache_backend: + from ..settings import DEFAULT_IMAGE_CACHE_BACKEND as import_path + try: + dot = import_path.rindex('.') + except ValueError: + raise ImproperlyConfigured("%s isn't an image cache backend module." % \ + import_path) + module, classname = import_path[:dot], import_path[dot + 1:] + try: + mod = import_module(module) + except ImportError, e: + raise ImproperlyConfigured('Error importing image cache backend module %s: "%s"' % (module, e)) + try: + cls = getattr(mod, classname) + _default_image_cache_backend = cls() + except AttributeError: + raise ImproperlyConfigured('Image cache backend module "%s" does not define a "%s" class.' % (module, classname)) + return _default_image_cache_backend diff --git a/imagekit/imagecache.py b/imagekit/imagecache/base.py similarity index 53% rename from imagekit/imagecache.py rename to imagekit/imagecache/base.py index 0edc8d6..b7e3bc2 100644 --- a/imagekit/imagecache.py +++ b/imagekit/imagecache/base.py @@ -1,7 +1,3 @@ -from django.core.exceptions import ImproperlyConfigured -from django.utils.importlib import import_module - - class PessimisticImageCacheBackend(object): """ A very safe image cache backend. Guarantees that files will always be @@ -55,33 +51,3 @@ class NonValidatingImageCacheBackend(object): def clear(self, file): file.delete(save=False) - - -_default_image_cache_backend = None - - -def get_default_image_cache_backend(): - """ - Get the default image cache backend. Uses the same method as - django.core.file.storage.get_storage_class - - """ - global _default_image_cache_backend - if not _default_image_cache_backend: - from .settings import DEFAULT_IMAGE_CACHE_BACKEND as import_path - try: - dot = import_path.rindex('.') - except ValueError: - raise ImproperlyConfigured("%s isn't an image cache backend module." % \ - import_path) - module, classname = import_path[:dot], import_path[dot + 1:] - try: - mod = import_module(module) - except ImportError, e: - raise ImproperlyConfigured('Error importing image cache backend module %s: "%s"' % (module, e)) - try: - cls = getattr(mod, classname) - _default_image_cache_backend = cls() - except AttributeError: - raise ImproperlyConfigured('Image cache backend module "%s" does not define a "%s" class.' % (module, classname)) - return _default_image_cache_backend diff --git a/imagekit/imagecache/celery.py b/imagekit/imagecache/celery.py new file mode 100644 index 0000000..77dac73 --- /dev/null +++ b/imagekit/imagecache/celery.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from celery.task import task + +from imagekit.imagecache.base import PessimisticImageCacheBackend + + +@task +def generate(model, pk, attr): + try: + instance = model._default_manager.get(pk=pk) + except model.DoesNotExist: + pass # The model was deleted since the task was scheduled. NEVER MIND! + else: + getattr(instance, attr).generate(save=True) + + +class CeleryCacheStateBackend(PessimisticImageCacheBackend): + """ + A pessimistic cache state backend that uses celery to generate its spec + images. Like PessimisticCacheStateBackend, this one checks to see if the + file exists on validation, so the storage is hit fairly frequently, but an + image is guaranteed to exist. However, while validation guarantees the + existence of *an* image, it does not necessarily guarantee that you will get + the correct image, as the spec may be pending regeneration. In other words, + while there are `generate` tasks in the queue, it is possible to get a + stale spec image. The tradeoff is that calling `invalidate()` won't block + to interact with file storage. + + """ + + def invalidate(self, file): + generate.delay(file.instance.__class__, file.instance.pk, file.attname) + + def clear(self, file): + file.delete(save=False) diff --git a/imagekit/settings.py b/imagekit/settings.py index d84030d..ca6f4c8 100644 --- a/imagekit/settings.py +++ b/imagekit/settings.py @@ -2,4 +2,4 @@ from django.conf import settings DEFAULT_IMAGE_CACHE_BACKEND = getattr(settings, 'IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND', - 'imagekit.imagecache.PessimisticImageCacheBackend') + 'imagekit.imagecache.base.PessimisticImageCacheBackend') From 0fa86f7da82dafd3b9f4473e1a748c594c2120b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 13:58:09 +0100 Subject: [PATCH 2/8] rename celery module to async --- imagekit/imagecache/{celery.py => async.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename imagekit/imagecache/{celery.py => async.py} (100%) diff --git a/imagekit/imagecache/celery.py b/imagekit/imagecache/async.py similarity index 100% rename from imagekit/imagecache/celery.py rename to imagekit/imagecache/async.py From 9fbdd7bef4ee9d34200311df148a638f0ceb100f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 16:55:40 +0100 Subject: [PATCH 3/8] import base backends in imagecache module --- imagekit/imagecache/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imagekit/imagecache/__init__.py b/imagekit/imagecache/__init__.py index 05414cc..dab06cd 100644 --- a/imagekit/imagecache/__init__.py +++ b/imagekit/imagecache/__init__.py @@ -1,6 +1,8 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module +from imagekit.imagecache.base import PessimisticImageCacheBackend, NonValidatingImageCacheBackend + _default_image_cache_backend = None From 983bceff6288a6d240a0a06dfd49c194d4ed30b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 17:00:35 +0100 Subject: [PATCH 4/8] rename CeleryCacheStateBackend as CeleryImageCacheBackend --- imagekit/imagecache/async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagekit/imagecache/async.py b/imagekit/imagecache/async.py index 77dac73..213e679 100644 --- a/imagekit/imagecache/async.py +++ b/imagekit/imagecache/async.py @@ -14,7 +14,7 @@ def generate(model, pk, attr): getattr(instance, attr).generate(save=True) -class CeleryCacheStateBackend(PessimisticImageCacheBackend): +class CeleryImageCacheBackend(PessimisticImageCacheBackend): """ A pessimistic cache state backend that uses celery to generate its spec images. Like PessimisticCacheStateBackend, this one checks to see if the From f3976a5c68c0ec2d5cc1677666ff3edff3b529c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 17:36:08 +0100 Subject: [PATCH 5/8] change import path --- imagekit/imagecache/async.py | 2 +- imagekit/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imagekit/imagecache/async.py b/imagekit/imagecache/async.py index 213e679..f5dc7a2 100644 --- a/imagekit/imagecache/async.py +++ b/imagekit/imagecache/async.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from celery.task import task -from imagekit.imagecache.base import PessimisticImageCacheBackend +from imagekit.imagecache import PessimisticImageCacheBackend @task diff --git a/imagekit/settings.py b/imagekit/settings.py index ca6f4c8..d84030d 100644 --- a/imagekit/settings.py +++ b/imagekit/settings.py @@ -2,4 +2,4 @@ from django.conf import settings DEFAULT_IMAGE_CACHE_BACKEND = getattr(settings, 'IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND', - 'imagekit.imagecache.base.PessimisticImageCacheBackend') + 'imagekit.imagecache.PessimisticImageCacheBackend') From 7af4940914587161c2792475a2c489ed0e3614c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Mon, 5 Mar 2012 17:47:19 +0100 Subject: [PATCH 6/8] try to import celery on init --- imagekit/imagecache/__init__.py | 2 +- imagekit/imagecache/async.py | 35 ++++++++++++++++++++------------- imagekit/imagecache/base.py | 7 +++++++ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/imagekit/imagecache/__init__.py b/imagekit/imagecache/__init__.py index dab06cd..3c582c8 100644 --- a/imagekit/imagecache/__init__.py +++ b/imagekit/imagecache/__init__.py @@ -1,7 +1,7 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module -from imagekit.imagecache.base import PessimisticImageCacheBackend, NonValidatingImageCacheBackend +from imagekit.imagecache.base import InvalidImageCacheBackendError, PessimisticImageCacheBackend, NonValidatingImageCacheBackend _default_image_cache_backend = None diff --git a/imagekit/imagecache/async.py b/imagekit/imagecache/async.py index f5dc7a2..38426af 100644 --- a/imagekit/imagecache/async.py +++ b/imagekit/imagecache/async.py @@ -1,17 +1,5 @@ # -*- coding: utf-8 -*- -from celery.task import task - -from imagekit.imagecache import PessimisticImageCacheBackend - - -@task -def generate(model, pk, attr): - try: - instance = model._default_manager.get(pk=pk) - except model.DoesNotExist: - pass # The model was deleted since the task was scheduled. NEVER MIND! - else: - getattr(instance, attr).generate(save=True) +from imagekit.imagecache import PessimisticImageCacheBackend, InvalidImageCacheBackendError class CeleryImageCacheBackend(PessimisticImageCacheBackend): @@ -27,9 +15,28 @@ class CeleryImageCacheBackend(PessimisticImageCacheBackend): to interact with file storage. """ + def __init__(self): + try: + import celery + except: + raise InvalidImageCacheBackendError("Celery image cache backend requires either the 'celery' library") + + @property + def _task(self): + from celery.task import task + + @task + def generate(model, pk, attr): + try: + instance = model._default_manager.get(pk=pk) + except model.DoesNotExist: + pass # The model was deleted since the task was scheduled. NEVER MIND! + else: + getattr(instance, attr).generate(save=True) + return generate def invalidate(self, file): - generate.delay(file.instance.__class__, file.instance.pk, file.attname) + self._task.delay(file.instance.__class__, file.instance.pk, file.attname) def clear(self, file): file.delete(save=False) diff --git a/imagekit/imagecache/base.py b/imagekit/imagecache/base.py index b7e3bc2..f06c9b5 100644 --- a/imagekit/imagecache/base.py +++ b/imagekit/imagecache/base.py @@ -1,3 +1,10 @@ +from django.core.exceptions import ImproperlyConfigured + + +class InvalidImageCacheBackendError(ImproperlyConfigured): + pass + + class PessimisticImageCacheBackend(object): """ A very safe image cache backend. Guarantees that files will always be From 1e4ac109f79bc4d54e5faaf39276a404f3e51f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Thu, 8 Mar 2012 19:58:36 +0100 Subject: [PATCH 7/8] rename async to celery --- imagekit/imagecache/{async.py => celery.py} | 2 ++ 1 file changed, 2 insertions(+) rename imagekit/imagecache/{async.py => celery.py} (97%) diff --git a/imagekit/imagecache/async.py b/imagekit/imagecache/celery.py similarity index 97% rename from imagekit/imagecache/async.py rename to imagekit/imagecache/celery.py index 38426af..973f2f3 100644 --- a/imagekit/imagecache/async.py +++ b/imagekit/imagecache/celery.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import + from imagekit.imagecache import PessimisticImageCacheBackend, InvalidImageCacheBackendError From d1af56ba3e44e44b9ebb5423357d17bc16ab87a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timothe=CC=81e=20Peignier?= Date: Thu, 15 Mar 2012 10:37:28 +0100 Subject: [PATCH 8/8] ensure task is correctly created --- imagekit/imagecache/celery.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/imagekit/imagecache/celery.py b/imagekit/imagecache/celery.py index 973f2f3..7e3cf46 100644 --- a/imagekit/imagecache/celery.py +++ b/imagekit/imagecache/celery.py @@ -4,6 +4,15 @@ from __future__ import absolute_import from imagekit.imagecache import PessimisticImageCacheBackend, InvalidImageCacheBackendError +def generate(model, pk, attr): + try: + instance = model._default_manager.get(pk=pk) + except model.DoesNotExist: + pass # The model was deleted since the task was scheduled. NEVER MIND! + else: + getattr(instance, attr).generate(save=True) + + class CeleryImageCacheBackend(PessimisticImageCacheBackend): """ A pessimistic cache state backend that uses celery to generate its spec @@ -19,23 +28,11 @@ class CeleryImageCacheBackend(PessimisticImageCacheBackend): """ def __init__(self): try: - import celery + from celery.task import task except: raise InvalidImageCacheBackendError("Celery image cache backend requires either the 'celery' library") - - @property - def _task(self): - from celery.task import task - - @task - def generate(model, pk, attr): - try: - instance = model._default_manager.get(pk=pk) - except model.DoesNotExist: - pass # The model was deleted since the task was scheduled. NEVER MIND! - else: - getattr(instance, attr).generate(save=True) - return generate + if not getattr(CeleryImageCacheBackend, '_task', None): + CeleryImageCacheBackend._task = task(generate) def invalidate(self, file): self._task.delay(file.instance.__class__, file.instance.pk, file.attname)