diff --git a/imagekit/conf.py b/imagekit/conf.py index 4f3b95c..e43d04e 100644 --- a/imagekit/conf.py +++ b/imagekit/conf.py @@ -1,6 +1,10 @@ from appconf import AppConf +from .imagecache.actions import validate_now, clear_now class ImageKitConf(AppConf): - DEFAULT_IMAGE_CACHE_BACKEND = 'imagekit.imagecache.PessimisticImageCacheBackend' + DEFAULT_IMAGE_CACHE_BACKEND = 'imagekit.imagecache.backends.Simple' + CACHE_BACKEND = None CACHE_DIR = 'CACHE/images' + CACHE_PREFIX = 'ik-' + DEFAULT_SPEC_FIELD_IMAGE_CACHE_STRATEGY = 'imagekit.imagecache.strategies.Pessimistic' diff --git a/imagekit/generators.py b/imagekit/generators.py index 344fc67..e848981 100644 --- a/imagekit/generators.py +++ b/imagekit/generators.py @@ -49,18 +49,21 @@ class SpecFileGenerator(object): content = IKContentFile(filename, imgfile.read(), format=format) return img, content + def get_hash(self, source_file): + return md5(''.join([ + source_file.name, + pickle.dumps(self.get_processors(source_file)), + self.format, + pickle.dumps(self.options), + str(self.autoconvert), + ]).encode('utf-8')).hexdigest() + def generate_filename(self, source_file): source_filename = source_file.name filename = None if source_filename: - hash = md5(''.join([ - pickle.dumps(self.get_processors(source_file)), - self.format, - pickle.dumps(self.options), - str(self.autoconvert), - ])).hexdigest() + hash = self.get_hash(source_file) extension = suggest_extension(source_filename, self.format) - filename = os.path.normpath(os.path.join( settings.IMAGEKIT_CACHE_DIR, os.path.splitext(source_filename)[0], diff --git a/imagekit/imagecache/__init__.py b/imagekit/imagecache/__init__.py index cf98a9d..e69de29 100644 --- a/imagekit/imagecache/__init__.py +++ b/imagekit/imagecache/__init__.py @@ -1,34 +0,0 @@ -from django.core.exceptions import ImproperlyConfigured -from django.utils.importlib import import_module - -from imagekit.imagecache.base import InvalidImageCacheBackendError, PessimisticImageCacheBackend, NonValidatingImageCacheBackend - -_default_image_cache_backend = None - - -def get_default_image_cache_backend(): - """ - Get the default image cache backend. Uses the same method as - django.core.file.storage.get_storage_class - - """ - global _default_image_cache_backend - if not _default_image_cache_backend: - from django.conf import settings - import_path = settings.IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND - try: - dot = import_path.rindex('.') - except ValueError: - raise ImproperlyConfigured("%s isn't an image cache backend module." % \ - import_path) - module, classname = import_path[:dot], import_path[dot + 1:] - try: - mod = import_module(module) - except ImportError, e: - raise ImproperlyConfigured('Error importing image cache backend module %s: "%s"' % (module, e)) - try: - cls = getattr(mod, classname) - _default_image_cache_backend = cls() - except AttributeError: - raise ImproperlyConfigured('Image cache backend module "%s" does not define a "%s" class.' % (module, classname)) - return _default_image_cache_backend diff --git a/imagekit/imagecache/actions.py b/imagekit/imagecache/actions.py new file mode 100644 index 0000000..2e222b3 --- /dev/null +++ b/imagekit/imagecache/actions.py @@ -0,0 +1,22 @@ +def validate_now(file): + file.validate() + + +try: + from celery.task import task +except ImportError: + pass +else: + validate_now_task = task(validate_now) + + +def deferred_validate(file): + try: + import celery + except: + raise ImportError("Deferred validation requires the the 'celery' library") + validate_now_task.delay(file) + + +def clear_now(file): + file.clear() diff --git a/imagekit/imagecache/backends.py b/imagekit/imagecache/backends.py new file mode 100644 index 0000000..cbb7f5d --- /dev/null +++ b/imagekit/imagecache/backends.py @@ -0,0 +1,84 @@ +from ..utils import get_singleton +from django.core.cache import get_cache +from django.core.cache.backends.dummy import DummyCache +from django.core.exceptions import ImproperlyConfigured + + +def get_default_image_cache_backend(): + """ + Get the default image cache backend. + + """ + from django.conf import settings + return get_singleton(settings.IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND, + 'image cache backend') + + +class InvalidImageCacheBackendError(ImproperlyConfigured): + pass + + +class CachedValidationBackend(object): + @property + def cache(self): + if not getattr(self, '_cache', None): + from django.conf import settings + alias = settings.IMAGEKIT_CACHE_BACKEND + self._cache = get_cache(alias) if alias else DummyCache(None, {}) + return self._cache + + def get_key(self, file): + from django.conf import settings + return '%s%s-valid' % (settings.IMAGEKIT_CACHE_PREFIX, file.get_hash()) + + def is_invalid(self, file): + key = self.get_key(file) + cached_value = self.cache.get(key) + if cached_value is None: + cached_value = self._is_invalid(file) + self.cache.set(key, cached_value) + return cached_value + + def validate(self, file): + if self.is_invalid(file): + self._validate(file) + self.cache.set(self.get_key(file), True) + + def invalidate(self, file): + if not self.is_invalid(file): + self._invalidate(file) + self.cache.set(self.get_key(file), False) + + +class Simple(CachedValidationBackend): + """ + The most basic image cache backend. Files are considered valid if they + exist. To invalidate a file, it's deleted; to validate one, it's generated + immediately. + + """ + + def _is_invalid(self, file): + if not getattr(file, '_file', None): + # No file on object. Have to check storage. + return not file.storage.exists(file.name) + return False + + def _validate(self, file): + """ + Generates a new image by running the processors on the source file. + + """ + file.generate(save=True) + + def invalidate(self, file): + """ + Invalidate the file by deleting it. We override ``invalidate()`` + instead of ``_invalidate()`` because we don't really care to check + whether the file is invalid or not. + + """ + file.delete(save=False) + + def clear(self, file): + file.delete(save=False) diff --git a/imagekit/imagecache/base.py b/imagekit/imagecache/base.py deleted file mode 100644 index f06c9b5..0000000 --- a/imagekit/imagecache/base.py +++ /dev/null @@ -1,60 +0,0 @@ -from django.core.exceptions import ImproperlyConfigured - - -class InvalidImageCacheBackendError(ImproperlyConfigured): - pass - - -class PessimisticImageCacheBackend(object): - """ - A very safe image cache backend. Guarantees that files will always be - available, but at the cost of hitting the storage backend. - - """ - - def is_invalid(self, file): - if not getattr(file, '_file', None): - # No file on object. Have to check storage. - return not file.storage.exists(file.name) - return False - - def validate(self, file): - """ - Generates a new image by running the processors on the source file. - - """ - if self.is_invalid(file): - file.generate(save=True) - - def invalidate(self, file): - file.delete(save=False) - - def clear(self, file): - file.delete(save=False) - - -class NonValidatingImageCacheBackend(object): - """ - A backend that is super optimistic about the existence of spec files. It - will hit your file storage much less frequently than the pessimistic - backend, but it is technically possible for a cache file to be missing - after validation. - - """ - - def validate(self, file): - """ - NonValidatingImageCacheBackend has faith, so validate's a no-op. - - """ - pass - - def invalidate(self, file): - """ - Immediately generate a new spec file upon invalidation. - - """ - file.generate(save=True) - - def clear(self, file): - file.delete(save=False) diff --git a/imagekit/imagecache/celery.py b/imagekit/imagecache/celery.py deleted file mode 100644 index 9dee5ca..0000000 --- a/imagekit/imagecache/celery.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import - -from imagekit.imagecache import PessimisticImageCacheBackend, InvalidImageCacheBackendError - - -def generate(model, pk, attr): - try: - instance = model._default_manager.get(pk=pk) - except model.DoesNotExist: - pass # The model was deleted since the task was scheduled. NEVER MIND! - else: - field_file = getattr(instance, attr) - field_file.delete(save=False) - field_file.generate(save=True) - - -class CeleryImageCacheBackend(PessimisticImageCacheBackend): - """ - A pessimistic cache state backend that uses celery to generate its spec - images. Like PessimisticCacheStateBackend, this one checks to see if the - file exists on validation, so the storage is hit fairly frequently, but an - image is guaranteed to exist. However, while validation guarantees the - existence of *an* image, it does not necessarily guarantee that you will get - the correct image, as the spec may be pending regeneration. In other words, - while there are `generate` tasks in the queue, it is possible to get a - stale spec image. The tradeoff is that calling `invalidate()` won't block - to interact with file storage. - - """ - def __init__(self): - try: - from celery.task import task - except: - raise InvalidImageCacheBackendError("Celery image cache backend requires the 'celery' library") - if not getattr(CeleryImageCacheBackend, '_task', None): - CeleryImageCacheBackend._task = task(generate) - - def invalidate(self, file): - self._task.delay(file.instance.__class__, file.instance.pk, file.attname) - - def clear(self, file): - file.delete(save=False) diff --git a/imagekit/imagecache/strategies.py b/imagekit/imagecache/strategies.py new file mode 100644 index 0000000..7ec7252 --- /dev/null +++ b/imagekit/imagecache/strategies.py @@ -0,0 +1,57 @@ +from .actions import validate_now, clear_now +from ..utils import get_singleton + + +class Pessimistic(object): + """ + A caching strategy that validates the file every time it's accessed. + + """ + + def on_access(self, file): + validate_now(file) + + def on_source_delete(self, file): + clear_now(file) + + def on_source_change(self, file): + validate_now(file) + + +class Optimistic(object): + """ + A caching strategy that validates when the source file changes and assumes + that the cached file will persist. + + """ + + def on_source_create(self, file): + validate_now(file) + + def on_source_delete(self, file): + clear_now(file) + + def on_source_change(self, file): + validate_now(file) + + +class DictStrategy(object): + def __init__(self, callbacks): + for k, v in callbacks.items(): + setattr(self, k, v) + + +class StrategyWrapper(object): + def __init__(self, strategy): + if isinstance(strategy, basestring): + strategy = get_singleton(strategy, 'image cache strategy') + elif isinstance(strategy, dict): + strategy = DictStrategy(strategy) + elif callable(strategy): + strategy = strategy() + self._wrapped = strategy + + def invoke_callback(self, name, *args, **kwargs): + func = getattr(self._wrapped, 'on_%s' % name, None) + if func: + func(*args, **kwargs) diff --git a/imagekit/models/fields/__init__.py b/imagekit/models/fields/__init__.py index af9a591..fdb4475 100644 --- a/imagekit/models/fields/__init__.py +++ b/imagekit/models/fields/__init__.py @@ -1,18 +1,17 @@ import os +from django.conf import settings from django.db import models +from django.db.models.signals import post_init, post_save, post_delete -from ...imagecache import get_default_image_cache_backend +from ...imagecache.backends import get_default_image_cache_backend +from ...imagecache.strategies import StrategyWrapper from ...generators import SpecFileGenerator from .files import ImageSpecFieldFile, ProcessedImageFieldFile -from ..receivers import configure_receivers from .utils import ImageSpecFileDescriptor, ImageKitMeta, BoundImageKitMeta from ...utils import suggest_extension -configure_receivers() - - class ImageSpecField(object): """ The heart and soul of the ImageKit library, ImageSpecField allows you to add @@ -21,7 +20,7 @@ class ImageSpecField(object): """ def __init__(self, processors=None, format=None, options=None, image_field=None, pre_cache=None, storage=None, autoconvert=True, - image_cache_backend=None): + image_cache_backend=None, image_cache_strategy=None): """ :param processors: A list of processors to run on the original image. :param format: The format of the output file. If not provided, @@ -40,7 +39,10 @@ class ImageSpecField(object): ``prepare_image()`` should be performed prior to saving. :param image_cache_backend: An object responsible for managing the state of cached files. Defaults to an instance of - IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND + ``IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND`` + :param image_cache_strategy: A dictionary containing callbacks that + allow you to customize how and when the image cache is validated. + Defaults to ``IMAGEKIT_DEFAULT_SPEC_FIELD_IMAGE_CACHE_STRATEGY`` """ @@ -61,6 +63,9 @@ class ImageSpecField(object): self.storage = storage self.image_cache_backend = image_cache_backend or \ get_default_image_cache_backend() + if image_cache_strategy is None: + image_cache_strategy = settings.IMAGEKIT_DEFAULT_SPEC_FIELD_IMAGE_CACHE_STRATEGY + self.image_cache_strategy = StrategyWrapper(image_cache_strategy) def contribute_to_class(self, cls, name): setattr(cls, name, ImageSpecFileDescriptor(self, name)) @@ -78,12 +83,54 @@ class ImageSpecField(object): setattr(cls, '_ik', ik) ik.spec_fields.append(name) + # Connect to the signals only once for this class. + uid = '%s.%s' % (cls.__module__, cls.__name__) + post_init.connect(ImageSpecField._post_init_receiver, sender=cls, + dispatch_uid=uid) + post_save.connect(ImageSpecField._post_save_receiver, sender=cls, + dispatch_uid=uid) + post_delete.connect(ImageSpecField._post_delete_receiver, sender=cls, + dispatch_uid=uid) + # Register the field with the image_cache_backend try: self.image_cache_backend.register_field(cls, self, name) except AttributeError: pass + @staticmethod + def _post_save_receiver(sender, instance=None, created=False, raw=False, **kwargs): + if not raw: + old_hashes = instance._ik._source_hashes.copy() + new_hashes = ImageSpecField._update_source_hashes(instance) + for attname in instance._ik.spec_fields: + file = getattr(instance, attname) + if created: + file.field.image_cache_strategy.invoke_callback('source_create', file) + elif old_hashes[attname] != new_hashes[attname]: + file.field.image_cache_strategy.invoke_callback('source_change', file) + + @staticmethod + def _update_source_hashes(instance): + """ + Stores hashes of the source image files so that they can be compared + later to see whether the source image has changed (and therefore whether + the spec file needs to be regenerated). + + """ + instance._ik._source_hashes = dict((f.attname, hash(f.source_file)) \ + for f in instance._ik.spec_files) + return instance._ik._source_hashes + + @staticmethod + def _post_delete_receiver(sender, instance=None, **kwargs): + for spec_file in instance._ik.spec_files: + spec_file.field.image_cache_strategy.invoke_callback('source_delete', spec_file) + + @staticmethod + def _post_init_receiver(sender, instance, **kwargs): + ImageSpecField._update_source_hashes(instance) + class ProcessedImageField(models.ImageField): """ diff --git a/imagekit/models/fields/files.py b/imagekit/models/fields/files.py index c213ac3..2d58845 100644 --- a/imagekit/models/fields/files.py +++ b/imagekit/models/fields/files.py @@ -6,6 +6,9 @@ class ImageSpecFieldFile(ImageFieldFile): super(ImageSpecFieldFile, self).__init__(instance, field, None) self.attname = attname + def get_hash(self): + return self.field.generator.get_hash(self.source_file) + @property def source_file(self): field_name = getattr(self.field, 'image_field', None) @@ -31,8 +34,7 @@ class ImageSpecFieldFile(ImageFieldFile): def _require_file(self): if not self.source_file: raise ValueError("The '%s' attribute's image_field has no file associated with it." % self.attname) - else: - self.validate() + self.field.image_cache_strategy.invoke_callback('access', self) def clear(self): return self.field.image_cache_backend.clear(self) diff --git a/imagekit/utils.py b/imagekit/utils.py index 173f152..ef549f3 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -3,13 +3,13 @@ import mimetypes import sys import types +from django.core.exceptions import ImproperlyConfigured from django.core.files.base import ContentFile from django.db.models.loading import cache -from django.utils.functional import wraps from django.utils.encoding import smart_str, smart_unicode +from django.utils.functional import wraps +from django.utils.importlib import import_module -from .imagecache import get_default_image_cache_backend -from .lib import Image, ImageFile from .lib import Image, ImageFile, StringIO @@ -375,6 +375,37 @@ def prepare_image(img, format): return img, save_kwargs +def get_class(path, desc): + try: + dot = path.rindex('.') + except ValueError: + raise ImproperlyConfigured("%s isn't a %s module." % (path, desc)) + module, classname = path[:dot], path[dot + 1:] + try: + mod = import_module(module) + except ImportError, e: + raise ImproperlyConfigured('Error importing %s module %s: "%s"' % + (desc, module, e)) + try: + cls = getattr(mod, classname) + return cls + except AttributeError: + raise ImproperlyConfigured('%s module "%s" does not define a "%s"' + ' class.' % (desc[0].upper() + desc[1:], module, classname)) + + +_singletons = {} + + +def get_singleton(class_path, desc): + global _singletons + cls = get_class(class_path, desc) + instance = _singletons.get(cls) + if not instance: + instance = _singletons[cls] = cls() + return instance + + def ik_model_receiver(fn): @wraps(fn) def receiver(sender, **kwargs):