diff --git a/imagekit/imagecache.py b/imagekit/imagecache.py new file mode 100644 index 0000000..3784957 --- /dev/null +++ b/imagekit/imagecache.py @@ -0,0 +1,87 @@ +from django.core.exceptions import ImproperlyConfigured +from django.utils.importlib import import_module + + +class PessimisticImageCacheBackend(object): + """ + A very safe image cache backend. Guarantees that files will always be + available, but at the cost of hitting the storage backend. + + """ + + def is_invalid(self, file): + if not getattr(file, '_file', None): + # No file on object. Have to check storage. + return not file.storage.exists(file.name) + return False + + def validate(self, file): + """ + Generates a new image by running the processors on the source file. + + """ + if self.is_invalid(file): + file.generate(save=True) + + def invalidate(self, file): + file.delete(save=False) + + def clear(self, file): + file.delete(save=False) + + +class NonValidatingImageCacheBackend(object): + """ + A backend that is super optimistic about the existence of spec files. It + will hit your file storage much less frequently than the pessimistic + backend, but it is technically possible for a cache file to be missing + after validation. + + """ + + def validate(self, file): + """ + NonValidatingImageCacheBackend has faith, so validate's a no-op. + + """ + pass + + def invalidate(self, file): + """ + Immediately generate a new spec file upon invalidation. + + """ + file.generate(save=True) + + def clear(self, file): + file.delete(save=False) + + +_default_image_cache_backend = None + + +def get_default_image_cache_backend(): + """ + Get the default image cache backend. Uses the same method as + django.core.file.storage.get_storage_class + + """ + global _default_image_cache_backend + if not _default_image_cache_backend: + from .settings import DEFAULT_IMAGE_CACHE_BACKEND as import_path + try: + dot = import_path.rindex('.') + except ValueError: + raise ImproperlyConfigured("%s isn't an image cache backend module." % \ + import_path) + module, classname = import_path[:dot], import_path[dot+1:] + try: + mod = import_module(module) + except ImportError, e: + raise ImproperlyConfigured('Error importing image cache backend module %s: "%s"' % (module, e)) + try: + cls = getattr(mod, classname) + _default_image_cache_backend = cls() + except AttributeError: + raise ImproperlyConfigured('Image cache backend module "%s" does not define a "%s" class.' % (module, classname)) + return _default_image_cache_backend diff --git a/imagekit/management/commands/ikcacheinvalidate.py b/imagekit/management/commands/ikcacheinvalidate.py new file mode 100644 index 0000000..2b6e915 --- /dev/null +++ b/imagekit/management/commands/ikcacheinvalidate.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from django.db.models.loading import cache +from ...utils import invalidate_app_cache + + +class Command(BaseCommand): + help = ('Invalidates the image cache for a list of apps.') + args = '[apps]' + requires_model_validation = True + can_import_settings = True + + def handle(self, *args, **options): + apps = args or cache.app_models.keys() + invalidate_app_cache(apps) diff --git a/imagekit/management/commands/ikcachevalidate.py b/imagekit/management/commands/ikcachevalidate.py new file mode 100644 index 0000000..8e9fc6c --- /dev/null +++ b/imagekit/management/commands/ikcachevalidate.py @@ -0,0 +1,30 @@ +from optparse import make_option +from django.core.management.base import BaseCommand +from django.db.models.loading import cache +from ...utils import validate_app_cache + + +class Command(BaseCommand): + help = ('Validates the image cache for a list of apps.') + args = '[apps]' + requires_model_validation = True + can_import_settings = True + + option_list = BaseCommand.option_list + ( + make_option('--force-revalidation', + dest='force_revalidation', + action='store_true', + default=False, + help='Invalidate each image file before validating it, thereby' + ' ensuring its revalidation. This is very similar to' + ' running ikcacheinvalidate and then running' + ' ikcachevalidate; the difference being that this option' + ' causes files to be invalidated and validated' + ' one-at-a-time, whereas running the two commands in series' + ' would invalidate all images before validating any.' + ), + ) + + def handle(self, *args, **options): + apps = args or cache.app_models.keys() + validate_app_cache(apps, options['force_revalidation']) diff --git a/imagekit/management/commands/ikflush.py b/imagekit/management/commands/ikflush.py deleted file mode 100644 index 11d29f5..0000000 --- a/imagekit/management/commands/ikflush.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Flushes and re-caches all images under ImageKit. - -""" -from django.db.models.loading import cache -from django.core.management.base import BaseCommand - -from imagekit.utils import get_spec_files - - -class Command(BaseCommand): - help = ('Clears all ImageKit cached files.') - args = '[apps]' - requires_model_validation = True - can_import_settings = True - - def handle(self, *args, **options): - return flush_cache(args, options) - - -def flush_cache(apps, options): - apps = [a.strip(',') for a in apps] - if apps: - for app_label in apps: - app = cache.get_app(app_label) - for model in [m for m in cache.get_models(app)]: - print 'Flushing cache for "%s.%s"' % (app_label, model.__name__) - for obj in model.objects.order_by('-pk'): - for spec_file in get_spec_files(obj): - spec_file.delete(save=False) - if spec_file.field.pre_cache: - spec_file.generate(False) - else: - print 'Please specify one or more app names' diff --git a/imagekit/models/fields.py b/imagekit/models/fields.py index 038bf72..30f6411 100644 --- a/imagekit/models/fields.py +++ b/imagekit/models/fields.py @@ -5,14 +5,14 @@ from StringIO import StringIO from django.core.files.base import ContentFile from django.db import models from django.db.models.fields.files import ImageFieldFile -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_init, post_save, post_delete from django.utils.encoding import force_unicode, smart_str -from imagekit.utils import img_to_fobj, get_spec_files, open_image, \ +from imagekit.utils import img_to_fobj, open_image, \ format_to_extension, extension_to_format, UnknownFormatError, \ UnknownExtensionError from imagekit.processors import ProcessorPipeline, AutoConvert -import warnings +from .imagecache import get_default_image_cache_backend class _ImageSpecFieldMixin(object): @@ -31,7 +31,30 @@ class _ImageSpecFieldMixin(object): return processors.process(image.copy()) -class ImageSpecField(_ImageSpecFieldMixin): +class BoundImageKitMeta(object): + def __init__(self, instance, spec_fields): + self.instance = instance + self.spec_fields = spec_fields + + @property + def spec_files(self): + return [getattr(self.instance, n) for n in self.spec_fields] + + +class ImageKitMeta(object): + def __init__(self, spec_fields=None): + self.spec_fields = spec_fields or [] + + def __get__(self, instance, owner): + if instance is None: + return self + else: + ik = BoundImageKitMeta(instance, self.spec_fields) + setattr(instance, '_ik', ik) + return ik + + +class ImageSpec(_ImageSpecMixin): """ The heart and soul of the ImageKit library, ImageSpecField allows you to add variants of uploaded images to your models. @@ -40,8 +63,8 @@ class ImageSpecField(_ImageSpecFieldMixin): _upload_to_attr = 'cache_to' def __init__(self, processors=None, format=None, options={}, - image_field=None, pre_cache=False, storage=None, cache_to=None, - autoconvert=True): + image_field=None, pre_cache=None, storage=None, cache_to=None, + autoconvert=True, image_cache_backend=None): """ :param processors: A list of processors to run on the original image. :param format: The format of the output file. If not provided, @@ -54,8 +77,6 @@ class ImageSpecField(_ImageSpecFieldMixin): documentation for others. :param image_field: The name of the model property that contains the original image. - :param pre_cache: A boolean that specifies whether the image should - be generated immediately (True) or on demand (False). :param storage: A Django storage system to use to save the generated image. :param cache_to: Specifies the filename to use when saving the image @@ -75,31 +96,78 @@ class ImageSpecField(_ImageSpecFieldMixin): this extension, it's only a recommendation. :param autoconvert: Specifies whether the AutoConvert processor should be run before saving. + :param image_cache_backend: An object responsible for managing the state + of cached files. Defaults to an instance of + IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND """ - _ImageSpecFieldMixin.__init__(self, processors, format=format, + if pre_cache is not None: + raise Exception('The pre_cache argument has been removed in favor' + ' of cache state backends.') + + _ImageSpecMixin.__init__(self, processors, format=format, options=options, autoconvert=autoconvert) self.image_field = image_field self.pre_cache = pre_cache self.storage = storage self.cache_to = cache_to + self.image_cache_backend = image_cache_backend or \ + get_default_image_cache_backend() def contribute_to_class(self, cls, name): setattr(cls, name, _ImageSpecFieldDescriptor(self, name)) try: ik = getattr(cls, '_ik') except AttributeError: - ik = type('ImageKitMeta', (object,), {'spec_file_names': []}) + ik = ImageKitMeta() setattr(cls, '_ik', ik) - ik.spec_file_names.append(name) + ik.spec_fields.append(name) # Connect to the signals only once for this class. uid = '%s.%s' % (cls.__module__, cls.__name__) - post_save.connect(_post_save_handler, sender=cls, - dispatch_uid='%s_save' % uid) - post_delete.connect(_post_delete_handler, sender=cls, - dispatch_uid='%s.delete' % uid) + post_init.connect(ImageSpec._post_init_receiver, sender=cls, + dispatch_uid=uid) + post_save.connect(ImageSpec._post_save_receiver, sender=cls, + dispatch_uid=uid) + post_delete.connect(ImageSpec._post_delete_receiver, sender=cls, + dispatch_uid=uid) + + # Register the field with the image_cache_backend + try: + self.image_cache_backend.register_field(cls, self, name) + except AttributeError: + pass + + @staticmethod + def _post_save_receiver(sender, instance=None, created=False, raw=False, **kwargs): + if not raw: + old_hashes = instance._ik._source_hashes.copy() + new_hashes = ImageSpec._update_source_hashes(instance) + for attname in instance._ik.spec_fields: + if old_hashes[attname] != new_hashes[attname]: + getattr(instance, attname).invalidate() + + @staticmethod + def _update_source_hashes(instance): + """ + Stores hashes of the source image files so that they can be compared + later to see whether the source image has changed (and therefore whether + the spec file needs to be regenerated). + + """ + instance._ik._source_hashes = dict((f.attname, hash(f.source_file)) \ + for f in instance._ik.spec_files) + return instance._ik._source_hashes + + @staticmethod + def _post_delete_receiver(sender, instance=None, **kwargs): + for spec_file in instance._ik.spec_files: + spec_file.clear() + + @staticmethod + def _post_init_receiver(sender, instance, **kwargs): + ImageSpec._update_source_hashes(instance) def _get_suggested_extension(name, format): @@ -190,40 +258,47 @@ class ImageSpecFieldFile(_ImageSpecFieldFileMixin, ImageFieldFile): raise ValueError("The '%s' attribute's image_field has no file associated with it." % self.attname) def _get_file(self): - self.generate() + self.validate() return super(ImageFieldFile, self).file file = property(_get_file, ImageFieldFile._set_file, ImageFieldFile._del_file) - @property - def url(self): - self.generate() - return super(ImageFieldFile, self).url + def clear(self): + return self.field.image_cache_backend.clear(self) - def generate(self, lazy=True): + def invalidate(self): + return self.field.image_cache_backend.invalidate(self) + + def validate(self): + return self.field.image_cache_backend.validate(self) + + def generate(self, save=True): """ - Generates a new image by running the processors on the source file. - - Keyword Arguments: - lazy -- True if an already-existing image should be returned; - False if a new image should be created and the existing - one overwritten. + Generates a new image file by processing the source file and returns + the content of the result, ready for saving. """ - if lazy and (getattr(self, '_file', None) or self.storage.exists(self.name)): - return - - if self.source_file: # TODO: Should we error here or something if the source_file doesn't exist? + source_file = self.source_file + if source_file: # TODO: Should we error here or something if the source_file doesn't exist? # Process the original image file. try: - fp = self.source_file.storage.open(self.source_file.name) + fp = source_file.storage.open(source_file.name) except IOError: return fp.seek(0) fp = StringIO(fp.read()) img, content = self._process_content(self.name, fp) - self.storage.save(self.name, content) + + if save: + self.storage.save(self.name, content) + + return content + + @property + def url(self): + self.validate() + return super(ImageFieldFile, self).url def delete(self, save=False): """ @@ -342,7 +417,7 @@ def _post_delete_handler(sender, instance=None, **kwargs): spec_file.delete(save=False) -class ProcessedImageFieldFile(ImageFieldFile, _ImageSpecFieldFileMixin): +class ProcessedImageFieldFile(ImageFieldFile, _ImageSpecFileMixin): def save(self, name, content, save=True): new_filename = self.field.generate_filename(self.instance, name) img, content = self._process_content(new_filename, content) diff --git a/imagekit/settings.py b/imagekit/settings.py new file mode 100644 index 0000000..d84030d --- /dev/null +++ b/imagekit/settings.py @@ -0,0 +1,5 @@ +from django.conf import settings + +DEFAULT_IMAGE_CACHE_BACKEND = getattr(settings, + 'IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND', + 'imagekit.imagecache.PessimisticImageCacheBackend') diff --git a/imagekit/utils.py b/imagekit/utils.py index deadbbd..305bdd4 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -1,6 +1,7 @@ import tempfile import types +from django.db.models.loading import cache from django.utils.functional import wraps from imagekit.lib import Image, ImageFile @@ -26,11 +27,9 @@ def img_to_fobj(img, format, **kwargs): def get_spec_files(instance): try: - ik = getattr(instance, '_ik') + return instance._ik.spec_files except AttributeError: return [] - else: - return [getattr(instance, n) for n in ik.spec_file_names] def open_image(target): @@ -136,3 +135,33 @@ def format_to_extension(format): if not extension: raise UnknownFormatError(format) return extension + + +def _get_models(apps): + models = [] + for app_label in apps or []: + app = cache.get_app(app_label) + models += [m for m in cache.get_models(app)] + return models + + +def invalidate_app_cache(apps): + for model in _get_models(apps): + print 'Invalidating cache for "%s.%s"' % (model._meta.app_label, model.__name__) + for obj in model._default_manager.order_by('-pk'): + for f in get_spec_files(obj): + f.invalidate() + + +def validate_app_cache(apps, force_revalidation=False): + for model in _get_models(apps): + for obj in model._default_manager.order_by('-pk'): + model_name = '%s.%s' % (model._meta.app_label, model.__name__) + if force_revalidation: + print 'Invalidating & validating cache for "%s"' % model_name + else: + print 'Validating cache for "%s"' % model_name + for f in get_spec_files(obj): + if force_revalidation: + f.invalidate() + f.validate()