From 62ff07bf1f280e6c0a5c5e7da9ed99960d98d613 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Wed, 1 Feb 2012 22:37:58 -0500 Subject: [PATCH 01/13] First shot at cache state backend implementation --- imagekit/cachestate/__init__.py | 55 ++++++++++++++++++ imagekit/management/commands/ikflush.py | 4 +- imagekit/models.py | 75 ++++++++++++++----------- imagekit/settings.py | 5 ++ 4 files changed, 102 insertions(+), 37 deletions(-) create mode 100644 imagekit/cachestate/__init__.py create mode 100644 imagekit/settings.py diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py new file mode 100644 index 0000000..7f3d10d --- /dev/null +++ b/imagekit/cachestate/__init__.py @@ -0,0 +1,55 @@ +from imagekit.utils import get_spec_files +from django.core.exceptions import ImproperlyConfigured +from django.utils.importlib import import_module + + +class DefaultCacheStateBackend(object): + + def is_invalid(self, file): + if not getattr(file, '_file', None): + # No file on object. Have to check storage. + return not file.storage.exists(file.name) + return False + + def validate(self, file): + """ + Generates a new image by running the processors on the source file. + + """ + if self.is_invalid(file): + content = file.generate() + if content: + file.storage.save(file.name, content) + + def invalidate(self, file): + file.delete(save=False) + + +_default_cache_state_backend = None + + +def get_default_cache_state_backend(): + """ + Get the default cache state backend. Uses the same method as + django.core.file.storage.get_storage_class + + """ + global _default_cache_state_backend + if not _default_cache_state_backend: + from ..settings import DEFAULT_CACHE_STATE_BACKEND as import_path + try: + dot = import_path.rindex('.') + except ValueError: + raise ImproperlyConfigured("%s isn't a cache state backend module." % \ + import_path) + module, classname = import_path[:dot], import_path[dot+1:] + try: + mod = import_module(module) + except ImportError, e: + raise ImproperlyConfigured('Error importing cache state module %s: "%s"' % (module, e)) + try: + cls = getattr(mod, classname) + _default_cache_state_backend = cls() + except AttributeError: + raise ImproperlyConfigured('Cache state backend module "%s" does not define a "%s" class.' % (module, classname)) + return _default_cache_state_backend diff --git a/imagekit/management/commands/ikflush.py b/imagekit/management/commands/ikflush.py index 11d29f5..1609ed5 100644 --- a/imagekit/management/commands/ikflush.py +++ b/imagekit/management/commands/ikflush.py @@ -27,8 +27,6 @@ def flush_cache(apps, options): print 'Flushing cache for "%s.%s"' % (app_label, model.__name__) for obj in model.objects.order_by('-pk'): for spec_file in get_spec_files(obj): - spec_file.delete(save=False) - if spec_file.field.pre_cache: - spec_file.generate(False) + spec_file.invalidate() else: print 'Please specify one or more app names' diff --git a/imagekit/models.py b/imagekit/models.py index 01569fc..590b163 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -12,6 +12,7 @@ from imagekit.utils import img_to_fobj, get_spec_files, open_image, \ format_to_extension, extension_to_format, UnknownFormatError, \ UnknownExtensionError from imagekit.processors import ProcessorPipeline, AutoConvert +from .cachestate import get_default_cache_state_backend class _ImageSpecMixin(object): @@ -39,8 +40,8 @@ class ImageSpec(_ImageSpecMixin): _upload_to_attr = 'cache_to' def __init__(self, processors=None, format=None, options={}, - image_field=None, pre_cache=False, storage=None, cache_to=None, - autoconvert=True): + image_field=None, pre_cache=None, storage=None, cache_to=None, + autoconvert=True, cache_state_backend=None): """ :param processors: A list of processors to run on the original image. :param format: The format of the output file. If not provided, @@ -53,8 +54,6 @@ class ImageSpec(_ImageSpecMixin): documentation for others. :param image_field: The name of the model property that contains the original image. - :param pre_cache: A boolean that specifies whether the image should - be generated immediately (True) or on demand (False). :param storage: A Django storage system to use to save the generated image. :param cache_to: Specifies the filename to use when saving the image @@ -74,15 +73,24 @@ class ImageSpec(_ImageSpecMixin): this extension, it's only a recommendation. :param autoconvert: Specifies whether the AutoConvert processor should be run before saving. + :param cache_state_backend: An object responsible for managing the state + of cached files. Defaults to an instance of + IMAGEKIT_DEFAULT_CACHE_STATE_BACKEND """ + if pre_cache is not None: + raise Exception('The pre_cache argument has been removed in favor' + ' of cache state backends.') + _ImageSpecMixin.__init__(self, processors, format=format, options=options, autoconvert=autoconvert) self.image_field = image_field self.pre_cache = pre_cache self.storage = storage self.cache_to = cache_to + self.cache_state_backend = cache_state_backend or \ + get_default_cache_state_backend() def contribute_to_class(self, cls, name): setattr(cls, name, _ImageSpecDescriptor(self, name)) @@ -100,6 +108,12 @@ class ImageSpec(_ImageSpecMixin): post_delete.connect(_post_delete_handler, sender=cls, dispatch_uid='%s.delete' % uid) + # Register the field with the cache_state_backend + try: + self.cache_state_backend.register_field(cls, self, name) + except AttributeError: + pass + def _get_suggested_extension(name, format): original_extension = os.path.splitext(name)[1] @@ -189,40 +203,40 @@ class ImageSpecFile(_ImageSpecFileMixin, ImageFieldFile): raise ValueError("The '%s' attribute's image_field has no file associated with it." % self.attname) def _get_file(self): - self.generate() + self.validate() return super(ImageFieldFile, self).file file = property(_get_file, ImageFieldFile._set_file, ImageFieldFile._del_file) - @property - def url(self): - self.generate() - return super(ImageFieldFile, self).url + def invalidate(self): + return self.field.cache_state_backend.invalidate(self) - def generate(self, lazy=True): + def validate(self): + return self.field.cache_state_backend.validate(self) + + def generate(self): """ - Generates a new image by running the processors on the source file. - - Keyword Arguments: - lazy -- True if an already-existing image should be returned; - False if a new image should be created and the existing - one overwritten. + Generates a new image file by processing the source file and returns + the content of the result, ready for saving. """ - if lazy and (getattr(self, '_file', None) or self.storage.exists(self.name)): - return - - if self.source_file: # TODO: Should we error here or something if the source_file doesn't exist? + source_file = self.source_file + if source_file: # TODO: Should we error here or something if the source_file doesn't exist? # Process the original image file. try: - fp = self.source_file.storage.open(self.source_file.name) + fp = source_file.storage.open(source_file.name) except IOError: return fp.seek(0) fp = StringIO(fp.read()) img, content = self._process_content(self.name, fp) - self.storage.save(self.name, content) + return content + + @property + def url(self): + self.validate() + return super(ImageFieldFile, self).url def delete(self, save=False): """ @@ -323,21 +337,14 @@ class _ImageSpecDescriptor(object): def _post_save_handler(sender, instance=None, created=False, raw=False, **kwargs): - if raw: - return - spec_files = get_spec_files(instance) - for spec_file in spec_files: - if not created: - spec_file.delete(save=False) - if spec_file.field.pre_cache: - spec_file.generate(False) + if not raw: + for spec_file in get_spec_files(instance): + spec_file.invalidate() def _post_delete_handler(sender, instance=None, **kwargs): - assert instance._get_pk_val() is not None, "%s object can't be deleted because its %s attribute is set to None." % (instance._meta.object_name, instance._meta.pk.attname) - spec_files = get_spec_files(instance) - for spec_file in spec_files: - spec_file.delete(save=False) + for spec_file in get_spec_files(instance): + spec_file.invalidate() class ProcessedImageFieldFile(ImageFieldFile, _ImageSpecFileMixin): diff --git a/imagekit/settings.py b/imagekit/settings.py new file mode 100644 index 0000000..b6ca0b9 --- /dev/null +++ b/imagekit/settings.py @@ -0,0 +1,5 @@ +from django.conf import settings + +DEFAULT_CACHE_STATE_BACKEND = getattr(settings, + 'IMAGEKIT_DEFAULT_CACHE_STATE_BACKEND', + 'imagekit.cachestate.DefaultCacheStateBackend') From a1638127cfbdd53e172968ee3d4434f4c4c1f746 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Wed, 1 Feb 2012 22:41:33 -0500 Subject: [PATCH 02/13] Remove unused import --- imagekit/cachestate/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index 7f3d10d..c9fe316 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -1,4 +1,3 @@ -from imagekit.utils import get_spec_files from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module From a668df1c378a74823923714c0c00638e8413347b Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Wed, 1 Feb 2012 23:26:39 -0500 Subject: [PATCH 03/13] Spec files now accessible through _ik attr --- imagekit/management/commands/ikflush.py | 7 +++--- imagekit/models.py | 33 +++++++++++++++++++++---- imagekit/utils.py | 4 +-- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/imagekit/management/commands/ikflush.py b/imagekit/management/commands/ikflush.py index 1609ed5..de4be3f 100644 --- a/imagekit/management/commands/ikflush.py +++ b/imagekit/management/commands/ikflush.py @@ -4,8 +4,7 @@ Flushes and re-caches all images under ImageKit. """ from django.db.models.loading import cache from django.core.management.base import BaseCommand - -from imagekit.utils import get_spec_files +from .utils import get_spec_files class Command(BaseCommand): @@ -26,7 +25,7 @@ def flush_cache(apps, options): for model in [m for m in cache.get_models(app)]: print 'Flushing cache for "%s.%s"' % (app_label, model.__name__) for obj in model.objects.order_by('-pk'): - for spec_file in get_spec_files(obj): - spec_file.invalidate() + for f in get_spec_files(obj): + f.invalidate() else: print 'Please specify one or more app names' diff --git a/imagekit/models.py b/imagekit/models.py index 590b163..0bd175f 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -8,7 +8,7 @@ from django.db.models.fields.files import ImageFieldFile from django.db.models.signals import post_save, post_delete from django.utils.encoding import force_unicode, smart_str -from imagekit.utils import img_to_fobj, get_spec_files, open_image, \ +from imagekit.utils import img_to_fobj, open_image, \ format_to_extension, extension_to_format, UnknownFormatError, \ UnknownExtensionError from imagekit.processors import ProcessorPipeline, AutoConvert @@ -31,6 +31,29 @@ class _ImageSpecMixin(object): return processors.process(image.copy()) +class BoundImageKitMeta(object): + def __init__(self, instance, spec_fields): + self.instance = instance + self.spec_fields = spec_fields + + @property + def spec_files(self): + return [getattr(self.instance, n) for n in self.spec_fields] + + +class ImageKitMeta(object): + def __init__(self, spec_fields=None): + self.spec_fields = spec_fields or [] + + def __get__(self, instance, owner): + if instance is None: + return self + else: + ik = BoundImageKitMeta(instance, self.spec_fields) + setattr(instance, '_ik', ik) + return ik + + class ImageSpec(_ImageSpecMixin): """ The heart and soul of the ImageKit library, ImageSpec allows you to add @@ -97,9 +120,9 @@ class ImageSpec(_ImageSpecMixin): try: ik = getattr(cls, '_ik') except AttributeError: - ik = type('ImageKitMeta', (object,), {'spec_file_names': []}) + ik = ImageKitMeta() setattr(cls, '_ik', ik) - ik.spec_file_names.append(name) + ik.spec_fields.append(name) # Connect to the signals only once for this class. uid = '%s.%s' % (cls.__module__, cls.__name__) @@ -338,12 +361,12 @@ class _ImageSpecDescriptor(object): def _post_save_handler(sender, instance=None, created=False, raw=False, **kwargs): if not raw: - for spec_file in get_spec_files(instance): + for spec_file in instance._ik.spec_files: spec_file.invalidate() def _post_delete_handler(sender, instance=None, **kwargs): - for spec_file in get_spec_files(instance): + for spec_file in instance._ik.spec_files: spec_file.invalidate() diff --git a/imagekit/utils.py b/imagekit/utils.py index deadbbd..7a7592c 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -26,11 +26,9 @@ def img_to_fobj(img, format, **kwargs): def get_spec_files(instance): try: - ik = getattr(instance, '_ik') + return obj._ik.spec_files except AttributeError: return [] - else: - return [getattr(instance, n) for n in ik.spec_file_names] def open_image(target): From c4fc09c688b7671d213e474b9126dd2cde82daff Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Thu, 2 Feb 2012 00:08:22 -0500 Subject: [PATCH 04/13] Fix var name typo --- imagekit/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagekit/utils.py b/imagekit/utils.py index 7a7592c..02e71c1 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -26,7 +26,7 @@ def img_to_fobj(img, format, **kwargs): def get_spec_files(instance): try: - return obj._ik.spec_files + return instance._ik.spec_files except AttributeError: return [] From 8b6ba687e4ce0d3306cced4ef93be2438d7c0934 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Thu, 2 Feb 2012 00:19:46 -0500 Subject: [PATCH 05/13] Commands for validating and invalidating the cache --- .../management/commands/ikcacheinvalidate.py | 14 +++++++++ .../management/commands/ikcachevalidate.py | 30 ++++++++++++++++++ imagekit/management/commands/ikflush.py | 31 ------------------- imagekit/utils.py | 31 +++++++++++++++++++ 4 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 imagekit/management/commands/ikcacheinvalidate.py create mode 100644 imagekit/management/commands/ikcachevalidate.py delete mode 100644 imagekit/management/commands/ikflush.py diff --git a/imagekit/management/commands/ikcacheinvalidate.py b/imagekit/management/commands/ikcacheinvalidate.py new file mode 100644 index 0000000..2b6e915 --- /dev/null +++ b/imagekit/management/commands/ikcacheinvalidate.py @@ -0,0 +1,14 @@ +from django.core.management.base import BaseCommand +from django.db.models.loading import cache +from ...utils import invalidate_app_cache + + +class Command(BaseCommand): + help = ('Invalidates the image cache for a list of apps.') + args = '[apps]' + requires_model_validation = True + can_import_settings = True + + def handle(self, *args, **options): + apps = args or cache.app_models.keys() + invalidate_app_cache(apps) diff --git a/imagekit/management/commands/ikcachevalidate.py b/imagekit/management/commands/ikcachevalidate.py new file mode 100644 index 0000000..fc9a9c0 --- /dev/null +++ b/imagekit/management/commands/ikcachevalidate.py @@ -0,0 +1,30 @@ +from optparse import make_option +from django.core.management.base import BaseCommand +from django.db.models.loading import cache +from ...utils import validate_app_cache + + +class Command(BaseCommand): + help = ('Validates the image cache for a list of apps.') + args = '[apps]' + requires_model_validation = True + can_import_settings = True + + option_list = BaseCommand.option_list + ( + make_option('--force', + dest='force_revalidation', + action='store_true', + default=False, + help='Invalidate each image file before validating it, thereby' + ' ensuring its revalidation. This is very similar to' + ' running ikcacheinvalidate and then running' + ' ikcachevalidate; the difference being that this option' + ' causes files to be invalidated and validated' + ' one-at-a-time, whereas running the two commands in series' + ' would invalidate all images before validating any.' + ), + ) + + def handle(self, *args, **options): + apps = args or cache.app_models.keys() + validate_app_cache(apps, options['force_revalidation']) diff --git a/imagekit/management/commands/ikflush.py b/imagekit/management/commands/ikflush.py deleted file mode 100644 index de4be3f..0000000 --- a/imagekit/management/commands/ikflush.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Flushes and re-caches all images under ImageKit. - -""" -from django.db.models.loading import cache -from django.core.management.base import BaseCommand -from .utils import get_spec_files - - -class Command(BaseCommand): - help = ('Clears all ImageKit cached files.') - args = '[apps]' - requires_model_validation = True - can_import_settings = True - - def handle(self, *args, **options): - return flush_cache(args, options) - - -def flush_cache(apps, options): - apps = [a.strip(',') for a in apps] - if apps: - for app_label in apps: - app = cache.get_app(app_label) - for model in [m for m in cache.get_models(app)]: - print 'Flushing cache for "%s.%s"' % (app_label, model.__name__) - for obj in model.objects.order_by('-pk'): - for f in get_spec_files(obj): - f.invalidate() - else: - print 'Please specify one or more app names' diff --git a/imagekit/utils.py b/imagekit/utils.py index 02e71c1..305bdd4 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -1,6 +1,7 @@ import tempfile import types +from django.db.models.loading import cache from django.utils.functional import wraps from imagekit.lib import Image, ImageFile @@ -134,3 +135,33 @@ def format_to_extension(format): if not extension: raise UnknownFormatError(format) return extension + + +def _get_models(apps): + models = [] + for app_label in apps or []: + app = cache.get_app(app_label) + models += [m for m in cache.get_models(app)] + return models + + +def invalidate_app_cache(apps): + for model in _get_models(apps): + print 'Invalidating cache for "%s.%s"' % (model._meta.app_label, model.__name__) + for obj in model._default_manager.order_by('-pk'): + for f in get_spec_files(obj): + f.invalidate() + + +def validate_app_cache(apps, force_revalidation=False): + for model in _get_models(apps): + for obj in model._default_manager.order_by('-pk'): + model_name = '%s.%s' % (model._meta.app_label, model.__name__) + if force_revalidation: + print 'Invalidating & validating cache for "%s"' % model_name + else: + print 'Validating cache for "%s"' % model_name + for f in get_spec_files(obj): + if force_revalidation: + f.invalidate() + f.validate() From 011c0c2e5fbc72f3289c9da88f3e653a5e0c8a70 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Thu, 2 Feb 2012 23:20:09 -0500 Subject: [PATCH 06/13] Add clear method for when future validation is unwanted --- imagekit/cachestate/__init__.py | 3 +++ imagekit/models.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index c9fe316..165314a 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -23,6 +23,9 @@ class DefaultCacheStateBackend(object): def invalidate(self, file): file.delete(save=False) + def clear(self, file): + file.delete(save=False) + _default_cache_state_backend = None diff --git a/imagekit/models.py b/imagekit/models.py index 0bd175f..fcc3692 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -231,6 +231,9 @@ class ImageSpecFile(_ImageSpecFileMixin, ImageFieldFile): file = property(_get_file, ImageFieldFile._set_file, ImageFieldFile._del_file) + def clear(self): + return self.field.cache_state_backend.clear(self) + def invalidate(self): return self.field.cache_state_backend.invalidate(self) @@ -367,7 +370,7 @@ def _post_save_handler(sender, instance=None, created=False, raw=False, **kwargs def _post_delete_handler(sender, instance=None, **kwargs): for spec_file in instance._ik.spec_files: - spec_file.invalidate() + spec_file.clear() class ProcessedImageFieldFile(ImageFieldFile, _ImageSpecFileMixin): From 513b23b169e87a92b2fdf0bd6b33778ea68b9b24 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Thu, 2 Feb 2012 23:23:20 -0500 Subject: [PATCH 07/13] Rename force flag to force-revalidation --- imagekit/management/commands/ikcachevalidate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagekit/management/commands/ikcachevalidate.py b/imagekit/management/commands/ikcachevalidate.py index fc9a9c0..8e9fc6c 100644 --- a/imagekit/management/commands/ikcachevalidate.py +++ b/imagekit/management/commands/ikcachevalidate.py @@ -11,7 +11,7 @@ class Command(BaseCommand): can_import_settings = True option_list = BaseCommand.option_list + ( - make_option('--force', + make_option('--force-revalidation', dest='force_revalidation', action='store_true', default=False, From 35b807cfa929c50131afefb4838fa7ccbf866cde Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Fri, 3 Feb 2012 09:16:50 -0500 Subject: [PATCH 08/13] generate method (optionally) saves file This way, there's a creation counterpart to `delete()`. The user shouldn't have to deal with storage backends to create and delete the files, and now they don't. --- imagekit/cachestate/__init__.py | 4 +--- imagekit/models.py | 6 +++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index 165314a..32d43c6 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -16,9 +16,7 @@ class DefaultCacheStateBackend(object): """ if self.is_invalid(file): - content = file.generate() - if content: - file.storage.save(file.name, content) + file.generate(save=True) def invalidate(self, file): file.delete(save=False) diff --git a/imagekit/models.py b/imagekit/models.py index fcc3692..a742719 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -240,7 +240,7 @@ class ImageSpecFile(_ImageSpecFileMixin, ImageFieldFile): def validate(self): return self.field.cache_state_backend.validate(self) - def generate(self): + def generate(self, save=True): """ Generates a new image file by processing the source file and returns the content of the result, ready for saving. @@ -257,6 +257,10 @@ class ImageSpecFile(_ImageSpecFileMixin, ImageFieldFile): fp = StringIO(fp.read()) img, content = self._process_content(self.name, fp) + + if save: + self.storage.save(self.name, content) + return content @property From 60a8c0633637a5716422ecad0841b8ba1d23fc75 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Fri, 10 Feb 2012 22:08:13 -0500 Subject: [PATCH 09/13] Rename DefaultCacheStateBackend PessimisticCacheStateBackend more accurately describes what is is, instead of how it's used. --- imagekit/cachestate/__init__.py | 2 +- imagekit/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index 32d43c6..779e229 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -2,7 +2,7 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module -class DefaultCacheStateBackend(object): +class PessimisticCacheStateBackend(object): def is_invalid(self, file): if not getattr(file, '_file', None): diff --git a/imagekit/settings.py b/imagekit/settings.py index b6ca0b9..d02f718 100644 --- a/imagekit/settings.py +++ b/imagekit/settings.py @@ -2,4 +2,4 @@ from django.conf import settings DEFAULT_CACHE_STATE_BACKEND = getattr(settings, 'IMAGEKIT_DEFAULT_CACHE_STATE_BACKEND', - 'imagekit.cachestate.DefaultCacheStateBackend') + 'imagekit.cachestate.PessimisticCacheStateBackend') From 1956e16b4bebcd23018e60369889df505eeb787e Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Sat, 11 Feb 2012 00:04:56 -0500 Subject: [PATCH 10/13] Only invalidate spec file if source changes --- imagekit/models.py | 53 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/imagekit/models.py b/imagekit/models.py index a742719..a6fadfb 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -5,7 +5,7 @@ from StringIO import StringIO from django.core.files.base import ContentFile from django.db import models from django.db.models.fields.files import ImageFieldFile -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_init, post_save, post_delete from django.utils.encoding import force_unicode, smart_str from imagekit.utils import img_to_fobj, open_image, \ @@ -126,10 +126,12 @@ class ImageSpec(_ImageSpecMixin): # Connect to the signals only once for this class. uid = '%s.%s' % (cls.__module__, cls.__name__) - post_save.connect(_post_save_handler, sender=cls, - dispatch_uid='%s_save' % uid) - post_delete.connect(_post_delete_handler, sender=cls, - dispatch_uid='%s.delete' % uid) + post_init.connect(ImageSpec._post_init_receiver, sender=cls, + dispatch_uid=uid) + post_save.connect(ImageSpec._post_save_receiver, sender=cls, + dispatch_uid=uid) + post_delete.connect(ImageSpec._post_delete_receiver, sender=cls, + dispatch_uid=uid) # Register the field with the cache_state_backend try: @@ -137,6 +139,36 @@ class ImageSpec(_ImageSpecMixin): except AttributeError: pass + @staticmethod + def _post_save_receiver(sender, instance=None, created=False, raw=False, **kwargs): + if not raw: + old_hashes = instance._ik._source_hashes.copy() + new_hashes = ImageSpec._update_source_hashes(instance) + for attname in instance._ik.spec_fields: + if old_hashes[attname] != new_hashes[attname]: + getattr(instance, attname).invalidate() + + @staticmethod + def _update_source_hashes(instance): + """ + Stores hashes of the source image files so that they can be compared + later to see whether the source image has changed (and therefore whether + the spec file needs to be regenerated). + + """ + instance._ik._source_hashes = dict((f.attname, hash(f.source_file)) \ + for f in instance._ik.spec_files) + return instance._ik._source_hashes + + @staticmethod + def _post_delete_receiver(sender, instance=None, **kwargs): + for spec_file in instance._ik.spec_files: + spec_file.clear() + + @staticmethod + def _post_init_receiver(sender, instance, **kwargs): + ImageSpec._update_source_hashes(instance) + def _get_suggested_extension(name, format): original_extension = os.path.splitext(name)[1] @@ -366,17 +398,6 @@ class _ImageSpecDescriptor(object): return img_spec_file -def _post_save_handler(sender, instance=None, created=False, raw=False, **kwargs): - if not raw: - for spec_file in instance._ik.spec_files: - spec_file.invalidate() - - -def _post_delete_handler(sender, instance=None, **kwargs): - for spec_file in instance._ik.spec_files: - spec_file.clear() - - class ProcessedImageFieldFile(ImageFieldFile, _ImageSpecFileMixin): def save(self, name, content, save=True): new_filename = self.field.generate_filename(self.instance, name) From 164a4e11fc0e0d4fb7cbf5d533cd464a73638e97 Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Sat, 11 Feb 2012 00:15:58 -0500 Subject: [PATCH 11/13] Some documentation --- imagekit/cachestate/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index 779e229..f5a01d8 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -3,6 +3,11 @@ from django.utils.importlib import import_module class PessimisticCacheStateBackend(object): + """ + A very safe cache state backend. Guarantees that files will always be + available, but at the cost of hitting the storage backend. + + """ def is_invalid(self, file): if not getattr(file, '_file', None): From 15b15afe2c94c1873736e02236ccc14422e1e5ee Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Sat, 11 Feb 2012 00:42:56 -0500 Subject: [PATCH 12/13] Add NonValidatingCacheStateBackend --- imagekit/cachestate/__init__.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/imagekit/cachestate/__init__.py b/imagekit/cachestate/__init__.py index f5a01d8..e9ee56c 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/cachestate/__init__.py @@ -30,6 +30,33 @@ class PessimisticCacheStateBackend(object): file.delete(save=False) +class NonValidatingCacheStateBackend(object): + """ + A backend that is super optimistic about the existence of spec files. It + will hit your file storage much less frequently than the pessimistic + backend, but it is technically possible for a cache file to be missing + after validation. + + """ + + def validate(self, file): + """ + NonValidatingCacheStateBackend has faith, so validate's a no-op. + + """ + pass + + def invalidate(self, file): + """ + Immediately generate a new spec file upon invalidation. + + """ + file.generate(save=True) + + def clear(self, file): + file.delete(save=False) + + _default_cache_state_backend = None From e9e364eedda9e11aa65df690bee2dc1b83c4843d Mon Sep 17 00:00:00 2001 From: Matthew Tretter Date: Sun, 12 Feb 2012 16:18:34 -0500 Subject: [PATCH 13/13] Rename cache_state_backend to image_cache_backend Related names (like the package and class names) have also been updated. --- .../{cachestate/__init__.py => imagecache.py} | 30 +++++++++---------- imagekit/models.py | 22 +++++++------- imagekit/settings.py | 6 ++-- 3 files changed, 29 insertions(+), 29 deletions(-) rename imagekit/{cachestate/__init__.py => imagecache.py} (67%) diff --git a/imagekit/cachestate/__init__.py b/imagekit/imagecache.py similarity index 67% rename from imagekit/cachestate/__init__.py rename to imagekit/imagecache.py index e9ee56c..3784957 100644 --- a/imagekit/cachestate/__init__.py +++ b/imagekit/imagecache.py @@ -2,9 +2,9 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.importlib import import_module -class PessimisticCacheStateBackend(object): +class PessimisticImageCacheBackend(object): """ - A very safe cache state backend. Guarantees that files will always be + A very safe image cache backend. Guarantees that files will always be available, but at the cost of hitting the storage backend. """ @@ -30,7 +30,7 @@ class PessimisticCacheStateBackend(object): file.delete(save=False) -class NonValidatingCacheStateBackend(object): +class NonValidatingImageCacheBackend(object): """ A backend that is super optimistic about the existence of spec files. It will hit your file storage much less frequently than the pessimistic @@ -41,7 +41,7 @@ class NonValidatingCacheStateBackend(object): def validate(self, file): """ - NonValidatingCacheStateBackend has faith, so validate's a no-op. + NonValidatingImageCacheBackend has faith, so validate's a no-op. """ pass @@ -57,31 +57,31 @@ class NonValidatingCacheStateBackend(object): file.delete(save=False) -_default_cache_state_backend = None +_default_image_cache_backend = None -def get_default_cache_state_backend(): +def get_default_image_cache_backend(): """ - Get the default cache state backend. Uses the same method as + Get the default image cache backend. Uses the same method as django.core.file.storage.get_storage_class """ - global _default_cache_state_backend - if not _default_cache_state_backend: - from ..settings import DEFAULT_CACHE_STATE_BACKEND as import_path + global _default_image_cache_backend + if not _default_image_cache_backend: + from .settings import DEFAULT_IMAGE_CACHE_BACKEND as import_path try: dot = import_path.rindex('.') except ValueError: - raise ImproperlyConfigured("%s isn't a cache state backend module." % \ + raise ImproperlyConfigured("%s isn't an image cache backend module." % \ import_path) module, classname = import_path[:dot], import_path[dot+1:] try: mod = import_module(module) except ImportError, e: - raise ImproperlyConfigured('Error importing cache state module %s: "%s"' % (module, e)) + raise ImproperlyConfigured('Error importing image cache backend module %s: "%s"' % (module, e)) try: cls = getattr(mod, classname) - _default_cache_state_backend = cls() + _default_image_cache_backend = cls() except AttributeError: - raise ImproperlyConfigured('Cache state backend module "%s" does not define a "%s" class.' % (module, classname)) - return _default_cache_state_backend + raise ImproperlyConfigured('Image cache backend module "%s" does not define a "%s" class.' % (module, classname)) + return _default_image_cache_backend diff --git a/imagekit/models.py b/imagekit/models.py index a6fadfb..1ba7405 100755 --- a/imagekit/models.py +++ b/imagekit/models.py @@ -12,7 +12,7 @@ from imagekit.utils import img_to_fobj, open_image, \ format_to_extension, extension_to_format, UnknownFormatError, \ UnknownExtensionError from imagekit.processors import ProcessorPipeline, AutoConvert -from .cachestate import get_default_cache_state_backend +from .imagecache import get_default_image_cache_backend class _ImageSpecMixin(object): @@ -64,7 +64,7 @@ class ImageSpec(_ImageSpecMixin): def __init__(self, processors=None, format=None, options={}, image_field=None, pre_cache=None, storage=None, cache_to=None, - autoconvert=True, cache_state_backend=None): + autoconvert=True, image_cache_backend=None): """ :param processors: A list of processors to run on the original image. :param format: The format of the output file. If not provided, @@ -96,9 +96,9 @@ class ImageSpec(_ImageSpecMixin): this extension, it's only a recommendation. :param autoconvert: Specifies whether the AutoConvert processor should be run before saving. - :param cache_state_backend: An object responsible for managing the state + :param image_cache_backend: An object responsible for managing the state of cached files. Defaults to an instance of - IMAGEKIT_DEFAULT_CACHE_STATE_BACKEND + IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND """ @@ -112,8 +112,8 @@ class ImageSpec(_ImageSpecMixin): self.pre_cache = pre_cache self.storage = storage self.cache_to = cache_to - self.cache_state_backend = cache_state_backend or \ - get_default_cache_state_backend() + self.image_cache_backend = image_cache_backend or \ + get_default_image_cache_backend() def contribute_to_class(self, cls, name): setattr(cls, name, _ImageSpecDescriptor(self, name)) @@ -133,9 +133,9 @@ class ImageSpec(_ImageSpecMixin): post_delete.connect(ImageSpec._post_delete_receiver, sender=cls, dispatch_uid=uid) - # Register the field with the cache_state_backend + # Register the field with the image_cache_backend try: - self.cache_state_backend.register_field(cls, self, name) + self.image_cache_backend.register_field(cls, self, name) except AttributeError: pass @@ -264,13 +264,13 @@ class ImageSpecFile(_ImageSpecFileMixin, ImageFieldFile): file = property(_get_file, ImageFieldFile._set_file, ImageFieldFile._del_file) def clear(self): - return self.field.cache_state_backend.clear(self) + return self.field.image_cache_backend.clear(self) def invalidate(self): - return self.field.cache_state_backend.invalidate(self) + return self.field.image_cache_backend.invalidate(self) def validate(self): - return self.field.cache_state_backend.validate(self) + return self.field.image_cache_backend.validate(self) def generate(self, save=True): """ diff --git a/imagekit/settings.py b/imagekit/settings.py index d02f718..d84030d 100644 --- a/imagekit/settings.py +++ b/imagekit/settings.py @@ -1,5 +1,5 @@ from django.conf import settings -DEFAULT_CACHE_STATE_BACKEND = getattr(settings, - 'IMAGEKIT_DEFAULT_CACHE_STATE_BACKEND', - 'imagekit.cachestate.PessimisticCacheStateBackend') +DEFAULT_IMAGE_CACHE_BACKEND = getattr(settings, + 'IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND', + 'imagekit.imagecache.PessimisticImageCacheBackend')