diff --git a/imagekit/files.py b/imagekit/files.py index 1dd8976..150cfa0 100644 --- a/imagekit/files.py +++ b/imagekit/files.py @@ -2,7 +2,9 @@ from django.conf import settings from django.core.files.base import ContentFile, File from django.core.files.images import ImageFile from django.utils.encoding import smart_str, smart_unicode +from django.utils.functional import LazyObject import os +from .registry import generator_registry from .signals import before_access from .utils import (format_to_mimetype, extension_to_mimetype, get_logger, get_singleton, generate) @@ -101,7 +103,7 @@ class GeneratedImageCacheFile(BaseIKFile, ImageFile): super(GeneratedImageCacheFile, self).__init__(storage=storage) def _require_file(self): - before_access.send(sender=self, generator=self.generator, file=self) + before_access.send(sender=self, file=self) return super(GeneratedImageCacheFile, self)._require_file() def clear(self): @@ -158,3 +160,14 @@ class IKContentFile(ContentFile): def __unicode__(self): return smart_unicode(self.file.name or u'') + + +class LazyGeneratedImageCacheFile(LazyObject): + def __init__(self, generator_id, *args, **kwargs): + super(LazyGeneratedImageCacheFile, self).__init__() + + def setup(): + generator = generator_registry.get(generator_id, *args, **kwargs) + self._wrapped = GeneratedImageCacheFile(generator) + + self.__dict__['_setup'] = setup diff --git a/imagekit/generatorlibrary.py b/imagekit/generatorlibrary.py index bd9da9a..977e960 100644 --- a/imagekit/generatorlibrary.py +++ b/imagekit/generatorlibrary.py @@ -10,4 +10,4 @@ class Thumbnail(ImageSpec): super(Thumbnail, self).__init__(**kwargs) -register.spec('ik:thumbnail', Thumbnail) +register.generator('ik:thumbnail', Thumbnail) diff --git a/imagekit/imagecache/strategies.py b/imagekit/imagecache/strategies.py index 630b77c..28068f0 100644 --- a/imagekit/imagecache/strategies.py +++ b/imagekit/imagecache/strategies.py @@ -1,3 +1,4 @@ +from django.utils.functional import LazyObject from .actions import validate_now, clear_now from ..utils import get_singleton @@ -14,9 +15,9 @@ class JustInTime(object): class Optimistic(object): """ - A caching strategy that acts immediately when the source file chages and - assumes that the cache files will not be removed (i.e. doesn't revalidate - on access). + A caching strategy that acts immediately when the cacheable file changes + and assumes that the cache files will not be removed (i.e. doesn't + revalidate on access). """ @@ -36,7 +37,7 @@ class DictStrategy(object): setattr(self, k, v) -class StrategyWrapper(object): +class StrategyWrapper(LazyObject): def __init__(self, strategy): if isinstance(strategy, basestring): strategy = get_singleton(strategy, 'image cache strategy') @@ -46,10 +47,11 @@ class StrategyWrapper(object): strategy = strategy() self._wrapped = strategy - def invoke_callback(self, name, *args, **kwargs): - func = getattr(self._wrapped, name, None) - if func: - func(*args, **kwargs) + def __getstate__(self): + return {'_wrapped': self._wrapped} + + def __setstate__(self, state): + self._wrapped = state['_wrapped'] def __unicode__(self): return unicode(self._wrapped) diff --git a/imagekit/management/commands/warmimagecache.py b/imagekit/management/commands/warmimagecache.py index 2d147f5..9f2a8c8 100644 --- a/imagekit/management/commands/warmimagecache.py +++ b/imagekit/management/commands/warmimagecache.py @@ -1,35 +1,46 @@ from django.core.management.base import BaseCommand import re -from ...files import GeneratedImageCacheFile -from ...registry import generator_registry, source_group_registry +from ...registry import generator_registry, cacheable_registry class Command(BaseCommand): - help = ('Warm the image cache for the specified specs (or all specs if none' - ' was provided). Simple wildcard matching (using asterisks) is' - ' supported.') - args = '[spec_ids]' + help = ("""Warm the image cache for the specified generators (or all generators if +none was provided). Simple, fnmatch-like wildcards are allowed, with * +matching all characters within a segment, and ** matching across segments. +(Segments are separated with colons.) So, for example, "a:*:c" will match +"a:b:c", but not "a:b:x:c", whereas "a:**:c" will match both. Subsegments +are always matched, so "a" will match "a" as well as "a:b" and "a:b:c".""") + args = '[generator_ids]' def handle(self, *args, **options): - specs = generator_registry.get_ids() + generators = generator_registry.get_ids() if args: patterns = self.compile_patterns(args) - specs = (id for id in specs if any(p.match(id) for p in patterns)) + generators = (id for id in generators if any(p.match(id) for p in patterns)) - for spec_id in specs: - self.stdout.write('Validating spec: %s\n' % spec_id) - for source_group in source_group_registry.get(spec_id): - for source in source_group.files(): - if source: - spec = generator_registry.get(spec_id, source=source) - self.stdout.write(' %s\n' % source) - try: - # TODO: Allow other validation actions through command option - GeneratedImageCacheFile(spec).validate() - except Exception, err: - # TODO: How should we handle failures? Don't want to error, but should call it out more than this. - self.stdout.write(' FAILED: %s\n' % err) + for generator_id in generators: + self.stdout.write('Validating generator: %s\n' % generator_id) + for cacheable in cacheable_registry.get(generator_id): + self.stdout.write(' %s\n' % cacheable) + try: + # TODO: Allow other validation actions through command option + cacheable.validate() + except Exception, err: + # TODO: How should we handle failures? Don't want to error, but should call it out more than this. + self.stdout.write(' FAILED: %s\n' % err) - def compile_patterns(self, spec_ids): - return [re.compile('%s$' % '.*'.join(re.escape(part) for part in id.split('*'))) for id in spec_ids] + def compile_patterns(self, generator_ids): + return [self.compile_pattern(id) for id in generator_ids] + + def compile_pattern(self, generator_id): + parts = re.split(r'(\*{1,2})', generator_id) + pattern = '' + for part in parts: + if part == '*': + pattern += '[^:]*' + elif part == '**': + pattern += '.*' + else: + pattern += re.escape(part) + return re.compile('^%s(:.*)?$' % pattern) diff --git a/imagekit/models/fields/__init__.py b/imagekit/models/fields/__init__.py index 131ff33..35c78da 100644 --- a/imagekit/models/fields/__init__.py +++ b/imagekit/models/fields/__init__.py @@ -11,7 +11,7 @@ class SpecHostField(SpecHost): # Generate a spec_id to register the spec with. The default spec id is # ":_" if not getattr(self, 'spec_id', None): - spec_id = (u'%s:%s_%s' % (cls._meta.app_label, + spec_id = (u'%s:%s:%s' % (cls._meta.app_label, cls._meta.object_name, name)).lower() # Register the spec with the id. This allows specs to be overridden @@ -45,8 +45,8 @@ class ImageSpecField(SpecHostField): self.set_spec_id(cls, name) # Add the model and field as a source for this spec id - register.sources(self.spec_id, - [ImageFieldSourceGroup(cls, self.source)]) + register.source_group(self.spec_id, + ImageFieldSourceGroup(cls, self.source)) class ProcessedImageField(models.ImageField, SpecHostField): diff --git a/imagekit/registry.py b/imagekit/registry.py index 9c0f4b8..764ba24 100644 --- a/imagekit/registry.py +++ b/imagekit/registry.py @@ -1,21 +1,22 @@ from .exceptions import AlreadyRegistered, NotRegistered -from .signals import (before_access, source_created, source_changed, - source_deleted) +from .signals import before_access, source_created, source_changed, source_deleted +from .utils import call_strategy_method class GeneratorRegistry(object): """ - An object for registering generators (specs). This registry provides + An object for registering generators. This registry provides a convenient way for a distributable app to define default generators without locking the users of the app into it. """ def __init__(self): self._generators = {} + before_access.connect(self.before_access_receiver) def register(self, id, generator): if id in self._generators: - raise AlreadyRegistered('The spec or generator with id %s is' + raise AlreadyRegistered('The generator with id %s is' ' already registered' % id) self._generators[id] = generator @@ -24,14 +25,14 @@ class GeneratorRegistry(object): try: del self._generators[id] except KeyError: - raise NotRegistered('The spec or generator with id %s is not' + raise NotRegistered('The generator with id %s is not' ' registered' % id) def get(self, id, **kwargs): try: generator = self._generators[id] except KeyError: - raise NotRegistered('The spec or generator with id %s is not' + raise NotRegistered('The generator with id %s is not' ' registered' % id) if callable(generator): return generator(**kwargs) @@ -41,109 +42,150 @@ class GeneratorRegistry(object): def get_ids(self): return self._generators.keys() + def before_access_receiver(self, sender, file, **kwargs): + generator = file.generator + if generator in self._generators.values(): + # Only invoke the strategy method for registered generators. + call_strategy_method(generator, 'before_access', file=file) + class SourceGroupRegistry(object): """ - An object for registering source groups with specs. The two are - associated with each other via a string id. We do this (as opposed to - associating them directly by, for example, putting a ``source_groups`` - attribute on specs) so that specs can be overridden without losing the - associated sources. That way, a distributable app can define its own - specs without locking the users of the app into it. + The source group registry is responsible for listening to source_* signals + on source groups, and relaying them to the image cache strategies of the + appropriate generators. + + In addition, registering a new source group also registers its cacheables + generator with the cacheable registry. """ - - _signals = [ - source_created, - source_changed, - source_deleted, - ] + _signals = { + source_created: 'on_source_created', + source_changed: 'on_source_changed', + source_deleted: 'on_source_deleted', + } def __init__(self): self._source_groups = {} - for signal in self._signals: + for signal in self._signals.keys(): signal.connect(self.source_group_receiver) - before_access.connect(self.before_access_receiver) - def register(self, spec_id, source_groups): + def register(self, generator_id, source_group): + from .specs.sourcegroups import SourceGroupCacheablesGenerator + generator_ids = self._source_groups.setdefault(source_group, set()) + generator_ids.add(generator_id) + cacheable_registry.register(generator_id, + SourceGroupCacheablesGenerator(source_group, generator_id)) + + def unregister(self, generator_id, source_group): + from .specs.sourcegroups import SourceGroupCacheablesGenerator + generator_ids = self._source_groups.setdefault(source_group, set()) + if generator_id in generator_ids: + generator_ids.remove(generator_id) + cacheable_registry.unregister(generator_id, + SourceGroupCacheablesGenerator(source_group, generator_id)) + + def source_group_receiver(self, sender, source, signal, **kwargs): """ - Associates source groups with a spec id - - """ - for source_group in source_groups: - if source_group not in self._source_groups: - self._source_groups[source_group] = set() - self._source_groups[source_group].add(spec_id) - - def unregister(self, spec_id, source_groups): - """ - Disassociates sources with a spec id - - """ - for source_group in source_groups: - try: - self._source_groups[source_group].remove(spec_id) - except KeyError: - continue - - def get(self, spec_id): - return [source_group for source_group in self._source_groups - if spec_id in self._source_groups[source_group]] - - def before_access_receiver(self, sender, generator, file, **kwargs): - generator.image_cache_strategy.invoke_callback('before_access', file) - - def source_group_receiver(self, sender, source, signal, info, **kwargs): - """ - Redirects signals dispatched on sources to the appropriate specs. + Relay source group signals to the appropriate spec strategy. """ + from .files import GeneratedImageCacheFile source_group = sender + + # Ignore signals from unregistered groups. if source_group not in self._source_groups: return - for spec in (generator_registry.get(id, source=source) - for id in self._source_groups[source_group]): - event_name = { - source_created: 'source_created', - source_changed: 'source_changed', - source_deleted: 'source_deleted', - } - spec._handle_source_event(event_name, source) + specs = [generator_registry.get(id, source=source) for id in + self._source_groups[source_group]] + callback_name = self._signals[signal] + + for spec in specs: + file = GeneratedImageCacheFile(spec) + call_strategy_method(spec, callback_name, file=file) + + +class CacheableRegistry(object): + """ + An object for registering cacheables with generators. The two are + associated with each other via a string id. We do this (as opposed to + associating them directly by, for example, putting a ``cacheables`` + attribute on generators) so that generators can be overridden without + losing the associated cacheables. That way, a distributable app can define + its own generators without locking the users of the app into it. + + """ + + def __init__(self): + self._cacheables = {} + + def register(self, generator_id, cacheables): + """ + Associates cacheables with a generator id + + """ + if cacheables not in self._cacheables: + self._cacheables[cacheables] = set() + self._cacheables[cacheables].add(generator_id) + + def unregister(self, generator_id, cacheables): + """ + Disassociates cacheables with a generator id + + """ + try: + self._cacheables[cacheables].remove(generator_id) + except KeyError: + pass + + def get(self, generator_id): + for k, v in self._cacheables.items(): + if generator_id in v: + for cacheable in k(): + yield cacheable class Register(object): """ - Register specs and sources. + Register generators and cacheables. """ - def spec(self, id, spec=None): - if spec is None: + def generator(self, id, generator=None): + if generator is None: # Return a decorator def decorator(cls): - self.spec(id, cls) + self.generator(id, cls) return cls return decorator - generator_registry.register(id, spec) + generator_registry.register(id, generator) - def sources(self, spec_id, sources): - source_group_registry.register(spec_id, sources) + # iterable that returns kwargs or callable that returns iterable of kwargs + def cacheables(self, generator_id, cacheables): + cacheable_registry.register(generator_id, cacheables) + + def source_group(self, generator_id, source_group): + source_group_registry.register(generator_id, source_group) class Unregister(object): """ - Unregister specs and sources. + Unregister generators and cacheables. """ - def spec(self, id, spec): - generator_registry.unregister(id, spec) + def generator(self, id, generator): + generator_registry.unregister(id, generator) - def sources(self, spec_id, sources): - source_group_registry.unregister(spec_id, sources) + def cacheables(self, generator_id, cacheables): + cacheable_registry.unregister(generator_id, cacheables) + + def source_group(self, generator_id, source_group): + source_group_registry.unregister(generator_id, source_group) generator_registry = GeneratorRegistry() +cacheable_registry = CacheableRegistry() source_group_registry = SourceGroupRegistry() register = Register() unregister = Unregister() diff --git a/imagekit/signals.py b/imagekit/signals.py index 4c7aefa..c01e30e 100644 --- a/imagekit/signals.py +++ b/imagekit/signals.py @@ -1,6 +1,10 @@ from django.dispatch import Signal + +# "Cacheables" (cache file) signals before_access = Signal() -source_created = Signal(providing_args=[]) + +# Source group signals +source_created = Signal() source_changed = Signal() source_deleted = Signal() diff --git a/imagekit/specs/__init__.py b/imagekit/specs/__init__.py index d2b154b..7b92845 100644 --- a/imagekit/specs/__init__.py +++ b/imagekit/specs/__init__.py @@ -3,7 +3,6 @@ from django.db.models.fields.files import ImageFieldFile from hashlib import md5 import os import pickle -from ..files import GeneratedImageCacheFile from ..imagecache.backends import get_default_image_cache_backend from ..imagecache.strategies import StrategyWrapper from ..processors import ProcessorPipeline @@ -43,11 +42,6 @@ class BaseImageSpec(object): def generate(self): raise NotImplementedError - # TODO: I don't like this interface. Is there a standard Python one? pubsub? - def _handle_source_event(self, event_name, source): - file = GeneratedImageCacheFile(self) - self.image_cache_strategy.invoke_callback('on_%s' % event_name, file) - class ImageSpec(BaseImageSpec): """ @@ -208,7 +202,7 @@ class SpecHost(object): """ self.spec_id = id - register.spec(id, self._original_spec) + register.generator(id, self._original_spec) def get_spec(self, source): """ diff --git a/imagekit/specs/sourcegroups.py b/imagekit/specs/sourcegroups.py index b36b211..3c23d64 100644 --- a/imagekit/specs/sourcegroups.py +++ b/imagekit/specs/sourcegroups.py @@ -1,12 +1,26 @@ +""" +Source groups are the means by which image spec sources are identified. They +have two responsibilities: + +1. To dispatch ``source_created``, ``source_changed``, and ``source_deleted`` + signals. (These will be relayed to the corresponding specs' image cache + strategies.) +2. To provide the source files that they represent, via a generator method named + ``files()``. (This is used by the warmimagecache management command for + "pre-caching" image files.) + +""" + from django.db.models.signals import post_init, post_save, post_delete from django.utils.functional import wraps +from ..files import LazyGeneratedImageCacheFile from ..signals import source_created, source_changed, source_deleted def ik_model_receiver(fn): """ A method decorator that filters out signals coming from models that don't - have fields that function as ImageFieldSourceGroup + have fields that function as ImageFieldSourceGroup sources. """ @wraps(fn) @@ -18,8 +32,15 @@ def ik_model_receiver(fn): class ModelSignalRouter(object): """ - Handles signals dispatched by models and relays them to the spec source - groups that represent those models. + Normally, ``ImageFieldSourceGroup`` would be directly responsible for + watching for changes on the model field it represents. However, Django does + not dispatch events for abstract base classes. Therefore, we must listen for + the signals on all models and filter out those that aren't represented by + ``ImageFieldSourceGroup``s. This class encapsulates that functionality. + + Related: + https://github.com/jdriscoll/django-imagekit/issues/126 + https://code.djangoproject.com/ticket/9318 """ @@ -89,35 +110,55 @@ class ModelSignalRouter(object): """ for source_group in self._source_groups: if source_group.model_class is model_class and source_group.image_field == attname: - info = dict( - source_group=source_group, - instance=instance, - field_name=attname, - ) - signal.send(sender=source_group, source=file, info=info) + signal.send(sender=source_group, source=file) class ImageFieldSourceGroup(object): + """ + A source group that repesents a particular field across all instances of a + model. + + """ def __init__(self, model_class, image_field): - """ - Good design would dictate that this instance would be responsible for - watching for changes for the provided field. However, due to a bug in - Django, we can't do that without leaving abstract base models (which - don't trigger signals) in the lurch. So instead, we do all signal - handling through the signal router. - - Related: - https://github.com/jdriscoll/django-imagekit/issues/126 - https://code.djangoproject.com/ticket/9318 - - """ self.model_class = model_class self.image_field = image_field signal_router.add(self) def files(self): + """ + A generator that returns the source files that this source group + represents; in this case, a particular field of every instance of a + particular model. + + """ for instance in self.model_class.objects.all(): yield getattr(instance, self.image_field) +class SourceGroupCacheablesGenerator(object): + """ + A cacheables generator for source groups. The purpose of this class is to + generate cacheables (cache files) from a source group. + + """ + def __init__(self, source_group, generator_id): + self.source_group = source_group + self.generator_id = generator_id + + def __eq__(self, other): + return (isinstance(other, self.__class__) + and self.__dict__ == other.__dict__) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.source_group, self.generator_id)) + + def __call__(self): + for source_file in self.source_group.files(): + yield LazyGeneratedImageCacheFile(self.generator_id, + source=source_file) + + signal_router = ModelSignalRouter() diff --git a/imagekit/utils.py b/imagekit/utils.py index 82a2c02..adeb377 100644 --- a/imagekit/utils.py +++ b/imagekit/utils.py @@ -422,3 +422,10 @@ def generate(generator): content = f return File(content) + + +def call_strategy_method(generator, method_name, *args, **kwargs): + strategy = getattr(generator, 'image_cache_strategy', None) + fn = getattr(strategy, method_name, None) + if fn is not None: + fn(*args, **kwargs) diff --git a/tests/imagespecs.py b/tests/imagespecs.py index 06d2dab..11e87b3 100644 --- a/tests/imagespecs.py +++ b/tests/imagespecs.py @@ -12,5 +12,5 @@ class ResizeTo1PixelSquare(ImageSpec): super(ResizeTo1PixelSquare, self).__init__(**kwargs) -register.spec('testspec', TestSpec) -register.spec('1pxsq', ResizeTo1PixelSquare) +register.generator('testspec', TestSpec) +register.generator('1pxsq', ResizeTo1PixelSquare)