Rename cache things (it isn't cachine)

https://twitter.com/alex_gaynor/statuses/257558176965206016
This commit is contained in:
Matthew Tretter 2013-01-31 03:51:29 -05:00
parent 7f6188623c
commit 04aa72c1f9
16 changed files with 189 additions and 215 deletions

View file

@ -3,11 +3,11 @@ from django.conf import settings
class ImageKitConf(AppConf):
DEFAULT_IMAGE_CACHE_BACKEND = 'imagekit.imagecache.backends.Simple'
DEFAULT_GENERATEDFILE_BACKEND = 'imagekit.generators.filebackends.Simple'
CACHE_BACKEND = None
CACHE_DIR = 'CACHE/images'
GENERATED_FILE_DIR = 'generated/images'
CACHE_PREFIX = 'imagekit:'
DEFAULT_IMAGE_CACHE_STRATEGY = 'imagekit.imagecache.strategies.JustInTime'
DEFAULT_IMAGE_GENERATOR_STRATEGY = 'imagekit.generators.strategies.JustInTime'
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
def configure_cache_backend(self, value):

View file

@ -72,48 +72,41 @@ class BaseIKFile(File):
file.close()
class GeneratedImageCacheFile(BaseIKFile, ImageFile):
class GeneratedImageFile(BaseIKFile, ImageFile):
"""
A cache file that represents the result of a generator. Creating an instance
of this class is not enough to trigger the creation of the cache file. In
fact, one of the main points of this class is to allow the creation of the
file to be deferred until the time that the image cache strategy requires
it.
A file that represents the result of a generator. Creating an instance of
this class is not enough to trigger the generation of the file. In fact,
one of the main points of this class is to allow the creation of the file
to be deferred until the time that the image generator strategy requires it.
"""
def __init__(self, generator, name=None, storage=None, image_cache_backend=None):
def __init__(self, generator, name=None, storage=None, generatedfile_backend=None):
"""
:param generator: The object responsible for generating a new image.
:param name: The filename
:param storage: A Django storage object that will be used to save the
file.
:param image_cache_backend: The object responsible for managing the
state of the cache file.
:param generatedfile_backend: The object responsible for managing the
state of the file.
"""
self.generator = generator
self.name = name or getattr(generator, 'cache_file_name', None)
storage = storage or getattr(generator, 'cache_file_storage',
self.name = name or getattr(generator, 'generatedfile_name', None)
storage = storage or getattr(generator, 'generatedfile_storage',
None) or get_singleton(settings.IMAGEKIT_DEFAULT_FILE_STORAGE,
'file storage backend')
self.image_cache_backend = image_cache_backend or getattr(generator,
'image_cache_backend', None)
self.generatedfile_backend = generatedfile_backend or getattr(generator,
'generatedfile_backend', None)
super(GeneratedImageCacheFile, self).__init__(storage=storage)
super(GeneratedImageFile, self).__init__(storage=storage)
def _require_file(self):
before_access.send(sender=self, file=self)
return super(GeneratedImageCacheFile, self)._require_file()
return super(GeneratedImageFile, self)._require_file()
def clear(self):
return self.image_cache_backend.clear(self)
def invalidate(self):
return self.image_cache_backend.invalidate(self)
def validate(self):
return self.image_cache_backend.validate(self)
def ensure_exists(self):
return self.generatedfile_backend.ensure_exists(self)
def generate(self):
# Generate the file
@ -126,11 +119,11 @@ class GeneratedImageCacheFile(BaseIKFile, ImageFile):
' with the requested name ("%s") and instead used'
' "%s". This may be because a file already existed with'
' the requested name. If so, you may have meant to call'
' validate() instead of generate(), or there may be a'
' race condition in the image cache backend %s. The'
' saved file will not be used.' % (self.storage,
' ensure_exists() instead of generate(), or there may be a'
' race condition in the file backend %s. The saved file'
' will not be used.' % (self.storage,
self.name, actual_name,
self.image_cache_backend))
self.generatedfile_backend))
class IKContentFile(ContentFile):
@ -162,12 +155,12 @@ class IKContentFile(ContentFile):
return smart_unicode(self.file.name or u'')
class LazyGeneratedImageCacheFile(LazyObject):
class LazyGeneratedImageFile(LazyObject):
def __init__(self, generator_id, *args, **kwargs):
super(LazyGeneratedImageCacheFile, self).__init__()
super(LazyGeneratedImageFile, self).__init__()
def setup():
generator = generator_registry.get(generator_id, *args, **kwargs)
self._wrapped = GeneratedImageCacheFile(generator)
self._wrapped = GeneratedImageFile(generator)
self.__dict__['_setup'] = setup

View file

@ -1,5 +1,5 @@
def validate_now(file):
file.validate()
def ensure_exists(file):
file.ensure_exists()
try:
@ -7,15 +7,15 @@ try:
except ImportError:
pass
else:
validate_now_task = task(validate_now)
ensure_exists_task = task(ensure_exists)
def deferred_validate(file):
def ensure_exists_deferred(file):
try:
import celery # NOQA
except:
raise ImportError("Deferred validation requires the the 'celery' library")
validate_now_task.delay(file)
ensure_exists_task.delay(file)
def clear_now(file):

View file

@ -0,0 +1,64 @@
from ..utils import get_singleton
from django.core.cache import get_cache
from django.core.exceptions import ImproperlyConfigured
def get_default_generatedfile_backend():
"""
Get the default file backend.
"""
from django.conf import settings
return get_singleton(settings.IMAGEKIT_DEFAULT_GENERATEDFILE_BACKEND,
'file backend')
class InvalidFileBackendError(ImproperlyConfigured):
pass
class CachedFileBackend(object):
@property
def cache(self):
if not getattr(self, '_cache', None):
from django.conf import settings
self._cache = get_cache(settings.IMAGEKIT_CACHE_BACKEND)
return self._cache
def get_key(self, file):
from django.conf import settings
return '%s%s-exists' % (settings.IMAGEKIT_CACHE_PREFIX, file.name)
def file_exists(self, file):
key = self.get_key(file)
exists = self.cache.get(key)
if exists is None:
exists = self._file_exists(file)
self.cache.set(key, exists)
return exists
def ensure_exists(self, file):
if self.file_exists(file):
self.create(file)
self.cache.set(self.get_key(file), True)
class Simple(CachedFileBackend):
"""
The most basic file backend. The storage is consulted to see if the file
exists.
"""
def _file_exists(self, file):
if not getattr(file, '_file', None):
# No file on object. Have to check storage.
return not file.storage.exists(file.name)
return False
def create(self, file):
"""
Generates a new image by running the processors on the source file.
"""
file.generate()

View file

@ -1,34 +1,33 @@
from django.utils.functional import LazyObject
from .actions import validate_now, clear_now
from ..utils import get_singleton
class JustInTime(object):
"""
A caching strategy that validates the file right before it's needed.
A strategy that ensures the file exists right before it's needed.
"""
def before_access(self, file):
validate_now(file)
file.ensure_exists()
class Optimistic(object):
"""
A caching strategy that acts immediately when the cacheable file changes
and assumes that the cache files will not be removed (i.e. doesn't
revalidate on access).
A strategy that acts immediately when the source file changes and assumes
that the generated files will not be removed (i.e. it doesn't ensure the
generated file exists when it's accessed).
"""
def on_source_created(self, file):
validate_now(file)
file.ensure_exists()
def on_source_deleted(self, file):
clear_now(file)
file.delete()
def on_source_changed(self, file):
validate_now(file)
file.ensure_exists()
class DictStrategy(object):
@ -40,7 +39,7 @@ class DictStrategy(object):
class StrategyWrapper(LazyObject):
def __init__(self, strategy):
if isinstance(strategy, basestring):
strategy = get_singleton(strategy, 'image cache strategy')
strategy = get_singleton(strategy, 'generator strategy')
elif isinstance(strategy, dict):
strategy = DictStrategy(strategy)
elif callable(strategy):

View file

@ -1,82 +0,0 @@
from ..utils import get_singleton
from django.core.cache import get_cache
from django.core.exceptions import ImproperlyConfigured
def get_default_image_cache_backend():
"""
Get the default image cache backend.
"""
from django.conf import settings
return get_singleton(settings.IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND,
'image cache backend')
class InvalidImageCacheBackendError(ImproperlyConfigured):
pass
class CachedValidationBackend(object):
@property
def cache(self):
if not getattr(self, '_cache', None):
from django.conf import settings
self._cache = get_cache(settings.IMAGEKIT_CACHE_BACKEND)
return self._cache
def get_key(self, file):
from django.conf import settings
return '%s%s-valid' % (settings.IMAGEKIT_CACHE_PREFIX, file.name)
def is_invalid(self, file):
key = self.get_key(file)
cached_value = self.cache.get(key)
if cached_value is None:
cached_value = self._is_invalid(file)
self.cache.set(key, cached_value)
return cached_value
def validate(self, file):
if self.is_invalid(file):
self._validate(file)
self.cache.set(self.get_key(file), True)
def invalidate(self, file):
if not self.is_invalid(file):
self._invalidate(file)
self.cache.set(self.get_key(file), False)
class Simple(CachedValidationBackend):
"""
The most basic image cache backend. Files are considered valid if they
exist. To invalidate a file, it's deleted; to validate one, it's generated
immediately.
"""
def _is_invalid(self, file):
if not getattr(file, '_file', None):
# No file on object. Have to check storage.
return not file.storage.exists(file.name)
return False
def _validate(self, file):
"""
Generates a new image by running the processors on the source file.
"""
file.generate()
def invalidate(self, file):
"""
Invalidate the file by deleting it. We override ``invalidate()``
instead of ``_invalidate()`` because we don't really care to check
whether the file is invalid or not.
"""
file.delete(save=False)
def clear(self, file):
file.delete(save=False)

View file

@ -1,15 +1,16 @@
from django.core.management.base import BaseCommand
import re
from ...registry import generator_registry, cacheable_registry
from ...registry import generator_registry, generatedfile_registry
class Command(BaseCommand):
help = ("""Warm the image cache for the specified generators (or all generators if
none was provided). Simple, fnmatch-like wildcards are allowed, with *
matching all characters within a segment, and ** matching across segments.
(Segments are separated with colons.) So, for example, "a:*:c" will match
"a:b:c", but not "a:b:x:c", whereas "a:**:c" will match both. Subsegments
are always matched, so "a" will match "a" as well as "a:b" and "a:b:c".""")
help = ("""Generate files for the specified image generators (or all of them if
none was provided). Simple, glob-like wildcards are allowed, with *
matching all characters within a segment, and ** matching across
segments. (Segments are separated with colons.) So, for example,
"a:*:c" will match "a:b:c", but not "a:b:x:c", whereas "a:**:c" will
match both. Subsegments are always matched, so "a" will match "a" as
well as "a:b" and "a:b:c".""")
args = '[generator_ids]'
def handle(self, *args, **options):
@ -21,11 +22,11 @@ are always matched, so "a" will match "a" as well as "a:b" and "a:b:c".""")
for generator_id in generators:
self.stdout.write('Validating generator: %s\n' % generator_id)
for cacheable in cacheable_registry.get(generator_id):
self.stdout.write(' %s\n' % cacheable)
for file in generatedfile_registry.get(generator_id):
self.stdout.write(' %s\n' % file)
try:
# TODO: Allow other validation actions through command option
cacheable.validate()
file.ensure_exists()
except Exception, err:
# TODO: How should we handle failures? Don't want to error, but should call it out more than this.
self.stdout.write(' FAILED: %s\n' % err)

View file

@ -26,15 +26,15 @@ class ImageSpecField(SpecHostField):
"""
def __init__(self, processors=None, format=None, options=None,
source=None, cache_file_storage=None, autoconvert=None,
image_cache_backend=None, image_cache_strategy=None, spec=None,
source=None, generatedfile_storage=None, autoconvert=None,
generatedfile_backend=None, generator_strategy=None, spec=None,
id=None):
SpecHost.__init__(self, processors=processors, format=format,
options=options, cache_file_storage=cache_file_storage,
options=options, generatedfile_storage=generatedfile_storage,
autoconvert=autoconvert,
image_cache_backend=image_cache_backend,
image_cache_strategy=image_cache_strategy, spec=spec,
generatedfile_backend=generatedfile_backend,
generator_strategy=generator_strategy, spec=spec,
spec_id=id)
# TODO: Allow callable for source. See https://github.com/jdriscoll/django-imagekit/issues/158#issuecomment-10921664

View file

@ -1,4 +1,4 @@
from ...files import GeneratedImageCacheFile
from ...files import GeneratedImageFile
from django.db.models.fields.files import ImageField
@ -30,7 +30,7 @@ class ImageSpecFileDescriptor(object):
else:
source = image_fields[0]
spec = self.field.get_spec(source=source)
file = GeneratedImageCacheFile(spec)
file = GeneratedImageFile(spec)
instance.__dict__[self.attname] = file
return file

View file

@ -52,11 +52,11 @@ class GeneratorRegistry(object):
class SourceGroupRegistry(object):
"""
The source group registry is responsible for listening to source_* signals
on source groups, and relaying them to the image cache strategies of the
on source groups, and relaying them to the image generator strategies of the
appropriate generators.
In addition, registering a new source group also registers its cacheables
generator with the cacheable registry.
In addition, registering a new source group also registers its generated
files with that registry.
"""
_signals = {
@ -71,26 +71,26 @@ class SourceGroupRegistry(object):
signal.connect(self.source_group_receiver)
def register(self, generator_id, source_group):
from .specs.sourcegroups import SourceGroupCacheablesGenerator
from .specs.sourcegroups import SourceGroupFilesGenerator
generator_ids = self._source_groups.setdefault(source_group, set())
generator_ids.add(generator_id)
cacheable_registry.register(generator_id,
SourceGroupCacheablesGenerator(source_group, generator_id))
generatedfile_registry.register(generator_id,
SourceGroupFilesGenerator(source_group, generator_id))
def unregister(self, generator_id, source_group):
from .specs.sourcegroups import SourceGroupCacheablesGenerator
from .specs.sourcegroups import SourceGroupFilesGenerator
generator_ids = self._source_groups.setdefault(source_group, set())
if generator_id in generator_ids:
generator_ids.remove(generator_id)
cacheable_registry.unregister(generator_id,
SourceGroupCacheablesGenerator(source_group, generator_id))
generatedfile_registry.unregister(generator_id,
SourceGroupFilesGenerator(source_group, generator_id))
def source_group_receiver(self, sender, source, signal, **kwargs):
"""
Relay source group signals to the appropriate spec strategy.
"""
from .files import GeneratedImageCacheFile
from .files import GeneratedImageFile
source_group = sender
# Ignore signals from unregistered groups.
@ -102,53 +102,53 @@ class SourceGroupRegistry(object):
callback_name = self._signals[signal]
for spec in specs:
file = GeneratedImageCacheFile(spec)
file = GeneratedImageFile(spec)
call_strategy_method(spec, callback_name, file=file)
class CacheableRegistry(object):
class GeneratedFileRegistry(object):
"""
An object for registering cacheables with generators. The two are
An object for registering generated files with image generators. The two are
associated with each other via a string id. We do this (as opposed to
associating them directly by, for example, putting a ``cacheables``
attribute on generators) so that generators can be overridden without
losing the associated cacheables. That way, a distributable app can define
its own generators without locking the users of the app into it.
associating them directly by, for example, putting a ``generatedfiles``
attribute on image generators) so that image generators can be overridden
without losing the associated files. That way, a distributable app can
define its own generators without locking the users of the app into it.
"""
def __init__(self):
self._cacheables = {}
self._generatedfiles = {}
def register(self, generator_id, cacheables):
def register(self, generator_id, generatedfiles):
"""
Associates cacheables with a generator id
Associates generated files with a generator id
"""
if cacheables not in self._cacheables:
self._cacheables[cacheables] = set()
self._cacheables[cacheables].add(generator_id)
if generatedfiles not in self._generatedfiles:
self._generatedfiles[generatedfiles] = set()
self._generatedfiles[generatedfiles].add(generator_id)
def unregister(self, generator_id, cacheables):
def unregister(self, generator_id, generatedfiles):
"""
Disassociates cacheables with a generator id
Disassociates generated files with a generator id
"""
try:
self._cacheables[cacheables].remove(generator_id)
self._generatedfiles[generatedfiles].remove(generator_id)
except KeyError:
pass
def get(self, generator_id):
for k, v in self._cacheables.items():
for k, v in self._generatedfiles.items():
if generator_id in v:
for cacheable in k():
yield cacheable
for file in k():
yield file
class Register(object):
"""
Register generators and cacheables.
Register generators and generated files.
"""
def generator(self, id, generator=None):
@ -162,8 +162,8 @@ class Register(object):
generator_registry.register(id, generator)
# iterable that returns kwargs or callable that returns iterable of kwargs
def cacheables(self, generator_id, cacheables):
cacheable_registry.register(generator_id, cacheables)
def generatedfiles(self, generator_id, generatedfiles):
generatedfile_registry.register(generator_id, generatedfiles)
def source_group(self, generator_id, source_group):
source_group_registry.register(generator_id, source_group)
@ -171,21 +171,21 @@ class Register(object):
class Unregister(object):
"""
Unregister generators and cacheables.
Unregister generators and generated files.
"""
def generator(self, id, generator):
generator_registry.unregister(id, generator)
def cacheables(self, generator_id, cacheables):
cacheable_registry.unregister(generator_id, cacheables)
def generatedfiles(self, generator_id, generatedfiles):
generatedfile_registry.unregister(generator_id, generatedfiles)
def source_group(self, generator_id, source_group):
source_group_registry.unregister(generator_id, source_group)
generator_registry = GeneratorRegistry()
cacheable_registry = CacheableRegistry()
generatedfile_registry = GeneratedFileRegistry()
source_group_registry = SourceGroupRegistry()
register = Register()
unregister = Unregister()

View file

@ -1,7 +1,7 @@
from django.dispatch import Signal
# "Cacheables" (cache file) signals
# Generated file signals
before_access = Signal()
# Source group signals

View file

@ -3,8 +3,8 @@ from django.db.models.fields.files import ImageFieldFile
from hashlib import md5
import os
import pickle
from ..imagecache.backends import get_default_image_cache_backend
from ..imagecache.strategies import StrategyWrapper
from ..generators.filebackends import get_default_generatedfile_backend
from ..generators.strategies import StrategyWrapper
from ..processors import ProcessorPipeline
from ..utils import open_image, img_to_fobj, suggest_extension
from ..registry import generator_registry, register
@ -17,27 +17,27 @@ class BaseImageSpec(object):
"""
cache_file_storage = None
"""A Django storage system to use to save a generated cache file."""
generatedfile_storage = None
"""A Django storage system to use to save a generated file."""
image_cache_backend = None
generatedfile_backend = None
"""
An object responsible for managing the state of cached files. Defaults to an
instance of ``IMAGEKIT_DEFAULT_IMAGE_CACHE_BACKEND``
An object responsible for managing the state of generated files. Defaults to
an instance of ``IMAGEKIT_DEFAULT_GENERATEDFILE_BACKEND``
"""
image_cache_strategy = settings.IMAGEKIT_DEFAULT_IMAGE_CACHE_STRATEGY
generator_strategy = settings.IMAGEKIT_DEFAULT_IMAGE_GENERATOR_STRATEGY
"""
A dictionary containing callbacks that allow you to customize how and when
the image cache is validated. Defaults to
``IMAGEKIT_DEFAULT_SPEC_FIELD_IMAGE_CACHE_STRATEGY``.
the image file is created. Defaults to
``IMAGEKIT_DEFAULT_IMAGE_GENERATOR_STRATEGY``.
"""
def __init__(self):
self.image_cache_backend = self.image_cache_backend or get_default_image_cache_backend()
self.image_cache_strategy = StrategyWrapper(self.image_cache_strategy)
self.generatedfile_backend = self.generatedfile_backend or get_default_generatedfile_backend()
self.generator_strategy = StrategyWrapper(self.generator_strategy)
def generate(self):
raise NotImplementedError
@ -83,16 +83,16 @@ class ImageSpec(BaseImageSpec):
super(ImageSpec, self).__init__()
@property
def cache_file_name(self):
def generatedfile_name(self):
source_filename = getattr(self.source, 'name', None)
if source_filename is None or os.path.isabs(source_filename):
# Generally, we put the file right in the cache directory.
dir = settings.IMAGEKIT_CACHE_DIR
# Generally, we put the file right in the generated file directory.
dir = settings.IMAGEKIT_GENERATED_FILE_DIR
else:
# For source files with relative names (like Django media files),
# use the source's name to create the new filename.
dir = os.path.join(settings.IMAGEKIT_CACHE_DIR,
dir = os.path.join(settings.IMAGEKIT_GENERATED_FILE_DIR,
os.path.splitext(source_filename)[0])
ext = suggest_extension(source_filename or '', self.format)

View file

@ -3,17 +3,17 @@ Source groups are the means by which image spec sources are identified. They
have two responsibilities:
1. To dispatch ``source_created``, ``source_changed``, and ``source_deleted``
signals. (These will be relayed to the corresponding specs' image cache
signals. (These will be relayed to the corresponding specs' generator
strategies.)
2. To provide the source files that they represent, via a generator method named
``files()``. (This is used by the warmimagecache management command for
``files()``. (This is used by the generateimages management command for
"pre-caching" image files.)
"""
from django.db.models.signals import post_init, post_save, post_delete
from django.utils.functional import wraps
from ..files import LazyGeneratedImageCacheFile
from ..files import LazyGeneratedImageFile
from ..signals import source_created, source_changed, source_deleted
@ -135,10 +135,9 @@ class ImageFieldSourceGroup(object):
yield getattr(instance, self.image_field)
class SourceGroupCacheablesGenerator(object):
class SourceGroupFilesGenerator(object):
"""
A cacheables generator for source groups. The purpose of this class is to
generate cacheables (cache files) from a source group.
A Python generator that yields generated file objects for source groups.
"""
def __init__(self, source_group, generator_id):
@ -157,7 +156,7 @@ class SourceGroupCacheablesGenerator(object):
def __call__(self):
for source_file in self.source_group.files():
yield LazyGeneratedImageCacheFile(self.generator_id,
yield LazyGeneratedImageFile(self.generator_id,
source=source_file)

View file

@ -2,7 +2,7 @@ from django import template
from django.utils.html import escape
from django.utils.safestring import mark_safe
from .compat import parse_bits
from ..files import GeneratedImageCacheFile
from ..files import GeneratedImageFile
from ..registry import generator_registry
@ -19,12 +19,12 @@ _kwarg_map = {
}
def get_cache_file(context, generator_id, generator_kwargs, source=None):
def get_generatedfile(context, generator_id, generator_kwargs, source=None):
generator_id = generator_id.resolve(context)
kwargs = dict((_kwarg_map.get(k, k), v.resolve(context)) for k,
v in generator_kwargs.items())
generator = generator_registry.get(generator_id, **kwargs)
return GeneratedImageCacheFile(generator)
return GeneratedImageFile(generator)
def parse_dimensions(dimensions):
@ -53,7 +53,7 @@ class GenerateImageAssignmentNode(template.Node):
autodiscover()
variable_name = self.get_variable_name(context)
context[variable_name] = get_cache_file(context, self._generator_id,
context[variable_name] = get_generatedfile(context, self._generator_id,
self._generator_kwargs)
return ''
@ -69,7 +69,7 @@ class GenerateImageTagNode(template.Node):
from ..utils import autodiscover
autodiscover()
file = get_cache_file(context, self._generator_id,
file = get_generatedfile(context, self._generator_id,
self._generator_kwargs)
attrs = dict((k, v.resolve(context)) for k, v in
self._html_attrs.items())
@ -110,7 +110,7 @@ class ThumbnailAssignmentNode(template.Node):
kwargs.update(parse_dimensions(self._dimensions.resolve(context)))
generator = generator_registry.get(generator_id, **kwargs)
context[variable_name] = GeneratedImageCacheFile(generator)
context[variable_name] = GeneratedImageFile(generator)
return ''
@ -136,7 +136,7 @@ class ThumbnailImageTagNode(template.Node):
kwargs.update(dimensions)
generator = generator_registry.get(generator_id, **kwargs)
file = GeneratedImageCacheFile(generator)
file = GeneratedImageFile(generator)
attrs = dict((k, v.resolve(context)) for k, v in
self._html_attrs.items())

View file

@ -425,7 +425,7 @@ def generate(generator):
def call_strategy_method(generator, method_name, *args, **kwargs):
strategy = getattr(generator, 'image_cache_strategy', None)
strategy = getattr(generator, 'generator_strategy', None)
fn = getattr(strategy, method_name, None)
if fn is not None:
fn(*args, **kwargs)