From d959f3e42fd60c9f762cd1ea8deae4665dc74fdf Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Wed, 28 Oct 2015 12:54:39 +0100 Subject: [PATCH] Waits until the end of transaction before triggering the signal, and trigger the signal on all invalidations. --- cachalot/api.py | 33 +++++++++++---------- cachalot/cache.py | 9 +++++- cachalot/tests/signals.py | 61 +++++++++++++++++++++++++++++++++++++-- cachalot/transaction.py | 13 ++++++--- cachalot/utils.py | 20 ++++++++----- docs/quickstart.rst | 8 +++++ 6 files changed, 114 insertions(+), 30 deletions(-) diff --git a/cachalot/api.py b/cachalot/api.py index 91f8a50..8813357 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -7,24 +7,23 @@ from django.db import connections from django.utils.six import string_types from .cache import cachalot_caches -from .utils import _get_table_cache_key, _invalidate_table_cache_keys +from .signals import post_invalidation +from .utils import _get_table_cache_key, _invalidate_tables __all__ = ('invalidate', 'get_last_invalidation') -def _get_table_cache_keys_per_cache_and_db(tables, cache_alias, db_alias): +def _cache_db_tables_iterator(tables, cache_alias, db_alias): no_tables = not tables cache_aliases = settings.CACHES if cache_alias is None else (cache_alias,) db_aliases = settings.DATABASES if db_alias is None else (db_alias,) for db_alias in db_aliases: if no_tables: tables = connections[db_alias].introspection.table_names() - for cache_alias in cache_aliases: - table_cache_keys = [ - _get_table_cache_key(db_alias, t) for t in tables] - if table_cache_keys: - yield cache_alias, db_alias, table_cache_keys + if tables: + for cache_alias in cache_aliases: + yield cache_alias, db_alias, tables def _get_tables(tables_or_models): @@ -61,11 +60,15 @@ def invalidate(*tables_or_models, **kwargs): raise TypeError( "invalidate() got an unexpected keyword argument '%s'" % k) - table_cache_keys_per_cache = _get_table_cache_keys_per_cache_and_db( - _get_tables(tables_or_models), cache_alias, db_alias) - for cache_alias, db_alias, table_cache_keys in table_cache_keys_per_cache: - _invalidate_table_cache_keys( - cachalot_caches.get_cache(cache_alias, db_alias), table_cache_keys) + invalidated = set() + for cache_alias, db_alias, tables in _cache_db_tables_iterator( + _get_tables(tables_or_models), cache_alias, db_alias): + _invalidate_tables( + cachalot_caches.get_cache(cache_alias, db_alias), db_alias, tables) + invalidated.update(tables) + + for table in invalidated: + post_invalidation.send(table, db_alias=db_alias) def get_last_invalidation(*tables_or_models, **kwargs): @@ -97,9 +100,9 @@ def get_last_invalidation(*tables_or_models, **kwargs): "keyword argument '%s'" % k) last_invalidation = 0.0 - table_cache_keys_per_cache = _get_table_cache_keys_per_cache_and_db( - _get_tables(tables_or_models), cache_alias, db_alias) - for cache_alias, db_alias, table_cache_keys in table_cache_keys_per_cache: + for cache_alias, db_alias, tables in _cache_db_tables_iterator( + _get_tables(tables_or_models), cache_alias, db_alias): + table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables] invalidations = cachalot_caches.get_cache( cache_alias, db_alias).get_many(table_cache_keys).values() if invalidations: diff --git a/cachalot/cache.py b/cachalot/cache.py index 6a35350..3fd64dd 100644 --- a/cachalot/cache.py +++ b/cachalot/cache.py @@ -8,6 +8,7 @@ from django.core.cache import caches from django.db import DEFAULT_DB_ALIAS from .settings import cachalot_settings +from .signals import post_invalidation from .transaction import AtomicCache @@ -21,7 +22,7 @@ class CacheHandler(local): def get_atomic_cache(self, cache_alias, db_alias, level): if cache_alias not in self.atomic_caches[db_alias][level]: self.atomic_caches[db_alias][level][cache_alias] = AtomicCache( - self.get_cache(cache_alias, db_alias, level-1)) + self.get_cache(cache_alias, db_alias, level-1), db_alias) return self.atomic_caches[db_alias][level][cache_alias] def get_cache(self, cache_alias=None, db_alias=None, atomic_level=-1): @@ -45,8 +46,14 @@ class CacheHandler(local): db_alias = DEFAULT_DB_ALIAS atomic_caches = self.atomic_caches[db_alias].pop().values() if commit: + to_be_invalidated = set() for atomic_cache in atomic_caches: atomic_cache.commit() + to_be_invalidated.update(atomic_cache.to_be_invalidated) + # This happens when committing the outermost atomic block. + if not self.atomic_caches[db_alias]: + for table in to_be_invalidated: + post_invalidation.send(table, db_alias=db_alias) cachalot_caches = CacheHandler() diff --git a/cachalot/tests/signals.py b/cachalot/tests/signals.py index 20431e3..89ebf10 100644 --- a/cachalot/tests/signals.py +++ b/cachalot/tests/signals.py @@ -5,10 +5,11 @@ from unittest import skipIf from django.conf import settings from django.contrib.auth.models import User -from django.db import DEFAULT_DB_ALIAS +from django.db import DEFAULT_DB_ALIAS, transaction from django.test import TransactionTestCase -from cachalot.signals import post_invalidation +from ..api import invalidate +from ..signals import post_invalidation from .models import Test @@ -35,6 +36,62 @@ class SignalsTestCase(TransactionTestCase): self.assertListEqual(l, []) User.objects.create_user('user') self.assertListEqual(l, [('auth_user', DEFAULT_DB_ALIAS)]) + post_invalidation.disconnect(receiver, sender=User._meta.db_table) + + def test_table_invalidated_in_transaction(self): + """ + Checks that the ``post_invalidation`` signal is triggered only after + the end of a transaction. + """ + l = [] + + def receiver(sender, **kwargs): + db_alias = kwargs['db_alias'] + l.append((sender, db_alias)) + + post_invalidation.connect(receiver) + self.assertListEqual(l, []) + with transaction.atomic(): + Test.objects.create(name='test1') + self.assertListEqual(l, []) + self.assertListEqual(l, [('cachalot_test', DEFAULT_DB_ALIAS)]) + + del l[:] # Empties the list + self.assertListEqual(l, []) + with transaction.atomic(): + Test.objects.create(name='test2') + with transaction.atomic(): + Test.objects.create(name='test3') + self.assertListEqual(l, []) + self.assertListEqual(l, []) + self.assertListEqual(l, [('cachalot_test', DEFAULT_DB_ALIAS)]) + post_invalidation.disconnect(receiver) + + def test_table_invalidated_once_per_transaction_or_invalidate(self): + """ + Checks that the ``post_invalidation`` signal is triggered only after + the end of a transaction. + """ + l = [] + + def receiver(sender, **kwargs): + db_alias = kwargs['db_alias'] + l.append((sender, db_alias)) + + post_invalidation.connect(receiver) + self.assertListEqual(l, []) + with transaction.atomic(): + Test.objects.create(name='test1') + self.assertListEqual(l, []) + Test.objects.create(name='test2') + self.assertListEqual(l, []) + self.assertListEqual(l, [('cachalot_test', DEFAULT_DB_ALIAS)]) + + del l[:] # Empties the list + self.assertListEqual(l, []) + invalidate(Test, db_alias=DEFAULT_DB_ALIAS) + self.assertListEqual(l, [('cachalot_test', DEFAULT_DB_ALIAS)]) + post_invalidation.disconnect(receiver) @skipIf(len(settings.DATABASES) == 1, 'We can’t change the DB used since there’s only one configured') diff --git a/cachalot/transaction.py b/cachalot/transaction.py index 1955afe..a89bb01 100644 --- a/cachalot/transaction.py +++ b/cachalot/transaction.py @@ -2,13 +2,12 @@ from __future__ import unicode_literals -from .utils import _invalidate_table_cache_keys - class AtomicCache(dict): - def __init__(self, parent_cache): + def __init__(self, parent_cache, db_alias): super(AtomicCache, self).__init__() self.parent_cache = parent_cache + self.db_alias = db_alias self.to_be_invalidated = set() def set(self, k, v, timeout): @@ -31,4 +30,10 @@ class AtomicCache(dict): self.parent_cache.set_many(self, None) # The previous `set_many` is not enough. The parent cache needs to be # invalidated in case another transaction occurred in the meantime. - _invalidate_table_cache_keys(self.parent_cache, self.to_be_invalidated) + _invalidate_tables(self.parent_cache, self.db_alias, + self.to_be_invalidated) + + +# We import this after AtomicCache to avoid a circular import issue and +# avoid importing this locally, which degrades performance. +from .utils import _invalidate_tables diff --git a/cachalot/utils.py b/cachalot/utils.py index 2e4980b..65371b9 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -13,6 +13,7 @@ from django.utils.six import text_type, binary_type from .settings import cachalot_settings from .signals import post_invalidation +from .transaction import AtomicCache class UncachableQuery(Exception): @@ -145,18 +146,21 @@ def _get_table_cache_keys(compiler): return [_get_table_cache_key(db_alias, t) for t in tables] -def _invalidate_table_cache_keys(cache, table_cache_keys): - if hasattr(cache, 'to_be_invalidated'): - cache.to_be_invalidated.update(table_cache_keys) +def _invalidate_tables(cache, db_alias, tables): now = time() d = {} - for k in table_cache_keys: - d[k] = now + for table in tables: + d[_get_table_cache_key(db_alias, table)] = now cache.set_many(d, None) + if isinstance(cache, AtomicCache): + cache.to_be_invalidated.update(tables) + def _invalidate_table(cache, db_alias, table): - table_cache_key = _get_table_cache_key(db_alias, table) - _invalidate_table_cache_keys(cache, (table_cache_key,)) + cache.set(_get_table_cache_key(db_alias, table), time(), None) - post_invalidation.send(table, db_alias=db_alias) + if isinstance(cache, AtomicCache): + cache.to_be_invalidated.add(table) + else: + post_invalidation.send(table, db_alias=db_alias) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 42a7840..38169ba 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -204,6 +204,14 @@ just after a cache invalidation (when you modify something in a SQL table). Be careful when you specify ``sender``, as it is sensible to string type. To be sure, use ``Model._meta.db_table``. +This signal is not directly triggered during transactions, +it waits until the current transaction ends. This signal is also triggered +when invalidating using the API or the ``manage.py`` command. Be careful +when using multiple databases, if you invalidate all databases by simply +calling ``invalidate()``, this signal will be triggered one time +for each database and for each model. If you have 3 databases and 20 models, +``invalidate()`` will trigger the signal 60 times. + Example: .. code:: python