diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eabfca4..2b487fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ master ------ - Adds a simple API containing: - ``invalidate_tables``, ``invalidate_models``, ``clear`` + ``invalidate_tables``, ``invalidate_models``, ``invalidate_all`` - Fixes a stale cache issue occurring when an invalidation is done exactly during a SQL request on the invalidated table(s) - Uses infinite timeout @@ -13,6 +13,7 @@ master - Caches all queries implying ``Queryset.extra`` - Invalidates raw queries - Adds 2 settings to customize how cache keys are generated +- Rewrites invalidation for a better speed & memory performance 0.8.1 diff --git a/benchmark.py b/benchmark.py index b847747..9a2de16 100644 --- a/benchmark.py +++ b/benchmark.py @@ -17,7 +17,7 @@ from django.test.utils import CaptureQueriesContext import matplotlib.pyplot as plt import pandas as pd -from cachalot.api import clear +from cachalot.api import invalidate_all from cachalot.settings import cachalot_settings from cachalot.tests.models import Test diff --git a/cachalot/api.py b/cachalot/api.py index 90e23d2..3674f11 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -6,10 +6,10 @@ from django.conf import settings from django.db import connection from .cache import cachalot_caches -from .utils import _get_table_cache_key, _invalidate_tables_cache_keys +from .utils import _get_table_cache_key, _invalidate_table_cache_keys -__all__ = ('invalidate_tables', 'invalidate_models', 'clear') +__all__ = ('invalidate_tables', 'invalidate_models', 'invalidate_all') def invalidate_tables(tables, cache_alias=None, db_alias=None): @@ -36,10 +36,9 @@ def invalidate_tables(tables, cache_alias=None, db_alias=None): if db_alias is None: db_alias = connection.alias - tables_cache_keys = [_get_table_cache_key(db_alias, table) - for table in tables] + table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables] cache = cachalot_caches.get_cache(cache_alias) - _invalidate_tables_cache_keys(cache, tables_cache_keys) + _invalidate_table_cache_keys(cache, table_cache_keys) def invalidate_models(models, cache_alias=None, db_alias=None): @@ -61,7 +60,7 @@ def invalidate_models(models, cache_alias=None, db_alias=None): cache_alias, db_alias) -def clear(cache_alias=None, db_alias=None): +def invalidate_all(cache_alias=None, db_alias=None): """ Clears everything that was cached by django-cachalot. @@ -83,4 +82,4 @@ def clear(cache_alias=None, db_alias=None): db_aliases = settings.DATABASES if db_alias is None else (db_alias,) for cache_alias in cache_aliases: for db_alias in db_aliases: - cachalot_caches.clear(cache_alias, db_alias) + cachalot_caches.invalidate_all(cache_alias, db_alias) diff --git a/cachalot/cache.py b/cachalot/cache.py index 401c756..fa9b8ed 100644 --- a/cachalot/cache.py +++ b/cachalot/cache.py @@ -9,7 +9,7 @@ from django.db import connections from .settings import cachalot_settings from .transaction import AtomicCache -from .utils import _get_table_cache_key, _invalidate_tables_cache_keys +from .utils import _get_table_cache_key, _invalidate_table_cache_keys class CacheHandler(local): @@ -43,11 +43,11 @@ class CacheHandler(local): for atomic_cache in atomic_caches: atomic_cache.commit() - def clear(self, cache_alias, db_alias): + def invalidate_all(self, cache_alias, db_alias): tables = connections[db_alias].introspection.table_names() - tables_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables] - _invalidate_tables_cache_keys(cachalot_caches.get_cache(cache_alias), - tables_cache_keys) + table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables] + _invalidate_table_cache_keys(cachalot_caches.get_cache(cache_alias), + table_cache_keys) cachalot_caches = CacheHandler() diff --git a/cachalot/monkey_patch.py b/cachalot/monkey_patch.py index b136bbd..909754c 100644 --- a/cachalot/monkey_patch.py +++ b/cachalot/monkey_patch.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from collections import Iterable from functools import wraps +from time import time from django import VERSION as django_version from django.conf import settings @@ -20,12 +21,12 @@ from django.db.models.sql.compiler import ( from django.db.transaction import Atomic, get_connection from django.test import TransactionTestCase -from .api import clear, invalidate_tables +from .api import invalidate_all, invalidate_tables from .cache import cachalot_caches from .settings import cachalot_settings from .utils import ( - _get_query_cache_key, _update_tables_queries, _invalidate_tables, - _get_tables_cache_keys, _get_tables_from_sql, _still_in_cache) + _get_query_cache_key, _invalidate_tables, + _get_table_cache_keys, _get_tables_from_sql) WRITE_COMPILERS = (SQLInsertCompiler, SQLUpdateCompiler, SQLDeleteCompiler) @@ -63,26 +64,26 @@ def _patch_compiler(original): return original(compiler, *args, **kwargs) cache = cachalot_caches.get_cache() - tables_cache_keys = _get_tables_cache_keys(compiler) - tables_queries = cache.get_many(tables_cache_keys + [cache_key]) + table_cache_keys = _get_table_cache_keys(compiler) + data = cache.get_many(table_cache_keys + [cache_key]) - if cache_key in tables_queries: - result = tables_queries[cache_key] - del tables_queries[cache_key] - if _still_in_cache(tables_cache_keys, tables_queries, cache_key): + new_table_cache_keys = frozenset(table_cache_keys) - frozenset(data) + + if new_table_cache_keys: + for table_cache_key in new_table_cache_keys: + cache.add(table_cache_key, time(), None) + elif cache_key in data: + timestamp, result = data[cache_key] + del data[cache_key] + if timestamp > max(data.values()): return result - _update_tables_queries(cache, tables_cache_keys, tables_queries, - cache_key) - result = original(compiler, *args, **kwargs) if isinstance(result, Iterable) \ and not isinstance(result, (tuple, list)): result = list(result) - tables_queries = cache.get_many(tables_cache_keys) - if _still_in_cache(tables_cache_keys, tables_queries, cache_key): - cache.set(cache_key, result, None) + cache.set(cache_key, (time(), result), None) return result @@ -159,7 +160,7 @@ def _patch_tests(): @wraps(original) def inner(*args, **kwargs): out = original(*args, **kwargs) - clear() + invalidate_all() return out inner.original = original @@ -171,7 +172,7 @@ def _patch_tests(): def _invalidate_on_migration(sender, **kwargs): db_alias = kwargs['using'] if django_version >= (1, 7) else kwargs['db'] - clear(db_alias=db_alias) + invalidate_all(db_alias=db_alias) def patch(): diff --git a/cachalot/transaction.py b/cachalot/transaction.py index 184c77e..133e23b 100644 --- a/cachalot/transaction.py +++ b/cachalot/transaction.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals -from .utils import _invalidate_tables_cache_keys +from .utils import _invalidate_table_cache_keys class AtomicCache(dict): @@ -24,6 +24,10 @@ class AtomicCache(dict): self.deleted.remove(k) self[k] = v + def add(self, k, v, timeout): + if k not in self: + self.set(k, v, timeout) + def get_many(self, keys): data = dict([(k, self[k]) for k in keys if k in self]) missing_keys = set(keys) @@ -43,6 +47,6 @@ class AtomicCache(dict): del self[k] def commit(self): - _invalidate_tables_cache_keys(self.parent_cache, + _invalidate_table_cache_keys(self.parent_cache, list(self.to_be_invalidated)) self.parent_cache.set_many(self, None) diff --git a/cachalot/utils.py b/cachalot/utils.py index cb29182..cff30b5 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from hashlib import md5 +from time import time import django from django.db import connections @@ -83,34 +84,18 @@ def _get_tables(compiler): return tables -def _get_tables_cache_keys(compiler): +def _get_table_cache_keys(compiler): using = compiler.using return [_get_table_cache_key(using, t) for t in _get_tables(compiler)] -def _update_tables_queries(cache, tables_cache_keys, tables_queries, - cache_key): - new_tables_queries = {} - for k in tables_cache_keys: - new_tables_queries[k] = tables_queries.get(k, set()) - new_tables_queries[k].add(cache_key) - cache.set_many(new_tables_queries, None) - - -def _invalidate_tables_cache_keys(cache, tables_cache_keys): +def _invalidate_table_cache_keys(cache, table_cache_keys): if hasattr(cache, 'to_be_invalidated'): - cache.to_be_invalidated.update(tables_cache_keys) - tables_queries = cache.get_many(tables_cache_keys) - queries = [q for q_list in tables_queries.values() for q in q_list] - cache.delete_many(queries + tables_cache_keys) + cache.to_be_invalidated.update(table_cache_keys) + now = time() + cache.set_many(dict((k, now) for k in table_cache_keys), None) def _invalidate_tables(cache, compiler): - tables_cache_keys = _get_tables_cache_keys(compiler) - _invalidate_tables_cache_keys(cache, tables_cache_keys) - - -def _still_in_cache(tables_cache_keys, tables_queries, cache_key): - return ( - frozenset(tables_queries) == frozenset(tables_cache_keys) - and all(cache_key in queries for queries in tables_queries.values())) + table_cache_keys = _get_table_cache_keys(compiler) + _invalidate_table_cache_keys(cache, table_cache_keys)