Hopiu/django-cachalot
1
0
Fork 0
mirror of https://github.com/Hopiu/django-cachalot.git synced 2026-03-16 21:30:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrites invalidation for a better speed & memory performance.

Browse source
This commit is contained in:
Bertrand Bordage 2014-11-24 12:05:00 +01:00
parent 73d09b2e5f
commit 45e4cd543b
7 changed files with 46 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.rst
View file

@ -5,7 +5,7 @@ master
------
- Adds a simple API containing:
``invalidate_tables``, ``invalidate_models``, ``clear``
``invalidate_tables``, ``invalidate_models``, ``invalidate_all``
- Fixes a stale cache issue occurring when an invalidation is done
exactly during a SQL request on the invalidated table(s)
- Uses infinite timeout
@ -13,6 +13,7 @@ master
- Caches all queries implying ``Queryset.extra``
- Invalidates raw queries
- Adds 2 settings to customize how cache keys are generated
- Rewrites invalidation for a better speed & memory performance
0.8.1

2
benchmark.py
View file

@ -17,7 +17,7 @@ from django.test.utils import CaptureQueriesContext
import matplotlib.pyplot as plt
import pandas as pd
from cachalot.api import clear
from cachalot.api import invalidate_all
from cachalot.settings import cachalot_settings
from cachalot.tests.models import Test

13
cachalot/api.py
View file

@ -6,10 +6,10 @@ from django.conf import settings
from django.db import connection
from .cache import cachalot_caches
from .utils import _get_table_cache_key, _invalidate_tables_cache_keys
from .utils import _get_table_cache_key, _invalidate_table_cache_keys
__all__ = ('invalidate_tables', 'invalidate_models', 'clear')
__all__ = ('invalidate_tables', 'invalidate_models', 'invalidate_all')
def invalidate_tables(tables, cache_alias=None, db_alias=None):
@ -36,10 +36,9 @@ def invalidate_tables(tables, cache_alias=None, db_alias=None):
if db_alias is None:
db_alias = connection.alias
tables_cache_keys = [_get_table_cache_key(db_alias, table)
for table in tables]
table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables]
cache = cachalot_caches.get_cache(cache_alias)
_invalidate_tables_cache_keys(cache, tables_cache_keys)
_invalidate_table_cache_keys(cache, table_cache_keys)
def invalidate_models(models, cache_alias=None, db_alias=None):
@ -61,7 +60,7 @@ def invalidate_models(models, cache_alias=None, db_alias=None):
cache_alias, db_alias)
def clear(cache_alias=None, db_alias=None):
def invalidate_all(cache_alias=None, db_alias=None):
"""
Clears everything that was cached by django-cachalot.
@ -83,4 +82,4 @@ def clear(cache_alias=None, db_alias=None):
db_aliases = settings.DATABASES if db_alias is None else (db_alias,)
for cache_alias in cache_aliases:
for db_alias in db_aliases:
cachalot_caches.clear(cache_alias, db_alias)
cachalot_caches.invalidate_all(cache_alias, db_alias)

10
cachalot/cache.py
View file

@ -9,7 +9,7 @@ from django.db import connections
from .settings import cachalot_settings
from .transaction import AtomicCache
from .utils import _get_table_cache_key, _invalidate_tables_cache_keys
from .utils import _get_table_cache_key, _invalidate_table_cache_keys
class CacheHandler(local):
@ -43,11 +43,11 @@ class CacheHandler(local):
for atomic_cache in atomic_caches:
atomic_cache.commit()
def clear(self, cache_alias, db_alias):
def invalidate_all(self, cache_alias, db_alias):
tables = connections[db_alias].introspection.table_names()
tables_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables]
_invalidate_tables_cache_keys(cachalot_caches.get_cache(cache_alias),
tables_cache_keys)
table_cache_keys = [_get_table_cache_key(db_alias, t) for t in tables]
_invalidate_table_cache_keys(cachalot_caches.get_cache(cache_alias),
table_cache_keys)
cachalot_caches = CacheHandler()

35
cachalot/monkey_patch.py
View file

@ -3,6 +3,7 @@
from __future__ import unicode_literals
from collections import Iterable
from functools import wraps
from time import time
from django import VERSION as django_version
from django.conf import settings
@ -20,12 +21,12 @@ from django.db.models.sql.compiler import (
from django.db.transaction import Atomic, get_connection
from django.test import TransactionTestCase
from .api import clear, invalidate_tables
from .api import invalidate_all, invalidate_tables
from .cache import cachalot_caches
from .settings import cachalot_settings
from .utils import (
_get_query_cache_key, _update_tables_queries, _invalidate_tables,
_get_tables_cache_keys, _get_tables_from_sql, _still_in_cache)
_get_query_cache_key, _invalidate_tables,
_get_table_cache_keys, _get_tables_from_sql)
WRITE_COMPILERS = (SQLInsertCompiler, SQLUpdateCompiler, SQLDeleteCompiler)
@ -63,26 +64,26 @@ def _patch_compiler(original):
return original(compiler, *args, **kwargs)
cache = cachalot_caches.get_cache()
tables_cache_keys = _get_tables_cache_keys(compiler)
tables_queries = cache.get_many(tables_cache_keys + [cache_key])
table_cache_keys = _get_table_cache_keys(compiler)
data = cache.get_many(table_cache_keys + [cache_key])
if cache_key in tables_queries:
result = tables_queries[cache_key]
del tables_queries[cache_key]
if _still_in_cache(tables_cache_keys, tables_queries, cache_key):
new_table_cache_keys = frozenset(table_cache_keys) - frozenset(data)
if new_table_cache_keys:
for table_cache_key in new_table_cache_keys:
cache.add(table_cache_key, time(), None)
elif cache_key in data:
timestamp, result = data[cache_key]
del data[cache_key]
if timestamp > max(data.values()):
return result
_update_tables_queries(cache, tables_cache_keys, tables_queries,
cache_key)
result = original(compiler, *args, **kwargs)
if isinstance(result, Iterable) \
and not isinstance(result, (tuple, list)):
result = list(result)
tables_queries = cache.get_many(tables_cache_keys)
if _still_in_cache(tables_cache_keys, tables_queries, cache_key):
cache.set(cache_key, result, None)
cache.set(cache_key, (time(), result), None)
return result
@ -159,7 +160,7 @@ def _patch_tests():
@wraps(original)
def inner(*args, **kwargs):
out = original(*args, **kwargs)
clear()
invalidate_all()
return out
inner.original = original
@ -171,7 +172,7 @@ def _patch_tests():
def _invalidate_on_migration(sender, **kwargs):
db_alias = kwargs['using'] if django_version >= (1, 7) else kwargs['db']
clear(db_alias=db_alias)
invalidate_all(db_alias=db_alias)
def patch():

8
cachalot/transaction.py
View file

@ -2,7 +2,7 @@
from __future__ import unicode_literals
from .utils import _invalidate_tables_cache_keys
from .utils import _invalidate_table_cache_keys
class AtomicCache(dict):
@ -24,6 +24,10 @@ class AtomicCache(dict):
self.deleted.remove(k)
self[k] = v
def add(self, k, v, timeout):
if k not in self:
self.set(k, v, timeout)
def get_many(self, keys):
data = dict([(k, self[k]) for k in keys if k in self])
missing_keys = set(keys)
@ -43,6 +47,6 @@ class AtomicCache(dict):
del self[k]
def commit(self):
_invalidate_tables_cache_keys(self.parent_cache,
_invalidate_table_cache_keys(self.parent_cache,
list(self.to_be_invalidated))
self.parent_cache.set_many(self, None)

31
cachalot/utils.py
View file

@ -2,6 +2,7 @@
from __future__ import unicode_literals
from hashlib import md5
from time import time
import django
from django.db import connections
@ -83,34 +84,18 @@ def _get_tables(compiler):
return tables
def _get_tables_cache_keys(compiler):
def _get_table_cache_keys(compiler):
using = compiler.using
return [_get_table_cache_key(using, t) for t in _get_tables(compiler)]
def _update_tables_queries(cache, tables_cache_keys, tables_queries,
cache_key):
new_tables_queries = {}
for k in tables_cache_keys:
new_tables_queries[k] = tables_queries.get(k, set())
new_tables_queries[k].add(cache_key)
cache.set_many(new_tables_queries, None)
def _invalidate_tables_cache_keys(cache, tables_cache_keys):
def _invalidate_table_cache_keys(cache, table_cache_keys):
if hasattr(cache, 'to_be_invalidated'):
cache.to_be_invalidated.update(tables_cache_keys)
tables_queries = cache.get_many(tables_cache_keys)
queries = [q for q_list in tables_queries.values() for q in q_list]
cache.delete_many(queries + tables_cache_keys)
cache.to_be_invalidated.update(table_cache_keys)
now = time()
cache.set_many(dict((k, now) for k in table_cache_keys), None)
def _invalidate_tables(cache, compiler):
tables_cache_keys = _get_tables_cache_keys(compiler)
_invalidate_tables_cache_keys(cache, tables_cache_keys)
def _still_in_cache(tables_cache_keys, tables_queries, cache_key):
return (
frozenset(tables_queries) == frozenset(tables_cache_keys)
and all(cache_key in queries for queries in tables_queries.values()))
table_cache_keys = _get_table_cache_keys(compiler)
_invalidate_table_cache_keys(cache, table_cache_keys)