Multiple Python optimisations.

This commit is contained in:
Bertrand Bordage 2017-06-07 08:59:35 +02:00
parent 78c4345364
commit bc49fd18ea
6 changed files with 62 additions and 23 deletions

View file

@ -87,11 +87,6 @@ def check_databases_compatibility(app_configs, **kwargs):
class CachalotConfig(AppConfig):
name = 'cachalot'
patched = False
def ready(self):
cachalot_settings.load()
if not self.patched:
patch()
self.patched = True

View file

@ -42,21 +42,19 @@ def _get_result_or_execute_query(execute_query_func, cache,
new_table_cache_keys = set(table_cache_keys)
new_table_cache_keys.difference_update(data)
if new_table_cache_keys:
now = time()
cache.set_many({k: now for k in new_table_cache_keys},
cachalot_settings.CACHALOT_TIMEOUT)
elif cache_key in data:
if not new_table_cache_keys and cache_key in data:
timestamp, result = data.pop(cache_key)
table_times = data.values()
if table_times and timestamp > max(table_times):
if timestamp >= max(data.values()):
return result
result = execute_query_func()
if result.__class__ not in ITERABLES and isinstance(result, Iterable):
result = list(result)
cache.set(cache_key, (time(), result), cachalot_settings.CACHALOT_TIMEOUT)
now = time()
to_be_set = {k: now for k in new_table_cache_keys}
to_be_set[cache_key] = (now, result)
cache.set_many(to_be_set, cachalot_settings.CACHALOT_TIMEOUT)
return result
@ -67,8 +65,7 @@ def _patch_compiler(original):
def inner(compiler, *args, **kwargs):
execute_query_func = lambda: original(compiler, *args, **kwargs)
db_alias = compiler.using
if not cachalot_settings.CACHALOT_ENABLED \
or db_alias not in cachalot_settings.CACHALOT_DATABASES \
if db_alias not in cachalot_settings.CACHALOT_DATABASES \
or isinstance(compiler, WRITE_COMPILERS):
return execute_query_func()
@ -101,19 +98,26 @@ def _patch_write_compiler(original):
def _patch_orm():
SQLCompiler.execute_sql = _patch_compiler(SQLCompiler.execute_sql)
if cachalot_settings.CACHALOT_ENABLED:
SQLCompiler.execute_sql = _patch_compiler(SQLCompiler.execute_sql)
for compiler in WRITE_COMPILERS:
compiler.execute_sql = _patch_write_compiler(compiler.execute_sql)
def _unpatch_orm():
if hasattr(SQLCompiler.execute_sql, '__wrapped__'):
SQLCompiler.execute_sql = SQLCompiler.execute_sql.__wrapped__
for compiler in WRITE_COMPILERS:
compiler.execute_sql = compiler.execute_sql.__wrapped__
def _patch_cursor():
def _patch_cursor_execute(original):
@wraps(original)
def inner(cursor, sql, *args, **kwargs):
out = original(cursor, sql, *args, **kwargs)
connection = cursor.db
if getattr(connection, 'raw', True) \
and cachalot_settings.CACHALOT_INVALIDATE_RAW:
if getattr(connection, 'raw', True):
if isinstance(sql, binary_type):
sql = sql.decode('utf-8')
sql = sql.lower()
@ -128,8 +132,16 @@ def _patch_cursor():
return inner
CursorWrapper.execute = _patch_cursor_execute(CursorWrapper.execute)
CursorWrapper.executemany = _patch_cursor_execute(CursorWrapper.executemany)
if cachalot_settings.CACHALOT_INVALIDATE_RAW:
CursorWrapper.execute = _patch_cursor_execute(CursorWrapper.execute)
CursorWrapper.executemany = \
_patch_cursor_execute(CursorWrapper.executemany)
def _unpatch_cursor():
if hasattr(CursorWrapper.execute, '__wrapped__'):
CursorWrapper.execute = CursorWrapper.execute.__wrapped__
CursorWrapper.executemany = CursorWrapper.executemany.__wrapped__
def _patch_atomic():
@ -155,6 +167,11 @@ def _patch_atomic():
Atomic.__exit__ = patch_exit(Atomic.__exit__)
def _unpatch_atomic():
Atomic.__enter__ = Atomic.__enter__.__wrapped__
Atomic.__exit__ = Atomic.__exit__.__wrapped__
def _invalidate_on_migration(sender, **kwargs):
invalidate(*sender.get_models(), db_alias=kwargs['using'],
cache_alias=cachalot_settings.CACHALOT_CACHE)
@ -166,3 +183,11 @@ def patch():
_patch_cursor()
_patch_atomic()
_patch_orm()
def unpatch():
post_migrate.disconnect(_invalidate_on_migration)
_unpatch_cursor()
_unpatch_atomic()
_unpatch_orm()

View file

@ -33,9 +33,11 @@ class CachalotPanel(Panel):
def enable_instrumentation(self):
settings.CACHALOT_ENABLED = True
cachalot_settings.reload()
def disable_instrumentation(self):
settings.CACHALOT_ENABLED = False
cachalot_settings.reload()
def process_response(self, request, response):
self.collect_invalidations()

View file

@ -35,6 +35,7 @@ ITERABLES = {tuple, list, frozenset, set}
class Settings(object):
patched = False
converters = {}
CACHALOT_ENABLED = True
@ -68,6 +69,21 @@ class Settings(object):
value = converter(value)
setattr(self, name, value)
if not self.patched:
from .monkey_patch import patch
patch()
self.patched = True
def unload(self):
if self.patched:
from .monkey_patch import unpatch
unpatch()
self.patched = False
def reload(self):
self.unload()
self.load()
@Settings.add_converter('CACHALOT_DATABASES')
def convert(value):

View file

@ -16,4 +16,4 @@ from .debug_toolbar import DebugToolbarTestCase
@receiver(setting_changed)
def reload_settings(sender, **kwargs):
cachalot_settings.load()
cachalot_settings.reload()

View file

@ -114,6 +114,7 @@ def _find_subqueries(children):
if child_class is WhereNode:
for grand_child in _find_subqueries(child.children):
yield grand_child
# TODO: Remove this condition when we drop Django 1.8 support.
elif child_class is SubqueryConstraint:
query_object = child.query_object
yield (query_object if query_object.__class__ is Query
@ -155,8 +156,8 @@ def filter_cachable(tables):
def _get_tables(db_alias, query):
if query.select_for_update or (
'?' in query.order_by
and not cachalot_settings.CACHALOT_CACHE_RANDOM):
not cachalot_settings.CACHALOT_CACHE_RANDOM
and '?' in query.order_by):
raise UncachableQuery
try: