diff --git a/.travis.yml b/.travis.yml index 7834a31..3d4021d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,436 +3,61 @@ language: python services: - memcached - redis-server + - mysql + - postgresql -addons: - postgresql: 9.6 - -cache: pip +python: + - "2.7" + - "3.5" + - "3.6" + - "3.7" + - "3.8" +env: + - DJANGO="1.11" + - DJANGO="2.0" + - DJANGO="2.1" + - DJANGO="2.2" + - DJANGO="3.0" matrix: - include: + exclude: - python: 2.7 - env: TOXENV=py2.7-django1.11-sqlite3-redis + env: DJANGO=2.0 - python: 2.7 - env: TOXENV=py2.7-django1.11-sqlite3-memcached + env: DJANGO=2.1 - python: 2.7 - env: TOXENV=py2.7-django1.11-sqlite3-pylibmc + env: DJANGO=2.2 - python: 2.7 - env: TOXENV=py2.7-django1.11-sqlite3-locmem - - python: 2.7 - env: TOXENV=py2.7-django1.11-sqlite3-filebased - - - python: 2.7 - env: TOXENV=py2.7-django1.11-postgresql-redis - - python: 2.7 - env: TOXENV=py2.7-django1.11-postgresql-memcached - - python: 2.7 - env: TOXENV=py2.7-django1.11-postgresql-pylibmc - - python: 2.7 - env: TOXENV=py2.7-django1.11-postgresql-locmem - - python: 2.7 - env: TOXENV=py2.7-django1.11-postgresql-filebased - - - python: 2.7 - env: TOXENV=py2.7-django1.11-mysql-redis - - python: 2.7 - env: TOXENV=py2.7-django1.11-mysql-memcached - - python: 2.7 - env: TOXENV=py2.7-django1.11-mysql-pylibmc - - python: 2.7 - env: TOXENV=py2.7-django1.11-mysql-locmem - - python: 2.7 - env: TOXENV=py2.7-django1.11-mysql-filebased - - - python: 3.4 - env: TOXENV=py3.4-django1.11-sqlite3-redis - - python: 3.4 - env: TOXENV=py3.4-django1.11-sqlite3-memcached - - python: 3.4 - env: TOXENV=py3.4-django1.11-sqlite3-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django1.11-sqlite3-locmem - - python: 3.4 - env: TOXENV=py3.4-django1.11-sqlite3-filebased - - - python: 3.4 - env: TOXENV=py3.4-django1.11-postgresql-redis - - python: 3.4 - env: TOXENV=py3.4-django1.11-postgresql-memcached - - python: 3.4 - env: TOXENV=py3.4-django1.11-postgresql-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django1.11-postgresql-locmem - - python: 3.4 - env: TOXENV=py3.4-django1.11-postgresql-filebased - - - python: 3.4 - env: TOXENV=py3.4-django1.11-mysql-redis - - python: 3.4 - env: TOXENV=py3.4-django1.11-mysql-memcached - - python: 3.4 - env: TOXENV=py3.4-django1.11-mysql-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django1.11-mysql-locmem - - python: 3.4 - env: TOXENV=py3.4-django1.11-mysql-filebased + env: DJANGO=3.0 - python: 3.5 - env: TOXENV=py3.5-django1.11-sqlite3-redis - - python: 3.5 - env: TOXENV=py3.5-django1.11-sqlite3-memcached - - python: 3.5 - env: TOXENV=py3.5-django1.11-sqlite3-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django1.11-sqlite3-locmem - - python: 3.5 - env: TOXENV=py3.5-django1.11-sqlite3-filebased + env: DJANGO=3.0 - - python: 3.5 - env: TOXENV=py3.5-django1.11-postgresql-redis - - python: 3.5 - env: TOXENV=py3.5-django1.11-postgresql-memcached - - python: 3.5 - env: TOXENV=py3.5-django1.11-postgresql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django1.11-postgresql-locmem - - python: 3.5 - env: TOXENV=py3.5-django1.11-postgresql-filebased - - - python: 3.5 - env: TOXENV=py3.5-django1.11-mysql-redis - - python: 3.5 - env: TOXENV=py3.5-django1.11-mysql-memcached - - python: 3.5 - env: TOXENV=py3.5-django1.11-mysql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django1.11-mysql-locmem - - python: 3.5 - env: TOXENV=py3.5-django1.11-mysql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django1.11-sqlite3-redis - - python: 3.6 - env: TOXENV=py3.6-django1.11-sqlite3-memcached - - python: 3.6 - env: TOXENV=py3.6-django1.11-sqlite3-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django1.11-sqlite3-locmem - - python: 3.6 - env: TOXENV=py3.6-django1.11-sqlite3-filebased - - - python: 3.6 - env: TOXENV=py3.6-django1.11-postgresql-redis - - python: 3.6 - env: TOXENV=py3.6-django1.11-postgresql-memcached - - python: 3.6 - env: TOXENV=py3.6-django1.11-postgresql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django1.11-postgresql-locmem - - python: 3.6 - env: TOXENV=py3.6-django1.11-postgresql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django1.11-mysql-redis - - python: 3.6 - env: TOXENV=py3.6-django1.11-mysql-memcached - - python: 3.6 - env: TOXENV=py3.6-django1.11-mysql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django1.11-mysql-locmem - - python: 3.6 - env: TOXENV=py3.6-django1.11-mysql-filebased - - - python: 3.4 - env: TOXENV=py3.4-django2.0-sqlite3-redis - - python: 3.4 - env: TOXENV=py3.4-django2.0-sqlite3-memcached - - python: 3.4 - env: TOXENV=py3.4-django2.0-sqlite3-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django2.0-sqlite3-locmem - - python: 3.4 - env: TOXENV=py3.4-django2.0-sqlite3-filebased - - - python: 3.4 - env: TOXENV=py3.4-django2.0-postgresql-redis - - python: 3.4 - env: TOXENV=py3.4-django2.0-postgresql-memcached - - python: 3.4 - env: TOXENV=py3.4-django2.0-postgresql-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django2.0-postgresql-locmem - - python: 3.4 - env: TOXENV=py3.4-django2.0-postgresql-filebased - - - python: 3.4 - env: TOXENV=py3.4-django2.0-mysql-redis - - python: 3.4 - env: TOXENV=py3.4-django2.0-mysql-memcached - - python: 3.4 - env: TOXENV=py3.4-django2.0-mysql-pylibmc - - python: 3.4 - env: TOXENV=py3.4-django2.0-mysql-locmem - - python: 3.4 - env: TOXENV=py3.4-django2.0-mysql-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.0-sqlite3-redis - - python: 3.5 - env: TOXENV=py3.5-django2.0-sqlite3-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.0-sqlite3-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.0-sqlite3-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.0-sqlite3-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.0-postgresql-redis - - python: 3.5 - env: TOXENV=py3.5-django2.0-postgresql-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.0-postgresql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.0-postgresql-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.0-postgresql-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.0-mysql-redis - - python: 3.5 - env: TOXENV=py3.5-django2.0-mysql-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.0-mysql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.0-mysql-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.0-mysql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.0-sqlite3-redis - - python: 3.6 - env: TOXENV=py3.6-django2.0-sqlite3-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.0-sqlite3-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.0-sqlite3-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.0-sqlite3-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.0-postgresql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.0-postgresql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.0-postgresql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.0-postgresql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.0-postgresql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.0-mysql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.0-mysql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.0-mysql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.0-mysql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.0-mysql-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.1-sqlite3-redis - - python: 3.5 - env: TOXENV=py3.5-django2.1-sqlite3-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.1-sqlite3-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.1-sqlite3-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.1-sqlite3-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.1-postgresql-redis - - python: 3.5 - env: TOXENV=py3.5-django2.1-postgresql-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.1-postgresql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.1-postgresql-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.1-postgresql-filebased - - - python: 3.5 - env: TOXENV=py3.5-django2.1-mysql-redis - - python: 3.5 - env: TOXENV=py3.5-django2.1-mysql-memcached - - python: 3.5 - env: TOXENV=py3.5-django2.1-mysql-pylibmc - - python: 3.5 - env: TOXENV=py3.5-django2.1-mysql-locmem - - python: 3.5 - env: TOXENV=py3.5-django2.1-mysql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.1-sqlite3-redis - - python: 3.6 - env: TOXENV=py3.6-django2.1-sqlite3-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.1-sqlite3-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.1-sqlite3-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.1-sqlite3-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.1-postgresql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.1-postgresql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.1-postgresql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.1-postgresql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.1-postgresql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.1-mysql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.1-mysql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.1-mysql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.1-mysql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.1-mysql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.2-sqlite3-redis - - python: 3.6 - env: TOXENV=py3.6-django2.2-sqlite3-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.2-sqlite3-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.2-sqlite3-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.2-sqlite3-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.2-postgresql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.2-postgresql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.2-postgresql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.2-postgresql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.2-postgresql-filebased - - - python: 3.6 - env: TOXENV=py3.6-django2.2-mysql-redis - - python: 3.6 - env: TOXENV=py3.6-django2.2-mysql-memcached - - python: 3.6 - env: TOXENV=py3.6-django2.2-mysql-pylibmc - - python: 3.6 - env: TOXENV=py3.6-django2.2-mysql-locmem - - python: 3.6 - env: TOXENV=py3.6-django2.2-mysql-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.1-sqlite3-redis - - python: 3.7 - env: TOXENV=py3.7-django2.1-sqlite3-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.1-sqlite3-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.1-sqlite3-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.1-sqlite3-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.1-postgresql-redis - - python: 3.7 - env: TOXENV=py3.7-django2.1-postgresql-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.1-postgresql-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.1-postgresql-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.1-postgresql-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.1-mysql-redis - - python: 3.7 - env: TOXENV=py3.7-django2.1-mysql-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.1-mysql-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.1-mysql-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.1-mysql-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.2-sqlite3-redis - - python: 3.7 - env: TOXENV=py3.7-django2.2-sqlite3-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.2-sqlite3-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.2-sqlite3-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.2-sqlite3-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.2-postgresql-redis - - python: 3.7 - env: TOXENV=py3.7-django2.2-postgresql-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.2-postgresql-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.2-postgresql-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.2-postgresql-filebased - - - python: 3.7 - env: TOXENV=py3.7-django2.2-mysql-redis - - python: 3.7 - env: TOXENV=py3.7-django2.2-mysql-memcached - - python: 3.7 - env: TOXENV=py3.7-django2.2-mysql-pylibmc - - python: 3.7 - env: TOXENV=py3.7-django2.2-mysql-locmem - - python: 3.7 - env: TOXENV=py3.7-django2.2-mysql-filebased - - allow_failures: - - env: TOXENV=py3.7-django2.1-sqlite3-redis - - env: TOXENV=py3.7-django2.1-sqlite3-memcached - - env: TOXENV=py3.7-django2.1-sqlite3-pylibmc - - env: TOXENV=py3.7-django2.1-sqlite3-locmem - - env: TOXENV=py3.7-django2.1-sqlite3-filebased - - env: TOXENV=py3.7-django2.1-postgresql-redis - - env: TOXENV=py3.7-django2.1-postgresql-memcached - - env: TOXENV=py3.7-django2.1-postgresql-pylibmc - - env: TOXENV=py3.7-django2.1-postgresql-locmem - - env: TOXENV=py3.7-django2.1-postgresql-filebased - - env: TOXENV=py3.7-django2.1-mysql-redis - - env: TOXENV=py3.7-django2.1-mysql-memcached - - env: TOXENV=py3.7-django2.1-mysql-pylibmc - - env: TOXENV=py3.7-django2.1-mysql-locmem - - env: TOXENV=py3.7-django2.1-mysql-filebased + - python: 3.8 + env: DJANGO=1.11 + - python: 3.8 + env: DJANGO=2.0 + - python: 3.8 + env: DJANGO=2.1 sudo: false -install: pip install tox coveralls +cache: pip + +install: pip install tox tox-travis coveralls before_script: - psql -c 'CREATE USER cachalot SUPERUSER;' -U postgres - psql -c 'CREATE DATABASE cachalot OWNER cachalot;' -U postgres - mysql -u root -e 'CREATE DATABASE cachalot;' -script: tox -e $TOXENV +script: tox after_success: coveralls + +notifications: + email: + recipients: + - acwangpython@gmail.com + on_success: change + on_failure: always diff --git a/README.rst b/README.rst index 783bf60..5c732e4 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,13 @@ Documentation: http://django-cachalot.readthedocs.io .. image:: https://img.shields.io/badge/cachalot-Chat%20on%20Slack-green?style=flat&logo=slack :target: https://join.slack.com/t/cachalotdjango/shared_invite/enQtOTMyNzI0NTQzOTA3LWViYmYwMWY3MmU0OTZkYmNiMjBhN2NjNjc4OWVlZDNiMjMxN2Y3YzljYmNiYTY4ZTRjOGQxZDRiMTM0NWE3NGI +Quickstart +---------- + +Cachalot officially supports Python 2.7, 3.4-3.8 and Django 1.11, 2.0-2.2, 3.0 with the databases PostgreSQL, SQLite, and MySQL. + +Note: Python 3.4 with MySQL fails on tests. If you're MySQL is configured correctly, + Third-Party Cache Comparison ---------------------------- @@ -46,7 +53,10 @@ Cachalot is good when there are <50 modifications per second on a hot cached tab which is why we suggest you use cache-machine for hot caches. Cache-machine caches individual objects, taking up more in the memory store but invalidates those individual objects instead of the entire table like cachalot. -Yes, the bane of our entire existence lies in cache invalidation and naming variables. Why does cachalot suck when stuck with a huge table that's accessed rapidly? Since you've mixed your cold (90% of) with your hot (10% of) records, you're caching and invalidating an entire table. It's like trying to boil 1 ton of noodles inside ONE pot instead of 100 pots boiling 1 ton of noodles. Which is more efficient? The splitting up of them. +Yes, the bane of our entire existence lies in cache invalidation and naming variables. Why does cachalot suck when +stuck with a huge table that's modified rapidly? Since you've mixed your cold (90% of) with your hot (10% of) records, +you're caching and invalidating an entire table. It's like trying to boil 1 ton of noodles inside ONE pot instead of +100 pots boiling 1 ton of noodles. Which is more efficient? The splitting up of them. Note 1: My personal experience with caches stems from Reddit's: https://redditblog.com/2017/01/17/caching-at-reddit/ diff --git a/cachalot/__init__.py b/cachalot/__init__.py index 5cdd3ce..903b668 100644 --- a/cachalot/__init__.py +++ b/cachalot/__init__.py @@ -1,4 +1,4 @@ -VERSION = (2, 1, 0) +VERSION = (2, 2, 0) __version__ = '.'.join(map(str, VERSION)) default_app_config = 'cachalot.apps.CachalotConfig' diff --git a/cachalot/api.py b/cachalot/api.py index 079a877..90df8a5 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -5,7 +5,10 @@ from __future__ import unicode_literals from django.apps import apps from django.conf import settings from django.db import connections -from six import string_types +try: + from django.utils.six import string_types +except ImportError: + from six import string_types from .cache import cachalot_caches from .settings import cachalot_settings diff --git a/cachalot/monkey_patch.py b/cachalot/monkey_patch.py index e09870b..77db96e 100644 --- a/cachalot/monkey_patch.py +++ b/cachalot/monkey_patch.py @@ -11,7 +11,11 @@ from django.db.models.sql.compiler import ( SQLCompiler, SQLInsertCompiler, SQLUpdateCompiler, SQLDeleteCompiler, ) from django.db.transaction import Atomic, get_connection -from six import binary_type, wraps + +try: + from django.utils.six import binary_type, wraps +except ImportError: + from six import binary_type, wraps from .api import invalidate from .cache import cachalot_caches diff --git a/cachalot/panels.py b/cachalot/panels.py index f2d4437..3a7033d 100644 --- a/cachalot/panels.py +++ b/cachalot/panels.py @@ -39,8 +39,9 @@ class CachalotPanel(Panel): settings.CACHALOT_ENABLED = False cachalot_settings.reload() - def process_response(self, request, response): + def process_request(self, request): self.collect_invalidations() + return super(CachalotPanel, self).process_request(request) def collect_invalidations(self): models = apps.get_models() diff --git a/cachalot/tests/api.py b/cachalot/tests/api.py index 1d99968..b31c6b3 100644 --- a/cachalot/tests/api.py +++ b/cachalot/tests/api.py @@ -19,6 +19,8 @@ from .test_utils import TestUtilsMixin class APITestCase(TestUtilsMixin, TransactionTestCase): + databases = set(settings.DATABASES.keys()) + def setUp(self): super(APITestCase, self).setUp() self.t1 = Test.objects.create(name='test1') diff --git a/cachalot/tests/debug_toolbar.py b/cachalot/tests/debug_toolbar.py index 3467f67..addbaa2 100644 --- a/cachalot/tests/debug_toolbar.py +++ b/cachalot/tests/debug_toolbar.py @@ -1,10 +1,13 @@ from uuid import UUID from bs4 import BeautifulSoup +from django.conf import settings from django.test import LiveServerTestCase, override_settings @override_settings(DEBUG=True) class DebugToolbarTestCase(LiveServerTestCase): + databases = set(settings.DATABASES.keys()) + def test_rendering(self): # # Rendering toolbar diff --git a/cachalot/tests/multi_db.py b/cachalot/tests/multi_db.py index c189456..e98aaeb 100644 --- a/cachalot/tests/multi_db.py +++ b/cachalot/tests/multi_db.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from unittest import skipIf +from django import VERSION as DJANGO_VERSION from django.conf import settings from django.db import DEFAULT_DB_ALIAS, connections, transaction from django.test import TransactionTestCase @@ -28,6 +29,24 @@ class MultiDatabaseTestCase(TransactionTestCase): # will execute an extra SQL request below. connection2.cursor() + def is_django_21_below_and_sqlite2(self): + """ + Note: See test_utils.py with this function name + Checks if Django 2.1 or below and SQLite2 + """ + django_version = DJANGO_VERSION + if not self.is_sqlite2: + # Immediately know if SQLite + return False + if django_version[0] < 2: + # Takes Django 0 and 1 out of the picture + return True + else: + if django_version[0] == 2 and django_version[1] < 2: + # Takes Django 2.0-2.1 out + return True + return False + def test_read(self): with self.assertNumQueries(1): data1 = list(Test.objects.all()) @@ -49,7 +68,7 @@ class MultiDatabaseTestCase(TransactionTestCase): data1 = list(Test.objects.using(self.db_alias2)) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite2 else 1, + with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, using=self.db_alias2): t3 = Test.objects.using(self.db_alias2).create(name='test3') @@ -65,7 +84,7 @@ class MultiDatabaseTestCase(TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, [self.t1, self.t2]) - with self.assertNumQueries(2 if self.is_sqlite2 else 1, + with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, using=self.db_alias2): Test.objects.using(self.db_alias2).create(name='test3') diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index c0e93ea..44420f5 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -707,7 +707,9 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): r'Planning time: [\d\.]+ ms\n' r'Execution time: [\d\.]+ ms$') % (operation_detail, operation_detail) - with self.assertNumQueries(2 if self.is_mysql else 1): + with self.assertNumQueries( + 2 if self.is_mysql and django_version[0] < 3 + else 1): explanation1 = Test.objects.explain(**explain_kwargs) self.assertRegex(explanation1, expected) with self.assertNumQueries(0): @@ -915,9 +917,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs, after=1 if self.is_sqlite else 0) def test_float(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', a_float=0.123456789) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', a_float=12345.6789) with self.assertNumQueries(1): data1 = list(Test.objects.values_list('a_float', flat=True).filter( @@ -936,9 +938,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(a_float=0.123456789) def test_decimal(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', a_decimal=Decimal('123.45')) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', a_decimal=Decimal('12.3')) qs = Test.objects.values_list('a_decimal', flat=True).filter( @@ -952,9 +954,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(a_decimal=Decimal('123.45')) def test_ipv4_address(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', ip='127.0.0.1') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', ip='192.168.0.1') qs = Test.objects.values_list('ip', flat=True).filter( @@ -968,9 +970,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(ip='127.0.0.1') def test_ipv6_address(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1/64') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', ip='2001:db8:0:85a3::ac1f:8001') qs = Test.objects.values_list('ip', flat=True).filter( @@ -985,9 +987,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(ip='2001:db8:0:85a3::ac1f:8001') def test_duration(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', duration=datetime.timedelta(30)) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', duration=datetime.timedelta(60)) qs = Test.objects.values_list('duration', flat=True).filter( @@ -1002,10 +1004,10 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(duration=datetime.timedelta(30)) def test_uuid(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1', uuid='1cc401b7-09f4-4520-b8d0-c267576d196b') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', uuid='ebb3b6e1-1737-4321-93e3-4c35d61ff491') diff --git a/cachalot/tests/settings.py b/cachalot/tests/settings.py index 9c57f28..4709c66 100644 --- a/cachalot/tests/settings.py +++ b/cachalot/tests/settings.py @@ -43,7 +43,7 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase): list(Test.objects.all()) with self.settings(CACHALOT_ENABLED=False): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t = Test.objects.create(name='test') with self.assertNumQueries(1): data = list(Test.objects.all()) diff --git a/cachalot/tests/signals.py b/cachalot/tests/signals.py index 879336d..1ee5f0d 100644 --- a/cachalot/tests/signals.py +++ b/cachalot/tests/signals.py @@ -15,6 +15,8 @@ from .models import Test class SignalsTestCase(TransactionTestCase): + databases = set(settings.DATABASES.keys()) + def test_table_invalidated(self): l = [] diff --git a/cachalot/tests/test_utils.py b/cachalot/tests/test_utils.py index 5929c27..7b3530d 100644 --- a/cachalot/tests/test_utils.py +++ b/cachalot/tests/test_utils.py @@ -1,6 +1,10 @@ +from django import VERSION as DJANGO_VERSION from django.core.management.color import no_style from django.db import connection, transaction -from django.utils.six import string_types +try: + from django.utils.six import string_types +except ImportError: + from six import string_types from .models import PostgresModel from ..utils import _get_tables @@ -58,3 +62,30 @@ class TestUtilsMixin: assert_function(data2, data1) if result is not None: assert_function(data2, result) + + def is_dj_21_below_and_is_sqlite(self): + """ + Checks if Django 2.1 or lower and if SQLite is the DB + Django 2.1 and lower had two queries on SQLite DBs: + + After an insertion, e.g. Test.objects.create(name="asdf"), + SQLite returns the queries: + [{'sql': 'INSERT INTO "cachalot_test" ("name") VALUES (\'asd\')', 'time': '0.001'}, {'sql': 'BEGIN', 'time': '0.000'}] + + This can be seen with django.db import connection; print(connection.queries) + In Django 2.2 and above, the latter was removed. + + :return: bool is Django 2.1 or below and is SQLite the DB + """ + django_version = DJANGO_VERSION + if not self.is_sqlite: + # Immediately know if SQLite + return False + if django_version[0] < 2: + # Takes Django 0 and 1 out of the picture + return True + else: + if django_version[0] == 2 and django_version[1] < 2: + # Takes Django 2.0-2.1 out + return True + return False diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 86b5b7b..f3dff91 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from unittest import skipIf, skipUnless +from django import VERSION as DJANGO_VERSION from django.contrib.auth.models import User, Permission, Group from django.contrib.contenttypes.models import ContentType from django.core.exceptions import MultipleObjectsReturned @@ -28,21 +29,21 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t1 = Test.objects.create(name='test1') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data2 = list(Test.objects.all()) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t3 = Test.objects.create(name='test3') with self.assertNumQueries(1): data3 = list(Test.objects.all()) self.assertListEqual(data2, [t1, t2]) self.assertListEqual(data3, [t1, t2, t3]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t3_copy = Test.objects.create(name='test3') self.assertNotEqual(t3_copy, t3) with self.assertNumQueries(1): @@ -128,12 +129,12 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['test%02d' % (i // 2) for i in range(2, 22)]) def test_update(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t = Test.objects.create(name='test1') with self.assertNumQueries(1): t1 = Test.objects.get() - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t.name = 'test2' t.save() with self.assertNumQueries(1): @@ -141,21 +142,21 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(t1.name, 'test1') self.assertEqual(t2.name, 'test2') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.update(name='test3') with self.assertNumQueries(1): t3 = Test.objects.get() self.assertEqual(t3.name, 'test3') def test_delete(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t1 = Test.objects.create(name='test1') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data1 = list(Test.objects.values_list('name', flat=True)) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t2.delete() with self.assertNumQueries(1): data2 = list(Test.objects.values_list('name', flat=True)) @@ -178,7 +179,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.create(name='test') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): self.assertTrue(Test.objects.create()) def test_invalidate_count(self): @@ -316,22 +317,22 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): u = User.objects.create_user('test') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', owner=u) with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 1) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test3') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 1) @@ -341,13 +342,13 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(User.objects.annotate(n=Count('test')).order_by('pk')) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(User.objects.annotate(n=Count('test')).order_by('pk')) self.assertListEqual(data2, []) - with self.assertNumQueries(4 if self.is_sqlite else 2): + with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): user1 = User.objects.create_user('user1') user2 = User.objects.create_user('user2') with self.assertNumQueries(1): @@ -355,7 +356,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data3, [user1, user2]) self.assertListEqual([u.n for u in data3], [0, 0]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test2', owner=user1) with self.assertNumQueries(1): data4 = list(User.objects.annotate(n=Count('test')).order_by('pk')) @@ -583,7 +584,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.select_related('owner')) self.assertListEqual(data1, []) - with self.assertNumQueries(4 if self.is_sqlite else 2): + with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): u1 = User.objects.create_user('test1') u2 = User.objects.create_user('test2') with self.assertNumQueries(1): @@ -617,7 +618,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): .prefetch_related('owner__groups__permissions')) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.select_related('owner') @@ -625,7 +626,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, [t1]) self.assertEqual(data2[0].owner, None) - with self.assertNumQueries(4 if self.is_sqlite else 2): + with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): u = User.objects.create_user('user') t1.owner = u t1.save() @@ -636,7 +637,13 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data3[0].owner, u) self.assertListEqual(list(data3[0].owner.groups.all()), []) - with self.assertNumQueries(9 if self.is_sqlite else 6): + with self.assertNumQueries( + 9 if self.is_dj_21_below_and_is_sqlite() + else 8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2 + else 4 if self.is_postgresql and DJANGO_VERSION[0] > 2 + else 4 if self.is_mysql and DJANGO_VERSION[0] > 2 + else 6 + ): group = Group.objects.create(name='test_group') permissions = list(Permission.objects.all()[:5]) group.permissions.add(*permissions) @@ -652,7 +659,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(list(groups[0].permissions.all()), permissions) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data5 = list(Test.objects.select_related('owner') @@ -666,13 +673,13 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): for p in g.permissions.all()] self.assertListEqual(data5_permissions, permissions) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): permissions[0].save() with self.assertNumQueries(1): list(Test.objects.select_related('owner') .prefetch_related('owner__groups__permissions')) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): group.name = 'modified_test_group' group.save() with self.assertNumQueries(2): @@ -681,7 +688,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): g = list(data6[0].owner.groups.all())[0] self.assertEqual(g.name, 'modified_test_group') - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): User.objects.update(username='modified_user') with self.assertNumQueries(2): @@ -818,26 +825,26 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data4, []) def test_invalidate_extra_tables(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): User.objects.create_user('user1') with self.assertNumQueries(1): data1 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data2, [t1]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data3 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data3, [t1, t2]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): User.objects.create_user('user2') with self.assertNumQueries(1): data4 = list(Test.objects.all().extra(tables=['auth_user'])) @@ -867,7 +874,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertEqual(TestChild.objects.get(), t_child) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): TestParent.objects.filter(pk=t_child.pk).update(name='modified') with self.assertNumQueries(1): @@ -875,7 +882,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(modified_t_child.pk, t_child.pk) self.assertEqual(modified_t_child.name, 'modified') - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries(3 if self.is_dj_21_below_and_is_sqlite() else 2): TestChild.objects.filter(pk=t_child.pk).update(name='modified2') with self.assertNumQueries(1): @@ -893,7 +900,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.execute( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test1', %s)", [1 if self.is_sqlite else True]) + "VALUES ('test1', %s)", [1 if self.is_dj_21_below_and_is_sqlite() else True]) with self.assertNumQueries(1): self.assertListEqual( @@ -904,7 +911,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.execute( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test2', %s)", [1 if self.is_sqlite else True]) + "VALUES ('test2', %s)", [1 if self.is_dj_21_below_and_is_sqlite() else True]) with self.assertNumQueries(1): self.assertListEqual( @@ -915,7 +922,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.executemany( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test3', %s)", [[1 if self.is_sqlite else True]]) + "VALUES ('test3', %s)", [[1 if self.is_dj_21_below_and_is_sqlite() else True]]) with self.assertNumQueries(1): self.assertListEqual( @@ -923,7 +930,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['test1', 'test2', 'test3']) def test_raw_update(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test') with self.assertNumQueries(1): self.assertListEqual( @@ -940,7 +947,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['new name']) def test_raw_delete(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): Test.objects.create(name='test') with self.assertNumQueries(1): self.assertListEqual( diff --git a/cachalot/utils.py b/cachalot/utils.py index 1149c0e..fc42d36 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -13,7 +13,10 @@ from django.db.models import QuerySet, Subquery, Exists from django.db.models.functions import Now from django.db.models.sql import Query, AggregateQuery from django.db.models.sql.where import ExtraWhere, WhereNode, NothingNode -from six import text_type, binary_type, integer_types +try: + from django.utils.six import text_type, binary_type, integer_types +except ImportError: + from six import text_type, binary_type, integer_types from .settings import ITERABLES, cachalot_settings from .transaction import AtomicCache @@ -160,7 +163,11 @@ def _get_tables(db_alias, query): # Gets tables in subquery annotations. for annotation in query.annotations.values(): if isinstance(annotation, Subquery): - tables.update(_get_tables(db_alias, annotation.queryset.query)) + # Django 2.2+ removed queryset in favor of simply using query + try: + tables.update(_get_tables(db_alias, annotation.queryset.query)) + except AttributeError: + tables.update(_get_tables(db_alias, annotation.query)) # Gets tables in WHERE subqueries. for subquery in _find_subqueries_in_where(query.where.children): tables.update(_get_tables(db_alias, subquery)) diff --git a/docs/index.rst b/docs/index.rst index 7a6d980..233df62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,9 +20,57 @@ Caches your Django ORM queries and automatically invalidates them. .. image:: http://img.shields.io/scrutinizer/g/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 :target: https://scrutinizer-ci.com/g/noripyt/django-cachalot/ -.. image:: https://img.shields.io/gitter/room/django-cachalot/Lobby.svg?style=flat-square&maxAge=3600 - :target: https://gitter.im/django-cachalot/Lobby +.. image:: https://img.shields.io/badge/cachalot-Chat%20on%20Slack-green?style=flat&logo=slack + :target: https://join.slack.com/t/cachalotdjango/shared_invite/enQtOTMyNzI0NTQzOTA3LWViYmYwMWY3MmU0OTZkYmNiMjBhN2NjNjc4OWVlZDNiMjMxN2Y3YzljYmNiYTY4ZTRjOGQxZDRiMTM0NWE3NGI +Usage +..... + +#. ``pip install django-cachalot`` +#. Add ``'cachalot',`` to your ``INSTALLED_APPS`` +#. If you use multiple servers with a common cache server, + :ref:`double check their clock synchronisation `_ +#. If you modify data outside Django + – typically after restoring a SQL database –, + use the :ref:`manage.py command `_ +#. Be aware of :ref:`the few other limits `_ +#. If you use + `django-debug-toolbar `_, + you can add ``'cachalot.panels.CachalotPanel',`` + to your ``DEBUG_TOOLBAR_PANELS`` +#. Enjoy! + +Note: In settings, you can use `CACHALOT_UNCACHABLE_TABLES `_ as a frozenset of table names (e.g. "public_test" if public was the app name and test is a model name). + +Why use cachalot? `Check out our comparison `_ + +In-depth opinion (from new maintainer): + +There are three main third party caches: cachalot, cache-machine, and cache-ops. Which do you use? We suggest a mix: + +TL;DR Use cachalot for cold or modified <50 times per seconds (Most people should stick with only cachalot since you +most likely won't need to scale to the point of needing cache-machine added to the bowl). If you're an enterprise that +already has huge statistics, then mixing cold caches for cachalot and your hot caches with cache-machine is the best +mix. + +Recall, cachalot caches THE ENTIRE TABLE. That's where its inefficiency stems from: if you keep updating the records, +then the cachalot constantly invalidates the table and re-caches. Luckily caching is very efficient, it's just the cache +invalidation part that kills all our systems. Look at Note 1 below to see how Reddit deals with it. + +Cachalot is more-or-less intended for cold caches or "just-right" conditions. If you find a partition library for +Django (also authored but work-in-progress by `Andrew Chen Wang `_), +then the caching will work better since sharding the cold/accessed-the-least records aren't invalidated as much. + +Cachalot is good when there are <50 modifications per second on a hot cached table. This is mostly due to cache invalidation. It's the same with any cache, +which is why we suggest you use cache-machine for hot caches. Cache-machine caches individual objects, taking up more in the memory store but +invalidates those individual objects instead of the entire table like cachalot. + +Yes, the bane of our entire existence lies in cache invalidation and naming variables. Why does cachalot suck when stuck +with a huge table that's modified rapidly? Since you've mixed your cold (90% of) with your hot (10% of) records, you're +caching and invalidating an entire table. It's like trying to boil 1 ton of noodles inside ONE pot instead of 100 pots +boiling 1 ton of noodles. Which is more efficient? The splitting up of them. + +Note 1: My personal experience with caches stems from Reddit's: https://redditblog.com/2017/01/17/caching-at-reddit/ .. toctree:: :maxdepth: 2 diff --git a/docs/introduction.rst b/docs/introduction.rst index 4b4d184..488ff89 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -16,8 +16,8 @@ since it’s unfortunately badly optimised (use foreign keys in ``list_editable` if you need to be convinced). However, it’s not suited for projects where there is **a high number -of modifications per minute** on each table, like a social network with -more than a 50 messages per minute. Django-cachalot may still give a small +of modifications per second** on each table, like a social network with +more than a 50 messages per second. Django-cachalot may still give a small speedup in such cases, but it may also slow things a bit (in the worst case scenario, a 20% slowdown, according to :ref:`the benchmark `). diff --git a/docs/quickstart.rst b/docs/quickstart.rst index a55b238..fde8092 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,8 +4,8 @@ Quick start Requirements ............ -- Django 1.11, 2.0 or 2.1 -- Python 2.7, 3.4, 3.5, 3.6 or 3.7 +- Django 1.11, 2.0-2.2, or 3.0 +- Python 2.7, 3.4-3.8 - a cache configured as ``'default'`` with one of these backends: - `django-redis `_ diff --git a/requirements.txt b/requirements.txt index ee092dc..37d10bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -Django>=1.11,<3.1 -tox==3.14.3 \ No newline at end of file +Django>=1.11 +six>=1.13 \ No newline at end of file diff --git a/runtests.py b/runtests.py index 08718be..05fd51b 100755 --- a/runtests.py +++ b/runtests.py @@ -11,7 +11,7 @@ if __name__ == '__main__': os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') django.setup() from django.test.runner import DiscoverRunner - test_runner = DiscoverRunner(verbosity=2) + test_runner = DiscoverRunner(verbosity=2, interactive=False) failures = test_runner.run_tests(['cachalot.tests']) if failures: sys.exit(failures) diff --git a/setup.py b/setup.py index a1b8521..a730564 100755 --- a/setup.py +++ b/setup.py @@ -14,8 +14,8 @@ with open(os.path.join(CURRENT_PATH, 'requirements.txt')) as f: setup( name='django-cachalot', version=__version__, - author='Bertrand Bordage', - author_email='bordage.bertrand@gmail.com', + author='Bertrand Bordage, Andrew Chen Wang', + author_email='acwangpython@gmail.com', url='https://github.com/noripyt/django-cachalot', description='Caches your Django ORM queries ' 'and automatically invalidates them.', diff --git a/tox.ini b/tox.ini index 80c8fbe..239f434 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,24 @@ [tox] envlist = - py{2.7,3.4,3.5,3.6}-django1.11-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{3.4,3.5,3.6}-django2.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{3.5,3.6,3.7}-django2.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{27,35,36,37}-django1.11-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{35,36,37}-django2.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{35,36,37}-django2.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{35,36,37,38}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{36,37,38}-django3.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] basepython = - py2.7: python2.7 - py3.4: python3.4 - py3.5: python3.5 - py3.6: python3.6 + py27: python2.7 + py35: python3.5 + py36: python3.6 + py37: python3.7 + py38: python3.8 deps = django1.11: Django>=1.11,<1.12 django2.0: Django>=2.0,<2.1 - django2.1: Django>=2.0,<2.2 + django2.1: Django>=2.1,<2.2 + django2.2: Django>=2.2,<2.3 + django3.0: Django>=3.0,<3.1 psycopg2-binary mysqlclient django-redis @@ -24,6 +29,7 @@ deps = django-debug-toolbar beautifulsoup4 coverage + six setenv = sqlite3: DB_ENGINE=sqlite3 postgresql: DB_ENGINE=postgresql @@ -35,3 +41,11 @@ setenv = pylibmc: CACHE_BACKEND=pylibmc commands = coverage run -a --source=cachalot ./runtests.py + +[travis:env] +DJANGO = + 1.11: django1.11 + 2.0: django2.0 + 2.1: django2.1 + 2.2: django2.2 + 3.0: django3.0