Merge branch 'andrew-wang/dj4.2'

* Increment version to 4.5.0

.coveragerc

@@ -1,2 +1,13 @@
 [run]
 omit = */tests/*
+
+[report]
+exclude_lines = 
+    pragma: no cover
+    pragma: nocover
+    def __repr__
+
+    if __name__ == .__main__.:
+    if TYPE_CHECKING:
+    except ImportError:
+

.github/workflows/ci.yml

@@ -6,18 +6,30 @@ on:
   pull_request:
     branches: [ master ]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-        django-version: ['2.2', '3.0', '3.1', '3.2']
+        django-version: ['3.2', '4.1', '4.2']
-
+        exclude:
+          - python-version: '3.11'
+            django-version: '3.2'
+          - python-version: '3.11'
+            django-version: '4.1'
+          - python-version: '3.7'
+            django-version: '4.1'
+          - python-version: '3.7'
+            django-version: '4.2'
     services:
       redis:
-        image: redis:6.0
+        image: redis:6
         ports:
           - 6379:6379
       postgres:
@@ -51,11 +63,23 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key:
+          ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ matrix.python-version }}-v1-
     - name: Install dependencies
       run: |
-        sudo apt-get install -y libmemcached-dev zlib1g-dev
+        sudo apt-get install -y libmemcached-dev zlib1g-dev libpq-dev
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip wheel
-        pip install tox tox-gh-actions coveralls
+        python -m pip install tox tox-gh-actions coveralls
     - name: Tox Test
       run: tox
       env:

.github/workflows/main-ci.yml

@@ -0,0 +1,74 @@
+name: Django Main Testing CI
+
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9']
+
+    services:
+      redis:
+        image: redis:6
+        ports:
+          - 6379:6379
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: cachalot
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: cachalot
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mysql:
+        image: mysql
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_DATABASE: cachalot
+        ports:
+          - 3306:3306
+      memcached:
+        image: memcached
+        ports:
+          - 11211:11211
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: Cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key:
+          ${{ matrix.python-version }}-v1-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ matrix.python-version }}-v1-
+    - name: Install dependencies
+      run: |
+        sudo apt-get install -y libmemcached-dev zlib1g-dev
+        python -m pip install --upgrade pip wheel
+        python -m pip install tox tox-gh-actions coveralls
+    - name: Tox Test
+      run: tox
+      env:
+        TOX_TESTENV_PASSENV: POSTGRES_PASSWORD
+        POSTGRES_PASSWORD: password
+        DJANGO: 'main'

.gitignore

@@ -57,6 +57,7 @@ coverage.xml
 # Django stuff:
 *.log
 local_settings.py
+*.sqlite3
 db.sqlite3
 db.sqlite3-journal
 

CHANGELOG.rst

@@ -1,6 +1,74 @@
 What’s new in django-cachalot?
 ==============================
 
+2.6.0
+-----
+
+- Dropped Django 2.2 and 4.0 support
+- Added Django 4.2 and Python 3.11 support
+- Added psycopg support (#229)
+- Updated tests to account for the `BEGIN` and `COMMIT` query changes in Django 4.2
+- Standardized django version comparisons in tests
+
+2.5.3
+-----
+
+- Verify get_meta isn't none before requesting db_table (#225 #226)
+
+2.5.2
+-----
+
+- Added Django 4.1 support (#217)
+
+2.5.1
+-----
+
+- Table invalidation condition enhanced (#213)
+- Add test settings to sdist (#203)
+- Include docs in sdist (#202)
+
+2.5.0
+-----
+
+- Add final SQL check to include potentially overlooked tables when looking up involved tables (#199)
+- Add ``CACHALOT_FINAL_SQL_CHECK`` for enabling Final SQL check
+
+2.4.5
+-----
+
+- Dropped Python 3.6 and Django 3.1 support. Added Django 4.0 support (#208)
+
+2.4.4
+-----
+
+- Handle queryset implementations without lhs/rhs attribute (#204)
+- Add Python 3.10 support (#206)
+- (Internal) Omit additional unnecessary code in coverage
+
+2.4.3
+-----
+
+- Fix annotated Now being cached (#195)
+- Fix conditional annotated expressions not being cached (#196)
+- Simplify annotation handling by using the flatten method (#197)
+- Fix Django 3.2 default_app_config deprecation (#198)
+- (Internal) Pinned psycopg2 to <2.9 due to Django 2.2 incompatibility
+
+2.4.2
+-----
+
+- Add convenience settings `CACHALOT_ONLY_CACHABLE_APPS`
+  and `CACHALOT_UNCACHABLE_APPS` (#187)
+- Drop support for Django 3.0 (#189)
+- (Internal) Added Django main-branch CI on cron job
+- (Internal) Removed duplicate code (#190)
+
+2.4.1
+-----
+
+- Fix Django requirement constraint to include 3.2.X not just 3.2
+- (Internal) Deleted obsolete travis-matrix.py file
+
 2.4.0
 -----
 

MANIFEST.in

@@ -1,2 +1,3 @@
-include README.rst LICENSE CHANGELOG.rst requirements.txt
+include README.rst LICENSE CHANGELOG.rst requirements.txt tox.ini runtests.py runtests_urls.py settings.py
 recursive-include cachalot *.json *.html
+graft docs

README.rst

@@ -39,10 +39,9 @@ Table of Contents:
 Quickstart
 ----------
 
-Cachalot officially supports Python 3.6-3.9 and Django 2.2 and 3.0-3.2 with the databases PostgreSQL, SQLite, and MySQL.
+Cachalot officially supports Python 3.7-3.11 and Django 3.2, 4.1, 4.2 with the databases PostgreSQL, SQLite, and MySQL.
 
-No upper limit is imposed by cachalot. However, if you use a Django version that is not officially supported,
+Note: an upper limit on Django version is set for your safety. Please do not ignore it.
-you may end up battling mounting errors/exceptions and cache misses. Please stay within compatibility.
 
 Usage
 -----

cachalot/__init__.py

@@ -1,4 +1,4 @@
-VERSION = (2, 4, 0)
+VERSION = (2, 6, 0)
 __version__ = ".".join(map(str, VERSION))
 
 default_app_config = "cachalot.apps.CachalotConfig"

cachalot/admin_tests/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from .models import TestModel
+
+@admin.register(TestModel)
+class TestModelAdmin(admin.ModelAdmin):
+    list_display = ('name', 'owner')

cachalot/admin_tests/migrations/0001_initial.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.1.7 on 2023-03-10 19:33
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.db.models.functions.text
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="TestModel",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=20)),
+                (
+                    "owner",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ("name",),
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="testmodel",
+            constraint=models.UniqueConstraint(
+                fields=["name"],
+                condition=models.Q(owner=None),
+                name="unique_name",
+            ),
+        ),
+    ]

cachalot/admin_tests/models.py

@@ -0,0 +1,18 @@
+from django.conf import settings
+from django.db.models import Q, UniqueConstraint, Model, CharField, ForeignKey, SET_NULL
+
+
+class TestModel(Model):
+    name = CharField(max_length=20)
+    owner = ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True,
+                       on_delete=SET_NULL)
+
+    class Meta:
+        ordering = ('name',)
+        constraints = [
+            UniqueConstraint(
+                fields=["name"],
+                condition=Q(owner=None),
+                name="unique_name",
+            )
+        ]

cachalot/admin_tests/test_admin.py

@@ -0,0 +1,19 @@
+from django.test import TestCase
+from django.contrib.auth.models import User
+from .models import TestModel
+from django.test import Client
+
+
+class AdminTestCase(TestCase):
+    def setUp(self):
+        self.client = Client()
+        self.user = User.objects.create(username='admin', is_staff=True, is_superuser=True)
+
+    def test_save_test_model(self):
+        """
+        Model 'TestModel' has UniqueConstraint which caused problems when saving TestModelAdmin in Django >= 4.1
+        """
+        self.client.force_login(self.user)
+        response = self.client.post('/admin/admin_tests/testmodel/add/', {'name': 'test', 'public': True})
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(TestModel.objects.count(), 1)

cachalot/api.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+from typing import Any, Optional, Tuple, Union
 
 from django.apps import apps
 from django.conf import settings
@@ -45,7 +46,11 @@ def _get_tables(tables_or_models):
                else table_or_model._meta.db_table)
 
 
-def invalidate(*tables_or_models, **kwargs):
+def invalidate(
+    *tables_or_models: Tuple[Union[str, Any], ...],
+    cache_alias: Optional[str] = None,
+    db_alias: Optional[str] = None,
+) -> None:
     """
     Clears what was cached by django-cachalot implying one or more SQL tables
     or models from ``tables_or_models``.
@@ -62,19 +67,9 @@ def invalidate(*tables_or_models, **kwargs):
                            (or a combination)
     :type tables_or_models: tuple of strings or models
     :arg cache_alias: Alias from the Django ``CACHES`` setting
-    :type cache_alias: string or NoneType
     :arg db_alias: Alias from the Django ``DATABASES`` setting
-    :type db_alias: string or NoneType
     :returns: Nothing
-    :rtype: NoneType
     """
-    # TODO: Replace with positional arguments when we drop Python 2 support.
-    cache_alias = kwargs.pop('cache_alias', None)
-    db_alias = kwargs.pop('db_alias', None)
-    for k in kwargs:
-        raise TypeError(
-            "invalidate() got an unexpected keyword argument '%s'" % k)
-
     send_signal = False
     invalidated = set()
     for cache_alias, db_alias, tables in _cache_db_tables_iterator(
@@ -90,7 +85,11 @@ def invalidate(*tables_or_models, **kwargs):
             post_invalidation.send(table, db_alias=db_alias)
 
 
-def get_last_invalidation(*tables_or_models, **kwargs):
+def get_last_invalidation(
+    *tables_or_models: Tuple[Union[str, Any], ...],
+    cache_alias: Optional[str] = None,
+    db_alias: Optional[str] = None,
+) -> float:
     """
     Returns the timestamp of the most recent invalidation of the given
     ``tables_or_models``.  If ``tables_or_models`` is not specified,
@@ -106,19 +105,9 @@ def get_last_invalidation(*tables_or_models, **kwargs):
                            (or a combination)
     :type tables_or_models: tuple of strings or models
     :arg cache_alias: Alias from the Django ``CACHES`` setting
-    :type cache_alias: string or NoneType
     :arg db_alias: Alias from the Django ``DATABASES`` setting
-    :type db_alias: string or NoneType
     :returns: The timestamp of the most recent invalidation
-    :rtype: float
     """
-    # TODO: Replace with positional arguments when we drop Python 2 support.
-    cache_alias = kwargs.pop('cache_alias', None)
-    db_alias = kwargs.pop('db_alias', None)
-    for k in kwargs:
-        raise TypeError("get_last_invalidation() got an unexpected "
-                        "keyword argument '%s'" % k)
-
     last_invalidation = 0.0
     for cache_alias, db_alias, tables in _cache_db_tables_iterator(
             list(_get_tables(tables_or_models)), cache_alias, db_alias):
@@ -134,7 +123,7 @@ def get_last_invalidation(*tables_or_models, **kwargs):
 
 
 @contextmanager
-def cachalot_disabled(all_queries=False):
+def cachalot_disabled(all_queries: bool = False):
     """
     Context manager for temporarily disabling cachalot.
     If you evaluate the same queryset a second time,
@@ -158,7 +147,6 @@ def cachalot_disabled(all_queries=False):
     the original and duplicate query.
 
     :arg all_queries: Any query, including already evaluated queries, are re-evaluated.
-    :type all_queries: bool
     """
     was_enabled = getattr(LOCAL_STORAGE, "cachalot_enabled", cachalot_settings.CACHALOT_ENABLED)
     LOCAL_STORAGE.cachalot_enabled = False

cachalot/monkey_patch.py

@@ -1,3 +1,4 @@
+import re
 from collections.abc import Iterable
 from functools import wraps
 from time import time
@@ -21,6 +22,13 @@ from .utils import (
 
 WRITE_COMPILERS = (SQLInsertCompiler, SQLUpdateCompiler, SQLDeleteCompiler)
 
+SQL_DATA_CHANGE_RE = re.compile(
+    '|'.join([
+        fr'(\W|\A){re.escape(keyword)}(\W|\Z)'
+        for keyword in ['update', 'insert', 'delete', 'alter', 'create', 'drop']
+    ]),
+    flags=re.IGNORECASE,
+)
 
 def _unset_raw_connection(original):
     def inner(compiler, *args, **kwargs):
@@ -133,9 +141,7 @@ def _patch_cursor():
                     if isinstance(sql, bytes):
                         sql = sql.decode('utf-8')
                     sql = sql.lower()
-                    if 'update' in sql or 'insert' in sql or 'delete' in sql \
+                    if SQL_DATA_CHANGE_RE.search(sql):
-                            or 'alter' in sql or 'create' in sql \
-                            or 'drop' in sql:
                         tables = filter_cachable(
                             _get_tables_from_sql(connection, sql))
                         if tables:

cachalot/settings.py

@@ -1,3 +1,6 @@
+from itertools import chain
+
+from django.apps import apps
 from django.conf import settings
 from django.utils.module_loading import import_string
 
@@ -6,8 +9,6 @@ SUPPORTED_DATABASE_ENGINES = {
     'django.db.backends.sqlite3',
     'django.db.backends.postgresql',
     'django.db.backends.mysql',
-    # TODO: Remove when we drop Django 2.x support.
-    'django.db.backends.postgresql_psycopg2',
 
     # GeoDjango
     'django.contrib.gis.db.backends.spatialite',
@@ -17,8 +18,6 @@ SUPPORTED_DATABASE_ENGINES = {
     # django-transaction-hooks
     'transaction_hooks.backends.sqlite3',
     'transaction_hooks.backends.postgis',
-    # TODO: Remove when we drop Django 2.x support.
-    'transaction_hooks.backends.postgresql_psycopg2',
     'transaction_hooks.backends.mysql',
 
     # django-prometheus wrapped engines
@@ -52,10 +51,13 @@ class Settings(object):
     CACHALOT_CACHE_RANDOM = False
     CACHALOT_INVALIDATE_RAW = True
     CACHALOT_ONLY_CACHABLE_TABLES = ()
+    CACHALOT_ONLY_CACHABLE_APPS = ()
     CACHALOT_UNCACHABLE_TABLES = ('django_migrations',)
+    CACHALOT_UNCACHABLE_APPS = ()
     CACHALOT_ADDITIONAL_TABLES = ()
     CACHALOT_QUERY_KEYGEN = 'cachalot.utils.get_query_cache_key'
     CACHALOT_TABLE_KEYGEN = 'cachalot.utils.get_table_cache_key'
+    CACHALOT_FINAL_SQL_CHECK = False
 
     @classmethod
     def add_converter(cls, setting):
@@ -103,14 +105,24 @@ def convert(value):
     return value
 
 
+def convert_tables(value, setting_app_name):
+    dj_apps = getattr(settings, setting_app_name, ())
+    if dj_apps:
+        dj_apps = tuple(model._meta.db_table for model in chain.from_iterable(
+            apps.all_models[_app].values() for _app in dj_apps
+        ))  # Use [] lookup to make sure app is loaded (via INSTALLED_APP's order)
+        return frozenset(tuple(value) + dj_apps)
+    return frozenset(value)
+
+
 @Settings.add_converter('CACHALOT_ONLY_CACHABLE_TABLES')
 def convert(value):
-    return frozenset(value)
+    return convert_tables(value, 'CACHALOT_ONLY_CACHABLE_APPS')
 
 
 @Settings.add_converter('CACHALOT_UNCACHABLE_TABLES')
 def convert(value):
-    return frozenset(value)
+    return convert_tables(value, 'CACHALOT_UNCACHABLE_APPS')
 
 
 @Settings.add_converter('CACHALOT_ADDITIONAL_TABLES')

cachalot/tests/__init__.py

@@ -4,7 +4,7 @@ from django.dispatch import receiver
 from ..settings import cachalot_settings
 from .read import ReadTestCase, ParameterTypeTestCase
 from .write import WriteTestCase, DatabaseCommandTestCase
-from .transaction import AtomicTestCase
+from .transaction import AtomicCacheTestCase, AtomicTestCase
 from .thread_safety import ThreadSafetyTestCase
 from .multi_db import MultiDatabaseTestCase
 from .settings import SettingsTestCase

cachalot/tests/migrations/0001_initial.py

@@ -1,7 +1,8 @@
+from django import VERSION as DJANGO_VERSION
 from django.conf import settings
 from django.contrib.postgres.fields import (
     ArrayField, HStoreField, IntegerRangeField,
-    DateRangeField, DateTimeRangeField)
+    DateRangeField, DateTimeRangeField, DecimalRangeField)
 from django.contrib.postgres.operations import (
     HStoreExtension, UnaccentExtension)
 from django.db import models, migrations
@@ -10,11 +11,8 @@ from django.db import models, migrations
 def extra_regular_available_fields():
     fields = []
     try:
-        # TODO Add to module import when Dj40 dropped
+        from django.db.models import JSONField
-        from django import VERSION as DJANGO_VERSION
+        fields.append(('json', JSONField(null=True, blank=True)))
-        from django.contrib.postgres.fields import JSONField
-        if float(".".join(map(str, DJANGO_VERSION[:2]))) > 3.0:
-            fields.append(('json', JSONField(null=True, blank=True)))
     except ImportError:
         pass
 
@@ -23,27 +21,12 @@ def extra_regular_available_fields():
 
 def extra_postgres_available_fields():
     fields = []
-    try:
-        # TODO Remove when Dj31 support is dropped
-        from django.contrib.postgres.fields import FloatRangeField
-        fields.append(('float_range', FloatRangeField(null=True, blank=True)))
-    except ImportError:
-        pass
-
-    try:
-        # TODO Add to module import when Dj31 is dropped
-        from django.contrib.postgres.fields import DecimalRangeField
-        fields.append(('decimal_range', DecimalRangeField(null=True, blank=True)))
-    except ImportError:
-        pass
 
     # Future proofing with Django 40 deprecation
-    try:
+    if DJANGO_VERSION[0] < 4:
         # TODO Remove when Dj40 support is dropped
         from django.contrib.postgres.fields import JSONField
         fields.append(('json', JSONField(null=True, blank=True)))
-    except ImportError:
-        pass
 
     return fields
 
@@ -107,6 +90,7 @@ class Migration(migrations.Migration):
                 ('int_range', IntegerRangeField(null=True, blank=True)),
                 ('date_range', DateRangeField(null=True, blank=True)),
                 ('datetime_range', DateTimeRangeField(null=True, blank=True)),
+                ('decimal_range', DecimalRangeField(null=True, blank=True))
             ] + extra_postgres_available_fields(),
         ),
         migrations.RunSQL('CREATE TABLE cachalot_unmanagedmodel '

cachalot/tests/models.py

@@ -1,3 +1,4 @@
+from django import VERSION as DJANGO_VERSION
 from django.conf import settings
 from django.contrib.postgres.fields import (
     ArrayField, HStoreField,
@@ -44,6 +45,10 @@ class TestParent(Model):
 
 
 class TestChild(TestParent):
+    """
+    A OneToOneField to TestParent is automatically added here.
+    https://docs.djangoproject.com/en/3.2/topics/db/models/#multi-table-inheritance
+    """
     public = BooleanField(default=False)
     permissions = ManyToManyField('auth.Permission', blank=True)
 
@@ -53,11 +58,9 @@ class PostgresModel(Model):
                            null=True, blank=True)
 
     hstore = HStoreField(null=True, blank=True)
-    try:
+    if DJANGO_VERSION < (4, 0):
         from django.contrib.postgres.fields import JSONField
         json = JSONField(null=True, blank=True)
-    except ImportError:
-        pass
 
     int_range = IntegerRangeField(null=True, blank=True)
     try:

cachalot/tests/multi_db.py

@@ -1,6 +1,5 @@
 from unittest import skipIf
 
-from django import VERSION as DJANGO_VERSION
 from django.conf import settings
 from django.db import DEFAULT_DB_ALIAS, connections, transaction
 from django.test import TransactionTestCase
@@ -27,24 +26,6 @@ class MultiDatabaseTestCase(TransactionTestCase):
             # will execute an extra SQL request below.
             connection2.cursor()
 
-    def is_django_21_below_and_sqlite2(self):
-        """
-        Note: See test_utils.py with this function name
-        Checks if Django 2.1 or below and SQLite2
-        """
-        django_version = DJANGO_VERSION
-        if not self.is_sqlite2:
-            # Immediately know if SQLite
-            return False
-        if django_version[0] < 2:
-            # Takes Django 0 and 1 out of the picture
-            return True
-        else:
-            if django_version[0] == 2 and django_version[1] < 2:
-                # Takes Django 2.0-2.1 out
-                return True
-            return False
-
     def test_read(self):
         with self.assertNumQueries(1):
             data1 = list(Test.objects.all())
@@ -66,8 +47,7 @@ class MultiDatabaseTestCase(TransactionTestCase):
             data1 = list(Test.objects.using(self.db_alias2))
             self.assertListEqual(data1, [])
 
-        with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1,
+        with self.assertNumQueries(1, using=self.db_alias2):
-                                   using=self.db_alias2):
             t3 = Test.objects.using(self.db_alias2).create(name='test3')
 
         with self.assertNumQueries(1, using=self.db_alias2):
@@ -82,8 +62,7 @@ class MultiDatabaseTestCase(TransactionTestCase):
             data1 = list(Test.objects.all())
             self.assertListEqual(data1, [self.t1, self.t2])
 
-        with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1,
+        with self.assertNumQueries(1, using=self.db_alias2):
-                                   using=self.db_alias2):
             Test.objects.using(self.db_alias2).create(name='test3')
 
         with self.assertNumQueries(0):

cachalot/tests/postgres.py

@@ -5,17 +5,18 @@ from unittest import skipUnless
 from django.contrib.postgres.functions import TransactionNow
 from django.db import connection
 from django.test import TransactionTestCase, override_settings
-from psycopg2.extras import NumericRange, DateRange, DateTimeTZRange
+from psycopg2.extras import DateRange, DateTimeTZRange, NumericRange
 from pytz import timezone
 
 from ..utils import UncachableQuery
 from .api import invalidate
 from .models import PostgresModel, Test
 from .test_utils import TestUtilsMixin
-
+from .tests_decorators import all_final_sql_checks, no_final_sql_check, with_final_sql_check
 
 # FIXME: Add tests for aggregations.
 
+
 def is_pg_field_available(name):
     fields = []
     try:
@@ -91,14 +92,18 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.obj1.save()
         self.obj2.save()
 
+    @all_final_sql_checks
     def test_unaccent(self):
-        Test.objects.create(name='Clémentine')
+        obj1 = Test.objects.create(name='Clémentine')
-        Test.objects.create(name='Clementine')
+        obj2 = Test.objects.create(name='Clementine')
         qs = (Test.objects.filter(name__unaccent='Clémentine')
               .values_list('name', flat=True))
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, ['Clementine', 'Clémentine'])
+        obj1.delete()
+        obj2.delete()
 
+    @all_final_sql_checks
     def test_int_array(self):
         with self.assertNumQueries(1):
             data1 = [o.int_array for o in PostgresModel.objects.all()]
@@ -145,6 +150,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, PostgresModel)
         self.assert_query_cached(qs, [[1, 2, 3]])
 
+    @all_final_sql_checks
     def test_hstore(self):
         with self.assertNumQueries(1):
             data1 = [o.hstore for o in PostgresModel.objects.all()]
@@ -198,6 +204,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, PostgresModel)
         self.assert_query_cached(qs, [{'a': '1', 'b': '2'}])
 
+    @all_final_sql_checks
     @skipUnless(is_pg_field_available("JSONField"),
                 "JSONField was removed in Dj 4.0")
     def test_json(self):
@@ -309,6 +316,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(list(qs.all()),
                              [self.obj1.json, self.obj2.json])
 
+    @all_final_sql_checks
     def test_int_range(self):
         with self.assertNumQueries(1):
             data1 = [o.int_range for o in PostgresModel.objects.all()]
@@ -378,13 +386,16 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, PostgresModel)
         self.assert_query_cached(qs, [NumericRange(1900, 2000)])
 
-        PostgresModel.objects.create(int_range=[1900, 1900])
+        obj = PostgresModel.objects.create(int_range=[1900, 1900])
 
         qs = (PostgresModel.objects.filter(int_range__isempty=True)
               .values_list('int_range', flat=True))
         self.assert_tables(qs, PostgresModel)
         self.assert_query_cached(qs, [NumericRange(empty=True)])
 
+        obj.delete()
+
+    @all_final_sql_checks
     @skipUnless(is_pg_field_available("FloatRangeField"),
                 "FloatRangeField was removed in Dj 3.1")
     def test_float_range(self):
@@ -398,6 +409,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
             NumericRange(Decimal('-1000.0'), Decimal('9.87654321')),
             NumericRange(Decimal('0.0'))])
 
+    @all_final_sql_checks
     @skipUnless(is_pg_field_available("DecimalRangeField"),
                 "DecimalRangeField was added in Dj 2.2")
     def test_decimal_range(self):
@@ -407,6 +419,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
             NumericRange(Decimal('-1000.0'), Decimal('9.87654321')),
             NumericRange(Decimal('0.0'))])
 
+    @all_final_sql_checks
     def test_date_range(self):
         qs = PostgresModel.objects.values_list('date_range', flat=True)
         self.assert_tables(qs, PostgresModel)
@@ -414,6 +427,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
             DateRange(date(1678, 3, 4), date(1741, 7, 28)),
             DateRange(date(1989, 1, 30))])
 
+    @all_final_sql_checks
     def test_datetime_range(self):
         qs = PostgresModel.objects.values_list('datetime_range', flat=True)
         self.assert_tables(qs, PostgresModel)
@@ -422,6 +436,7 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
                                      tzinfo=timezone('Europe/Paris'))),
             DateTimeTZRange(bounds='()')])
 
+    @all_final_sql_checks
     def test_transaction_now(self):
         """
         Checks that queries with a TransactionNow() parameter are not cached.
@@ -431,3 +446,5 @@ class PostgresReadTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertRaises(UncachableQuery):
             self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [obj], after=1)
+
+        obj.delete()

cachalot/tests/read.py

@@ -3,25 +3,27 @@ from unittest import skipIf
 from uuid import UUID
 from decimal import Decimal
 
-from django import VERSION as django_version
+from django import VERSION as DJANGO_VERSION
+from django.conf import settings
 from django.contrib.auth.models import Group, Permission, User
 from django.contrib.contenttypes.models import ContentType
 from django.db import (
     connection, transaction, DEFAULT_DB_ALIAS, ProgrammingError,
     OperationalError)
-from django.db.models import Count, Q
+from django.db.models import Case, Count, Q, Value, When
 from django.db.models.expressions import RawSQL, Subquery, OuterRef, Exists
-from django.db.models.functions import Now
+from django.db.models.functions import Coalesce, Now
 from django.db.transaction import TransactionManagementError
-from django.test import (
+from django.test import TransactionTestCase, skipUnlessDBFeature, override_settings
-    TransactionTestCase, skipUnlessDBFeature, override_settings)
 from pytz import UTC
 
 from cachalot.cache import cachalot_caches
 from ..settings import cachalot_settings
 from ..utils import UncachableQuery
 from .models import Test, TestChild, TestParent, UnmanagedModel
-from .test_utils import TestUtilsMixin
+from .test_utils import TestUtilsMixin, FilteredTransactionTestCase
+
+from .tests_decorators import all_final_sql_checks, with_final_sql_check, no_final_sql_check
 
 
 def is_field_available(name):
@@ -34,7 +36,7 @@ def is_field_available(name):
     return name in fields
 
 
-class ReadTestCase(TestUtilsMixin, TransactionTestCase):
+class ReadTestCase(TestUtilsMixin, FilteredTransactionTestCase):
     """
     Tests if every SQL request that only reads data is cached.
 
@@ -50,12 +52,13 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.group__permissions = list(Permission.objects.all()[:3])
         self.group.permissions.add(*self.group__permissions)
         self.user = User.objects.create_user('user')
-        self.user__permissions = list(Permission.objects.all()[3:6])
+        self.user__permissions = list(Permission.objects.filter(content_type__app_label='auth')[3:6])
         self.user.groups.add(self.group)
         self.user.user_permissions.add(*self.user__permissions)
         self.admin = User.objects.create_superuser('admin', 'admin@test.me',
                                                    'password')
-        self.t1__permission = (Permission.objects.order_by('?')
+        self.t1__permission = (Permission.objects
+                               .order_by('?')
                                .select_related('content_type')[0])
         self.t1 = Test.objects.create(
             name='test1', owner=self.user,
@@ -126,6 +129,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data2, data1)
         self.assertListEqual(data2, [self.t1, self.t2])
 
+    @all_final_sql_checks
     def test_filter(self):
         qs = Test.objects.filter(public=True)
         self.assert_tables(qs, Test)
@@ -143,11 +147,13 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t1])
 
+    @all_final_sql_checks
     def test_filter_empty(self):
         qs = Test.objects.filter(public=True, name='user')
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [])
 
+    @all_final_sql_checks
     def test_exclude(self):
         qs = Test.objects.exclude(public=True)
         self.assert_tables(qs, Test)
@@ -157,11 +163,13 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t1])
 
+    @all_final_sql_checks
     def test_slicing(self):
         qs = Test.objects.all()[:1]
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t1])
 
+    @all_final_sql_checks
     def test_order_by(self):
         qs = Test.objects.order_by('pk')
         self.assert_tables(qs, Test)
@@ -171,12 +179,38 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t2, self.t1])
 
+    @all_final_sql_checks
     def test_random_order_by(self):
         qs = Test.objects.order_by('?')
         with self.assertRaises(UncachableQuery):
             self.assert_tables(qs, Test)
         self.assert_query_cached(qs, after=1, compare_results=False)
 
+    @with_final_sql_check
+    def test_order_by_field_of_another_table_with_check(self):
+        qs = Test.objects.order_by('owner__username')
+        self.assert_tables(qs, Test, User)
+        self.assert_query_cached(qs, [self.t2, self.t1])
+
+    @no_final_sql_check
+    def test_order_by_field_of_another_table_no_check(self):
+        qs = Test.objects.order_by('owner__username')
+        self.assert_tables(qs, Test)
+        self.assert_query_cached(qs, [self.t2, self.t1])
+
+    @with_final_sql_check
+    def test_order_by_field_of_another_table_with_expression_with_check(self):
+        qs = Test.objects.order_by(Coalesce('name', 'owner__username'))
+        self.assert_tables(qs, Test, User)
+        self.assert_query_cached(qs, [self.t1, self.t2])
+
+    @no_final_sql_check
+    def test_order_by_field_of_another_table_with_expression_no_check(self):
+        qs = Test.objects.order_by(Coalesce('name', 'owner__username'))
+        self.assert_tables(qs, Test)
+        self.assert_query_cached(qs, [self.t1, self.t2])
+
+    @all_final_sql_checks
     @skipIf(connection.vendor == 'mysql',
             'MySQL does not support limit/offset on a subquery. '
             'Since Django only applies ordering in subqueries when they are '
@@ -188,11 +222,13 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             self.assert_tables(qs, Test)
         self.assert_query_cached(qs, after=1, compare_results=False)
 
+    @all_final_sql_checks
     def test_reverse(self):
         qs = Test.objects.reverse()
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t2, self.t1])
 
+    @all_final_sql_checks
     def test_distinct(self):
         # We ensure that the query without distinct should return duplicate
         # objects, in order to have a real-world example.
@@ -223,12 +259,14 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertDictEqual(data2, data1)
         self.assertDictEqual(data2, {self.t2.pk: self.t2})
 
+    @all_final_sql_checks
     def test_values(self):
         qs = Test.objects.values('name', 'public')
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [{'name': 'test1', 'public': False},
                                       {'name': 'test2', 'public': True}])
 
+    @all_final_sql_checks
     def test_values_list(self):
         qs = Test.objects.values_list('name', flat=True)
         self.assert_tables(qs, Test)
@@ -250,18 +288,21 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertEqual(data2, data1)
         self.assertEqual(data2, self.t2)
 
+    @all_final_sql_checks
     def test_dates(self):
         qs = Test.objects.dates('date', 'year')
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [datetime.date(1789, 1, 1),
                                       datetime.date(1944, 1, 1)])
 
+    @all_final_sql_checks
     def test_datetimes(self):
         qs = Test.objects.datetimes('datetime', 'hour')
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [datetime.datetime(1789, 7, 14, 16),
                                       datetime.datetime(1944, 6, 6, 6)])
 
+    @all_final_sql_checks
     @skipIf(connection.vendor == 'mysql',
             'Time zones are not supported by MySQL.')
     @override_settings(USE_TZ=True)
@@ -272,6 +313,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             datetime.datetime(1789, 7, 14, 16, tzinfo=UTC),
             datetime.datetime(1944, 6, 6, 6, tzinfo=UTC)])
 
+    @all_final_sql_checks
     def test_foreign_key(self):
         with self.assertNumQueries(3):
             data1 = [t.owner for t in Test.objects.all()]
@@ -284,7 +326,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test, User)
         self.assert_query_cached(qs, [self.user.pk, self.admin.pk])
 
-    def test_many_to_many(self):
+    def _test_many_to_many(self):
         u = User.objects.create_user('test_user')
         ct = ContentType.objects.get_for_model(User)
         u.user_permissions.add(
@@ -294,50 +336,93 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
                 name='Can touch', content_type=ct, codename='touch'),
             Permission.objects.create(
                 name='Can cuddle', content_type=ct, codename='cuddle'))
-        qs = u.user_permissions.values_list('codename', flat=True)
+        return u.user_permissions.values_list('codename', flat=True)
+
+    @with_final_sql_check
+    def test_many_to_many_when_sql_check(self):
+        qs = self._test_many_to_many()
+        self.assert_tables(qs, User, User.user_permissions.through, Permission, ContentType)
+        self.assert_query_cached(qs, ['cuddle', 'discuss', 'touch'])
+
+    @no_final_sql_check
+    def test_many_to_many_when_no_sql_check(self):
+        qs = self._test_many_to_many()
         self.assert_tables(qs, User, User.user_permissions.through, Permission)
         self.assert_query_cached(qs, ['cuddle', 'discuss', 'touch'])
 
+    @all_final_sql_checks
     def test_subquery(self):
+        additional_tables = []
+        if DJANGO_VERSION >= (4, 0) and DJANGO_VERSION < (4, 1) and settings.CACHALOT_FINAL_SQL_CHECK:
+            # with Django 4.0 comes some query optimalizations that do selects little differently.
+            additional_tables.append('django_content_type')
         qs = Test.objects.filter(owner__in=User.objects.all())
         self.assert_tables(qs, Test, User)
         self.assert_query_cached(qs, [self.t1, self.t2])
 
         qs = Test.objects.filter(
-            owner__groups__permissions__in=Permission.objects.all())
+            owner__groups__permissions__in=Permission.objects.all()
-        self.assert_tables(qs, Test, User, User.groups.through, Group,
+        )
-                           Group.permissions.through, Permission)
+        self.assert_tables(
+            qs, Test, User, User.groups.through, Group,
+            Group.permissions.through, Permission,
+            *additional_tables
+        )
         self.assert_query_cached(qs, [self.t1, self.t1, self.t1])
 
         qs = Test.objects.filter(
             owner__groups__permissions__in=Permission.objects.all()
         ).distinct()
-        self.assert_tables(qs, Test, User, User.groups.through, Group,
+        self.assert_tables(
-                           Group.permissions.through, Permission)
+            qs, Test, User, User.groups.through, Group,
+            Group.permissions.through, Permission,
+            *additional_tables
+        )
         self.assert_query_cached(qs, [self.t1])
 
         qs = TestChild.objects.exclude(permissions__isnull=True)
-        self.assert_tables(qs, TestParent, TestChild,
+        self.assert_tables(
-                           TestChild.permissions.through, Permission)
+            qs, TestParent, TestChild,
+            TestChild.permissions.through, Permission
+        )
         self.assert_query_cached(qs, [])
 
         qs = TestChild.objects.exclude(permissions__name='')
-        self.assert_tables(qs, TestParent, TestChild,
+        self.assert_tables(
-                           TestChild.permissions.through, Permission)
+            qs, TestParent, TestChild,
+            TestChild.permissions.through, Permission
+        )
         self.assert_query_cached(qs, [])
 
-    def test_custom_subquery(self):
+    @with_final_sql_check
+    def test_custom_subquery_with_check(self):
+        tests = Test.objects.filter(permission=OuterRef('pk')).values('name')
+        qs = Permission.objects.annotate(first_permission=Subquery(tests[:1]))
+        self.assert_tables(qs, Permission, Test, ContentType)
+        self.assert_query_cached(qs, list(Permission.objects.all()))
+
+    @no_final_sql_check
+    def test_custom_subquery_no_check(self):
         tests = Test.objects.filter(permission=OuterRef('pk')).values('name')
         qs = Permission.objects.annotate(first_permission=Subquery(tests[:1]))
         self.assert_tables(qs, Permission, Test)
         self.assert_query_cached(qs, list(Permission.objects.all()))
 
+    @with_final_sql_check
+    def test_custom_subquery_exists(self):
+        tests = Test.objects.filter(permission=OuterRef('pk'))
+        qs = Permission.objects.annotate(has_tests=Exists(tests))
+        self.assert_tables(qs, Permission, Test, ContentType)
+        self.assert_query_cached(qs, list(Permission.objects.all()))
+
+    @no_final_sql_check
     def test_custom_subquery_exists(self):
         tests = Test.objects.filter(permission=OuterRef('pk'))
         qs = Permission.objects.annotate(has_tests=Exists(tests))
         self.assert_tables(qs, Permission, Test)
         self.assert_query_cached(qs, list(Permission.objects.all()))
 
+    @all_final_sql_checks
     def test_raw_subquery(self):
         with self.assertNumQueries(0):
             raw_sql = RawSQL('SELECT id FROM auth_permission WHERE id = %s',
@@ -351,28 +436,79 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test, Permission)
         self.assert_query_cached(qs, [self.t1])
 
+    @all_final_sql_checks
     def test_aggregate(self):
-        Test.objects.create(name='test3', owner=self.user)
+        test3 = Test.objects.create(name='test3', owner=self.user)
         with self.assertNumQueries(1):
             n1 = User.objects.aggregate(n=Count('test'))['n']
         with self.assertNumQueries(0):
             n2 = User.objects.aggregate(n=Count('test'))['n']
         self.assertEqual(n2, n1)
         self.assertEqual(n2, 3)
+        test3.delete()
 
+    @all_final_sql_checks
     def test_annotate(self):
-        Test.objects.create(name='test3', owner=self.user)
+        test3 = Test.objects.create(name='test3', owner=self.user)
         qs = (User.objects.annotate(n=Count('test')).order_by('pk')
               .values_list('n', flat=True))
         self.assert_tables(qs, User, Test)
         self.assert_query_cached(qs, [2, 1])
+        test3.delete()
 
+    @all_final_sql_checks
     def test_annotate_subquery(self):
         tests = Test.objects.filter(owner=OuterRef('pk')).values('name')
         qs = User.objects.annotate(first_test=Subquery(tests[:1]))
         self.assert_tables(qs, User, Test)
         self.assert_query_cached(qs, [self.user, self.admin])
 
+    @all_final_sql_checks
+    def test_annotate_case_with_when_and_query_in_default(self):
+        tests = Test.objects.filter(owner=OuterRef('pk')).values('name')
+        qs = User.objects.annotate(
+            first_test=Case(
+                When(Q(pk=1), then=Value('noname')),
+                default=Subquery(tests[:1])
+            )
+        )
+        self.assert_tables(qs, User, Test)
+        self.assert_query_cached(qs, [self.user, self.admin])
+
+    @all_final_sql_checks
+    def test_annotate_case_with_when(self):
+        tests = Test.objects.filter(owner=OuterRef('pk')).values('name')
+        qs = User.objects.annotate(
+            first_test=Case(
+                When(Q(pk=1), then=Subquery(tests[:1])),
+                default=Value('noname')
+            )
+        )
+        self.assert_tables(qs, User, Test)
+        self.assert_query_cached(qs, [self.user, self.admin])
+
+    @all_final_sql_checks
+    def test_annotate_coalesce(self):
+        tests = Test.objects.filter(owner=OuterRef('pk')).values('name')
+        qs = User.objects.annotate(
+            name=Coalesce(
+                Subquery(tests[:1]),
+                Value('notest')
+            )
+        )
+        self.assert_tables(qs, User, Test)
+        self.assert_query_cached(qs, [self.user, self.admin])
+
+    @all_final_sql_checks
+    def test_annotate_raw(self):
+        qs = User.objects.annotate(
+            perm_id=RawSQL('SELECT id FROM auth_permission WHERE id = %s',
+                           (self.t1__permission.pk,))
+        )
+        self.assert_tables(qs, User, Permission)
+        self.assert_query_cached(qs, [self.user, self.admin])
+
+    @all_final_sql_checks
     def test_only(self):
         with self.assertNumQueries(1):
             t1 = Test.objects.only('name').first()
@@ -388,6 +524,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertEqual(t2.name, t1.name)
         self.assertEqual(t2.public, t1.public)
 
+    @all_final_sql_checks
     def test_defer(self):
         with self.assertNumQueries(1):
             t1 = Test.objects.defer('name').first()
@@ -403,6 +540,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertEqual(t2.name, t1.name)
         self.assertEqual(t2.public, t1.public)
 
+    @all_final_sql_checks
     def test_select_related(self):
         # Simple select_related
         with self.assertNumQueries(1):
@@ -428,6 +566,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertEqual(t4, t3)
         self.assertEqual(t4, self.t1)
 
+    @all_final_sql_checks
     def test_prefetch_related(self):
         # Simple prefetch_related
         with self.assertNumQueries(2):
@@ -490,35 +629,74 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(permissions8, permissions7)
         self.assertListEqual(permissions8, self.group__permissions)
 
-    def test_filtered_relation(self):
+    @all_final_sql_checks
+    def test_test_parent(self):
+        child = TestChild.objects.create(name='child')
+        qs = TestChild.objects.filter(name='child')
+        self.assert_query_cached(qs)
+
+        parent = TestParent.objects.all().first()
+        parent.name = 'another name'
+        parent.save()
+
+        child = TestChild.objects.all().first()
+        self.assertEqual(child.name, 'another name')
+
+    def _filtered_relation(self):
+        """
+        Resulting query:
+            SELECT "cachalot_testparent"."id", "cachalot_testparent"."name",
+            "cachalot_testchild"."testparent_ptr_id", "cachalot_testchild"."public"
+            FROM "cachalot_testchild" INNER JOIN "cachalot_testparent" ON
+            ("cachalot_testchild"."testparent_ptr_id" = "cachalot_testparent"."id")
+        """
         from django.db.models import FilteredRelation
 
         qs = TestChild.objects.annotate(
             filtered_permissions=FilteredRelation(
-                'permissions', condition=Q(permissions__pk__gt=1)))
+                'permissions', condition=Q(permissions__pk__gt=1))
-        self.assert_tables(qs, TestChild)
+        )
+        return qs
+
+    def _filtered_relation_common_asserts(self, qs):
         self.assert_query_cached(qs)
 
         values_qs = qs.values('filtered_permissions')
         self.assert_tables(
-            values_qs, TestChild, TestChild.permissions.through, Permission)
+            values_qs, TestParent, TestChild, TestChild.permissions.through, Permission
+        )
         self.assert_query_cached(values_qs)
 
         filtered_qs = qs.filter(filtered_permissions__pk__gt=2)
         self.assert_tables(
-            values_qs, TestChild, TestChild.permissions.through, Permission)
+            values_qs, TestParent, TestChild, TestChild.permissions.through, Permission
+        )
         self.assert_query_cached(filtered_qs)
 
-    @skipUnlessDBFeature('supports_select_union')
+    @with_final_sql_check
-    def test_union(self):
+    def test_filtered_relation_with_check(self):
-        qs = (Test.objects.filter(pk__lt=5)
+        qs = self._filtered_relation()
-              | Test.objects.filter(permission__name__contains='a'))
+        self.assert_tables(qs, TestParent, TestChild)
+        self._filtered_relation_common_asserts(qs)
+
+    @no_final_sql_check
+    def test_filtered_relation_no_check(self):
+        qs = self._filtered_relation()
+        self.assert_tables(qs, TestChild)
+        self._filtered_relation_common_asserts(qs)
+
+    def _test_union(self, check: bool):
+        qs = (
+            Test.objects.filter(pk__lt=5)
+            | Test.objects.filter(permission__name__contains='a')
+        )
         self.assert_tables(qs, Test, Permission)
         self.assert_query_cached(qs)
 
         with self.assertRaisesMessage(
-                AssertionError,
+            AssertionError if DJANGO_VERSION < (4, 0) else TypeError,
-                'Cannot combine queries on two different base models.'):
+            'Cannot combine queries on two different base models.'
+        ):
             Test.objects.all() | Permission.objects.all()
 
         qs = Test.objects.filter(pk__lt=5)
@@ -536,19 +714,32 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             qs = qs.order_by()
             sub_qs = sub_qs.order_by()
         qs = qs.union(sub_qs)
-        self.assert_tables(qs, Test, Permission)
+        tables = {Test, Permission}
+        # Sqlite does not do an ORDER BY django_content_type
+        if not self.is_sqlite and check:
+            tables.add(ContentType)
+        self.assert_tables(qs, *tables)
         with self.assertRaises((ProgrammingError, OperationalError)):
             self.assert_query_cached(qs)
 
-    @skipUnlessDBFeature('supports_select_intersection')
+    @with_final_sql_check
-    def test_intersection(self):
+    @skipUnlessDBFeature('supports_select_union')
+    def test_union_with_sql_check(self):
+        self._test_union(check=True)
+
+    @no_final_sql_check
+    @skipUnlessDBFeature('supports_select_union')
+    def test_union_with_sql_check(self):
+        self._test_union(check=False)
+
+    def _test_intersection(self, check: bool):
         qs = (Test.objects.filter(pk__lt=5)
               & Test.objects.filter(permission__name__contains='a'))
         self.assert_tables(qs, Test, Permission)
         self.assert_query_cached(qs)
 
         with self.assertRaisesMessage(
-                AssertionError,
+                AssertionError if DJANGO_VERSION < (4, 0) else TypeError,
                 'Cannot combine queries on two different base models.'):
             Test.objects.all() & Permission.objects.all()
 
@@ -567,12 +758,24 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             qs = qs.order_by()
             sub_qs = sub_qs.order_by()
         qs = qs.intersection(sub_qs)
-        self.assert_tables(qs, Test, Permission)
+        tables = {Test, Permission}
+        if not self.is_sqlite and check:
+            tables.add(ContentType)
+        self.assert_tables(qs, *tables)
         with self.assertRaises((ProgrammingError, OperationalError)):
             self.assert_query_cached(qs)
 
-    @skipUnlessDBFeature('supports_select_difference')
+    @with_final_sql_check
-    def test_difference(self):
+    @skipUnlessDBFeature('supports_select_intersection')
+    def test_intersection_with_check(self):
+        self._test_intersection(check=True)
+
+    @no_final_sql_check
+    @skipUnlessDBFeature('supports_select_intersection')
+    def test_intersection_with_check(self):
+        self._test_intersection(check=False)
+
+    def _test_difference(self, check: bool):
         qs = Test.objects.filter(pk__lt=5)
         sub_qs = Test.objects.filter(permission__name__contains='a')
         if self.is_sqlite:
@@ -588,10 +791,23 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             qs = qs.order_by()
             sub_qs = sub_qs.order_by()
         qs = qs.difference(sub_qs)
-        self.assert_tables(qs, Test, Permission)
+        tables = {Test, Permission}
+        if not self.is_sqlite and check:
+            tables.add(ContentType)
+        self.assert_tables(qs, *tables)
         with self.assertRaises((ProgrammingError, OperationalError)):
             self.assert_query_cached(qs)
 
+    @with_final_sql_check
+    @skipUnlessDBFeature('supports_select_difference')
+    def test_difference_with_check(self):
+        self._test_difference(check=True)
+
+    @no_final_sql_check
+    @skipUnlessDBFeature('supports_select_difference')
+    def test_difference_with_check(self):
+        self._test_difference(check=False)
+
     @skipUnlessDBFeature('has_select_for_update')
     def test_select_for_update(self):
         """
@@ -625,6 +841,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
                 self.assertListEqual([t.name for t in data4],
                                      ['test1', 'test2'])
 
+    @all_final_sql_checks
     def test_having(self):
         qs = (User.objects.annotate(n=Count('user_permissions')).filter(n__gte=1))
         self.assert_tables(qs, User, User.user_permissions.through, Permission)
@@ -657,6 +874,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             self.assertListEqual(data2, [self.t1, self.t2])
             self.assertListEqual([o.username_length for o in data2], [4, 5])
 
+    @all_final_sql_checks
     def test_extra_where(self):
         sql_condition = ("owner_id IN "
                          "(SELECT id FROM auth_user WHERE username = 'admin')")
@@ -664,19 +882,21 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test, User)
         self.assert_query_cached(qs, [self.t2])
 
+    @all_final_sql_checks
     def test_extra_tables(self):
         qs = Test.objects.extra(tables=['auth_user'],
                                 select={'extra_id': 'auth_user.id'})
         self.assert_tables(qs, Test, User)
         self.assert_query_cached(qs)
 
+    @all_final_sql_checks
     def test_extra_order_by(self):
         qs = Test.objects.extra(order_by=['-cachalot_test.name'])
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs, [self.t2, self.t1])
 
     def test_table_inheritance(self):
-        with self.assertNumQueries(3 if self.is_sqlite else 2):
+        with self.assertNumQueries(2):
             t_child = TestChild.objects.create(name='test_child')
 
         with self.assertNumQueries(1):
@@ -688,18 +908,13 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
     def test_explain(self):
         explain_kwargs = {}
         if self.is_sqlite:
-            expected = (r'\d+ 0 0 SCAN TABLE cachalot_test\n'
+            expected = (r'\d+ 0 0 SCAN cachalot_test\n'
                         r'\d+ 0 0 USE TEMP B-TREE FOR ORDER BY')
         elif self.is_mysql:
-            if self.django_version < (3, 1):
+            expected = (
-                expected = (
+                r'-> Sort row IDs: cachalot_test.`name`  \(cost=[\d\.]+ rows=\d\)\n    '
-                    r'1 SIMPLE cachalot_test '
+                r'-> Table scan on cachalot_test  \(cost=[\d\.]+ rows=\d\)\n'
-                    r'(?:None )?ALL None None None None 2 100\.0 Using filesort')
+            )
-            else:
-                expected = (
-                    r'-> Sort row IDs: cachalot_test.`name`  \(cost=[\d\.]+ rows=\d\)\n    '
-                    r'-> Table scan on cachalot_test  \(cost=[\d\.]+ rows=\d\)\n'
-                )
         else:
             explain_kwargs.update(
                 analyze=True,
@@ -715,9 +930,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
                 r'Planning Time: [\d\.]+ ms\n'
                 r'Execution Time: [\d\.]+ ms$') % (operation_detail,
                                                    operation_detail)
-        with self.assertNumQueries(
+        with self.assertNumQueries(1):
-                2 if self.is_mysql and django_version[0] < 3
-                else 1):
             explanation1 = Test.objects.explain(**explain_kwargs)
         self.assertRegex(explanation1, expected)
         with self.assertNumQueries(0):
@@ -810,6 +1023,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data2, data1)
         self.assertListEqual(data2, [(1,), (2,)])
 
+    @all_final_sql_checks
     def test_missing_table_cache_key(self):
         qs = Test.objects.all()
         self.assert_tables(qs, Test)
@@ -821,6 +1035,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
 
         self.assert_query_cached(qs)
 
+    @all_final_sql_checks
     def test_broken_query_cache_value(self):
         """
         In some undetermined cases, cache.get_many return wrong values such
@@ -849,6 +1064,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
             with self.assertRaises(Test.DoesNotExist):
                 Test.objects.get(name='Clémentine')
 
+    @all_final_sql_checks
     def test_unicode_table_name(self):
         """
         Tests if using unicode in table names does not break caching.
@@ -868,13 +1084,20 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase):
         with connection.cursor() as cursor:
             cursor.execute('DROP TABLE %s;' % table_name)
 
+    @all_final_sql_checks
     def test_unmanaged_model(self):
         qs = UnmanagedModel.objects.all()
         self.assert_tables(qs, UnmanagedModel)
         self.assert_query_cached(qs)
 
+    def test_now_annotate(self):
+        """Check that queries with a Now() annotation are not cached #193"""
+        qs = Test.objects.annotate(now=Now())
+        self.assert_query_cached(qs, after=1)
+
 
 class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
+    @all_final_sql_checks
     def test_tuple(self):
         qs = Test.objects.filter(pk__in=(1, 2, 3))
         self.assert_tables(qs, Test)
@@ -884,6 +1107,7 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs)
 
+    @all_final_sql_checks
     def test_list(self):
         qs = Test.objects.filter(pk__in=[1, 2, 3])
         self.assert_tables(qs, Test)
@@ -904,6 +1128,7 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         self.assert_tables(qs, Test)
         self.assert_query_cached(qs)
 
+    @all_final_sql_checks
     def test_binary(self):
         """
         Binary data should be cached on PostgreSQL & MySQL, but not on SQLite,
@@ -945,11 +1170,12 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(a_float=0.123456789)
 
+    @all_final_sql_checks
     def test_decimal(self):
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1', a_decimal=Decimal('123.45'))
+            test1 = Test.objects.create(name='test1', a_decimal=Decimal('123.45'))
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1', a_decimal=Decimal('12.3'))
+            test2 = Test.objects.create(name='test2', a_decimal=Decimal('12.3'))
 
         qs = Test.objects.values_list('a_decimal', flat=True).filter(
             a_decimal__isnull=False).order_by('a_decimal')
@@ -961,11 +1187,15 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(a_decimal=Decimal('123.45'))
 
+        test1.delete()
+        test2.delete()
+
+    @all_final_sql_checks
     def test_ipv4_address(self):
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1', ip='127.0.0.1')
+            test1 = Test.objects.create(name='test1', ip='127.0.0.1')
         with self.assertNumQueries(1):
-            Test.objects.create(name='test2', ip='192.168.0.1')
+            test2 = Test.objects.create(name='test2', ip='192.168.0.1')
 
         qs = Test.objects.values_list('ip', flat=True).filter(
             ip__isnull=False).order_by('ip')
@@ -977,11 +1207,15 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(ip='127.0.0.1')
 
+        test1.delete()
+        test2.delete()
+
+    @all_final_sql_checks
     def test_ipv6_address(self):
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1/64')
+            test1 = Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1/64')
         with self.assertNumQueries(1):
-            Test.objects.create(name='test2', ip='2001:db8:0:85a3::ac1f:8001')
+            test2 = Test.objects.create(name='test2', ip='2001:db8:0:85a3::ac1f:8001')
 
         qs = Test.objects.values_list('ip', flat=True).filter(
             ip__isnull=False).order_by('ip')
@@ -994,11 +1228,15 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(ip='2001:db8:0:85a3::ac1f:8001')
 
+        test1.delete()
+        test2.delete()
+
+    @all_final_sql_checks
     def test_duration(self):
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1', duration=datetime.timedelta(30))
+            test1 = Test.objects.create(name='test1', duration=datetime.timedelta(30))
         with self.assertNumQueries(1):
-            Test.objects.create(name='test2', duration=datetime.timedelta(60))
+            test2 = Test.objects.create(name='test2', duration=datetime.timedelta(60))
 
         qs = Test.objects.values_list('duration', flat=True).filter(
             duration__isnull=False).order_by('duration')
@@ -1011,12 +1249,16 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(duration=datetime.timedelta(30))
 
+        test1.delete()
+        test2.delete()
+
+    @all_final_sql_checks
     def test_uuid(self):
         with self.assertNumQueries(1):
-            Test.objects.create(name='test1',
+            test1 = Test.objects.create(name='test1',
                                 uuid='1cc401b7-09f4-4520-b8d0-c267576d196b')
         with self.assertNumQueries(1):
-            Test.objects.create(name='test2',
+            test2 = Test.objects.create(name='test2',
                                 uuid='ebb3b6e1-1737-4321-93e3-4c35d61ff491')
 
         qs = Test.objects.values_list('uuid', flat=True).filter(
@@ -1031,13 +1273,15 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(0):
             Test.objects.get(uuid=UUID('1cc401b7-09f4-4520-b8d0-c267576d196b'))
 
+        test1.delete()
+        test2.delete()
+
     def test_now(self):
         """
         Checks that queries with a Now() parameter are not cached.
         """
         obj = Test.objects.create(datetime='1992-07-02T12:00:00')
-        qs = Test.objects.filter(
+        qs = Test.objects.filter(datetime__lte=Now())
-            datetime__lte=Now())
         with self.assertNumQueries(1):
             obj1 = qs.get()
         with self.assertNumQueries(1):

cachalot/tests/settings.py

@@ -1,17 +1,19 @@
 from time import sleep
 from unittest import skipIf
+from unittest.mock import MagicMock, patch
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.cache import DEFAULT_CACHE_ALIAS
-from django.core.checks import run_checks, Tags, Warning, Error
+from django.core.checks import Error, Tags, Warning, run_checks
 from django.db import connection
 from django.test import TransactionTestCase
 from django.test.utils import override_settings
 
 from ..api import invalidate
-from ..settings import SUPPORTED_ONLY, SUPPORTED_DATABASE_ENGINES
+from ..settings import SUPPORTED_DATABASE_ENGINES, SUPPORTED_ONLY
-from .models import Test, TestParent, TestChild, UnmanagedModel
+from ..utils import _get_tables
+from .models import Test, TestChild, TestParent, UnmanagedModel
 from .test_utils import TestUtilsMixin
 
 
@@ -152,6 +154,21 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase):
             # table, it’s cachable.
             self.assert_query_cached(TestChild.objects.values('public'))
 
+    @override_settings(CACHALOT_ONLY_CACHABLE_APPS=('cachalot',))
+    def test_only_cachable_apps(self):
+        self.assert_query_cached(Test.objects.all())
+        self.assert_query_cached(TestParent.objects.all())
+        self.assert_query_cached(Test.objects.select_related('owner'), after=1)
+
+    # Must use override_settings to get the correct effect. Using the cm doesn't
+    # reload settings on cachalot's side
+    @override_settings(CACHALOT_ONLY_CACHABLE_TABLES=('cachalot_test', 'auth_user'),
+                       CACHALOT_ONLY_CACHABLE_APPS=('cachalot',))
+    def test_only_cachable_apps_set_combo(self):
+        self.assert_query_cached(Test.objects.all())
+        self.assert_query_cached(TestParent.objects.all())
+        self.assert_query_cached(Test.objects.select_related('owner'))
+
     def test_uncachable_tables(self):
         qs = Test.objects.all()
 
@@ -163,6 +180,17 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase):
         with self.settings(CACHALOT_UNCACHABLE_TABLES=('cachalot_test',)):
             self.assert_query_cached(qs, after=1)
 
+    @override_settings(CACHALOT_UNCACHABLE_APPS=('cachalot',))
+    def test_uncachable_apps(self):
+        self.assert_query_cached(Test.objects.all(), after=1)
+        self.assert_query_cached(TestParent.objects.all(), after=1)
+
+    @override_settings(CACHALOT_UNCACHABLE_TABLES=('cachalot_test',),
+                       CACHALOT_UNCACHABLE_APPS=('cachalot',))
+    def test_uncachable_apps_set_combo(self):
+        self.assert_query_cached(Test.objects.all(), after=1)
+        self.assert_query_cached(TestParent.objects.all(), after=1)
+
     def test_only_cachable_and_uncachable_table(self):
         with self.settings(
                 CACHALOT_ONLY_CACHABLE_TABLES=('cachalot_test',
@@ -288,3 +316,29 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase):
         with self.settings(CACHALOT_DATABASES='invalid value'):
             errors = run_checks(tags=[Tags.compatibility])
             self.assertListEqual(errors, [error002])
+
+    def call_get_tables(self):
+        qs = Test.objects.all()
+        compiler_mock = MagicMock()
+        compiler_mock.__cachalot_generated_sql = ''
+        tables = _get_tables(qs.db, qs.query, compiler_mock)
+        self.assertTrue(tables)
+        return tables
+
+    @override_settings(CACHALOT_FINAL_SQL_CHECK=True)
+    @patch('cachalot.utils._get_tables_from_sql')
+    def test_cachalot_final_sql_check_when_true(self, _get_tables_from_sql):
+        _get_tables_from_sql.return_value = {'patched'}
+        tables = self.call_get_tables()
+        _get_tables_from_sql.assert_called_once()
+        self.assertIn('patched', tables)
+
+
+    @override_settings(CACHALOT_FINAL_SQL_CHECK=False)
+    @patch('cachalot.utils._get_tables_from_sql')
+    def test_cachalot_final_sql_check_when_false(self, _get_tables_from_sql):
+        _get_tables_from_sql.return_value = {'patched'}
+        tables = self.call_get_tables()
+        _get_tables_from_sql.assert_not_called()
+        self.assertNotIn('patched', tables)
+

cachalot/tests/test_utils.py

@@ -1,9 +1,11 @@
-from django import VERSION as DJANGO_VERSION
 from django.core.management.color import no_style
-from django.db import connection, transaction
+from django.db import DEFAULT_DB_ALIAS, connection, connections, transaction
+
+from django.test import TransactionTestCase
+from django.test.utils import CaptureQueriesContext
 
-from .models import PostgresModel
 from ..utils import _get_tables
+from .models import PostgresModel
 
 
 class TestUtilsMixin:
@@ -11,7 +13,6 @@ class TestUtilsMixin:
         self.is_sqlite = connection.vendor == 'sqlite'
         self.is_mysql = connection.vendor == 'mysql'
         self.is_postgresql = connection.vendor == 'postgresql'
-        self.django_version = DJANGO_VERSION
         self.force_reopen_connection()
 
     # TODO: Remove this workaround when this issue is fixed:
@@ -19,8 +20,6 @@ class TestUtilsMixin:
     def tearDown(self):
         if connection.vendor == 'postgresql':
             flush_args = [no_style(), (PostgresModel._meta.db_table,),]
-            if float(".".join(map(str, DJANGO_VERSION[:2]))) < 3.1:
-                flush_args.append(())
             flush_sql_list = connection.ops.sql_flush(*flush_args)
             with transaction.atomic():
                 for sql in flush_sql_list:
@@ -36,7 +35,7 @@ class TestUtilsMixin:
     def assert_tables(self, queryset, *tables):
         tables = {table if isinstance(table, str)
                   else table._meta.db_table for table in tables}
-        self.assertSetEqual(_get_tables(queryset.db, queryset.query), tables)
+        self.assertSetEqual(_get_tables(queryset.db, queryset.query), tables, str(queryset.query))
 
     def assert_query_cached(self, queryset, result=None, result_type=None,
                             compare_results=True, before=1, after=0):
@@ -61,3 +60,61 @@ class TestUtilsMixin:
         assert_function(data2, data1)
         if result is not None:
             assert_function(data2, result)
+
+class FilteredTransactionTestCase(TransactionTestCase):
+    """
+    TransactionTestCase with assertNumQueries that ignores BEGIN, COMMIT and ROLLBACK
+    queries.
+    """
+    def assertNumQueries(self, num, func=None, *args, using=DEFAULT_DB_ALIAS, **kwargs):
+        conn = connections[using]
+
+        context = FilteredAssertNumQueriesContext(self, num, conn)
+        if func is None:
+            return context
+
+        with context:
+            func(*args, **kwargs)
+
+
+class FilteredAssertNumQueriesContext(CaptureQueriesContext):
+    """
+    Capture queries and assert their number ignoring BEGIN, COMMIT and ROLLBACK queries.
+    """
+    EXCLUDE = ('BEGIN', 'COMMIT', 'ROLLBACK')
+
+    def __init__(self, test_case, num, connection):
+        self.test_case = test_case
+        self.num = num
+        super().__init__(connection)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super().__exit__(exc_type, exc_value, traceback)
+        if exc_type is not None:
+            return
+
+        filtered_queries = []
+        excluded_queries = []
+        for q in self.captured_queries:
+            if q['sql'].upper() not in self.EXCLUDE:
+                filtered_queries.append(q)
+            else:
+                excluded_queries.append(q)
+
+        executed = len(filtered_queries)
+
+        self.test_case.assertEqual(
+            executed,
+            self.num,
+            f"\n{executed} queries executed, {self.num} expected\n" +
+            "\nCaptured queries were:\n" +
+            "".join(
+                f"{i}. {query['sql']}\n"
+                for i, query in enumerate(filtered_queries, start=1)
+            ) +
+            "\nCaptured queries, that were excluded:\n" +
+            "".join(
+                f"{i}. {query['sql']}\n"
+                for i, query in enumerate(excluded_queries, start=1)
+            )
+        )

cachalot/tests/tests_decorators.py

@@ -0,0 +1,50 @@
+import logging
+from functools import wraps
+
+from django.core.cache import cache
+from django.test.utils import override_settings
+
+logger = logging.getLogger(__name__)
+
+
+def all_final_sql_checks(func):
+    """
+    Runs test as two sub-tests:
+    one with CACHALOT_FINAL_SQL_CHECK setting True, one with False
+    """
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        for final_sql_check in (True, False):
+            with self.subTest(msg=f'CACHALOT_FINAL_SQL_CHECK = {final_sql_check}'):
+                with override_settings(
+                        CACHALOT_FINAL_SQL_CHECK=final_sql_check
+                ):
+                    func(self, *args, **kwargs)
+            cache.clear()
+
+    return wrapper
+
+
+def no_final_sql_check(func):
+    """
+    Runs test with CACHALOT_FINAL_SQL_CHECK = False
+    """
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        with override_settings(CACHALOT_FINAL_SQL_CHECK=False):
+            func(self, *args, **kwargs)
+
+    return wrapper
+
+
+def with_final_sql_check(func):
+    """
+    Runs test with CACHALOT_FINAL_SQL_CHECK = True
+    """
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        with override_settings(CACHALOT_FINAL_SQL_CHECK=True):
+            func(self, *args, **kwargs)
+
+    return wrapper

cachalot/tests/thread_safety.py

@@ -1,10 +1,10 @@
 from threading import Thread
 
 from django.db import connection, transaction
-from django.test import TransactionTestCase, skipUnlessDBFeature
+from django.test import skipUnlessDBFeature
 
 from .models import Test
-from .test_utils import TestUtilsMixin
+from .test_utils import TestUtilsMixin, FilteredTransactionTestCase
 
 
 class TestThread(Thread):
@@ -19,7 +19,7 @@ class TestThread(Thread):
 
 
 @skipUnlessDBFeature('test_db_allows_multiple_connections')
-class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase):
+class ThreadSafetyTestCase(TestUtilsMixin, FilteredTransactionTestCase):
     def test_concurrent_caching(self):
         t1 = TestThread().start_and_join()
         t = Test.objects.create(name='test')

cachalot/tests/transaction.py

@@ -1,14 +1,17 @@
+from cachalot.transaction import AtomicCache
+
 from django.contrib.auth.models import User
+from django.core.cache import cache
 from django.db import transaction, connection, IntegrityError
-from django.test import TransactionTestCase, skipUnlessDBFeature
+from django.test import SimpleTestCase, skipUnlessDBFeature
 
 from .models import Test
-from .test_utils import TestUtilsMixin
+from .test_utils import TestUtilsMixin, FilteredTransactionTestCase
 
 
-class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
+class AtomicTestCase(TestUtilsMixin, FilteredTransactionTestCase):
     def test_successful_read_atomic(self):
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             with transaction.atomic():
                 data1 = list(Test.objects.all())
         self.assertListEqual(data1, [])
@@ -18,7 +21,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data2, [])
 
     def test_unsuccessful_read_atomic(self):
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             try:
                 with transaction.atomic():
                     data1 = list(Test.objects.all())
@@ -36,21 +39,21 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
             data1 = list(Test.objects.all())
         self.assertListEqual(data1, [])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             with transaction.atomic():
                 t1 = Test.objects.create(name='test1')
         with self.assertNumQueries(1):
             data2 = list(Test.objects.all())
         self.assertListEqual(data2, [t1])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             with transaction.atomic():
                 t2 = Test.objects.create(name='test2')
         with self.assertNumQueries(1):
             data3 = list(Test.objects.all())
         self.assertListEqual(data3, [t1, t2])
 
-        with self.assertNumQueries(4 if self.is_sqlite else 3):
+        with self.assertNumQueries(3):
             with transaction.atomic():
                 data4 = list(Test.objects.all())
                 t3 = Test.objects.create(name='test3')
@@ -65,7 +68,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
             data1 = list(Test.objects.all())
         self.assertListEqual(data1, [])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             try:
                 with transaction.atomic():
                     Test.objects.create(name='test')
@@ -80,7 +83,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
                 Test.objects.get(name='test')
 
     def test_cache_inside_atomic(self):
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             with transaction.atomic():
                 data1 = list(Test.objects.all())
                 data2 = list(Test.objects.all())
@@ -88,7 +91,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data2, [])
 
     def test_invalidation_inside_atomic(self):
-        with self.assertNumQueries(4 if self.is_sqlite else 3):
+        with self.assertNumQueries(3):
             with transaction.atomic():
                 data1 = list(Test.objects.all())
                 t = Test.objects.create(name='test')
@@ -97,7 +100,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data2, [t])
 
     def test_successful_nested_read_atomic(self):
-        with self.assertNumQueries(7 if self.is_sqlite else 6):
+        with self.assertNumQueries(6):
             with transaction.atomic():
                 list(Test.objects.all())
                 with transaction.atomic():
@@ -112,7 +115,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
             list(User.objects.all())
 
     def test_unsuccessful_nested_read_atomic(self):
-        with self.assertNumQueries(6 if self.is_sqlite else 5):
+        with self.assertNumQueries(5):
             with transaction.atomic():
                 try:
                     with transaction.atomic():
@@ -125,7 +128,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
                     list(Test.objects.all())
 
     def test_successful_nested_write_atomic(self):
-        with self.assertNumQueries(13 if self.is_sqlite else 12):
+        with self.assertNumQueries(12):
             with transaction.atomic():
                 t1 = Test.objects.create(name='test1')
                 with transaction.atomic():
@@ -142,7 +145,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data3, [t1, t2, t3, t4])
 
     def test_unsuccessful_nested_write_atomic(self):
-        with self.assertNumQueries(16 if self.is_sqlite else 15):
+        with self.assertNumQueries(15):
             with transaction.atomic():
                 t1 = Test.objects.create(name='test1')
                 try:
@@ -167,7 +170,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(1):
             data3 = list(Test.objects.all())
         self.assertListEqual(data3, [t1])
-
+    
     @skipUnlessDBFeature('can_defer_constraint_checks')
     def test_deferred_error(self):
         """
@@ -187,3 +190,13 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase):
                         '-- ' + Test._meta.db_table)  # Should invalidate Test.
         with self.assertNumQueries(1):
             list(Test.objects.all())
+
+
+class AtomicCacheTestCase(SimpleTestCase):
+    def setUp(self):
+        self.atomic_cache = AtomicCache(cache, 'db_alias')
+    
+    def test_set(self):
+        self.assertDictEqual(self.atomic_cache, {})
+        self.atomic_cache.set('key', 'value', None)
+        self.assertDictEqual(self.atomic_cache, {'key': 'value'})

cachalot/tests/write.py

@@ -1,6 +1,5 @@
 from unittest import skipIf, skipUnless
 
-from django import VERSION as DJANGO_VERSION
 from django.contrib.auth.models import User, Permission, Group
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import MultipleObjectsReturned
@@ -12,10 +11,10 @@ from django.db.models.expressions import RawSQL
 from django.test import TransactionTestCase, skipUnlessDBFeature
 
 from .models import Test, TestParent, TestChild
-from .test_utils import TestUtilsMixin
+from .test_utils import TestUtilsMixin, FilteredTransactionTestCase
 
 
-class WriteTestCase(TestUtilsMixin, TransactionTestCase):
+class WriteTestCase(TestUtilsMixin, FilteredTransactionTestCase):
     """
     Tests if every SQL request writing data is not cached and invalidates the
     implied data.
@@ -56,7 +55,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             data1 = list(Test.objects.all())
         self.assertListEqual(data1, [])
 
-        with self.assertNumQueries(3 if self.is_sqlite else 2):
+        with self.assertNumQueries(2):
             t, created = Test.objects.get_or_create(name='test')
         self.assertTrue(created)
 
@@ -78,14 +77,14 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
         with self.assertNumQueries(1):
             self.assertListEqual(list(Test.objects.all()), [])
 
-        with self.assertNumQueries(5 if self.is_sqlite else 4):
+        with self.assertNumQueries(4):
             t, created = Test.objects.update_or_create(
                 name='test', defaults={'public': True})
             self.assertTrue(created)
             self.assertEqual(t.name, 'test')
             self.assertEqual(t.public, True)
 
-        with self.assertNumQueries(3 if self.is_sqlite else 2):
+        with self.assertNumQueries(2):
             t, created = Test.objects.update_or_create(
                 name='test', defaults={'public': False})
             self.assertFalse(created)
@@ -94,7 +93,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
 
         # The number of SQL queries doesn’t decrease because update_or_create
         # always calls an UPDATE, even when data wasn’t changed.
-        with self.assertNumQueries(3 if self.is_sqlite else 2):
+        with self.assertNumQueries(2):
             t, created = Test.objects.update_or_create(
                 name='test', defaults={'public': False})
             self.assertFalse(created)
@@ -109,12 +108,12 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             data1 = list(Test.objects.all())
         self.assertListEqual(data1, [])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)]
             Test.objects.bulk_create(unsaved_tests)
         self.assertEqual(Test.objects.count(), 10)
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)]
             Test.objects.bulk_create(unsaved_tests)
         self.assertEqual(Test.objects.count(), 20)
@@ -160,12 +159,12 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data1, [t1.name, t2.name])
         self.assertListEqual(data2, [t1.name])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             Test.objects.bulk_create([Test(name='test%s' % i)
                                       for i in range(2, 11)])
         with self.assertNumQueries(1):
             self.assertEqual(Test.objects.count(), 10)
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             Test.objects.all().delete()
         with self.assertNumQueries(1):
             self.assertEqual(Test.objects.count(), 0)
@@ -360,7 +359,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
         self.assertListEqual(data4, [user1, user2])
         self.assertListEqual([u.n for u in data4], [1, 0])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             Test.objects.bulk_create([
                 Test(name='test3', owner=user1),
                 Test(name='test4', owner=user2),
@@ -588,7 +587,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             data2 = list(Test.objects.select_related('owner'))
         self.assertListEqual(data2, [])
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             Test.objects.bulk_create([
                 Test(name='test1', owner=u1),
                 Test(name='test2', owner=u2),
@@ -602,7 +601,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             self.assertEqual(data3[2].owner, u2)
             self.assertEqual(data3[3].owner, u1)
 
-        with self.assertNumQueries(2 if self.is_sqlite else 1):
+        with self.assertNumQueries(1):
             Test.objects.filter(name__in=['test1', 'test2']).delete()
         with self.assertNumQueries(1):
             data4 = list(Test.objects.select_related('owner'))
@@ -634,12 +633,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             self.assertEqual(data3[0].owner, u)
             self.assertListEqual(list(data3[0].owner.groups.all()), [])
 
-        with self.assertNumQueries(
+        with self.assertNumQueries(4):
-                8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2
-                else 4 if self.is_postgresql and DJANGO_VERSION[0] > 2
-                else 4 if self.is_mysql and DJANGO_VERSION[0] > 2
-                else 6
-        ):
             group = Group.objects.create(name='test_group')
             permissions = list(Permission.objects.all()[:5])
             group.permissions.add(*permissions)
@@ -852,7 +846,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase):
             with self.assertRaises(TestChild.DoesNotExist):
                 TestChild.objects.get()
 
-        with self.assertNumQueries(3 if self.is_sqlite else 2):
+        with self.assertNumQueries(2):
             t_child = TestChild.objects.create(name='test_child')
 
         with self.assertNumQueries(1):

cachalot/transaction.py

@@ -3,7 +3,7 @@ from .settings import cachalot_settings
 
 class AtomicCache(dict):
     def __init__(self, parent_cache, db_alias):
-        super(AtomicCache, self).__init__()
+        super().__init__()
         self.parent_cache = parent_cache
         self.db_alias = db_alias
         self.to_be_invalidated = set()

cachalot/utils.py

@@ -2,11 +2,13 @@ import datetime
 from decimal import Decimal
 from hashlib import sha1
 from time import time
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 from django.contrib.postgres.functions import TransactionNow
 from django.db import connections
-from django.db.models import QuerySet, Subquery, Exists
+from django.db.models import Exists, QuerySet, Subquery
+from django.db.models.expressions import RawSQL
 from django.db.models.functions import Now
 from django.db.models.sql import Query, AggregateQuery
 from django.db.models.sql.where import ExtraWhere, WhereNode, NothingNode
@@ -15,6 +17,10 @@ from .settings import ITERABLES, cachalot_settings
 from .transaction import AtomicCache
 
 
+if TYPE_CHECKING:
+    from django.db.models.expressions import BaseExpression 
+
+
 class UncachableQuery(Exception):
     pass
 
@@ -29,13 +35,6 @@ CACHABLE_PARAM_TYPES = {
 }
 UNCACHABLE_FUNCS = {Now, TransactionNow}
 
-try:
-    # TODO Drop after Dj30 drop
-    from django.contrib.postgres.fields.jsonb import JsonAdapter
-    CACHABLE_PARAM_TYPES.update((JsonAdapter,))
-except ImportError:
-    pass
-
 try:
     from psycopg2 import Binary
     from psycopg2.extras import (
@@ -77,6 +76,10 @@ def get_query_cache_key(compiler):
     check_parameter_types(params)
     cache_key = '%s:%s:%s' % (compiler.using, sql,
                               [str(p) for p in params])
+    # Set attribute on compiler for later access
+    # to the generated SQL. This prevents another as_sql() call!
+    compiler.__cachalot_generated_sql = sql.lower()
+
     return sha1(cache_key.encode('utf-8')).hexdigest()
 
 
@@ -95,9 +98,23 @@ def get_table_cache_key(db_alias, table):
     return sha1(cache_key.encode('utf-8')).hexdigest()
 
 
-def _get_tables_from_sql(connection, lowercased_sql):
+def _get_tables_from_sql(connection, lowercased_sql, enable_quote: bool = False):
-    return {t for t in connection.introspection.django_table_names()
+    """Returns names of involved tables after analyzing the final SQL query."""
-            + cachalot_settings.CACHALOT_ADDITIONAL_TABLES if t in lowercased_sql}
+    return {table for table in (connection.introspection.django_table_names()
+            + cachalot_settings.CACHALOT_ADDITIONAL_TABLES)
+            if _quote_table_name(table, connection, enable_quote) in lowercased_sql}
+
+
+def _quote_table_name(table_name, connection, enable_quote: bool):
+    """
+    Returns quoted table name.
+
+    Put database-specific quotation marks around the table name
+    to preven that tables with substrings of the table are considered.
+    E.g. cachalot_testparent must not return cachalot_test.
+    """
+    return f'{connection.ops.quote_name(table_name)}' \
+        if enable_quote else table_name
 
 
 def _find_rhs_lhs_subquery(side):
@@ -107,31 +124,7 @@ def _find_rhs_lhs_subquery(side):
     elif h_class is QuerySet:
         return side.query
     elif h_class in (Subquery, Exists):  # Subquery allows QuerySet & Query
-        try:
+        return side.query.query if side.query.__class__ is QuerySet else side.query
-            return side.query.query if side.query.__class__ is QuerySet else side.query
-        except AttributeError:  # TODO Remove try/except closure after drop Django 2.2
-            try:
-                return side.queryset.query
-            except AttributeError:
-                return None
-    elif h_class in UNCACHABLE_FUNCS:
-        raise UncachableQuery
-
-
-def _find_rhs_lhs_subquery(side):
-    h_class = side.__class__
-    if h_class is Query:
-        return side
-    elif h_class is QuerySet:
-        return side.query
-    elif h_class in (Subquery, Exists):  # Subquery allows QuerySet & Query
-        try:
-            return side.query.query if side.query.__class__ is QuerySet else side.query
-        except AttributeError:  # TODO Remove try/except closure after drop Django 2.2
-            try:
-                return side.queryset.query
-            except AttributeError:
-                return None
     elif h_class in UNCACHABLE_FUNCS:
         raise UncachableQuery
 
@@ -147,10 +140,15 @@ def _find_subqueries_in_where(children):
         elif child_class is NothingNode:
             pass
         else:
-            rhs = _find_rhs_lhs_subquery(child.rhs)
+            try:
+                child_rhs = child.rhs
+                child_lhs = child.lhs
+            except AttributeError:
+                raise UncachableQuery
+            rhs = _find_rhs_lhs_subquery(child_rhs)
             if rhs is not None:
                 yield rhs
-            lhs = _find_rhs_lhs_subquery(child.lhs)
+            lhs = _find_rhs_lhs_subquery(child_lhs)
             if lhs is not None:
                 yield lhs
 
@@ -177,7 +175,24 @@ def filter_cachable(tables):
     return tables
 
 
-def _get_tables(db_alias, query):
+def _flatten(expression: 'BaseExpression'):
+    """
+    Recursively yield this expression and all subexpressions, in
+    depth-first order.
+
+    Taken from Django 3.2 as the previous Django versions don’t check
+    for existence of flatten.
+    """
+    yield expression
+    for expr in expression.get_source_expressions():
+        if expr:
+            if hasattr(expr, 'flatten'):
+                yield from _flatten(expr)
+            else:
+                yield expr
+
+
+def _get_tables(db_alias, query, compiler=False):
     if query.select_for_update or (
             not cachalot_settings.CACHALOT_CACHE_RANDOM
             and '?' in query.order_by):
@@ -186,16 +201,27 @@ def _get_tables(db_alias, query):
     try:
         if query.extra_select:
             raise IsRawQuery
+
         # Gets all tables already found by the ORM.
         tables = set(query.table_map)
-        tables.add(query.get_meta().db_table)
+        if query.get_meta():
+            tables.add(query.get_meta().db_table)
+
         # Gets tables in subquery annotations.
         for annotation in query.annotations.values():
-            if isinstance(annotation, Subquery):
+            if type(annotation) in UNCACHABLE_FUNCS:
-                if hasattr(annotation, "queryset"):
+                raise UncachableQuery
-                    tables.update(_get_tables(db_alias, annotation.queryset.query))
+            for expression in _flatten(annotation):
-                else:
+                if isinstance(expression, Subquery):
-                    tables.update(_get_tables(db_alias, annotation.query))
+                    # Django 2.2 only: no query, only queryset
+                    if not hasattr(expression, 'query'):
+                        tables.update(_get_tables(db_alias, expression.queryset.query))
+                    # Django 3+
+                    else:
+                        tables.update(_get_tables(db_alias, expression.query))
+                elif isinstance(expression, RawSQL):
+                    sql = expression.as_sql(None, None)[0].lower()
+                    tables.update(_get_tables_from_sql(connections[db_alias], sql))
         # Gets tables in WHERE subqueries.
         for subquery in _find_subqueries_in_where(query.where.children):
             tables.update(_get_tables(db_alias, subquery))
@@ -213,6 +239,18 @@ def _get_tables(db_alias, query):
     except IsRawQuery:
         sql = query.get_compiler(db_alias).as_sql()[0].lower()
         tables = _get_tables_from_sql(connections[db_alias], sql)
+    else:
+        # Additional check of the final SQL.
+        # Potentially overlooked tables are added here. Tables may be overlooked by the regular checks
+        # as not all expressions are handled yet. This final check acts as safety net.
+        if cachalot_settings.CACHALOT_FINAL_SQL_CHECK:
+            if compiler:
+                # Access generated SQL stored when caching the query!
+                sql = compiler.__cachalot_generated_sql
+            else:
+                sql = query.get_compiler(db_alias).as_sql()[0].lower()
+            final_check_tables = _get_tables_from_sql(connections[db_alias], sql, enable_quote=True)
+            tables.update(final_check_tables)
 
     if not are_all_cachable(tables):
         raise UncachableQuery
@@ -223,7 +261,7 @@ def _get_table_cache_keys(compiler):
     db_alias = compiler.using
     get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
     return [get_table_cache_key(db_alias, t)
-            for t in _get_tables(db_alias, compiler.query)]
+            for t in _get_tables(db_alias, compiler.query, compiler)]
 
 
 def _invalidate_tables(cache, db_alias, tables):

docs/index.rst

@@ -11,8 +11,8 @@ Caches your Django ORM queries and automatically invalidates them.
 .. image:: http://img.shields.io/pypi/v/django-cachalot.svg?style=flat-square&maxAge=3600
    :target: https://pypi.python.org/pypi/django-cachalot
 
-.. image:: http://img.shields.io/travis/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600
+.. image:: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml/badge.svg
-   :target: https://travis-ci.org/noripyt/django-cachalot
+   :target: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml
 
 .. image:: http://img.shields.io/coveralls/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600
    :target: https://coveralls.io/r/noripyt/django-cachalot?branch=master

docs/limits.rst

@@ -65,11 +65,9 @@ in a multi-processes project, if you use RQ or Celery for instance.
 Filebased
 .........
 
-Filebased, a simple persistent cache implemented in Django, has a small bug
+Filebased, a simple persistent cache implemented in Django, had a small bug
 (`#25501 <https://code.djangoproject.com/ticket/25501>`_):
-it cannot cache some objects, like psycopg2 ranges.
+it cannot cache some objects, like psycopg2 ranges. This bug was fixed in 2015, if you sill use an affected Django version and you use range fields from `django.contrib.postgres`, you need to add the tables using range fields
-If you use range fields from `django.contrib.postgres` and your Django
-version is affected by this bug, you need to add the tables using range fields
 to :ref:`CACHALOT_UNCACHABLE_TABLES`.
 
 .. _MySQL:

docs/quickstart.rst

@@ -4,8 +4,8 @@ Quick start
 Requirements
 ............
 
-- Django 2.0-2.2, 3.0-3.1
+- Django 3.2, 4.1, 4.2
-- Python 3.5-3.9
+- Python 3.7-3.11
 - a cache configured as ``'default'`` with one of these backends:
 
   - `django-redis <https://github.com/niwinz/django-redis>`_
@@ -67,7 +67,7 @@ Settings
      change this setting, you end up on a cache that may contain stale data.
 
 .. |CACHES| replace:: ``CACHES``
-.. _CACHES: https://docs.djangoproject.com/en/2.0/ref/settings/#std:setting-CACHES
+.. _CACHES: https://docs.djangoproject.com/en/dev/ref/settings/#caches
 
 ``CACHALOT_DATABASES``
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -80,7 +80,7 @@ Settings
   engines.
 
 .. |DATABASES| replace:: ``DATABASES``
-.. _DATABASES: https://docs.djangoproject.com/en/2.0/ref/settings/#std:setting-DATABASES
+.. _DATABASES: https://docs.djangoproject.com/en/dev/ref/settings/#databases
 
 ``CACHALOT_TIMEOUT``
 ~~~~~~~~~~~~~~~~~~~~
@@ -117,6 +117,8 @@ Settings
   SQL queries – read :ref:`raw queries limits <Raw SQL queries>` for more info.
 
 
+.. _CACHALOT_ONLY_CACHABLE_TABLES:
+
 ``CACHALOT_ONLY_CACHABLE_TABLES``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -124,12 +126,23 @@ Settings
 :Description:
   Sequence of SQL table names that will be the only ones django-cachalot
   will cache. Only queries with a subset of these tables will be cached.
-  The sequence being empty (as it is by default) doesn’t mean that no table
+  The sequence being empty (as it is by default) does not mean that no table
   can be cached: it disables this setting, so any table can be cached.
   :ref:`CACHALOT_UNCACHABLE_TABLES` has more weight than this:
   if you add a table to both settings, it will never be cached.
   Run ``./manage.py invalidate_cachalot`` after changing this setting.
 
+``CACHALOT_ONLY_CACHABLE_APPS``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Default: ``frozenset()``
+:Description:
+  Sequence of Django apps whose associated models will be appended to
+  :ref:`CACHALOT_ONLY_CACHABLE_TABLES`. The rules between
+  :ref:`CACHALOT_UNCACHABLE_TABLES` and :ref:`CACHALOT_ONLY_CACHABLE_TABLES` still
+  apply as this setting only appends the given Django apps' tables on initial
+  Django setup.
+
 
 .. _CACHALOT_UNCACHABLE_TABLES:
 
@@ -144,6 +157,17 @@ Settings
   some issues, especially during tests.
   Run ``./manage.py invalidate_cachalot`` after changing this setting.
 
+``CACHALOT_UNCACHABLE_APPS``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Default: ``frozenset()``
+:Description:
+  Sequence of Django apps whose associated models will be appended to
+  :ref:`CACHALOT_UNCACHABLE_TABLES`. The rules between
+  :ref:`CACHALOT_UNCACHABLE_TABLES` and :ref:`CACHALOT_ONLY_CACHABLE_TABLES` still
+  apply as this setting only appends the given Django apps' tables on initial
+  Django setup.
+
 ``CACHALOT_ADDITIONAL_TABLES``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -172,6 +196,39 @@ Settings
               Clear your cache after changing this setting (it’s not enough
               to use ``./manage.py invalidate_cachalot``).
 
+``CACHALOT_FINAL_SQL_CHECK``
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Default: ``False``
+:Description:
+    If set to ``True``, the final SQL check will be performed.
+    The `Final SQL check` checks for potentially overlooked tables when looking up involved tables
+    (eg. Ordering by referenced table). See tests for more details
+    (eg. ``test_order_by_field_of_another_table_with_check``).
+
+    Enabling this setting comes with a small performance cost::
+
+        CACHALOT_FINAL_SQL_CHECK=False:
+            mysql      is 1.4× slower then 9.9× faster
+            postgresql is 1.3× slower then 11.7× faster
+            sqlite     is 1.4× slower then 3.0× faster
+            filebased  is 1.4× slower then 9.5× faster
+            locmem     is 1.3× slower then 11.3× faster
+            pylibmc    is 1.4× slower then 8.5× faster
+            pymemcache is 1.4× slower then 7.3× faster
+            redis      is 1.4× slower then 6.8× faster
+
+        CACHALOT_FINAL_SQL_CHECK=True:
+            mysql      is 1.5× slower then 9.0× faster
+            postgresql is 1.3× slower then 10.5× faster
+            sqlite     is 1.4× slower then 2.6× faster
+            filebased  is 1.4× slower then 9.1× faster
+            locmem     is 1.3× slower then 9.9× faster
+            pylibmc    is 1.4× slower then 7.5× faster
+            pymemcache is 1.4× slower then 6.5× faster
+            redis      is 1.5× slower then 6.2× faster
+
+
 
 .. _Command:
 
@@ -201,7 +258,7 @@ Examples:
 Template utils
 ..............
 
-`Caching template fragments <https://docs.djangoproject.com/en/2.0/topics/cache/#template-fragment-caching>`_
+`Caching template fragments <https://docs.djangoproject.com/en/dev/topics/cache/#template-fragment-caching>`_
 can be extremely powerful to speedup a Django application.  However, it often
 means you have to adapt your models to get a relevant cache key, typically
 by adding a timestamp that refers to the last modification of the object.

requirements.txt

@@ -1 +1 @@
-Django>=2.0,<=3.2
+Django>=3.2,<4.3

requirements/tests.txt

@@ -2,7 +2,8 @@
 
 django>=2
 
-psycopg2-binary
+psycopg2
+psycopg
 mysqlclient
 django-redis
 python-memcached

runtests.py

@@ -9,6 +9,6 @@ if __name__ == '__main__':
     django.setup()
     from django.test.runner import DiscoverRunner
     test_runner = DiscoverRunner(verbosity=2, interactive=False)
-    failures = test_runner.run_tests(['cachalot.tests'])
+    failures = test_runner.run_tests(['cachalot.tests', 'cachalot.admin_tests'])
     if failures:
         sys.exit(failures)

runtests_urls.py

@@ -1,6 +1,7 @@
 import debug_toolbar
-from django.conf.urls import url, include
+from django.urls import path, re_path, include
 from django.http import HttpResponse
+from django.contrib import admin
 
 
 def empty_page(request):
@@ -8,6 +9,7 @@ def empty_page(request):
 
 
 urlpatterns = [
-    url(r'^$', empty_page),
+    re_path(r'^$', empty_page),
-    url(r'^__debug__/', include(debug_toolbar.urls)),
+    re_path(r'^__debug__/', include(debug_toolbar.urls)),
+    path('admin/', admin.site.urls),
 ]

settings.py

@@ -12,6 +12,7 @@ DATABASES = {
         'ENGINE': 'django.db.backends.postgresql',
         'NAME': 'cachalot',
         'USER': 'cachalot',
+        'PASSWORD': 'password',
         'HOST': '127.0.0.1',
     },
     'mysql': {
@@ -90,9 +91,13 @@ elif DEFAULT_CACHE_ALIAS == 'pylibmc':
 
 INSTALLED_APPS = [
     'cachalot',
+    'cachalot.admin_tests',
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.postgres',  # Enables the unaccent lookup.
+    'django.contrib.sessions',
+    'django.contrib.admin',
+    'django.contrib.messages',
 ]
 
 MIGRATION_MODULES = {
@@ -104,6 +109,12 @@ TEMPLATES = [
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [],
         'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        }
     },
     {
         'BACKEND': 'django.template.backends.jinja2.Jinja2',
@@ -116,7 +127,11 @@ TEMPLATES = [
     }
 ]
 
-MIDDLEWARE = []
+MIDDLEWARE = [
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+]
 PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher']
 SECRET_KEY = 'it’s not important in tests but we have to set it'
 

setup.py

@@ -17,8 +17,7 @@ setup(
     author='Bertrand Bordage, Andrew Chen Wang',
     author_email='acwangpython@gmail.com',
     url='https://github.com/noripyt/django-cachalot',
-    description='Caches your Django ORM queries '
+    description='Caches your Django ORM queries and automatically invalidates them.',
-                'and automatically invalidates them.',
     long_description=open('README.rst').read(),
     classifiers=[
         'Development Status :: 5 - Production/Stable',
@@ -26,17 +25,15 @@ setup(
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
-        'Framework :: Django :: 2.0',
+        'Framework :: Django :: 3.2',
-        'Framework :: Django :: 2.1',
+        'Framework :: Django :: 4.1',
-        'Framework :: Django :: 2.2',
+        'Framework :: Django :: 4.2',
-        'Framework :: Django :: 3.0',
-        'Framework :: Django :: 3.1',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Internet :: WWW/HTTP',
     ],
     license='BSD',

tox.ini

@@ -1,22 +1,25 @@
 [tox]
 envlist =
-    py{36,37,38,39}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
+    py{37,38,39,310}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
-    py{36,37,38,39}-django3.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
+    py{38,39,310}-django4.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
-    py{36,37,38,39}-django3.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
+    py{38,39,310,311}-django4.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
-    py{36,37,38,39}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
+    py{38,39,310,311}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased},
 
 [testenv]
+passenv = *
 basepython =
-    py36: python3.6
     py37: python3.7
     py38: python3.8
     py39: python3.9
+    py310: python3.10
+    py311: python3.11
 deps =
-    django2.2: Django>=2.2,<2.3
+    django3.2: Django>=3.2,<4.0
-    django3.0: Django>=3.0,<3.1
+    django4.1: Django>=4.1,<4.2
-    django3.1: Django>=3.1,<3.2
+    django4.2: Django>=4.2,<4.3
-    django3.2: Django>=3.2,<3.3
+    djangomain: https://github.com/django/django/archive/main.tar.gz
-    psycopg2-binary
+    psycopg2>=2.9.5,<3.0
+    psycopg
     mysqlclient
     django-redis
     python-memcached
@@ -39,14 +42,17 @@ setenv =
 commands =
     coverage run -a --source=cachalot ./runtests.py
 
-[gh-actions:env]
+[gh-actions]
-PYTHON_VER =
+python =
-    3.6: py36
     3.7: py37
     3.8: py38
     3.9: py39
+    3.10: py310
+    3.11: py311
+
+[gh-actions:env]
 DJANGO =
-    2.2: django2.2
-    3.0: django3.0
-    3.1: django3.1
     3.2: django3.2
+    4.1: django4.1
+    4.2: django4.2
+    main: djangomain

travis-matrix.py

@@ -1,12 +0,0 @@
-#!/usr/bin/env python
-
-from subprocess import check_output
-
-
-envs = check_output(['tox', '-l'])
-
-print('matrix:')
-print('  include:')
-for env in filter(bool, envs.decode().split('\n')):
-    print('    - python: %s' % env[2:5])
-    print('      env: TOXENV=' + env)