diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1957129..6e0aa7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,21 +12,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.5', '3.6', '3.7', '3.8', '3.9'] - django-version: ['2.0', '2.1', '2.2', '3.0', '3.1'] - exclude: - - python-version: '3.5' - django-version: '3.0' - - python-version: '3.5' - django-version: '3.1' - - python-version: '3.8' - django-version: '2.0' - - python-version: '3.8' - django-version: '2.1' - - python-version: '3.9' - django-version: '2.0' - - python-version: '3.9' - django-version: '2.1' + python-version: ['3.6', '3.7', '3.8', '3.9'] + django-version: ['2.2', '3.0', '3.1', '3.2'] services: redis: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 19a86cf..0467b15 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ What’s new in django-cachalot? ============================== +2.4.0 +----- + +- Add support for Django 3.2 (#181) +- Remove enforced system check for Django version (#175) +- Drop support for Django 2.0-2.1 and Python 3.5 (#181) +- Add support for Pymemcache for Django 3.2+ (#181) +- Reverts #157 with proper fix. (#181) + 2.3.5 ----- @@ -286,6 +295,7 @@ Fixed: pk__in=User.objects.filter( pk__in=User.objects.filter( user_permissions__in=Permission.objects.all()))) + - Avoids setting useless cache keys by using table names instead of Django-generated table alias diff --git a/README.rst b/README.rst index aacb561..cbe5292 100644 --- a/README.rst +++ b/README.rst @@ -13,8 +13,8 @@ Documentation: http://django-cachalot.readthedocs.io .. image:: https://img.shields.io/pypi/pyversions/django-cachalot :target: https://django-cachalot.readthedocs.io/en/latest/ -.. image:: https://travis-ci.com/noripyt/django-cachalot.svg?branch=master - :target: https://travis-ci.com/noripyt/django-cachalot +.. image:: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml/badge.svg + :target: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml .. image:: http://img.shields.io/coveralls/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 :target: https://coveralls.io/r/noripyt/django-cachalot?branch=master @@ -39,7 +39,9 @@ Table of Contents: Quickstart ---------- -Cachalot officially supports Python 3.5-3.9 and Django 2.0-2.2, 3.0-3.1 with the databases PostgreSQL, SQLite, and MySQL. +Cachalot officially supports Python 3.6-3.9 and Django 2.2 and 3.0-3.2 with the databases PostgreSQL, SQLite, and MySQL. + +No upper limit is imposed by cachalot. When Django's async ORM or cache is rolled out, there may be huge changes and breakages due to cachalot monkey-patching. Usage ----- diff --git a/cachalot/__init__.py b/cachalot/__init__.py index 63521c1..78326a4 100644 --- a/cachalot/__init__.py +++ b/cachalot/__init__.py @@ -1,4 +1,4 @@ -VERSION = (2, 3, 5) +VERSION = (2, 4, 0) __version__ = ".".join(map(str, VERSION)) default_app_config = "cachalot.apps.CachalotConfig" diff --git a/cachalot/apps.py b/cachalot/apps.py index 87b0ab6..bfcd3d5 100644 --- a/cachalot/apps.py +++ b/cachalot/apps.py @@ -1,5 +1,4 @@ import copyreg -from django import __version__ as django__version__, VERSION as django_version from django.apps import AppConfig from django.conf import settings from django.core.checks import register, Tags, Warning, Error @@ -10,18 +9,6 @@ from .settings import ( SUPPORTED_ONLY) -@register(Tags.compatibility) -def check_django_version(app_configs, **kwargs): - if not (2, 0) <= django_version < (3, 2): - return [Error( - 'Django %s is not compatible with this version of django-cachalot.' - % django__version__, - hint='Refer to the django-cachalot documentation to find ' - 'which versions are compatible.', - id='cachalot.E003')] - return [] - - @register(Tags.caches, Tags.compatibility) def check_cache_compatibility(app_configs, **kwargs): cache = settings.CACHES[cachalot_settings.CACHALOT_CACHE] diff --git a/cachalot/monkey_patch.py b/cachalot/monkey_patch.py index c822877..f152864 100644 --- a/cachalot/monkey_patch.py +++ b/cachalot/monkey_patch.py @@ -147,8 +147,7 @@ def _patch_cursor(): if cachalot_settings.CACHALOT_INVALIDATE_RAW: CursorWrapper.execute = _patch_cursor_execute(CursorWrapper.execute) - CursorWrapper.executemany = \ - _patch_cursor_execute(CursorWrapper.executemany) + CursorWrapper.executemany = _patch_cursor_execute(CursorWrapper.executemany) def _unpatch_cursor(): diff --git a/cachalot/settings.py b/cachalot/settings.py index 4e2e4a2..e1d6383 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -34,6 +34,7 @@ SUPPORTED_CACHE_BACKENDS = { 'django_redis.cache.RedisCache', 'django.core.cache.backends.memcached.MemcachedCache', 'django.core.cache.backends.memcached.PyLibMCCache', + 'django.core.cache.backends.memcached.PyMemcacheCache', } SUPPORTED_ONLY = 'supported_only' diff --git a/cachalot/tests/api.py b/cachalot/tests/api.py index cd44442..88e3154 100644 --- a/cachalot/tests/api.py +++ b/cachalot/tests/api.py @@ -1,3 +1,4 @@ +import os from time import time, sleep from unittest import skipIf @@ -122,13 +123,14 @@ class APITestCase(TestUtilsMixin, TransactionTestCase): def test_get_last_invalidation(self): invalidate() timestamp = get_last_invalidation() - self.assertAlmostEqual(timestamp, time(), delta=0.1) + delta = 0.15 if os.environ.get("CACHE_BACKEND") == "filebased" else 0.1 + self.assertAlmostEqual(timestamp, time(), delta=delta) sleep(0.1) invalidate('cachalot_test') timestamp = get_last_invalidation('cachalot_test') - self.assertAlmostEqual(timestamp, time(), delta=0.1) + self.assertAlmostEqual(timestamp, time(), delta=delta) same_timestamp = get_last_invalidation('cachalot.Test') self.assertEqual(same_timestamp, timestamp) same_timestamp = get_last_invalidation(Test) @@ -136,9 +138,8 @@ class APITestCase(TestUtilsMixin, TransactionTestCase): timestamp = get_last_invalidation('cachalot_testparent') self.assertNotAlmostEqual(timestamp, time(), delta=0.1) - timestamp = get_last_invalidation('cachalot_testparent', - 'cachalot_test') - self.assertAlmostEqual(timestamp, time(), delta=0.1) + timestamp = get_last_invalidation('cachalot_testparent', 'cachalot_test') + self.assertAlmostEqual(timestamp, time(), delta=delta) def test_get_last_invalidation_template_tag(self): # Without arguments diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index cbdd209..d1ccfa2 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -490,8 +490,6 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(permissions8, permissions7) self.assertListEqual(permissions8, self.group__permissions) - @skipIf(django_version < (2, 0), - '`FilteredRelation` was introduced in Django 2.0.') def test_filtered_relation(self): from django.db.models import FilteredRelation @@ -628,8 +626,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): ['test1', 'test2']) def test_having(self): - qs = (User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1)) + qs = (User.objects.annotate(n=Count('user_permissions')).filter(n__gte=1)) self.assert_tables(qs, User, User.user_permissions.through, Permission) self.assert_query_cached(qs, [self.user]) @@ -688,17 +685,21 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(0): self.assertEqual(TestChild.objects.get(), t_child) - @skipIf(django_version < (2, 1), - '`QuerySet.explain()` was introduced in Django 2.1.') def test_explain(self): explain_kwargs = {} if self.is_sqlite: - expected = (r'0 0 0 SCAN TABLE cachalot_test\n' - r'0 0 0 USE TEMP B-TREE FOR ORDER BY') + expected = (r'\d+ 0 0 SCAN TABLE cachalot_test\n' + r'\d+ 0 0 USE TEMP B-TREE FOR ORDER BY') elif self.is_mysql: - expected = ( - r'1 SIMPLE cachalot_test ' - r'(?:None )?ALL None None None None 2 100\.0 Using filesort') + if self.django_version < (3, 1): + expected = ( + r'1 SIMPLE cachalot_test ' + r'(?:None )?ALL None None None None 2 100\.0 Using filesort') + else: + expected = ( + r'-> Sort row IDs: cachalot_test.`name` \(cost=[\d\.]+ rows=\d\)\n ' + r'-> Table scan on cachalot_test \(cost=[\d\.]+ rows=\d\)\n' + ) else: explain_kwargs.update( analyze=True, @@ -711,8 +712,8 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): r' Sort Key: name\n' r' Sort Method: quicksort Memory: \d+kB\n' r' -> Seq Scan on cachalot_test %s\n' - r'Planning time: [\d\.]+ ms\n' - r'Execution time: [\d\.]+ ms$') % (operation_detail, + r'Planning Time: [\d\.]+ ms\n' + r'Execution Time: [\d\.]+ ms$') % (operation_detail, operation_detail) with self.assertNumQueries( 2 if self.is_mysql and django_version[0] < 3 @@ -924,9 +925,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs, after=1 if self.is_sqlite else 0) def test_float(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', a_float=0.123456789) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', a_float=12345.6789) with self.assertNumQueries(1): data1 = list(Test.objects.values_list('a_float', flat=True).filter( @@ -945,9 +946,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(a_float=0.123456789) def test_decimal(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', a_decimal=Decimal('123.45')) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', a_decimal=Decimal('12.3')) qs = Test.objects.values_list('a_decimal', flat=True).filter( @@ -961,9 +962,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(a_decimal=Decimal('123.45')) def test_ipv4_address(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', ip='127.0.0.1') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', ip='192.168.0.1') qs = Test.objects.values_list('ip', flat=True).filter( @@ -977,9 +978,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(ip='127.0.0.1') def test_ipv6_address(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', ip='2001:db8:a0b:12f0::1/64') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', ip='2001:db8:0:85a3::ac1f:8001') qs = Test.objects.values_list('ip', flat=True).filter( @@ -994,9 +995,9 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(ip='2001:db8:0:85a3::ac1f:8001') def test_duration(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', duration=datetime.timedelta(30)) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', duration=datetime.timedelta(60)) qs = Test.objects.values_list('duration', flat=True).filter( @@ -1011,10 +1012,10 @@ class ParameterTypeTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(duration=datetime.timedelta(30)) def test_uuid(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1', uuid='1cc401b7-09f4-4520-b8d0-c267576d196b') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', uuid='ebb3b6e1-1737-4321-93e3-4c35d61ff491') diff --git a/cachalot/tests/settings.py b/cachalot/tests/settings.py index 9385bc1..09d4503 100644 --- a/cachalot/tests/settings.py +++ b/cachalot/tests/settings.py @@ -40,7 +40,7 @@ class SettingsTestCase(TestUtilsMixin, TransactionTestCase): list(Test.objects.all()) with self.settings(CACHALOT_ENABLED=False): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t = Test.objects.create(name='test') with self.assertNumQueries(1): data = list(Test.objects.all()) diff --git a/cachalot/tests/test_utils.py b/cachalot/tests/test_utils.py index 3de3c5a..4bd5fe4 100644 --- a/cachalot/tests/test_utils.py +++ b/cachalot/tests/test_utils.py @@ -11,7 +11,8 @@ class TestUtilsMixin: self.is_sqlite = connection.vendor == 'sqlite' self.is_mysql = connection.vendor == 'mysql' self.is_postgresql = connection.vendor == 'postgresql' - self.force_repoen_connection() + self.django_version = DJANGO_VERSION + self.force_reopen_connection() # TODO: Remove this workaround when this issue is fixed: # https://code.djangoproject.com/ticket/29494 @@ -26,7 +27,7 @@ class TestUtilsMixin: with connection.cursor() as cursor: cursor.execute(sql) - def force_repoen_connection(self): + def force_reopen_connection(self): if connection.vendor in ('mysql', 'postgresql'): # We need to reopen the connection or Django # will execute an extra SQL request below. @@ -60,29 +61,3 @@ class TestUtilsMixin: assert_function(data2, data1) if result is not None: assert_function(data2, result) - - def is_dj_21_below_and_is_sqlite(self): - """ - Checks if Django 2.1 or lower and if SQLite is the DB - Django 2.1 and lower had two queries on SQLite DBs: - - After an insertion, e.g. Test.objects.create(name="asdf"), - SQLite returns the queries: - [{'sql': 'INSERT INTO "cachalot_test" ("name") VALUES (\'asd\')', 'time': '0.001'}, {'sql': 'BEGIN', 'time': '0.000'}] - - This can be seen with django.db import connection; print(connection.queries) - In Django 2.2 and above, the latter was removed. - - :return: bool is Django 2.1 or below and is SQLite the DB - """ - if not self.is_sqlite: - # Immediately know if SQLite - return False - if DJANGO_VERSION[0] < 2: - # Takes Django 0 and 1 out of the picture - return True - else: - if DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] < 2: - # Takes Django 2.0-2.1 out - return True - return False diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 05c1b05..982bb63 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -26,21 +26,21 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t1 = Test.objects.create(name='test1') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data2 = list(Test.objects.all()) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t3 = Test.objects.create(name='test3') with self.assertNumQueries(1): data3 = list(Test.objects.all()) self.assertListEqual(data2, [t1, t2]) self.assertListEqual(data3, [t1, t2, t3]) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t3_copy = Test.objects.create(name='test3') self.assertNotEqual(t3_copy, t3) with self.assertNumQueries(1): @@ -126,12 +126,12 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['test%02d' % (i // 2) for i in range(2, 22)]) def test_update(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t = Test.objects.create(name='test1') with self.assertNumQueries(1): t1 = Test.objects.get() - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t.name = 'test2' t.save() with self.assertNumQueries(1): @@ -139,21 +139,21 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(t1.name, 'test1') self.assertEqual(t2.name, 'test2') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.update(name='test3') with self.assertNumQueries(1): t3 = Test.objects.get() self.assertEqual(t3.name, 'test3') def test_delete(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t1 = Test.objects.create(name='test1') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data1 = list(Test.objects.values_list('name', flat=True)) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t2.delete() with self.assertNumQueries(1): data2 = list(Test.objects.values_list('name', flat=True)) @@ -176,7 +176,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.create(name='test') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): self.assertTrue(Test.objects.create()) def test_invalidate_count(self): @@ -314,22 +314,22 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): u = User.objects.create_user('test') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 0) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', owner=u) with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 1) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test3') with self.assertNumQueries(1): self.assertEqual(User.objects.aggregate(n=Count('test'))['n'], 1) @@ -339,13 +339,13 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(User.objects.annotate(n=Count('test')).order_by('pk')) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(User.objects.annotate(n=Count('test')).order_by('pk')) self.assertListEqual(data2, []) - with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): + with self.assertNumQueries(2): user1 = User.objects.create_user('user1') user2 = User.objects.create_user('user2') with self.assertNumQueries(1): @@ -353,7 +353,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data3, [user1, user2]) self.assertListEqual([u.n for u in data3], [0, 0]) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test2', owner=user1) with self.assertNumQueries(1): data4 = list(User.objects.annotate(n=Count('test')).order_by('pk')) @@ -581,7 +581,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.select_related('owner')) self.assertListEqual(data1, []) - with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): + with self.assertNumQueries(2): u1 = User.objects.create_user('test1') u2 = User.objects.create_user('test2') with self.assertNumQueries(1): @@ -615,7 +615,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): .prefetch_related('owner__groups__permissions')) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.select_related('owner') @@ -623,7 +623,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, [t1]) self.assertEqual(data2[0].owner, None) - with self.assertNumQueries(4 if self.is_dj_21_below_and_is_sqlite() else 2): + with self.assertNumQueries(2): u = User.objects.create_user('user') t1.owner = u t1.save() @@ -635,8 +635,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(list(data3[0].owner.groups.all()), []) with self.assertNumQueries( - 9 if self.is_dj_21_below_and_is_sqlite() - else 8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2 + 8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2 else 4 if self.is_postgresql and DJANGO_VERSION[0] > 2 else 4 if self.is_mysql and DJANGO_VERSION[0] > 2 else 6 @@ -656,7 +655,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(list(groups[0].permissions.all()), permissions) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data5 = list(Test.objects.select_related('owner') @@ -670,13 +669,13 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): for p in g.permissions.all()] self.assertListEqual(data5_permissions, permissions) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): permissions[0].save() with self.assertNumQueries(1): list(Test.objects.select_related('owner') .prefetch_related('owner__groups__permissions')) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): group.name = 'modified_test_group' group.save() with self.assertNumQueries(2): @@ -685,7 +684,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): g = list(data6[0].owner.groups.all())[0] self.assertEqual(g.name, 'modified_test_group') - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): User.objects.update(username='modified_user') with self.assertNumQueries(2): @@ -739,8 +738,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, [t1]) self.assertListEqual([o.username_length for o in data2], [4]) - admin = User.objects.create_superuser('admin', - 'admin@test.me', 'password') + admin = User.objects.create_superuser('admin', 'admin@test.me', 'password') with self.assertNumQueries(1): data3 = list(Test.objects.extra( @@ -757,42 +755,35 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual([o.username_length for o in data4], [4, 5]) def test_invalidate_having(self): + def _query(): + return User.objects.annotate(n=Count('user_permissions')).filter(n__gte=1) + with self.assertNumQueries(1): - data1 = list(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1)) + data1 = list(_query()) self.assertListEqual(data1, []) u = User.objects.create_user('user') - with self.assertNumQueries(1): - data2 = list(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1)) + data2 = list(_query()) self.assertListEqual(data2, []) p = Permission.objects.first() p.save() - with self.assertNumQueries(1): - data3 = list(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1)) + data3 = list(_query()) self.assertListEqual(data3, []) u.user_permissions.add(p) - with self.assertNumQueries(1): - data3 = list(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1)) + data3 = list(_query()) self.assertListEqual(data3, [u]) with self.assertNumQueries(1): - self.assertEqual(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1).count(), 1) + self.assertEqual(_query().count(), 1) u.user_permissions.clear() - with self.assertNumQueries(1): - self.assertEqual(User.objects.annotate(n=Count('user_permissions')) - .filter(n__gte=1).count(), 0) + self.assertEqual(_query().count(), 0) def test_invalidate_extra_where(self): sql_condition = ("owner_id IN " @@ -801,47 +792,43 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.extra(where=[sql_condition])) self.assertListEqual(data1, []) - admin = User.objects.create_superuser('admin', - 'admin@test.me', 'password') - + admin = User.objects.create_superuser('admin', 'admin@test.me', 'password') with self.assertNumQueries(1): data2 = list(Test.objects.extra(where=[sql_condition])) self.assertListEqual(data2, []) t = Test.objects.create(name='test', owner=admin) - with self.assertNumQueries(1): data3 = list(Test.objects.extra(where=[sql_condition])) self.assertListEqual(data3, [t]) admin.username = 'modified' admin.save() - with self.assertNumQueries(1): data4 = list(Test.objects.extra(where=[sql_condition])) self.assertListEqual(data4, []) def test_invalidate_extra_tables(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): User.objects.create_user('user1') with self.assertNumQueries(1): data1 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data2, [t1]) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data3 = list(Test.objects.all().extra(tables=['auth_user'])) self.assertListEqual(data3, [t1, t2]) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): User.objects.create_user('user2') with self.assertNumQueries(1): data4 = list(Test.objects.all().extra(tables=['auth_user'])) @@ -871,7 +858,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertEqual(TestChild.objects.get(), t_child) - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): TestParent.objects.filter(pk=t_child.pk).update(name='modified') with self.assertNumQueries(1): @@ -879,7 +866,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(modified_t_child.pk, t_child.pk) self.assertEqual(modified_t_child.name, 'modified') - with self.assertNumQueries(3 if self.is_dj_21_below_and_is_sqlite() else 2): + with self.assertNumQueries(2): TestChild.objects.filter(pk=t_child.pk).update(name='modified2') with self.assertNumQueries(1): @@ -897,7 +884,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.execute( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test1', %s)", [1 if self.is_dj_21_below_and_is_sqlite() else True]) + "VALUES ('test1', %s)", [True]) with self.assertNumQueries(1): self.assertListEqual( @@ -908,7 +895,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.execute( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test2', %s)", [1 if self.is_dj_21_below_and_is_sqlite() else True]) + "VALUES ('test2', %s)", [True]) with self.assertNumQueries(1): self.assertListEqual( @@ -919,7 +906,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with connection.cursor() as cursor: cursor.executemany( "INSERT INTO cachalot_test (name, public) " - "VALUES ('test3', %s)", [[1 if self.is_dj_21_below_and_is_sqlite() else True]]) + "VALUES ('test3', %s)", [[True]]) with self.assertNumQueries(1): self.assertListEqual( @@ -927,7 +914,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['test1', 'test2', 'test3']) def test_raw_update(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test') with self.assertNumQueries(1): self.assertListEqual( @@ -944,7 +931,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): ['new name']) def test_raw_delete(self): - with self.assertNumQueries(2 if self.is_dj_21_below_and_is_sqlite() else 1): + with self.assertNumQueries(1): Test.objects.create(name='test') with self.assertNumQueries(1): self.assertListEqual( @@ -1026,7 +1013,7 @@ class DatabaseCommandTestCase(TestUtilsMixin, TransactionTestCase): call_command('flush', verbosity=0, interactive=False) - self.force_repoen_connection() + self.force_reopen_connection() with self.assertNumQueries(1): self.assertListEqual(list(Test.objects.all()), []) @@ -1038,7 +1025,7 @@ class DatabaseCommandTestCase(TestUtilsMixin, TransactionTestCase): call_command('loaddata', 'cachalot/tests/loaddata_fixture.json', verbosity=0) - self.force_repoen_connection() + self.force_reopen_connection() with self.assertNumQueries(1): self.assertListEqual([t.name for t in Test.objects.all()], diff --git a/cachalot/utils.py b/cachalot/utils.py index 962cf4d..1d19e41 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -100,6 +100,24 @@ def _get_tables_from_sql(connection, lowercased_sql): if t in lowercased_sql} +def _find_rhs_lhs_subquery(side): + h_class = side.__class__ + if h_class is Query: + return side + elif h_class is QuerySet: + return side.query + elif h_class in (Subquery, Exists): # Subquery allows QuerySet & Query + try: + return side.query.query if side.query.__class__ is QuerySet else side.query + except AttributeError: # TODO Remove try/except closure after drop Django 2.2 + try: + return side.queryset.query + except AttributeError: + return None + elif h_class in UNCACHABLE_FUNCS: + raise UncachableQuery + + def _find_subqueries_in_where(children): for child in children: child_class = child.__class__ @@ -108,17 +126,15 @@ def _find_subqueries_in_where(children): yield grand_child elif child_class is ExtraWhere: raise IsRawQuery - elif child_class in (NothingNode, Subquery, Exists): + elif child_class is NothingNode: pass else: - rhs = child.rhs - rhs_class = rhs.__class__ - if rhs_class is Query: + rhs = _find_rhs_lhs_subquery(child.rhs) + if rhs is not None: yield rhs - elif rhs_class is QuerySet: - yield rhs.query - elif rhs_class in UNCACHABLE_FUNCS: - raise UncachableQuery + lhs = _find_rhs_lhs_subquery(child.lhs) + if lhs is not None: + yield lhs def is_cachable(table): @@ -158,18 +174,19 @@ def _get_tables(db_alias, query): # Gets tables in subquery annotations. for annotation in query.annotations.values(): if isinstance(annotation, Subquery): - # Django 2.2+ removed queryset in favor of simply using query - try: + if hasattr(annotation, "queryset"): tables.update(_get_tables(db_alias, annotation.queryset.query)) - except AttributeError: + else: tables.update(_get_tables(db_alias, annotation.query)) # Gets tables in WHERE subqueries. for subquery in _find_subqueries_in_where(query.where.children): tables.update(_get_tables(db_alias, subquery)) # Gets tables in HAVING subqueries. if isinstance(query, AggregateQuery): - tables.update( - _get_tables_from_sql(connections[db_alias], query.subquery)) + try: + tables.update(_get_tables_from_sql(connections[db_alias], query.subquery)) + except TypeError: # For Django 3.2+ + tables.update(_get_tables(db_alias, query.inner_query)) # Gets tables in combined queries # using `.union`, `.intersection`, or `difference`. if query.combined_queries: diff --git a/requirements.txt b/requirements.txt index 82a39ab..afe2802 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Django>=2 \ No newline at end of file +Django>=2.0,<=3.2 diff --git a/requirements/tests.txt b/requirements/tests.txt index b8fc84a..4e17260 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -6,6 +6,7 @@ psycopg2-binary mysqlclient django-redis python-memcached +pymemcache pylibmc pytz diff --git a/settings.py b/settings.py index 13d4c6a..0c1a89f 100644 --- a/settings.py +++ b/settings.py @@ -1,5 +1,7 @@ import os +from django import VERSION as __DJ_V + DATABASES = { 'sqlite3': { @@ -29,11 +31,9 @@ for alias in DATABASES: DATABASES[alias]['TEST'] = {'NAME': test_db_name} DATABASES['default'] = DATABASES.pop(os.environ.get('DB_ENGINE', 'sqlite3')) - - +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" DATABASE_ROUTERS = ['cachalot.tests.db_router.PostgresRouter'] - CACHES = { 'redis': { 'BACKEND': 'django_redis.cache.RedisCache', @@ -46,7 +46,9 @@ CACHES = { }, }, 'memcached': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'BACKEND': 'django.core.cache.backends.memcached.' + + ('PyMemcacheCache' if __DJ_V[0] > 2 + and (__DJ_V[1] > 1 or __DJ_V[0] > 3) else 'MemcachedCache'), 'LOCATION': '127.0.0.1:11211', }, 'locmem': { @@ -86,7 +88,6 @@ if DEFAULT_CACHE_ALIAS == 'memcached' and 'pylibmc' in CACHES: elif DEFAULT_CACHE_ALIAS == 'pylibmc': del CACHES['memcached'] - INSTALLED_APPS = [ 'cachalot', 'django.contrib.auth', @@ -94,12 +95,10 @@ INSTALLED_APPS = [ 'django.contrib.postgres', # Enables the unaccent lookup. ] - MIGRATION_MODULES = { 'cachalot': 'cachalot.tests.migrations', } - TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', @@ -117,17 +116,13 @@ TEMPLATES = [ } ] - MIDDLEWARE = [] PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher'] SECRET_KEY = 'it’s not important in tests but we have to set it' - -USE_TZ = False # Time zones are not supported by MySQL, - # we only enable it in tests when needed. +USE_TZ = False # Time zones are not supported by MySQL, we only enable it in tests when needed. TIME_ZONE = 'UTC' - CACHALOT_ENABLED = True # diff --git a/tox.ini b/tox.ini index 11a5611..f5e13a0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,28 +1,26 @@ [tox] envlist = - py{35,36,37}-django2.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{35,36,37}-django2.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{35,36,37,38,39}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{36,37,38,39}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{36,37,38,39}-django3.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{36,37,38,39}-django3.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{36,37,38,39}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] basepython = - py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 deps = - django2.0: Django>=2.0,<2.1 - django2.1: Django>=2.1,<2.2 django2.2: Django>=2.2,<2.3 django3.0: Django>=3.0,<3.1 django3.1: Django>=3.1,<3.2 + django3.2: Django>=3.2,<3.3 psycopg2-binary mysqlclient django-redis python-memcached + pymemcache pylibmc pytz Jinja2 @@ -43,14 +41,12 @@ commands = [gh-actions:env] PYTHON_VER = - 3.5: py35 3.6: py36 3.7: py37 3.8: py38 3.9: py39 DJANGO = - 2.0: django2.0 - 2.1: django2.1 2.2: django2.2 3.0: django3.0 3.1: django3.1 + 3.2: django3.2