From d192bae22c07c5b0735cae19751bcdc23b05db4a Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Tue, 4 Apr 2023 15:14:29 -0400 Subject: [PATCH 1/6] Add Django 4.1-4.2 and Python 3.11 support (Fixes #228) * Increment version to 4.5.0 --- .github/workflows/ci.yml | 12 ++++++------ .gitignore | 1 + CHANGELOG.rst | 5 +++++ README.rst | 2 +- cachalot/__init__.py | 10 ++-------- docs/index.rst | 4 ++-- docs/quickstart.rst | 4 ++-- requirements.txt | 2 +- requirements/tests.txt | 3 ++- setup.py | 5 +++-- tox.ini | 16 ++++++++++------ 11 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd1d6d3..ed666d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,13 +12,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - django-version: ['2.2', '3.2', '4.0'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + django-version: ['3.2', '4.1', '4.2'] exclude: - - python-version: '3.10' - django-version: '2.2' - python-version: '3.7' - django-version: '4.0' + django-version: '4.1' + - python-version: '3.7' + django-version: '4.2' services: redis: @@ -70,7 +70,7 @@ jobs: ${{ matrix.python-version }}-v1- - name: Install dependencies run: | - sudo apt-get install -y libmemcached-dev zlib1g-dev + sudo apt-get install -y libmemcached-dev zlib1g-dev libpq-dev python -m pip install --upgrade pip wheel python -m pip install tox tox-gh-actions coveralls - name: Tox Test diff --git a/.gitignore b/.gitignore index c0c219b..45e07f2 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ coverage.xml # Django stuff: *.log local_settings.py +*.sqlite3 db.sqlite3 db.sqlite3-journal diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 16e8e81..fe0f555 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ What’s new in django-cachalot? ============================== +2.5.0 +----- + +- Dropped Django 3.2 support. Added Django 4.1-4.2 and Python 3.11 support (#229) + 2.4.5 ----- diff --git a/README.rst b/README.rst index 48fc720..1928f98 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ Table of Contents: Quickstart ---------- -Cachalot officially supports Python 3.7-3.10 and Django 2.2, 3.2, and 4.0 with the databases PostgreSQL, SQLite, and MySQL. +Cachalot officially supports Python 3.7-3.11 and Django 3.2, 4.1, 4.2 with the databases PostgreSQL, SQLite, and MySQL. Note: an upper limit on Django version is set for your safety. Please do not ignore it. diff --git a/cachalot/__init__.py b/cachalot/__init__.py index a14f00d..f40c7d4 100644 --- a/cachalot/__init__.py +++ b/cachalot/__init__.py @@ -1,10 +1,4 @@ -VERSION = (2, 4, 5) +VERSION = (2, 5, 0) __version__ = ".".join(map(str, VERSION)) -try: - from django import VERSION as DJANGO_VERSION - - if DJANGO_VERSION < (3, 2): - default_app_config = "cachalot.apps.CachalotConfig" -except ImportError: # pragma: no cover - default_app_config = "cachalot.apps.CachalotConfig" +default_app_config = "cachalot.apps.CachalotConfig" diff --git a/docs/index.rst b/docs/index.rst index fedf53f..46051d0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,8 +11,8 @@ Caches your Django ORM queries and automatically invalidates them. .. image:: http://img.shields.io/pypi/v/django-cachalot.svg?style=flat-square&maxAge=3600 :target: https://pypi.python.org/pypi/django-cachalot -.. image:: http://img.shields.io/travis/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 - :target: https://travis-ci.org/noripyt/django-cachalot +.. image:: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml/badge.svg + :target: https://github.com/noripyt/django-cachalot/actions/workflows/ci.yml .. image:: http://img.shields.io/coveralls/noripyt/django-cachalot/master.svg?style=flat-square&maxAge=3600 :target: https://coveralls.io/r/noripyt/django-cachalot?branch=master diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9f911cb..836f15b 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -4,8 +4,8 @@ Quick start Requirements ............ -- Django 2.2, 3.2, 4.0 -- Python 3.7-3.10 +- Django 3.2, 4.1, 4.2 +- Python 3.7-3.11 - a cache configured as ``'default'`` with one of these backends: - `django-redis `_ diff --git a/requirements.txt b/requirements.txt index 9e8cdd1..316ec43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Django>=2.2,<4.1 +Django>=3.2,<4.3 diff --git a/requirements/tests.txt b/requirements/tests.txt index 4e17260..8503267 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -2,7 +2,8 @@ django>=2 -psycopg2-binary +psycopg2 +psycopg mysqlclient django-redis python-memcached diff --git a/setup.py b/setup.py index 8f1a12a..a788af3 100755 --- a/setup.py +++ b/setup.py @@ -25,14 +25,15 @@ setup( 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', - 'Framework :: Django :: 2.2', 'Framework :: Django :: 3.2', - 'Framework :: Django :: 4.0', + 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Internet :: WWW/HTTP', ], license='BSD', diff --git a/tox.ini b/tox.ini index 780eb86..119530c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] envlist = - py{37,38,39}-django2.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{37,38,39,310}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{38,39,310}-django4.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{38,39,310}-django4.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{38,39,310,311}-django4.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{38,39,310}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] @@ -11,12 +11,15 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 deps = django2.2: Django>=2.2,<2.3 django3.2: Django>=3.2,<4.0 - django4.0: Django>=4.0,<4.1 + django4.1: Django>=4.1,<4.2 + django4.2: Django>=4.2,<4.3 djangomain: https://github.com/django/django/archive/main.tar.gz - psycopg2-binary>=2.8,<2.9 + psycopg2>=2.8,<2.9 + psycopg mysqlclient django-redis python-memcached @@ -45,10 +48,11 @@ python = 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 [gh-actions:env] DJANGO = - 2.2: django2.2 3.2: django3.2 - 4.0: django4.0 + 4.1: django4.1 + 4.2: django4.2 main: djangomain From 1d0a06a9ab9bdfbcfa1f22bbc86137f5596dde46 Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Tue, 4 Apr 2023 15:28:15 -0400 Subject: [PATCH 2/6] Remove Python 2 and Django 2.2 code --- CHANGELOG.rst | 2 +- cachalot/api.py | 36 ++++++++--------------- cachalot/settings.py | 4 --- cachalot/tests/migrations/0001_initial.py | 16 ++-------- cachalot/utils.py | 15 +--------- 5 files changed, 16 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d34b3d8..682cc55 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ What’s new in django-cachalot? 2.6.0 ----- -- Dropped Django 2.2 and 4.0 support. Added Django 4.2 and Python 3.11 support (#229) +- Dropped Django 2.2 and 4.0 support. Added Django 4.2 and Python 3.11 support. Added psycopg support (#229) 2.5.3 ----- diff --git a/cachalot/api.py b/cachalot/api.py index b451891..d50b277 100644 --- a/cachalot/api.py +++ b/cachalot/api.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from typing import Any, Optional, Tuple, Union from django.apps import apps from django.conf import settings @@ -45,7 +46,11 @@ def _get_tables(tables_or_models): else table_or_model._meta.db_table) -def invalidate(*tables_or_models, **kwargs): +def invalidate( + *tables_or_models: Tuple[Union[str, Any], ...], + cache_alias: Optional[str] = None, + db_alias: Optional[str] = None, +) -> None: """ Clears what was cached by django-cachalot implying one or more SQL tables or models from ``tables_or_models``. @@ -62,19 +67,9 @@ def invalidate(*tables_or_models, **kwargs): (or a combination) :type tables_or_models: tuple of strings or models :arg cache_alias: Alias from the Django ``CACHES`` setting - :type cache_alias: string or NoneType :arg db_alias: Alias from the Django ``DATABASES`` setting - :type db_alias: string or NoneType :returns: Nothing - :rtype: NoneType """ - # TODO: Replace with positional arguments when we drop Python 2 support. - cache_alias = kwargs.pop('cache_alias', None) - db_alias = kwargs.pop('db_alias', None) - for k in kwargs: - raise TypeError( - "invalidate() got an unexpected keyword argument '%s'" % k) - send_signal = False invalidated = set() for cache_alias, db_alias, tables in _cache_db_tables_iterator( @@ -90,7 +85,11 @@ def invalidate(*tables_or_models, **kwargs): post_invalidation.send(table, db_alias=db_alias) -def get_last_invalidation(*tables_or_models, **kwargs): +def get_last_invalidation( + *tables_or_models: Tuple[Union[str, Any], ...], + cache_alias: Optional[str] = None, + db_alias: Optional[str] = None, +) -> float: """ Returns the timestamp of the most recent invalidation of the given ``tables_or_models``. If ``tables_or_models`` is not specified, @@ -106,19 +105,9 @@ def get_last_invalidation(*tables_or_models, **kwargs): (or a combination) :type tables_or_models: tuple of strings or models :arg cache_alias: Alias from the Django ``CACHES`` setting - :type cache_alias: string or NoneType :arg db_alias: Alias from the Django ``DATABASES`` setting - :type db_alias: string or NoneType :returns: The timestamp of the most recent invalidation - :rtype: float """ - # TODO: Replace with positional arguments when we drop Python 2 support. - cache_alias = kwargs.pop('cache_alias', None) - db_alias = kwargs.pop('db_alias', None) - for k in kwargs: - raise TypeError("get_last_invalidation() got an unexpected " - "keyword argument '%s'" % k) - last_invalidation = 0.0 for cache_alias, db_alias, tables in _cache_db_tables_iterator( list(_get_tables(tables_or_models)), cache_alias, db_alias): @@ -134,7 +123,7 @@ def get_last_invalidation(*tables_or_models, **kwargs): @contextmanager -def cachalot_disabled(all_queries=False): +def cachalot_disabled(all_queries: bool = False): """ Context manager for temporarily disabling cachalot. If you evaluate the same queryset a second time, @@ -158,7 +147,6 @@ def cachalot_disabled(all_queries=False): the original and duplicate query. :arg all_queries: Any query, including already evaluated queries, are re-evaluated. - :type all_queries: bool """ was_enabled = getattr(LOCAL_STORAGE, "cachalot_enabled", cachalot_settings.CACHALOT_ENABLED) LOCAL_STORAGE.cachalot_enabled = False diff --git a/cachalot/settings.py b/cachalot/settings.py index a736465..2bf5c81 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -9,8 +9,6 @@ SUPPORTED_DATABASE_ENGINES = { 'django.db.backends.sqlite3', 'django.db.backends.postgresql', 'django.db.backends.mysql', - # TODO: Remove when we drop Django 2.x support. - 'django.db.backends.postgresql_psycopg2', # GeoDjango 'django.contrib.gis.db.backends.spatialite', @@ -20,8 +18,6 @@ SUPPORTED_DATABASE_ENGINES = { # django-transaction-hooks 'transaction_hooks.backends.sqlite3', 'transaction_hooks.backends.postgis', - # TODO: Remove when we drop Django 2.x support. - 'transaction_hooks.backends.postgresql_psycopg2', 'transaction_hooks.backends.mysql', # django-prometheus wrapped engines diff --git a/cachalot/tests/migrations/0001_initial.py b/cachalot/tests/migrations/0001_initial.py index 45b4f87..f2f97df 100644 --- a/cachalot/tests/migrations/0001_initial.py +++ b/cachalot/tests/migrations/0001_initial.py @@ -2,7 +2,7 @@ from django import VERSION as DJANGO_VERSION from django.conf import settings from django.contrib.postgres.fields import ( ArrayField, HStoreField, IntegerRangeField, - DateRangeField, DateTimeRangeField) + DateRangeField, DateTimeRangeField, DecimalRangeField) from django.contrib.postgres.operations import ( HStoreExtension, UnaccentExtension) from django.db import models, migrations @@ -21,19 +21,6 @@ def extra_regular_available_fields(): def extra_postgres_available_fields(): fields = [] - try: - # TODO Remove when Dj31 support is dropped - from django.contrib.postgres.fields import FloatRangeField - fields.append(('float_range', FloatRangeField(null=True, blank=True))) - except ImportError: - pass - - try: - # TODO Add to module import when Dj31 is dropped - from django.contrib.postgres.fields import DecimalRangeField - fields.append(('decimal_range', DecimalRangeField(null=True, blank=True))) - except ImportError: - pass # Future proofing with Django 40 deprecation if DJANGO_VERSION[0] < 4: @@ -103,6 +90,7 @@ class Migration(migrations.Migration): ('int_range', IntegerRangeField(null=True, blank=True)), ('date_range', DateRangeField(null=True, blank=True)), ('datetime_range', DateTimeRangeField(null=True, blank=True)), + ('decimal_range', DecimalRangeField(null=True, blank=True)) ] + extra_postgres_available_fields(), ), migrations.RunSQL('CREATE TABLE cachalot_unmanagedmodel ' diff --git a/cachalot/utils.py b/cachalot/utils.py index 9df2a74..e136e9e 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -35,13 +35,6 @@ CACHABLE_PARAM_TYPES = { } UNCACHABLE_FUNCS = {Now, TransactionNow} -try: - # TODO Drop after Dj30 drop - from django.contrib.postgres.fields.jsonb import JsonAdapter - CACHABLE_PARAM_TYPES.update((JsonAdapter,)) -except ImportError: - pass - try: from psycopg2 import Binary from psycopg2.extras import ( @@ -131,13 +124,7 @@ def _find_rhs_lhs_subquery(side): elif h_class is QuerySet: return side.query elif h_class in (Subquery, Exists): # Subquery allows QuerySet & Query - try: - return side.query.query if side.query.__class__ is QuerySet else side.query - except AttributeError: # TODO Remove try/except closure after drop Django 2.2 - try: - return side.queryset.query - except AttributeError: - return None + return side.query.query if side.query.__class__ is QuerySet else side.query elif h_class in UNCACHABLE_FUNCS: raise UncachableQuery From b2d9de29975f38445fc4af1db04e087fad733f98 Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Tue, 4 Apr 2023 15:30:23 -0400 Subject: [PATCH 3/6] Fix CI supported versions --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04eeaa0..c0e7e2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,10 @@ on: pull_request: branches: [ master ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest @@ -15,6 +19,10 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] django-version: ['3.2', '4.1', '4.2'] exclude: + - python-version: '3.11' + django-version: '3.2' + - python-version: '3.11' + django-version: '4.1' - python-version: '3.7' django-version: '4.1' - python-version: '3.7' From 27eaaa57cca219f77b04b97fbe45856e1c5009eb Mon Sep 17 00:00:00 2001 From: Jack Linke <73554672+jacklinke@users.noreply.github.com> Date: Wed, 24 May 2023 21:35:03 -0400 Subject: [PATCH 4/6] Correct problems with tests (#234) --- CHANGELOG.rst | 8 ++++- cachalot/tests/models.py | 2 +- cachalot/tests/multi_db.py | 25 ++------------ cachalot/tests/read.py | 35 +++++++++---------- cachalot/tests/test_utils.py | 4 --- cachalot/tests/thread_safety.py | 13 +++---- cachalot/tests/transaction.py | 50 ++++++++++++++++++++------- cachalot/tests/write.py | 60 +++++++++++++++++++++++---------- tox.ini | 5 ++- 9 files changed, 115 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 682cc55..b512e95 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,14 +4,20 @@ What’s new in django-cachalot? 2.6.0 ----- -- Dropped Django 2.2 and 4.0 support. Added Django 4.2 and Python 3.11 support. Added psycopg support (#229) +- Dropped Django 2.2 and 4.0 support +- Added Django 4.2 and Python 3.11 support +- Added psycopg support (#229) +- Updated tests to account for the `BEGIN` and `COMMIT` query changes in Django 4.2 +- Standardized django version comparisons in tests 2.5.3 ----- + - Verify get_meta isn't none before requesting db_table (#225 #226) 2.5.2 ----- + - Added Django 4.1 support (#217) 2.5.1 diff --git a/cachalot/tests/models.py b/cachalot/tests/models.py index 8c48640..5031328 100644 --- a/cachalot/tests/models.py +++ b/cachalot/tests/models.py @@ -58,7 +58,7 @@ class PostgresModel(Model): null=True, blank=True) hstore = HStoreField(null=True, blank=True) - if DJANGO_VERSION[0] < 4: + if DJANGO_VERSION < (4, 0): from django.contrib.postgres.fields import JSONField json = JSONField(null=True, blank=True) diff --git a/cachalot/tests/multi_db.py b/cachalot/tests/multi_db.py index 8f92362..e32ef49 100644 --- a/cachalot/tests/multi_db.py +++ b/cachalot/tests/multi_db.py @@ -1,6 +1,5 @@ from unittest import skipIf -from django import VERSION as DJANGO_VERSION from django.conf import settings from django.db import DEFAULT_DB_ALIAS, connections, transaction from django.test import TransactionTestCase @@ -27,24 +26,6 @@ class MultiDatabaseTestCase(TransactionTestCase): # will execute an extra SQL request below. connection2.cursor() - def is_django_21_below_and_sqlite2(self): - """ - Note: See test_utils.py with this function name - Checks if Django 2.1 or below and SQLite2 - """ - django_version = DJANGO_VERSION - if not self.is_sqlite2: - # Immediately know if SQLite - return False - if django_version[0] < 2: - # Takes Django 0 and 1 out of the picture - return True - else: - if django_version[0] == 2 and django_version[1] < 2: - # Takes Django 2.0-2.1 out - return True - return False - def test_read(self): with self.assertNumQueries(1): data1 = list(Test.objects.all()) @@ -66,8 +47,7 @@ class MultiDatabaseTestCase(TransactionTestCase): data1 = list(Test.objects.using(self.db_alias2)) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, - using=self.db_alias2): + with self.assertNumQueries(1, using=self.db_alias2): t3 = Test.objects.using(self.db_alias2).create(name='test3') with self.assertNumQueries(1, using=self.db_alias2): @@ -82,8 +62,7 @@ class MultiDatabaseTestCase(TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, [self.t1, self.t2]) - with self.assertNumQueries(2 if self.is_django_21_below_and_sqlite2() else 1, - using=self.db_alias2): + with self.assertNumQueries(1, using=self.db_alias2): Test.objects.using(self.db_alias2).create(name='test3') with self.assertNumQueries(0): diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index f06354a..badeec2 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -3,7 +3,7 @@ from unittest import skipIf from uuid import UUID from decimal import Decimal -from django import VERSION as django_version +from django import VERSION as DJANGO_VERSION from django.conf import settings from django.contrib.auth.models import Group, Permission, User from django.contrib.contenttypes.models import ContentType @@ -353,7 +353,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): @all_final_sql_checks def test_subquery(self): additional_tables = [] - if django_version[0] == 4 and django_version[1] < 1 and settings.CACHALOT_FINAL_SQL_CHECK: + if DJANGO_VERSION >= (4, 0) and DJANGO_VERSION < (4, 1) and settings.CACHALOT_FINAL_SQL_CHECK: # with Django 4.0 comes some query optimalizations that do selects little differently. additional_tables.append('django_content_type') qs = Test.objects.filter(owner__in=User.objects.all()) @@ -694,7 +694,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs) with self.assertRaisesMessage( - AssertionError if django_version[0] < 4 else TypeError, + AssertionError if DJANGO_VERSION < (4, 0) else TypeError, 'Cannot combine queries on two different base models.' ): Test.objects.all() | Permission.objects.all() @@ -739,7 +739,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs) with self.assertRaisesMessage( - AssertionError if django_version[0] < 4 else TypeError, + AssertionError if DJANGO_VERSION < (4, 0) else TypeError, 'Cannot combine queries on two different base models.'): Test.objects.all() & Permission.objects.all() @@ -816,21 +816,21 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): with self.assertRaises(TransactionManagementError): list(Test.objects.select_for_update()) - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): data1 = list(Test.objects.select_for_update()) self.assertListEqual(data1, [self.t1, self.t2]) self.assertListEqual([t.name for t in data1], ['test1', 'test2']) - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): data2 = list(Test.objects.select_for_update()) self.assertListEqual(data2, [self.t1, self.t2]) self.assertListEqual([t.name for t in data2], ['test1', 'test2']) - with self.assertNumQueries(2): + with self.assertNumQueries(4 if DJANGO_VERSION >= (4, 2) else 2): with transaction.atomic(): data3 = list(Test.objects.select_for_update()) data4 = list(Test.objects.select_for_update()) @@ -896,7 +896,9 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs, [self.t2, self.t1]) def test_table_inheritance(self): - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries( + 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) + ): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1): @@ -911,15 +913,10 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): expected = (r'\d+ 0 0 SCAN cachalot_test\n' r'\d+ 0 0 USE TEMP B-TREE FOR ORDER BY') elif self.is_mysql: - if self.django_version < (3, 1): - expected = ( - r'1 SIMPLE cachalot_test ' - r'(?:None )?ALL None None None None 2 100\.0 Using filesort') - else: - expected = ( - r'-> Sort row IDs: cachalot_test.`name` \(cost=[\d\.]+ rows=\d\)\n ' - r'-> Table scan on cachalot_test \(cost=[\d\.]+ rows=\d\)\n' - ) + expected = ( + r'-> Sort row IDs: cachalot_test.`name` \(cost=[\d\.]+ rows=\d\)\n ' + r'-> Table scan on cachalot_test \(cost=[\d\.]+ rows=\d\)\n' + ) else: explain_kwargs.update( analyze=True, @@ -935,9 +932,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): r'Planning Time: [\d\.]+ ms\n' r'Execution Time: [\d\.]+ ms$') % (operation_detail, operation_detail) - with self.assertNumQueries( - 2 if self.is_mysql and django_version[0] < 3 - else 1): + with self.assertNumQueries(1): explanation1 = Test.objects.explain(**explain_kwargs) self.assertRegex(explanation1, expected) with self.assertNumQueries(0): diff --git a/cachalot/tests/test_utils.py b/cachalot/tests/test_utils.py index d8db5f0..cb21774 100644 --- a/cachalot/tests/test_utils.py +++ b/cachalot/tests/test_utils.py @@ -1,4 +1,3 @@ -from django import VERSION as DJANGO_VERSION from django.core.management.color import no_style from django.db import connection, transaction @@ -11,7 +10,6 @@ class TestUtilsMixin: self.is_sqlite = connection.vendor == 'sqlite' self.is_mysql = connection.vendor == 'mysql' self.is_postgresql = connection.vendor == 'postgresql' - self.django_version = DJANGO_VERSION self.force_reopen_connection() # TODO: Remove this workaround when this issue is fixed: @@ -19,8 +17,6 @@ class TestUtilsMixin: def tearDown(self): if connection.vendor == 'postgresql': flush_args = [no_style(), (PostgresModel._meta.db_table,),] - if float(".".join(map(str, DJANGO_VERSION[:2]))) < 3.1: - flush_args.append(()) flush_sql_list = connection.ops.sql_flush(*flush_args) with transaction.atomic(): for sql in flush_sql_list: diff --git a/cachalot/tests/thread_safety.py b/cachalot/tests/thread_safety.py index cd7cd63..96400a2 100644 --- a/cachalot/tests/thread_safety.py +++ b/cachalot/tests/thread_safety.py @@ -1,5 +1,6 @@ from threading import Thread +from django import VERSION as DJANGO_VERSION from django.db import connection, transaction from django.test import TransactionTestCase, skipUnlessDBFeature @@ -29,7 +30,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(t2, t) def test_concurrent_caching_during_atomic(self): - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t1 = TestThread().start_and_join() t = Test.objects.create(name='test') @@ -45,7 +46,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): def test_concurrent_caching_before_and_during_atomic_1(self): t1 = TestThread().start_and_join() - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t2 = TestThread().start_and_join() t = Test.objects.create(name='test') @@ -60,7 +61,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): def test_concurrent_caching_before_and_during_atomic_2(self): t1 = TestThread().start_and_join() - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t = Test.objects.create(name='test') t2 = TestThread().start_and_join() @@ -73,7 +74,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data, t) def test_concurrent_caching_during_and_after_atomic_1(self): - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t1 = TestThread().start_and_join() t = Test.objects.create(name='test') @@ -88,7 +89,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data, t) def test_concurrent_caching_during_and_after_atomic_2(self): - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t = Test.objects.create(name='test') t1 = TestThread().start_and_join() @@ -103,7 +104,7 @@ class ThreadSafetyTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data, t) def test_concurrent_caching_during_and_after_atomic_3(self): - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): with transaction.atomic(): t1 = TestThread().start_and_join() t = Test.objects.create(name='test') diff --git a/cachalot/tests/transaction.py b/cachalot/tests/transaction.py index f55f41e..ceb7e55 100644 --- a/cachalot/tests/transaction.py +++ b/cachalot/tests/transaction.py @@ -1,4 +1,6 @@ from cachalot.transaction import AtomicCache + +from django import VERSION as DJANGO_VERSION from django.contrib.auth.models import User from django.core.cache import cache from django.db import transaction, connection, IntegrityError @@ -10,7 +12,9 @@ from .test_utils import TestUtilsMixin class AtomicTestCase(TestUtilsMixin, TransactionTestCase): def test_successful_read_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): with transaction.atomic(): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) @@ -20,7 +24,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, []) def test_unsuccessful_read_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): try: with transaction.atomic(): data1 = list(Test.objects.all()) @@ -38,21 +44,27 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): with transaction.atomic(): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.all()) self.assertListEqual(data2, [t1]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): with transaction.atomic(): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data3 = list(Test.objects.all()) self.assertListEqual(data3, [t1, t2]) - with self.assertNumQueries(4 if self.is_sqlite else 3): + with self.assertNumQueries( + 4 if self.is_sqlite else (5 if DJANGO_VERSION >= (4, 2) else 3) + ): with transaction.atomic(): data4 = list(Test.objects.all()) t3 = Test.objects.create(name='test3') @@ -67,7 +79,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): try: with transaction.atomic(): Test.objects.create(name='test') @@ -82,7 +96,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(name='test') def test_cache_inside_atomic(self): - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): with transaction.atomic(): data1 = list(Test.objects.all()) data2 = list(Test.objects.all()) @@ -90,7 +106,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, []) def test_invalidation_inside_atomic(self): - with self.assertNumQueries(4 if self.is_sqlite else 3): + with self.assertNumQueries( + 4 if self.is_sqlite else (5 if DJANGO_VERSION >= (4, 2) else 3) + ): with transaction.atomic(): data1 = list(Test.objects.all()) t = Test.objects.create(name='test') @@ -99,7 +117,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, [t]) def test_successful_nested_read_atomic(self): - with self.assertNumQueries(7 if self.is_sqlite else 6): + with self.assertNumQueries( + 7 if self.is_sqlite else (8 if DJANGO_VERSION >= (4, 2) else 6) + ): with transaction.atomic(): list(Test.objects.all()) with transaction.atomic(): @@ -114,7 +134,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): list(User.objects.all()) def test_unsuccessful_nested_read_atomic(self): - with self.assertNumQueries(6 if self.is_sqlite else 5): + with self.assertNumQueries( + 6 if self.is_sqlite else (7 if DJANGO_VERSION >= (4, 2) else 5) + ): with transaction.atomic(): try: with transaction.atomic(): @@ -127,7 +149,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): list(Test.objects.all()) def test_successful_nested_write_atomic(self): - with self.assertNumQueries(13 if self.is_sqlite else 12): + with self.assertNumQueries( + 13 if self.is_sqlite else (14 if DJANGO_VERSION >= (4, 2) else 12) + ): with transaction.atomic(): t1 = Test.objects.create(name='test1') with transaction.atomic(): @@ -144,7 +168,9 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data3, [t1, t2, t3, t4]) def test_unsuccessful_nested_write_atomic(self): - with self.assertNumQueries(16 if self.is_sqlite else 15): + with self.assertNumQueries( + 16 if self.is_sqlite else (17 if DJANGO_VERSION >= (4, 2) else 15) + ): with transaction.atomic(): t1 = Test.objects.create(name='test1') try: diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 982bb63..0ebcf3b 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -1,3 +1,5 @@ +import sys + from unittest import skipIf, skipUnless from django import VERSION as DJANGO_VERSION @@ -56,7 +58,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries( + 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) + ): t, created = Test.objects.get_or_create(name='test') self.assertTrue(created) @@ -78,14 +82,18 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertListEqual(list(Test.objects.all()), []) - with self.assertNumQueries(5 if self.is_sqlite else 4): + with self.assertNumQueries( + 5 if self.is_sqlite else (6 if DJANGO_VERSION >= (4, 2) else 4) + ): t, created = Test.objects.update_or_create( name='test', defaults={'public': True}) self.assertTrue(created) self.assertEqual(t.name, 'test') self.assertEqual(t.public, True) - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries( + 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) + ): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -94,7 +102,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): # The number of SQL queries doesn’t decrease because update_or_create # always calls an UPDATE, even when data wasn’t changed. - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries( + 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) + ): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -109,17 +119,21 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 20) - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else else 1): data2 = list(Test.objects.all()) self.assertEqual(len(data2), 20) self.assertListEqual([t.name for t in data2], @@ -160,12 +174,16 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data1, [t1.name, t2.name]) self.assertListEqual(data2, [t1.name]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): Test.objects.bulk_create([Test(name='test%s' % i) for i in range(2, 11)]) with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): Test.objects.all().delete() with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 0) @@ -360,7 +378,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data4, [user1, user2]) self.assertListEqual([u.n for u in data4], [1, 0]) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): Test.objects.bulk_create([ Test(name='test3', owner=user1), Test(name='test4', owner=user2), @@ -588,7 +608,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data2 = list(Test.objects.select_related('owner')) self.assertListEqual(data2, []) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): Test.objects.bulk_create([ Test(name='test1', owner=u1), Test(name='test2', owner=u2), @@ -602,7 +624,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data3[2].owner, u2) self.assertEqual(data3[3].owner, u1) - with self.assertNumQueries(2 if self.is_sqlite else 1): + with self.assertNumQueries( + 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) + ): Test.objects.filter(name__in=['test1', 'test2']).delete() with self.assertNumQueries(1): data4 = list(Test.objects.select_related('owner')) @@ -635,9 +659,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(list(data3[0].owner.groups.all()), []) with self.assertNumQueries( - 8 if self.is_sqlite and DJANGO_VERSION[0] == 2 and DJANGO_VERSION[1] == 2 - else 4 if self.is_postgresql and DJANGO_VERSION[0] > 2 - else 4 if self.is_mysql and DJANGO_VERSION[0] > 2 + 8 if self.is_postgresql and DJANGO_VERSION >= (4, 2) + else 4 if self.is_postgresql and DJANGO_VERSION >= (3, 0) + else 4 if self.is_mysql and DJANGO_VERSION >= (3, 0) else 6 ): group = Group.objects.create(name='test_group') @@ -694,7 +718,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): @skipUnlessDBFeature('has_select_for_update') def test_invalidate_select_for_update(self): - with self.assertNumQueries(1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): Test.objects.bulk_create([Test(name='test1'), Test(name='test2')]) with self.assertNumQueries(1): @@ -852,7 +876,9 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertRaises(TestChild.DoesNotExist): TestChild.objects.get() - with self.assertNumQueries(3 if self.is_sqlite else 2): + with self.assertNumQueries( + 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) + ): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1): diff --git a/tox.ini b/tox.ini index ff52608..c1b8fc8 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py{37,38,39,310}-django3.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{38,39,310}-django4.1-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, py{38,39,310,311}-django4.2-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, - py{38,39,310}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{38,39,310,311}-djangomain-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] passenv = * @@ -14,12 +14,11 @@ basepython = py310: python3.10 py311: python3.11 deps = - django2.2: Django>=2.2,<2.3 django3.2: Django>=3.2,<4.0 django4.1: Django>=4.1,<4.2 django4.2: Django>=4.2,<4.3 djangomain: https://github.com/django/django/archive/main.tar.gz - psycopg2>=2.8,<2.9 + psycopg2>=2.9.5,<3.0 psycopg mysqlclient django-redis From beff1e40504a5cf2e949bdcb39e972a3f4d6f30a Mon Sep 17 00:00:00 2001 From: Jack Linke Date: Wed, 24 May 2023 22:01:57 -0400 Subject: [PATCH 5/6] Fix simple error --- cachalot/tests/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 0ebcf3b..24d87f3 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -133,7 +133,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 20) - with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else else 1): + with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): data2 = list(Test.objects.all()) self.assertEqual(len(data2), 20) self.assertListEqual([t.name for t in data2], From fa1fbc1a5d7e1dbdfdf60e8562967ee624c935ea Mon Sep 17 00:00:00 2001 From: Benedikt Willi Date: Tue, 6 Jun 2023 10:05:44 +0200 Subject: [PATCH 6/6] Added a FilteredTransactionTestCase, updated tests. --- cachalot/tests/read.py | 8 ++--- cachalot/tests/test_utils.py | 63 ++++++++++++++++++++++++++++++++- cachalot/tests/transaction.py | 55 ++++++++--------------------- cachalot/tests/write.py | 66 +++++++++-------------------------- 4 files changed, 97 insertions(+), 95 deletions(-) diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index badeec2..b2b492f 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -21,7 +21,7 @@ from cachalot.cache import cachalot_caches from ..settings import cachalot_settings from ..utils import UncachableQuery from .models import Test, TestChild, TestParent, UnmanagedModel -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase from .tests_decorators import all_final_sql_checks, with_final_sql_check, no_final_sql_check @@ -36,7 +36,7 @@ def is_field_available(name): return name in fields -class ReadTestCase(TestUtilsMixin, TransactionTestCase): +class ReadTestCase(TestUtilsMixin, FilteredTransactionTestCase): """ Tests if every SQL request that only reads data is cached. @@ -896,9 +896,7 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_query_cached(qs, [self.t2, self.t1]) def test_table_inheritance(self): - with self.assertNumQueries( - 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) - ): + with self.assertNumQueries(2): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1): diff --git a/cachalot/tests/test_utils.py b/cachalot/tests/test_utils.py index cb21774..decb7ce 100644 --- a/cachalot/tests/test_utils.py +++ b/cachalot/tests/test_utils.py @@ -1,5 +1,8 @@ from django.core.management.color import no_style -from django.db import connection, transaction +from django.db import DEFAULT_DB_ALIAS, connection, connections, transaction + +from django.test import TransactionTestCase +from django.test.utils import CaptureQueriesContext from ..utils import _get_tables from .models import PostgresModel @@ -57,3 +60,61 @@ class TestUtilsMixin: assert_function(data2, data1) if result is not None: assert_function(data2, result) + +class FilteredTransactionTestCase(TransactionTestCase): + """ + TransactionTestCase with assertNumQueries that ignores BEGIN, COMMIT and ROLLBACK + queries. + """ + def assertNumQueries(self, num, func=None, *args, using=DEFAULT_DB_ALIAS, **kwargs): + conn = connections[using] + + context = FilteredAssertNumQueriesContext(self, num, conn) + if func is None: + return context + + with context: + func(*args, **kwargs) + + +class FilteredAssertNumQueriesContext(CaptureQueriesContext): + """ + Capture queries and assert their number ignoring BEGIN, COMMIT and ROLLBACK queries. + """ + EXCLUDE = ('BEGIN', 'COMMIT', 'ROLLBACK') + + def __init__(self, test_case, num, connection): + self.test_case = test_case + self.num = num + super().__init__(connection) + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + if exc_type is not None: + return + + filtered_queries = [] + excluded_queries = [] + for q in self.captured_queries: + if q['sql'].upper() not in self.EXCLUDE: + filtered_queries.append(q) + else: + excluded_queries.append(q) + + executed = len(filtered_queries) + + self.test_case.assertEqual( + executed, + self.num, + f"\n{executed} queries executed, {self.num} expected\n" + + "\nCaptured queries were:\n" + + "".join( + f"{i}. {query['sql']}\n" + for i, query in enumerate(filtered_queries, start=1) + ) + + "\nCaptured queries, that were excluded:\n" + + "".join( + f"{i}. {query['sql']}\n" + for i, query in enumerate(excluded_queries, start=1) + ) + ) diff --git a/cachalot/tests/transaction.py b/cachalot/tests/transaction.py index ceb7e55..38b3710 100644 --- a/cachalot/tests/transaction.py +++ b/cachalot/tests/transaction.py @@ -1,20 +1,17 @@ from cachalot.transaction import AtomicCache -from django import VERSION as DJANGO_VERSION from django.contrib.auth.models import User from django.core.cache import cache from django.db import transaction, connection, IntegrityError -from django.test import SimpleTestCase, TransactionTestCase, skipUnlessDBFeature +from django.test import SimpleTestCase, skipUnlessDBFeature from .models import Test -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase -class AtomicTestCase(TestUtilsMixin, TransactionTestCase): +class AtomicTestCase(TestUtilsMixin, FilteredTransactionTestCase): def test_successful_read_atomic(self): - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): with transaction.atomic(): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) @@ -24,9 +21,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, []) def test_unsuccessful_read_atomic(self): - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): try: with transaction.atomic(): data1 = list(Test.objects.all()) @@ -44,27 +39,21 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): with transaction.atomic(): t1 = Test.objects.create(name='test1') with self.assertNumQueries(1): data2 = list(Test.objects.all()) self.assertListEqual(data2, [t1]) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): with transaction.atomic(): t2 = Test.objects.create(name='test2') with self.assertNumQueries(1): data3 = list(Test.objects.all()) self.assertListEqual(data3, [t1, t2]) - with self.assertNumQueries( - 4 if self.is_sqlite else (5 if DJANGO_VERSION >= (4, 2) else 3) - ): + with self.assertNumQueries(3): with transaction.atomic(): data4 = list(Test.objects.all()) t3 = Test.objects.create(name='test3') @@ -79,9 +68,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): try: with transaction.atomic(): Test.objects.create(name='test') @@ -96,9 +83,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): Test.objects.get(name='test') def test_cache_inside_atomic(self): - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): with transaction.atomic(): data1 = list(Test.objects.all()) data2 = list(Test.objects.all()) @@ -106,9 +91,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, []) def test_invalidation_inside_atomic(self): - with self.assertNumQueries( - 4 if self.is_sqlite else (5 if DJANGO_VERSION >= (4, 2) else 3) - ): + with self.assertNumQueries(3): with transaction.atomic(): data1 = list(Test.objects.all()) t = Test.objects.create(name='test') @@ -117,9 +100,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data2, [t]) def test_successful_nested_read_atomic(self): - with self.assertNumQueries( - 7 if self.is_sqlite else (8 if DJANGO_VERSION >= (4, 2) else 6) - ): + with self.assertNumQueries(6): with transaction.atomic(): list(Test.objects.all()) with transaction.atomic(): @@ -134,9 +115,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): list(User.objects.all()) def test_unsuccessful_nested_read_atomic(self): - with self.assertNumQueries( - 6 if self.is_sqlite else (7 if DJANGO_VERSION >= (4, 2) else 5) - ): + with self.assertNumQueries(5): with transaction.atomic(): try: with transaction.atomic(): @@ -149,9 +128,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): list(Test.objects.all()) def test_successful_nested_write_atomic(self): - with self.assertNumQueries( - 13 if self.is_sqlite else (14 if DJANGO_VERSION >= (4, 2) else 12) - ): + with self.assertNumQueries(12): with transaction.atomic(): t1 = Test.objects.create(name='test1') with transaction.atomic(): @@ -168,9 +145,7 @@ class AtomicTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data3, [t1, t2, t3, t4]) def test_unsuccessful_nested_write_atomic(self): - with self.assertNumQueries( - 16 if self.is_sqlite else (17 if DJANGO_VERSION >= (4, 2) else 15) - ): + with self.assertNumQueries(15): with transaction.atomic(): t1 = Test.objects.create(name='test1') try: diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index 24d87f3..dd6459a 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -1,8 +1,5 @@ -import sys - from unittest import skipIf, skipUnless -from django import VERSION as DJANGO_VERSION from django.contrib.auth.models import User, Permission, Group from django.contrib.contenttypes.models import ContentType from django.core.exceptions import MultipleObjectsReturned @@ -14,10 +11,10 @@ from django.db.models.expressions import RawSQL from django.test import TransactionTestCase, skipUnlessDBFeature from .models import Test, TestParent, TestChild -from .test_utils import TestUtilsMixin +from .test_utils import TestUtilsMixin, FilteredTransactionTestCase -class WriteTestCase(TestUtilsMixin, TransactionTestCase): +class WriteTestCase(TestUtilsMixin, FilteredTransactionTestCase): """ Tests if every SQL request writing data is not cached and invalidates the implied data. @@ -58,9 +55,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries( - 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) - ): + with self.assertNumQueries(2): t, created = Test.objects.get_or_create(name='test') self.assertTrue(created) @@ -82,18 +77,14 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertNumQueries(1): self.assertListEqual(list(Test.objects.all()), []) - with self.assertNumQueries( - 5 if self.is_sqlite else (6 if DJANGO_VERSION >= (4, 2) else 4) - ): + with self.assertNumQueries(4): t, created = Test.objects.update_or_create( name='test', defaults={'public': True}) self.assertTrue(created) self.assertEqual(t.name, 'test') self.assertEqual(t.public, True) - with self.assertNumQueries( - 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) - ): + with self.assertNumQueries(2): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -102,9 +93,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): # The number of SQL queries doesn’t decrease because update_or_create # always calls an UPDATE, even when data wasn’t changed. - with self.assertNumQueries( - 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) - ): + with self.assertNumQueries(2): t, created = Test.objects.update_or_create( name='test', defaults={'public': False}) self.assertFalse(created) @@ -119,21 +108,17 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data1 = list(Test.objects.all()) self.assertListEqual(data1, []) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): unsaved_tests = [Test(name='test%02d' % i) for i in range(1, 11)] Test.objects.bulk_create(unsaved_tests) self.assertEqual(Test.objects.count(), 20) - with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): + with self.assertNumQueries(1): data2 = list(Test.objects.all()) self.assertEqual(len(data2), 20) self.assertListEqual([t.name for t in data2], @@ -174,16 +159,12 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data1, [t1.name, t2.name]) self.assertListEqual(data2, [t1.name]) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): Test.objects.bulk_create([Test(name='test%s' % i) for i in range(2, 11)]) with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 10) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): Test.objects.all().delete() with self.assertNumQueries(1): self.assertEqual(Test.objects.count(), 0) @@ -378,9 +359,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(data4, [user1, user2]) self.assertListEqual([u.n for u in data4], [1, 0]) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): Test.objects.bulk_create([ Test(name='test3', owner=user1), Test(name='test4', owner=user2), @@ -608,9 +587,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): data2 = list(Test.objects.select_related('owner')) self.assertListEqual(data2, []) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): Test.objects.bulk_create([ Test(name='test1', owner=u1), Test(name='test2', owner=u2), @@ -624,9 +601,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data3[2].owner, u2) self.assertEqual(data3[3].owner, u1) - with self.assertNumQueries( - 2 if self.is_sqlite else (3 if DJANGO_VERSION >= (4, 2) else 1) - ): + with self.assertNumQueries(1): Test.objects.filter(name__in=['test1', 'test2']).delete() with self.assertNumQueries(1): data4 = list(Test.objects.select_related('owner')) @@ -658,12 +633,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): self.assertEqual(data3[0].owner, u) self.assertListEqual(list(data3[0].owner.groups.all()), []) - with self.assertNumQueries( - 8 if self.is_postgresql and DJANGO_VERSION >= (4, 2) - else 4 if self.is_postgresql and DJANGO_VERSION >= (3, 0) - else 4 if self.is_mysql and DJANGO_VERSION >= (3, 0) - else 6 - ): + with self.assertNumQueries(4): group = Group.objects.create(name='test_group') permissions = list(Permission.objects.all()[:5]) group.permissions.add(*permissions) @@ -718,7 +688,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): @skipUnlessDBFeature('has_select_for_update') def test_invalidate_select_for_update(self): - with self.assertNumQueries(3 if DJANGO_VERSION >= (4, 2) else 1): + with self.assertNumQueries(1): Test.objects.bulk_create([Test(name='test1'), Test(name='test2')]) with self.assertNumQueries(1): @@ -876,9 +846,7 @@ class WriteTestCase(TestUtilsMixin, TransactionTestCase): with self.assertRaises(TestChild.DoesNotExist): TestChild.objects.get() - with self.assertNumQueries( - 3 if self.is_sqlite else (4 if DJANGO_VERSION >= (4, 2) else 2) - ): + with self.assertNumQueries(2): t_child = TestChild.objects.create(name='test_child') with self.assertNumQueries(1):