From 74e000b8e2b7f4cf9f31319a83d21620ea7b747d Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Fri, 4 May 2018 20:26:26 +0200 Subject: [PATCH] Adds Django 2.0 support. --- .travis.yml | 120 ++++++++++++++++++++++ cachalot/settings.py | 3 +- cachalot/templatetags/cachalot.py | 2 +- cachalot/tests/migrations/0001_initial.py | 7 +- cachalot/tests/models.py | 11 +- cachalot/tests/read.py | 27 ++++- cachalot/tests/write.py | 2 +- cachalot/utils.py | 34 +++--- runtests_requirements.txt | 3 +- settings.py | 4 +- tox.ini | 3 + 11 files changed, 186 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25e910e..6d6ddd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -131,6 +131,126 @@ matrix: env: TOXENV=py3.6-django1.11-mysql-locmem - python: 3.6 env: TOXENV=py3.6-django1.11-mysql-filebased + - python: 2.7 + env: TOXENV=py2.7-django2.0-sqlite3-redis + - python: 2.7 + env: TOXENV=py2.7-django2.0-sqlite3-memcached + - python: 2.7 + env: TOXENV=py2.7-django2.0-sqlite3-pylibmc + - python: 2.7 + env: TOXENV=py2.7-django2.0-sqlite3-locmem + - python: 2.7 + env: TOXENV=py2.7-django2.0-sqlite3-filebased + - python: 2.7 + env: TOXENV=py2.7-django2.0-postgresql-redis + - python: 2.7 + env: TOXENV=py2.7-django2.0-postgresql-memcached + - python: 2.7 + env: TOXENV=py2.7-django2.0-postgresql-pylibmc + - python: 2.7 + env: TOXENV=py2.7-django2.0-postgresql-locmem + - python: 2.7 + env: TOXENV=py2.7-django2.0-postgresql-filebased + - python: 2.7 + env: TOXENV=py2.7-django2.0-mysql-redis + - python: 2.7 + env: TOXENV=py2.7-django2.0-mysql-memcached + - python: 2.7 + env: TOXENV=py2.7-django2.0-mysql-pylibmc + - python: 2.7 + env: TOXENV=py2.7-django2.0-mysql-locmem + - python: 2.7 + env: TOXENV=py2.7-django2.0-mysql-filebased + - python: 3.4 + env: TOXENV=py3.4-django2.0-sqlite3-redis + - python: 3.4 + env: TOXENV=py3.4-django2.0-sqlite3-memcached + - python: 3.4 + env: TOXENV=py3.4-django2.0-sqlite3-pylibmc + - python: 3.4 + env: TOXENV=py3.4-django2.0-sqlite3-locmem + - python: 3.4 + env: TOXENV=py3.4-django2.0-sqlite3-filebased + - python: 3.4 + env: TOXENV=py3.4-django2.0-postgresql-redis + - python: 3.4 + env: TOXENV=py3.4-django2.0-postgresql-memcached + - python: 3.4 + env: TOXENV=py3.4-django2.0-postgresql-pylibmc + - python: 3.4 + env: TOXENV=py3.4-django2.0-postgresql-locmem + - python: 3.4 + env: TOXENV=py3.4-django2.0-postgresql-filebased + - python: 3.4 + env: TOXENV=py3.4-django2.0-mysql-redis + - python: 3.4 + env: TOXENV=py3.4-django2.0-mysql-memcached + - python: 3.4 + env: TOXENV=py3.4-django2.0-mysql-pylibmc + - python: 3.4 + env: TOXENV=py3.4-django2.0-mysql-locmem + - python: 3.4 + env: TOXENV=py3.4-django2.0-mysql-filebased + - python: 3.5 + env: TOXENV=py3.5-django2.0-sqlite3-redis + - python: 3.5 + env: TOXENV=py3.5-django2.0-sqlite3-memcached + - python: 3.5 + env: TOXENV=py3.5-django2.0-sqlite3-pylibmc + - python: 3.5 + env: TOXENV=py3.5-django2.0-sqlite3-locmem + - python: 3.5 + env: TOXENV=py3.5-django2.0-sqlite3-filebased + - python: 3.5 + env: TOXENV=py3.5-django2.0-postgresql-redis + - python: 3.5 + env: TOXENV=py3.5-django2.0-postgresql-memcached + - python: 3.5 + env: TOXENV=py3.5-django2.0-postgresql-pylibmc + - python: 3.5 + env: TOXENV=py3.5-django2.0-postgresql-locmem + - python: 3.5 + env: TOXENV=py3.5-django2.0-postgresql-filebased + - python: 3.5 + env: TOXENV=py3.5-django2.0-mysql-redis + - python: 3.5 + env: TOXENV=py3.5-django2.0-mysql-memcached + - python: 3.5 + env: TOXENV=py3.5-django2.0-mysql-pylibmc + - python: 3.5 + env: TOXENV=py3.5-django2.0-mysql-locmem + - python: 3.5 + env: TOXENV=py3.5-django2.0-mysql-filebased + - python: 3.6 + env: TOXENV=py3.6-django2.0-sqlite3-redis + - python: 3.6 + env: TOXENV=py3.6-django2.0-sqlite3-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.0-sqlite3-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.0-sqlite3-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.0-sqlite3-filebased + - python: 3.6 + env: TOXENV=py3.6-django2.0-postgresql-redis + - python: 3.6 + env: TOXENV=py3.6-django2.0-postgresql-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.0-postgresql-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.0-postgresql-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.0-postgresql-filebased + - python: 3.6 + env: TOXENV=py3.6-django2.0-mysql-redis + - python: 3.6 + env: TOXENV=py3.6-django2.0-mysql-memcached + - python: 3.6 + env: TOXENV=py3.6-django2.0-mysql-pylibmc + - python: 3.6 + env: TOXENV=py3.6-django2.0-mysql-locmem + - python: 3.6 + env: TOXENV=py3.6-django2.0-mysql-filebased sudo: false diff --git a/cachalot/settings.py b/cachalot/settings.py index 43e3104..b6424d1 100644 --- a/cachalot/settings.py +++ b/cachalot/settings.py @@ -6,7 +6,7 @@ SUPPORTED_DATABASE_ENGINES = { 'django.db.backends.sqlite3', 'django.db.backends.postgresql', 'django.db.backends.mysql', - # TODO: Remove when we drop Django 1.8 support. + # TODO: Remove when we drop Django 2.x support. 'django.db.backends.postgresql_psycopg2', # GeoDjango @@ -17,6 +17,7 @@ SUPPORTED_DATABASE_ENGINES = { # django-transaction-hooks 'transaction_hooks.backends.sqlite3', 'transaction_hooks.backends.postgis', + # TODO: Remove when we drop Django 2.x support. 'transaction_hooks.backends.postgresql_psycopg2', 'transaction_hooks.backends.mysql', } diff --git a/cachalot/templatetags/cachalot.py b/cachalot/templatetags/cachalot.py index 7c17b24..4c4ed5e 100644 --- a/cachalot/templatetags/cachalot.py +++ b/cachalot/templatetags/cachalot.py @@ -6,4 +6,4 @@ from ..api import get_last_invalidation register = Library() -register.assignment_tag(get_last_invalidation) +register.simple_tag(get_last_invalidation) diff --git a/cachalot/tests/migrations/0001_initial.py b/cachalot/tests/migrations/0001_initial.py index 6865e86..8b3f6e0 100644 --- a/cachalot/tests/migrations/0001_initial.py +++ b/cachalot/tests/migrations/0001_initial.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals -from django import VERSION as django_version from django.conf import settings from django.contrib.postgres.fields import ( ArrayField, HStoreField, @@ -26,8 +25,8 @@ class Migration(migrations.Migration): ('public', models.BooleanField(default=False)), ('date', models.DateField(null=True, blank=True)), ('datetime', models.DateTimeField(null=True, blank=True)), - ('owner', models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True)), - ('permission', models.ForeignKey(blank=True, to='auth.Permission', null=True)), + ('owner', models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL)), + ('permission', models.ForeignKey(blank=True, to='auth.Permission', null=True, on_delete=models.PROTECT)), ('a_float', models.FloatField(null=True, blank=True)), ('a_decimal', models.DecimalField(null=True, blank=True, max_digits=5, decimal_places=2)), ('bin', models.BinaryField(null=True, blank=True)), @@ -49,7 +48,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='TestChild', fields=[ - ('testparent_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='cachalot.TestParent')), + ('testparent_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='cachalot.TestParent', on_delete=models.CASCADE)), ('public', models.BooleanField(default=False)), ('permissions', models.ManyToManyField('auth.Permission', blank=True)) ], diff --git a/cachalot/tests/models.py b/cachalot/tests/models.py index d8d47aa..94d3c93 100644 --- a/cachalot/tests/models.py +++ b/cachalot/tests/models.py @@ -5,20 +5,23 @@ from __future__ import unicode_literals from django.conf import settings from django.contrib.postgres.fields import ( ArrayField, HStoreField, - IntegerRangeField, JSONField, FloatRangeField, DateRangeField, DateTimeRangeField) + IntegerRangeField, JSONField, FloatRangeField, DateRangeField, + DateTimeRangeField) from django.db.models import ( Model, CharField, ForeignKey, BooleanField, DateField, DateTimeField, ManyToManyField, BinaryField, IntegerField, GenericIPAddressField, - FloatField, DecimalField, DurationField, UUIDField) + FloatField, DecimalField, DurationField, UUIDField, SET_NULL, PROTECT) class Test(Model): name = CharField(max_length=20) - owner = ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True) + owner = ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True, + on_delete=SET_NULL) public = BooleanField(default=False) date = DateField(null=True, blank=True) datetime = DateTimeField(null=True, blank=True) - permission = ForeignKey('auth.Permission', null=True, blank=True) + permission = ForeignKey('auth.Permission', null=True, blank=True, + on_delete=PROTECT) # We can’t use the exact names `float` or `decimal` as database column name # since it fails on MySQL. diff --git a/cachalot/tests/read.py b/cachalot/tests/read.py index 530fd02..676ef30 100644 --- a/cachalot/tests/read.py +++ b/cachalot/tests/read.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import datetime -from unittest import skipIf, skipUnless +from unittest import skipIf from uuid import UUID from decimal import Decimal @@ -11,7 +11,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from django.db import connection, transaction from django.db.models import Count -from django.db.models.expressions import RawSQL +from django.db.models.expressions import RawSQL, Subquery, OuterRef, Exists from django.db.models.functions import Now from django.db.transaction import TransactionManagementError from django.test import ( @@ -311,6 +311,23 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): TestChild.permissions.through, Permission) self.assert_query_cached(qs, []) + qs = TestChild.objects.exclude(permissions__name='') + self.assert_tables(qs, TestParent, TestChild, + TestChild.permissions.through, Permission) + self.assert_query_cached(qs, []) + + def test_custom_subquery(self): + tests = Test.objects.filter(permission=OuterRef('pk')).values('name') + qs = Permission.objects.annotate(first_permission=Subquery(tests[:1])) + self.assert_tables(qs, Permission, Test) + self.assert_query_cached(qs, list(Permission.objects.all())) + + def test_custom_subquery_exists(self): + tests = Test.objects.filter(permission=OuterRef('pk')) + qs = Permission.objects.annotate(has_tests=Exists(tests)) + self.assert_tables(qs, Permission, Test) + self.assert_query_cached(qs, list(Permission.objects.all())) + def test_raw_subquery(self): with self.assertNumQueries(0): raw_sql = RawSQL('SELECT id FROM auth_permission WHERE id = %s', @@ -340,6 +357,12 @@ class ReadTestCase(TestUtilsMixin, TransactionTestCase): self.assert_tables(qs, User, Test) self.assert_query_cached(qs, [2, 1]) + def test_annotate_subquery(self): + tests = Test.objects.filter(owner=OuterRef('pk')).values('name') + qs = User.objects.annotate(first_test=Subquery(tests[:1])) + self.assert_tables(qs, User, Test) + self.assert_query_cached(qs, [self.user, self.admin]) + def test_only(self): with self.assertNumQueries(1): t1 = Test.objects.only('name').first() diff --git a/cachalot/tests/write.py b/cachalot/tests/write.py index a537aa0..86b5b7b 100644 --- a/cachalot/tests/write.py +++ b/cachalot/tests/write.py @@ -1032,7 +1032,7 @@ class DatabaseCommandTestCase(TestUtilsMixin, TransactionTestCase): self.assertListEqual(list(Test.objects.all()), [self.t]) call_command('loaddata', 'cachalot/tests/loaddata_fixture.json', - verbosity=0, interactive=False) + verbosity=0) self.force_repoen_connection() diff --git a/cachalot/utils.py b/cachalot/utils.py index b081ee5..1bf2ced 100644 --- a/cachalot/utils.py +++ b/cachalot/utils.py @@ -9,11 +9,10 @@ from uuid import UUID from django.contrib.postgres.functions import TransactionNow from django.db import connections -from django.db.models import QuerySet +from django.db.models import QuerySet, Subquery, Exists from django.db.models.functions import Now -from django.db.models.sql import Query -from django.db.models.sql.where import ( - ExtraWhere, SubqueryConstraint, WhereNode) +from django.db.models.sql import Query, AggregateQuery +from django.db.models.sql.where import ExtraWhere, WhereNode from django.utils.six import text_type, binary_type, integer_types from .settings import ITERABLES, cachalot_settings @@ -101,26 +100,23 @@ def _get_tables_from_sql(connection, lowercased_sql): if t in lowercased_sql} -def _find_subqueries(children): +def _find_subqueries_in_where(children): for child in children: child_class = child.__class__ if child_class is WhereNode: - for grand_child in _find_subqueries(child.children): + for grand_child in _find_subqueries_in_where(child.children): yield grand_child - # TODO: Remove this condition when we drop Django 1.8 support. - elif child_class is SubqueryConstraint: - query_object = child.query_object - yield (query_object if query_object.__class__ is Query - else query_object.query) elif child_class is ExtraWhere: raise IsRawQuery else: - rhs = getattr(child, 'rhs', None) + rhs = child.rhs rhs_class = rhs.__class__ if rhs_class is Query: yield rhs elif rhs_class is QuerySet: yield rhs.query + elif rhs_class is Subquery or rhs_class is Exists: + yield rhs.queryset.query elif rhs_class in UNCACHABLE_FUNCS: raise UncachableQuery @@ -154,12 +150,22 @@ def _get_tables(db_alias, query): raise UncachableQuery try: - if query.extra_select or getattr(query, 'subquery', False): + if query.extra_select: raise IsRawQuery + # Gets all tables already found by the ORM. tables = set(query.table_map) tables.add(query.get_meta().db_table) - for subquery in _find_subqueries(query.where.children): + # Gets tables in subquery annotations. + for annotation in query.annotations.values(): + if isinstance(annotation, Subquery): + tables.update(_get_tables(db_alias, annotation.queryset.query)) + # Gets tables in WHERE subqueries. + for subquery in _find_subqueries_in_where(query.where.children): tables.update(_get_tables(db_alias, subquery)) + # Gets tables in HAVING subqueries. + if isinstance(query, AggregateQuery): + tables.update( + _get_tables_from_sql(connections[db_alias], query.subquery)) except IsRawQuery: sql = query.get_compiler(db_alias).as_sql()[0].lower() tables = _get_tables_from_sql(connections[db_alias], sql) diff --git a/runtests_requirements.txt b/runtests_requirements.txt index 9723fef..9e9a724 100644 --- a/runtests_requirements.txt +++ b/runtests_requirements.txt @@ -1,6 +1,7 @@ -r requirements.txt -psycopg2-binary +# TODO: Switch to psycopg2-binary when psycopg/psycopg2#708 is fixed. +psycopg2 mysqlclient django-redis python-memcached diff --git a/settings.py b/settings.py index c9bd6bd..3a5ee47 100644 --- a/settings.py +++ b/settings.py @@ -117,7 +117,7 @@ TEMPLATES = [ ] -MIDDLEWARE_CLASSES = [] +MIDDLEWARE = [] PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher'] SECRET_KEY = 'it’s not important in tests but we have to set it' @@ -160,7 +160,7 @@ DEBUG_TOOLBAR_CONFIG = { 'RENDER_PANELS': False, } -MIDDLEWARE_CLASSES += [ +MIDDLEWARE += [ 'debug_toolbar.middleware.DebugToolbarMiddleware', ] diff --git a/tox.ini b/tox.ini index 7acb763..1c5942c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py{2.7,3.4,3.5,3.6}-django1.11-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, + py{3.4,3.5,3.6}-django2.0-{sqlite3,postgresql,mysql}-{redis,memcached,pylibmc,locmem,filebased}, [testenv] basepython = @@ -10,6 +11,8 @@ basepython = py3.6: python3.6 deps = django1.11: Django>=1.11,<1.12 + django2.0: Django>=2.0,<2.1 + # TODO: Switch to psycopg2-binary when psycopg/psycopg2#708 is fixed. psycopg2 mysqlclient django-redis