Add .specific() page queryset method

docs/reference/pages/queryset_reference.rst

@@ -196,3 +196,15 @@ Reference
 
             # Unpublish current_page and all of its children
             Page.objects.descendant_of(current_page, inclusive=True).unpublish()
+
+    .. automethod:: specific
+
+        Example:
+
+        .. code-block:: python
+
+            # Get the specific instance of all children of the hompage,
+            # in a minimum number of database queries.
+            homepage.get_children().specific()
+
+        See also: :py:attr:`Page.specific <wagtail.wagtailcore.models.Page.specific>`

wagtail/tests/testapp/fixtures/test_specific.json

@@ -0,0 +1,256 @@
+[
+{
+    "pk": 1,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Root",
+        "numchild": 1,
+        "show_in_menus": false,
+        "live": true,
+        "depth": 1,
+        "content_type": ["wagtailcore", "page"],
+        "path": "0001",
+        "url_path": "/",
+        "slug": "root"
+    }
+},
+
+{
+    "pk": 2,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Welcome to the Wagtail test site!",
+        "numchild": 5,
+        "show_in_menus": false,
+        "live": true,
+        "depth": 2,
+        "content_type": ["wagtailcore", "page"],
+        "path": "00010001",
+        "url_path": "/home/",
+        "slug": "home"
+    }
+},
+
+{
+    "pk": 3,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Events",
+        "numchild": 4,
+        "show_in_menus": true,
+        "live": true,
+        "depth": 3,
+        "content_type": ["tests", "eventindex"],
+        "path": "000100010001",
+        "url_path": "/home/events/",
+        "slug": "events"
+    }
+},
+{
+    "pk": 3,
+    "model": "tests.eventindex",
+    "fields": {
+        "intro": "Look at our lovely events."
+    }
+},
+
+{
+    "pk": 4,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Christmas",
+        "numchild": 0,
+        "show_in_menus": true,
+        "live": true,
+        "depth": 4,
+        "content_type": ["tests", "eventpage"],
+        "path": "0001000100010001",
+        "url_path": "/home/events/christmas/",
+        "slug": "christmas",
+        "owner": 1
+    }
+},
+{
+    "pk": 4,
+    "model": "tests.eventpage",
+    "fields": {
+        "date_from": "2014-12-25",
+        "audience": "public",
+        "location": "The North Pole",
+        "body": "<p>Chestnuts roasting on an open fire</p>",
+        "cost": "Free",
+        "feed_image": 1
+    }
+},
+
+{
+    "pk": 1,
+    "model": "wagtailimages.image",
+    "fields": {
+        "title": "A missing image",
+        "file": "original_images/missing.jpg",
+        "width": 1000,
+        "height": 1000,
+        "created_at": "2014-01-01T12:00:00.000Z"
+    }
+},
+
+{
+    "pk": 5,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Tentative Unpublished Event",
+        "numchild": 0,
+        "show_in_menus": true,
+        "live": false,
+        "depth": 4,
+        "content_type": ["tests", "eventpage"],
+        "path": "0001000100010002",
+        "url_path": "/home/events/tentative-unpublished-event/",
+        "slug": "tentative-unpublished-event",
+        "owner": 1
+    }
+},
+{
+    "pk": 5,
+    "model": "tests.eventpage",
+    "fields": {
+        "date_from": "2015-07-04",
+        "audience": "public",
+        "location": "The moon",
+        "body": "<p>I haven't worked out the details yet, but it's going to have cake and ponies</p>",
+        "cost": "Free"
+    }
+},
+
+{
+    "pk": 6,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Someone Else's Event",
+        "numchild": 0,
+        "show_in_menus": true,
+        "live": false,
+        "depth": 4,
+        "content_type": ["tests", "eventpage"],
+        "path": "0001000100010003",
+        "url_path": "/home/events/someone-elses-event/",
+        "slug": "someone-elses-event",
+        "owner": 1
+    }
+},
+{
+    "pk": 6,
+    "model": "tests.eventpage",
+    "fields": {
+        "date_from": "2015-07-04",
+        "audience": "private",
+        "location": "The moon",
+        "body": "<p>your name's not down, you're not coming in</p>",
+        "cost": "Free (but not for you)"
+    }
+},
+
+{
+    "pk": 7,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "About us",
+        "numchild": 0,
+        "show_in_menus": true,
+        "live": true,
+        "depth": 3,
+        "content_type": ["tests", "simplepage"],
+        "path": "000100010002",
+        "url_path": "/home/about-us/",
+        "slug": "about-us"
+    }
+},
+{
+    "pk": 7,
+    "model": "tests.simplepage",
+    "fields": {
+        "content": "<p>We are really good.</p>"
+    }
+},
+
+{
+    "pk": 11,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Other events",
+        "numchild": 1,
+        "show_in_menus": true,
+        "live": true,
+        "depth": 3,
+        "content_type": ["tests", "simplepage"],
+        "path": "000100010005",
+        "url_path": "/home/other/",
+        "slug": "other"
+    }
+},
+{
+    "pk": 11,
+    "model": "tests.simplepage",
+    "fields": {
+        "content": "<p>Other events</p>"
+    }
+},
+
+{
+    "pk": 12,
+    "model": "wagtailcore.page",
+    "fields": {
+        "title": "Special event",
+        "numchild": 0,
+        "show_in_menus": false,
+        "live": true,
+        "depth": 4,
+        "content_type": ["tests", "eventpage"],
+        "path": "0001000100050001",
+        "url_path": "/home/other/special-event/",
+        "slug": "special-event"
+    }
+},
+{
+    "pk": 12,
+    "model": "tests.eventpage",
+    "fields": {
+        "date_from": "2015-07-04",
+        "audience": "public",
+        "location": "Hobart",
+        "body": "<p>Party time</p>",
+        "cost": "free"
+    }
+},
+
+{
+    "pk": 1,
+    "model": "wagtailcore.site",
+    "fields": {
+        "root_page": 2,
+        "hostname": "localhost",
+        "port": 80,
+        "is_default_site": true
+    }
+},
+
+{
+    "pk": 1,
+    "model": "customuser.customuser",
+    "fields": {
+        "username": "superuser",
+        "first_name": "",
+        "last_name": "",
+        "is_active": true,
+        "is_superuser": true,
+        "is_staff": true,
+        "groups": [
+        ],
+        "user_permissions": [],
+        "password": "md5$seasalt$1e9bf2bf5606aa5c39852cc30f0f6f22",
+        "email": "superuser@example.com"
+    }
+}
+
+]

wagtail/wagtailcore/models.py

@@ -232,6 +232,9 @@ class PageManager(models.Manager):
     def search(self, query_string, fields=None, backend='default'):
         return self.get_queryset().search(query_string, fields=fields, backend=backend)
 
+    def specific(self):
+        return self.get_queryset().specific()
+
 
 class PageBase(models.base.ModelBase):
     """Metaclass for Page"""

wagtail/wagtailcore/query.py

@@ -1,3 +1,6 @@
+from collections import defaultdict
+
+from django import VERSION as DJANGO_VERSION
 from django.db.models import Q
 from django.contrib.contenttypes.models import ContentType
 from django.apps import apps
@@ -206,3 +209,59 @@ class PageQuerySet(MP_NodeQuerySet):
         This unpublishes all pages in the QuerySet
         """
         self.update(live=False, has_unpublished_changes=True)
+
+    def specific(self):
+        """
+        This efficiently gets all the specific pages for the queryset, using
+        the minimum number of queries.
+        """
+        if DJANGO_VERSION >= (1, 9):
+            clone = self._clone()
+            clone._iterator_class = SpecificIterator
+            return clone
+        else:
+            return self._clone(klass=SpecificQuerySet)
+
+
+def specific_iterator(qs):
+    """
+    This efficiently iterates all the specific pages in a queryset, using
+    the minimum number of queries.
+
+    This should be called from ``PageQuerySet.specific``
+    """
+    pks_and_types = qs.values_list('pk', 'content_type')
+    pks_by_type = defaultdict(list)
+    for pk, content_type in pks_and_types:
+        pks_by_type[content_type].append(pk)
+
+    # Content types are cached by ID, so this will not run any queries.
+    content_types = {pk: ContentType.objects.get_for_id(pk)
+                     for _, pk in pks_and_types}
+
+    # Get the specific instances of all pages, one model class at a time.
+    pages_by_type = {}
+    for content_type, pks in pks_by_type.items():
+        model = content_types[content_type].model_class()
+        pages = model.objects.filter(pk__in=pks)
+        pages_by_type[content_type] = {page.pk: page for page in pages}
+
+    # Yield all of the pages, in the order they occurred in the original query.
+    for pk, content_type in pks_and_types:
+        yield pages_by_type[content_type][pk]
+
+
+# Django 1.9 changed how extending QuerySets with different iterators behaved
+# considerably, in a way that is not easily compatible between the two versions
+if DJANGO_VERSION >= (1, 9):
+    # TODO Test this once Wagtail runs under Django 1.9.
+    from django.db.models.query import BaseIterator
+
+    class SpecificIterator(BaseIterator):
+        __iter__ = specific_iterator
+
+else:
+    from django.db.models.query import QuerySet
+
+    class SpecificQuerySet(QuerySet):
+        iterator = specific_iterator

wagtail/wagtailcore/tests/test_page_queryset.py

@@ -270,7 +270,6 @@ class TestPageQuerySet(TestCase):
         contact_us = Page.objects.get(url_path='/home/contact-us/')
         self.assertTrue(pages.filter(id=contact_us.id).exists())
 
-
     def test_not_type(self):
         pages = Page.objects.not_type(EventPage)
 
@@ -321,3 +320,93 @@ class TestPageQuerySet(TestCase):
 
         # Check that the event is in the results
         self.assertTrue(pages.filter(id=event.id).exists())
+
+
+class TestSpecificQuery(TestCase):
+    """
+    Test the .specific() queryset method. This is isolated in its own test case
+    because it is sensitive to database changes that might happen for other
+    tests.
+
+    The fixture sets up a page structure like:
+
+    =========== =========================================
+    Type        Path
+    =========== =========================================
+    Page        /
+    Page        /home/
+    SimplePage  /home/about-us/
+    EventIndex  /home/events/
+    EventPage   /home/events/christmas/
+    EventPage   /home/events/someone-elses-event/
+    EventPage   /home/events/tentative-unpublished-event/
+    SimplePage  /home/other/
+    EventPage   /home/other/special-event/
+    =========== =========================================
+    """
+
+    fixtures = ['test_specific.json']
+
+    def test_specific(self):
+        root = Page.objects.get(url_path='/home/')
+
+        with self.assertNumQueries(0):
+            # The query should be lazy.
+            qs = root.get_descendants().specific()
+
+        with self.assertNumQueries(4):
+            # One query to get page type and ID, one query per page type:
+            # EventIndex, EventPage, SimplePage
+            pages = list(qs)
+
+        self.assertIsInstance(pages, list)
+        self.assertEqual(len(pages), 7)
+
+        for page in pages:
+            # An instance of the specific page type should be returned,
+            # not wagtailcore.Page.
+            content_type = page.content_type
+            model = content_type.model_class()
+            self.assertIsInstance(page, model)
+
+            # The page should already be the specific type, so this should not
+            # need another database query.
+            with self.assertNumQueries(0):
+                self.assertIs(page, page.specific)
+
+    def test_filtering_before_specific(self):
+        # This will get the other events, and then christmas
+        # 'someone-elses-event' and the tentative event are unpublished.
+
+        with self.assertNumQueries(0):
+            qs = Page.objects.live().order_by('-url_path')[:3].specific()
+
+        with self.assertNumQueries(3):
+            # Metadata, EventIndex and EventPage
+            pages = list(qs)
+
+        self.assertEqual(len(pages), 3)
+
+        self.assertEqual(pages, [
+            Page.objects.get(url_path='/home/other/special-event/').specific,
+            Page.objects.get(url_path='/home/other/').specific,
+            Page.objects.get(url_path='/home/events/christmas/').specific])
+
+    def test_filtering_after_specific(self):
+        # This will get the other events, and then christmas
+        # 'someone-elses-event' and the tentative event are unpublished.
+
+        with self.assertNumQueries(0):
+            qs = Page.objects.specific().live().in_menu().order_by('-url_path')[:4]
+
+        with self.assertNumQueries(4):
+            # Metadata, EventIndex, EventPage, SimplePage.
+            pages = list(qs)
+
+        self.assertEqual(len(pages), 4)
+
+        self.assertEqual(pages, [
+            Page.objects.get(url_path='/home/other/').specific,
+            Page.objects.get(url_path='/home/events/christmas/').specific,
+            Page.objects.get(url_path='/home/events/').specific,
+            Page.objects.get(url_path='/home/about-us/').specific])