From a713895440657907fd3e39cfafda3a7ad8daf4df Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Tue, 17 Jun 2014 22:14:23 +0100
Subject: [PATCH 01/31] get_page_types no longer required in
 wagtailadmin.views.pages

---
 wagtail/wagtailadmin/views/pages.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wagtail/wagtailadmin/views/pages.py b/wagtail/wagtailadmin/views/pages.py
index 916c373eb..c60a1800e 100644
--- a/wagtail/wagtailadmin/views/pages.py
+++ b/wagtail/wagtailadmin/views/pages.py
@@ -12,7 +12,7 @@ from wagtail.wagtailadmin.edit_handlers import TabbedInterface, ObjectList
 from wagtail.wagtailadmin.forms import SearchForm
 from wagtail.wagtailadmin import tasks, hooks
 
-from wagtail.wagtailcore.models import Page, PageRevision, get_page_types
+from wagtail.wagtailcore.models import Page, PageRevision
 
 
 @permission_required('wagtailadmin.access_admin')

From a4b60715b99135ffe7bc452d312f043125a1d685 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Tue, 17 Jun 2014 22:27:11 +0100
Subject: [PATCH 02/31] Check that the content type passed to
 wagtailadmin.pages.create is valid according to subpage_types

---
 wagtail/wagtailadmin/tests/test_pages_views.py | 13 +++++++++++++
 wagtail/wagtailadmin/views/pages.py            | 12 ++++--------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/wagtail/wagtailadmin/tests/test_pages_views.py b/wagtail/wagtailadmin/tests/test_pages_views.py
index 09e6b6f5c..986bb260c 100644
--- a/wagtail/wagtailadmin/tests/test_pages_views.py
+++ b/wagtail/wagtailadmin/tests/test_pages_views.py
@@ -719,3 +719,16 @@ class TestSubpageBusinessRules(TestCase, WagtailTestUtils):
         self.assertEqual(response.status_code, 200)
         self.assertNotContains(response, 'Standard Child')
         self.assertEqual(0, len(response.context['page_types']))
+
+    def test_cannot_add_invalid_subpage_type(self):
+        # cannot add SimplePage as a child of BusinessIndex, as SimplePage is not present in subpage_types
+        response = self.client.get(reverse('wagtailadmin_pages_create', args=('tests', 'simplepage', self.business_index.id)))
+        self.assertEqual(response.status_code, 403)
+
+        # likewise for BusinessChild which has an empty subpage_types list
+        response = self.client.get(reverse('wagtailadmin_pages_create', args=('tests', 'simplepage', self.business_child.id)))
+        self.assertEqual(response.status_code, 403)
+
+        # but we can add a BusinessChild to BusinessIndex
+        response = self.client.get(reverse('wagtailadmin_pages_create', args=('tests', 'businesschild', self.business_index.id)))
+        self.assertEqual(response.status_code, 200)
diff --git a/wagtail/wagtailadmin/views/pages.py b/wagtail/wagtailadmin/views/pages.py
index c60a1800e..559adb922 100644
--- a/wagtail/wagtailadmin/views/pages.py
+++ b/wagtail/wagtailadmin/views/pages.py
@@ -111,15 +111,11 @@ def create(request, content_type_app_name, content_type_model_name, parent_page_
     except ContentType.DoesNotExist:
         raise Http404
 
-    page_class = content_type.model_class()
-
     # page must be in the list of allowed subpage types for this parent ID
-    # == Restriction temporarily relaxed so that as superusers we can add index pages and things -
-    # == TODO: reinstate this for regular editors when we have distinct user types
-    #
-    # if page_class not in parent_page.clean_subpage_types():
-    #     messages.error(request, "Sorry, you do not have access to create a page of type '%s' here." % content_type.name)
-    #     return redirect('wagtailadmin_pages_select_type')
+    if content_type not in parent_page.clean_subpage_types():
+        raise PermissionDenied
+
+    page_class = content_type.model_class()
 
     page = page_class(owner=request.user)
     edit_handler_class = get_page_edit_handler(page_class)

From 6bfe82f5e59ca375725b2769dc5b2564d3d19e2c Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Tue, 17 Jun 2014 22:53:54 +0100
Subject: [PATCH 03/31] Make can_add_subpage and can_publish_subpage permission
 checks return False for page models that disallow subpages; this ensures that
 'add child page' links are not shown

---
 .../wagtailadmin/tests/test_pages_views.py    | 32 ++++++++++++++++---
 wagtail/wagtailcore/models.py                 |  7 +++-
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/wagtail/wagtailadmin/tests/test_pages_views.py b/wagtail/wagtailadmin/tests/test_pages_views.py
index 986bb260c..5a2958fb5 100644
--- a/wagtail/wagtailadmin/tests/test_pages_views.py
+++ b/wagtail/wagtailadmin/tests/test_pages_views.py
@@ -703,22 +703,44 @@ class TestSubpageBusinessRules(TestCase, WagtailTestUtils):
         self.login()
 
     def test_standard_subpage(self):
-        response = self.client.get(reverse('wagtailadmin_pages_add_subpage', args=(self.standard_index.id, )))
+        add_subpage_url = reverse('wagtailadmin_pages_add_subpage', args=(self.standard_index.id, ))
+
+        # explorer should contain a link to 'add child page'
+        response = self.client.get(reverse('wagtailadmin_explore', args=(self.standard_index.id, )))
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, add_subpage_url)
+
+        # add_subpage should give us the full set of page types to choose
+        response = self.client.get(add_subpage_url)
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, 'Standard Child')
         self.assertContains(response, 'Business Child')
 
     def test_business_subpage(self):
-        response = self.client.get(reverse('wagtailadmin_pages_add_subpage', args=(self.business_index.id, )))
+        add_subpage_url = reverse('wagtailadmin_pages_add_subpage', args=(self.business_index.id, ))
+
+        # explorer should contain a link to 'add child page'
+        response = self.client.get(reverse('wagtailadmin_explore', args=(self.business_index.id, )))
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, add_subpage_url)
+
+        # add_subpage should give us a cut-down set of page types to choose
+        response = self.client.get(add_subpage_url)
         self.assertEqual(response.status_code, 200)
         self.assertNotContains(response, 'Standard Child')
         self.assertContains(response, 'Business Child')
 
     def test_business_child_subpage(self):
-        response = self.client.get(reverse('wagtailadmin_pages_add_subpage', args=(self.business_child.id, )))
+        add_subpage_url = reverse('wagtailadmin_pages_add_subpage', args=(self.business_child.id, ))
+
+        # explorer should not contain a link to 'add child page', as this page doesn't accept subpages
+        response = self.client.get(reverse('wagtailadmin_explore', args=(self.business_child.id, )))
         self.assertEqual(response.status_code, 200)
-        self.assertNotContains(response, 'Standard Child')
-        self.assertEqual(0, len(response.context['page_types']))
+        self.assertNotContains(response, add_subpage_url)
+
+        # this also means that fetching add_subpage is blocked at the permission-check level
+        response = self.client.get(reverse('wagtailadmin_pages_add_subpage', args=(self.business_child.id, )))
+        self.assertEqual(response.status_code, 403)
 
     def test_cannot_add_invalid_subpage_type(self):
         # cannot add SimplePage as a child of BusinessIndex, as SimplePage is not present in subpage_types
diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index 5c3f9d1c5..d7d23c988 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -843,6 +843,8 @@ class PagePermissionTester(object):
     def can_add_subpage(self):
         if not self.user.is_active:
             return False
+        if not self.page.specific_class.clean_subpage_types():  # this page model has an empty subpage_types list, so no subpages are allowed
+            return False
         return self.user.is_superuser or ('add' in self.permissions)
 
     def can_edit(self):
@@ -897,10 +899,13 @@ class PagePermissionTester(object):
         """
         Niggly special case for creating and publishing a page in one go.
         Differs from can_publish in that we want to be able to publish subpages of root, but not
-        to be able to publish root itself
+        to be able to publish root itself. (Also, can_publish_subpage returns false if the page
+        does not allow subpages at all.)
         """
         if not self.user.is_active:
             return False
+        if not self.page.specific_class.clean_subpage_types():  # this page model has an empty subpage_types list, so no subpages are allowed
+            return False
 
         return self.user.is_superuser or ('publish' in self.permissions)
 

From eddb060c8d3ea0701fc7ff651127a0f6611eb295 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Tue, 17 Jun 2014 23:08:30 +0100
Subject: [PATCH 04/31] Bypass 'choose a page type' screen when there is only
 one available choice in subpage_types

---
 wagtail/tests/models.py                       |  3 +++
 .../wagtailadmin/tests/test_pages_views.py    | 19 +++++++++++++++----
 wagtail/wagtailadmin/views/pages.py           |  6 ++++++
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/wagtail/tests/models.py b/wagtail/tests/models.py
index f798129dc..ca6d411da 100644
--- a/wagtail/tests/models.py
+++ b/wagtail/tests/models.py
@@ -303,6 +303,9 @@ class StandardChild(Page):
     pass
 
 class BusinessIndex(Page):
+    subpage_types = ['tests.BusinessChild', 'tests.BusinessSubIndex']
+
+class BusinessSubIndex(Page):
     subpage_types = ['tests.BusinessChild']
 
 class BusinessChild(Page):
diff --git a/wagtail/wagtailadmin/tests/test_pages_views.py b/wagtail/wagtailadmin/tests/test_pages_views.py
index 5a2958fb5..5f4b0021c 100644
--- a/wagtail/wagtailadmin/tests/test_pages_views.py
+++ b/wagtail/wagtailadmin/tests/test_pages_views.py
@@ -1,5 +1,5 @@
 from django.test import TestCase
-from wagtail.tests.models import SimplePage, EventPage, StandardIndex, StandardChild, BusinessIndex, BusinessChild
+from wagtail.tests.models import SimplePage, EventPage, StandardIndex, StandardChild, BusinessIndex, BusinessChild, BusinessSubIndex
 from wagtail.tests.utils import unittest, WagtailTestUtils
 from wagtail.wagtailcore.models import Page, PageRevision
 from django.core.urlresolvers import reverse
@@ -681,24 +681,30 @@ class TestSubpageBusinessRules(TestCase, WagtailTestUtils):
         # Find root page
         self.root_page = Page.objects.get(id=2)
 
-        # Add standard page
+        # Add standard page (allows subpages of any type)
         self.standard_index = StandardIndex()
         self.standard_index.title = "Standard Index"
         self.standard_index.slug = "standard-index"
         self.root_page.add_child(instance=self.standard_index)
 
-        # Add business page
+        # Add business page (allows BusinessChild and BusinessSubIndex as subpages)
         self.business_index = BusinessIndex()
         self.business_index.title = "Business Index"
         self.business_index.slug = "business-index"
         self.root_page.add_child(instance=self.business_index)
 
-        # Add business child
+        # Add business child (allows no subpages)
         self.business_child = BusinessChild()
         self.business_child.title = "Business Child"
         self.business_child.slug = "business-child"
         self.business_index.add_child(instance=self.business_child)
 
+        # Add business subindex (allows only BusinessChild as subpages)
+        self.business_subindex = BusinessSubIndex()
+        self.business_subindex.title = "Business Subindex"
+        self.business_subindex.slug = "business-subindex"
+        self.business_index.add_child(instance=self.business_subindex)
+
         # Login
         self.login()
 
@@ -754,3 +760,8 @@ class TestSubpageBusinessRules(TestCase, WagtailTestUtils):
         # but we can add a BusinessChild to BusinessIndex
         response = self.client.get(reverse('wagtailadmin_pages_create', args=('tests', 'businesschild', self.business_index.id)))
         self.assertEqual(response.status_code, 200)
+
+    def test_not_prompted_for_page_type_when_only_one_choice(self):
+        response = self.client.get(reverse('wagtailadmin_pages_add_subpage', args=(self.business_subindex.id, )))
+        # BusinessChild is the only valid subpage type of BusinessSubIndex, so redirect straight there
+        self.assertRedirects(response, reverse('wagtailadmin_pages_create', args=('tests', 'businesschild', self.business_subindex.id)))
diff --git a/wagtail/wagtailadmin/views/pages.py b/wagtail/wagtailadmin/views/pages.py
index 559adb922..f12a08fe6 100644
--- a/wagtail/wagtailadmin/views/pages.py
+++ b/wagtail/wagtailadmin/views/pages.py
@@ -59,6 +59,12 @@ def add_subpage(request, parent_page_id):
 
     page_types = sorted(parent_page.clean_subpage_types(), key=lambda pagetype: pagetype.name.lower())
 
+    if len(page_types) == 1:
+        # Only one page type is available - redirect straight to the create form rather than
+        # making the user choose
+        content_type = page_types[0]
+        return redirect('wagtailadmin_pages_create', content_type.app_label, content_type.model, parent_page.id)
+
     return render(request, 'wagtailadmin/pages/add_subpage.html', {
         'parent_page': parent_page,
         'page_types': page_types,

From 4081206c77fd5c9bcfc03dfe6ed923dfa4d93f23 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sat, 5 Apr 2014 20:10:58 +0100
Subject: [PATCH 05/31] Scrapped ElasticUtils, use Elasticsearch-py instead

Conflicts:
	wagtail/wagtailsearch/backends/elasticsearch.py
	wagtail/wagtailsearch/indexed.py
---
 runtests.py                                   |   2 +-
 .../wagtailsearch/backends/elasticsearch.py   | 187 +++++++++++++-----
 2 files changed, 142 insertions(+), 47 deletions(-)

diff --git a/runtests.py b/runtests.py
index 96653b40d..efa66910f 100755
--- a/runtests.py
+++ b/runtests.py
@@ -13,7 +13,7 @@ MEDIA_ROOT = os.path.join(WAGTAIL_ROOT, 'test-media')
 if not settings.configured:
 
     try:
-        import elasticutils
+        import elasticsearch
         has_elasticsearch = True
     except ImportError:
         has_elasticsearch = False
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index d2b58d564..7571db5ac 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -1,6 +1,8 @@
+from __future__ import absolute_import
+
 from django.db import models
 
-from elasticutils import get_es, S
+from elasticsearch import Elasticsearch, NotFoundError
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed
@@ -9,16 +11,131 @@ import string
 
 
 class ElasticSearchResults(object):
-    def __init__(self, model, query, prefetch_related=[]):
+    def __init__(self, backend, model, query_string, fields=None, filters={}, prefetch_related=[]):
+        self.backend = backend
         self.model = model
-        self.query = query
-        self.count = query.count()
+        self.query_string = query_string
+        self.fields = fields
+        self.filters = filters
         self.prefetch_related = prefetch_related
 
+    def _get_filters(self):
+        # Filters
+        filters = []
+
+        # Filter by content type
+        filters.append({
+            'prefix': {
+                'content_type': self.model.indexed_get_content_type()
+            }
+        })
+
+        # Extra filters
+        if self.filters:
+            for key, value in self.filters.items():
+                if '__' in key:
+                    field, lookup = key.split('__')
+                else:
+                    field = key
+                    lookup = None
+
+                if lookup is None:
+                    if value is None:
+                        filters.append({
+                            'missing': {
+                                'field': field,
+                            }
+                        })
+                    else:
+                        filters.append({
+                            'term': {
+                                field: value
+                            }
+                        })
+
+                if lookup in ['startswith', 'prefix']:
+                    filters.append({
+                        'prefix': {
+                            field: value
+                        }
+                    })
+
+                if lookup in ['gt', 'gte', 'lt', 'lte']:
+                    filters.append({
+                        'range': {
+                            field: {
+                                lookup: value,
+                            }
+                        }
+                    })
+
+                if lookup == 'range':
+                    lower, upper = value
+                    filters.append({
+                        'range': {
+                            field: {
+                                'gte': lower,
+                                'lte': upper,
+                            }
+                        }
+                    })
+
+        return filters
+
+    def _get_query(self):
+        # Query
+        query = {
+            'query_string': {
+                'query': self.query_string,
+            }
+        }
+
+        # Fields
+        if self.fields:
+            query['query_string']['fields'] = self.fields
+
+        # Filters
+        filters = self._get_filters()
+
+        return {
+            'query': {
+                'filtered': {
+                    'query': query,
+                    'filter': {
+                        'and': filters,
+                    }
+                }
+            }
+        }
+
+    def _get_results_pks(self, offset=0, limit=None):
+        query = self._get_query()
+        query['query']['from'] = offset
+        if limit is not None:
+            query['query']['size'] = limit
+
+        hits = self.backend.es.search(
+            index=self.backend.es_index,
+            body=query,
+            _source=False,
+            fields='pk',
+        )
+
+        return [hit['fields']['pk'][0] for hit in hits['hits']['hits']]
+
+    def _get_count(self):
+        query = self._get_query()
+        count = self.backend.es.count(
+            index=self.backend.es_index,
+            body=query,
+        )
+
+        return count['count']
+
     def __getitem__(self, key):
         if isinstance(key, slice):
             # Get primary keys
-            pk_list_unclean = [result._source["pk"] for result in self.query[key]]
+            pk_list_unclean = self._get_results_pks(key.start, key.stop - key.start)
 
             # Remove duplicate keys (and preserve order)
             seen_pks = set()
@@ -45,11 +162,11 @@ class ElasticSearchResults(object):
             return results_sorted
         else:
             # Return a single item
-            pk = self.query[key]._source["pk"]
+            pk = self._get_results_pks(key, key + 1)[0]
             return self.model.objects.get(pk=pk)
 
     def __len__(self):
-        return self.count
+        return self._get_count()
 
 
 class ElasticSearch(BaseSearch):
@@ -64,22 +181,17 @@ class ElasticSearch(BaseSearch):
 
         # Get ElasticSearch interface
         # Any remaining params are passed into the ElasticSearch constructor
-        self.es = get_es(
+        self.es = Elasticsearch(
             urls=self.es_urls,
             timeout=self.es_timeout,
             force_new=self.es_force_new,
             **params)
-        self.s = S().es(
-            urls=self.es_urls,
-            timeout=self.es_timeout,
-            force_new=self.es_force_new,
-            **params).indexes(self.es_index)
 
     def reset_index(self):
         # Delete old index
         try:
-            self.es.delete_index(self.es_index)
-        except:
+            self.es.indices.delete(self.es_index)
+        except NotFoundError:
             pass
 
         # Settings
@@ -128,7 +240,7 @@ class ElasticSearch(BaseSearch):
         }
 
         # Create new index
-        self.es.create_index(self.es_index, INDEX_SETTINGS)
+        self.es.indices.create(self.es_index, INDEX_SETTINGS)
 
     def add_type(self, model):
         # Get type name
@@ -144,14 +256,14 @@ class ElasticSearch(BaseSearch):
         }.items() + indexed_fields.items())
 
         # Put mapping
-        self.es.put_mapping(self.es_index, content_type, {
+        self.es.indices.put_mapping(index=self.es_index, doc_type=content_type, body={
             content_type: {
                 "properties": fields,
             }
         })
 
     def refresh_index(self):
-        self.es.refresh(self.es_index)
+        self.es.indices.refresh(self.es_index)
 
     def add(self, obj):
         # Make sure the object can be indexed
@@ -165,6 +277,10 @@ class ElasticSearch(BaseSearch):
         self.es.index(self.es_index, obj.indexed_get_content_type(), doc, id=doc["id"])
 
     def add_bulk(self, obj_list):
+        # TODO: Make this work with new elastic search module
+        for obj in obj_list:
+            self.add(obj)
+        return
         # Group all objects by their type
         type_set = {}
         for obj in obj_list:
@@ -194,13 +310,14 @@ class ElasticSearch(BaseSearch):
         if not isinstance(obj, Indexed) or not isinstance(obj, models.Model):
             return
 
-        # Get ID for document
-        doc_id = obj.indexed_get_document_id()
-
         # Delete document
         try:
-            self.es.delete(self.es_index, obj.indexed_get_content_type(), doc_id)
-        except:
+            self.es.delete(
+                self.es_index,
+                obj.indexed_get_content_type(),
+                obj.indexed_get_document_id(),
+            )
+        except NotFoundError:
             pass  # Document doesn't exist, ignore this exception
 
     def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
@@ -215,27 +332,5 @@ class ElasticSearch(BaseSearch):
         if not query_string:
             return []
 
-        # Query
-        if fields:
-            query = self.s.query_raw({
-                "query_string": {
-                    "query": query_string,
-                    "fields": fields,
-                }
-            })
-        else:
-            query = self.s.query_raw({
-                "query_string": {
-                    "query": query_string,
-                }
-            })
-
-        # Filter results by this content type
-        query = query.filter(content_type__prefix=model.indexed_get_content_type())
-
-        # Extra filters
-        if filters:
-            query = query.filter(**filters)
-
         # Return search results
-        return ElasticSearchResults(model, query, prefetch_related=prefetch_related)
+        return ElasticSearchResults(self, model, query_string, fields=fields, filters=filters, prefetch_related=prefetch_related)

From c00c4d24372079e8a07cc4276cfb2568643abd80 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 6 Apr 2014 14:35:57 +0100
Subject: [PATCH 06/31] New ElasticSearch module now supports bulk insert

---
 .../wagtailsearch/backends/elasticsearch.py   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 7571db5ac..78de0e94f 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from django.db import models
 
 from elasticsearch import Elasticsearch, NotFoundError
+from elasticsearch.helpers import bulk
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed
@@ -277,10 +278,6 @@ class ElasticSearch(BaseSearch):
         self.es.index(self.es_index, obj.indexed_get_content_type(), doc, id=doc["id"])
 
     def add_bulk(self, obj_list):
-        # TODO: Make this work with new elastic search module
-        for obj in obj_list:
-            self.add(obj)
-        return
         # Group all objects by their type
         type_set = {}
         for obj in obj_list:
@@ -299,11 +296,19 @@ class ElasticSearch(BaseSearch):
             type_set[obj_type].append(obj.indexed_build_document())
 
         # Loop through each type and bulk add them
-        results = []
         for type_name, type_objects in type_set.items():
-            results.append((type_name, len(type_objects)))
-            self.es.bulk_index(self.es_index, type_name, type_objects)
-        return results
+            # Get list of actions
+            actions = []
+            for obj in type_objects:
+                action = {
+                    '_index': self.es_index,
+                    '_type': type_name,
+                    '_id': obj['id'],
+                }
+                action.update(obj)
+                actions.append(action)
+
+            bulk(self.es, actions)
 
     def delete(self, obj):
         # Object must be a decendant of Indexed and be a django model

From 598e6193da90908241937fe1a3fe9ab27983df36 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 6 Apr 2014 18:07:53 +0100
Subject: [PATCH 07/31] Install elasticsearch instead of elasticutils on travis

Conflicts:
	.travis.yml
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3e0a4bb98..8764f2859 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ services:
 # Package installation
 install:
   - python setup.py install
-  - pip install psycopg2 pyelasticsearch elasticutils==0.8.2 wand embedly
+  - pip install psycopg2 elasticsearch wand embedly
   - pip install coveralls
 # Pre-test configuration
 before_script:

From 05cb87e3eb99c07764f176359d3ad488e020bb37 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Tue, 8 Apr 2014 10:47:13 +0100
Subject: [PATCH 08/31] Tests now succeed on elasticsearch 0.90.x

---
 .../wagtailsearch/backends/elasticsearch.py   | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 78de0e94f..d626d1d1c 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -2,7 +2,7 @@ from __future__ import absolute_import
 
 from django.db import models
 
-from elasticsearch import Elasticsearch, NotFoundError
+from elasticsearch import Elasticsearch, NotFoundError, RequestError
 from elasticsearch.helpers import bulk
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
@@ -99,38 +99,48 @@ class ElasticSearchResults(object):
         filters = self._get_filters()
 
         return {
-            'query': {
-                'filtered': {
-                    'query': query,
-                    'filter': {
-                        'and': filters,
-                    }
+            'filtered': {
+                'query': query,
+                'filter': {
+                    'and': filters,
                 }
             }
         }
 
     def _get_results_pks(self, offset=0, limit=None):
         query = self._get_query()
-        query['query']['from'] = offset
+        query['from'] = offset
         if limit is not None:
-            query['query']['size'] = limit
+            query['size'] = limit
 
         hits = self.backend.es.search(
             index=self.backend.es_index,
-            body=query,
+            body=dict(query=query),
             _source=False,
             fields='pk',
         )
 
-        return [hit['fields']['pk'][0] for hit in hits['hits']['hits']]
+        pks = [hit['fields']['pk'] for hit in hits['hits']['hits']]
+
+        # ElasticSearch 1.x likes to pack pks into lists, unpack them if this has happened
+        return [pk[0] if isinstance(pk, list) else pk for pk in pks]
 
     def _get_count(self):
         query = self._get_query()
+
+        # Elasticsearch 1.x
         count = self.backend.es.count(
             index=self.backend.es_index,
-            body=query,
+            body=dict(query=query),
         )
 
+        # ElasticSearch 0.90.x fallback
+        if not count['_shards']['successful'] and "No query registered for [query]]" in count['_shards']['failures'][0]['reason']:
+            count = self.backend.es.count(
+                index=self.backend.es_index,
+                body=query,
+            )
+
         return count['count']
 
     def __getitem__(self, key):

From 112ff57d20b079f4a80b66d0f255ce9532af9be9 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 10:58:44 +0100
Subject: [PATCH 09/31] Split ESResults into ESResults and ESQuery

---
 .../wagtailsearch/backends/elasticsearch.py   | 43 +++++++++++--------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index d626d1d1c..434a4e380 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -11,14 +11,12 @@ from wagtail.wagtailsearch.indexed import Indexed
 import string
 
 
-class ElasticSearchResults(object):
-    def __init__(self, backend, model, query_string, fields=None, filters={}, prefetch_related=[]):
-        self.backend = backend
+class ElasticSearchQuery(object):
+    def __init__(self, model, query_string, fields=None, filters={}):
         self.model = model
         self.query_string = query_string
-        self.fields = fields
+        self.fields = fields or ['_all']
         self.filters = filters
-        self.prefetch_related = prefetch_related
 
     def _get_filters(self):
         # Filters
@@ -83,7 +81,7 @@ class ElasticSearchResults(object):
 
         return filters
 
-    def _get_query(self):
+    def to_es(self):
         # Query
         query = {
             'query_string': {
@@ -107,26 +105,37 @@ class ElasticSearchResults(object):
             }
         }
 
-    def _get_results_pks(self, offset=0, limit=None):
-        query = self._get_query()
-        query['from'] = offset
-        if limit is not None:
-            query['size'] = limit
 
-        hits = self.backend.es.search(
+class ElasticSearchResults(object):
+    def __init__(self, backend, query, prefetch_related=[]):
+        self.backend = backend
+        self.query = query
+        self.prefetch_related = prefetch_related
+
+    def _get_results_pks(self, offset=0, limit=None):
+        # Params for elasticsearch query
+        params = dict(
             index=self.backend.es_index,
-            body=dict(query=query),
+            body=dict(query=self.query.to_es()),
             _source=False,
             fields='pk',
+            from_=offset,
         )
 
+        # Add limit if set
+        if limit is not None:
+            params['size'] = limit
+
+        # Send to ElasticSearch
+        hits = self.backend.es.search(**params)
+
         pks = [hit['fields']['pk'] for hit in hits['hits']['hits']]
 
         # ElasticSearch 1.x likes to pack pks into lists, unpack them if this has happened
         return [pk[0] if isinstance(pk, list) else pk for pk in pks]
 
     def _get_count(self):
-        query = self._get_query()
+        query = self.query.to_es()
 
         # Elasticsearch 1.x
         count = self.backend.es.count(
@@ -157,7 +166,7 @@ class ElasticSearchResults(object):
                     pk_list.append(pk)
 
             # Get results
-            results = self.model.objects.filter(pk__in=pk_list)
+            results = self.query.model.objects.filter(pk__in=pk_list)
 
             # Prefetch related
             for prefetch in self.prefetch_related:
@@ -174,7 +183,7 @@ class ElasticSearchResults(object):
         else:
             # Return a single item
             pk = self._get_results_pks(key, key + 1)[0]
-            return self.model.objects.get(pk=pk)
+            return self.query.model.objects.get(pk=pk)
 
     def __len__(self):
         return self._get_count()
@@ -348,4 +357,4 @@ class ElasticSearch(BaseSearch):
             return []
 
         # Return search results
-        return ElasticSearchResults(self, model, query_string, fields=fields, filters=filters, prefetch_related=prefetch_related)
+        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters), prefetch_related=prefetch_related)

From b2844a32252eebd72604a5a68c0f59ba3ca86d87 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:01:19 +0100
Subject: [PATCH 10/31] Renamed a couple of methods on ElasticSearchResults

---
 wagtail/wagtailsearch/backends/elasticsearch.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 434a4e380..c47427d35 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -112,7 +112,7 @@ class ElasticSearchResults(object):
         self.query = query
         self.prefetch_related = prefetch_related
 
-    def _get_results_pks(self, offset=0, limit=None):
+    def _do_search(self, offset=0, limit=None):
         # Params for elasticsearch query
         params = dict(
             index=self.backend.es_index,
@@ -134,7 +134,7 @@ class ElasticSearchResults(object):
         # ElasticSearch 1.x likes to pack pks into lists, unpack them if this has happened
         return [pk[0] if isinstance(pk, list) else pk for pk in pks]
 
-    def _get_count(self):
+    def _do_count(self):
         query = self.query.to_es()
 
         # Elasticsearch 1.x
@@ -155,7 +155,7 @@ class ElasticSearchResults(object):
     def __getitem__(self, key):
         if isinstance(key, slice):
             # Get primary keys
-            pk_list_unclean = self._get_results_pks(key.start, key.stop - key.start)
+            pk_list_unclean = self._do_search(key.start, key.stop - key.start)
 
             # Remove duplicate keys (and preserve order)
             seen_pks = set()
@@ -182,11 +182,11 @@ class ElasticSearchResults(object):
             return results_sorted
         else:
             # Return a single item
-            pk = self._get_results_pks(key, key + 1)[0]
+            pk = self._do_search(key, key + 1)[0]
             return self.query.model.objects.get(pk=pk)
 
     def __len__(self):
-        return self._get_count()
+        return self._do_count()
 
 
 class ElasticSearch(BaseSearch):

From f73d66908e32891ba9e49b1be2bf1cb4c3233508 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:02:12 +0100
Subject: [PATCH 11/31] Added __repr__ method to ElasticSearchQuery

---
 wagtail/wagtailsearch/backends/elasticsearch.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index c47427d35..7b301442e 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -1,5 +1,8 @@
 from __future__ import absolute_import
 
+import string
+import json
+
 from django.db import models
 
 from elasticsearch import Elasticsearch, NotFoundError, RequestError
@@ -8,7 +11,6 @@ from elasticsearch.helpers import bulk
 from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed
 
-import string
 
 
 class ElasticSearchQuery(object):
@@ -105,6 +107,9 @@ class ElasticSearchQuery(object):
             }
         }
 
+    def __repr__(self):
+        return json.dumps(self.to_es())
+
 
 class ElasticSearchResults(object):
     def __init__(self, backend, query, prefetch_related=[]):

From 3d3d0eb2bdc20a394eb316bcfcc13ea7b61453b2 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:09:45 +0100
Subject: [PATCH 12/31] Removed ability to use prefetch_related in search
 queries This will be replaced by search on queryset

---
 wagtail/wagtailsearch/backends/db.py            |  7 ++++---
 wagtail/wagtailsearch/backends/elasticsearch.py | 13 ++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
index 419d8a99b..0531bc3aa 100644
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -1,3 +1,5 @@
+import warnings
+
 from django.db import models
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
@@ -64,8 +66,7 @@ class DBSearch(BaseSearch):
         # Distinct
         query = query.distinct()
 
-        # Prefetch related
-        for prefetch in prefetch_related:
-            query = query.prefetch_related(prefetch)
+        # Give deprecation warning if prefetch_related was used
+        warnings.warn("prefetch_related on search queries is no longer implemented. ", DeprecationWarning)
 
         return query
\ No newline at end of file
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 7b301442e..e9fa739f8 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -2,6 +2,7 @@ from __future__ import absolute_import
 
 import string
 import json
+import warnings
 
 from django.db import models
 
@@ -112,10 +113,9 @@ class ElasticSearchQuery(object):
 
 
 class ElasticSearchResults(object):
-    def __init__(self, backend, query, prefetch_related=[]):
+    def __init__(self, backend, query):
         self.backend = backend
         self.query = query
-        self.prefetch_related = prefetch_related
 
     def _do_search(self, offset=0, limit=None):
         # Params for elasticsearch query
@@ -173,10 +173,6 @@ class ElasticSearchResults(object):
             # Get results
             results = self.query.model.objects.filter(pk__in=pk_list)
 
-            # Prefetch related
-            for prefetch in self.prefetch_related:
-                results = results.prefetch_related(prefetch)
-
             # Put results into a dictionary (using primary key as the key)
             results_dict = dict((str(result.pk), result) for result in results)
 
@@ -361,5 +357,8 @@ class ElasticSearch(BaseSearch):
         if not query_string:
             return []
 
+        # Give deprecation warning if prefetch_related was used
+        warnings.warn("prefetch_related on search queries is no longer implemented. ", DeprecationWarning)
+
         # Return search results
-        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters), prefetch_related=prefetch_related)
+        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters))

From 69c9863d4c60c512b08e2f9375791eb9dd295f51 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:18:18 +0100
Subject: [PATCH 13/31] Made finding objects for search results the
 responsibility of do_search

---
 .../wagtailsearch/backends/elasticsearch.py   | 40 +++++++------------
 1 file changed, 15 insertions(+), 25 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index e9fa739f8..eecbd2f1d 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -137,7 +137,18 @@ class ElasticSearchResults(object):
         pks = [hit['fields']['pk'] for hit in hits['hits']['hits']]
 
         # ElasticSearch 1.x likes to pack pks into lists, unpack them if this has happened
-        return [pk[0] if isinstance(pk, list) else pk for pk in pks]
+        pks = [pk[0] if isinstance(pk, list) else pk for pk in pks]
+
+        # Initialise results dictionary
+        results = dict((str(pk), None) for pk in pks)
+
+        # Find objects in database and add them to dict
+        queryset = self.query.model.objects.filter(pk__in=pks)
+        for obj in queryset:
+            results[str(obj.pk)] = obj
+
+        # Return results in order given by ElasticSearch
+        return [results[str(pk)] for pk in pks if results[str(pk)]]
 
     def _do_count(self):
         query = self.query.to_es()
@@ -159,32 +170,11 @@ class ElasticSearchResults(object):
 
     def __getitem__(self, key):
         if isinstance(key, slice):
-            # Get primary keys
-            pk_list_unclean = self._do_search(key.start, key.stop - key.start)
-
-            # Remove duplicate keys (and preserve order)
-            seen_pks = set()
-            pk_list = []
-            for pk in pk_list_unclean:
-                if pk not in seen_pks:
-                    seen_pks.add(pk)
-                    pk_list.append(pk)
-
-            # Get results
-            results = self.query.model.objects.filter(pk__in=pk_list)
-
-            # Put results into a dictionary (using primary key as the key)
-            results_dict = dict((str(result.pk), result) for result in results)
-
-            # Build new list with items in the correct order
-            results_sorted = [results_dict[str(pk)] for pk in pk_list if str(pk) in results_dict]
-
-            # Return the list
-            return results_sorted
+            # Run query
+            return self._do_search(key.start, key.stop - key.start)
         else:
             # Return a single item
-            pk = self._do_search(key, key + 1)[0]
-            return self.query.model.objects.get(pk=pk)
+            return self._do_search(key, key + 1)[0]
 
     def __len__(self):
         return self._do_count()

From 16b385da974402c20df7e7ef8a1a3b1595ba049e Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:35:19 +0100
Subject: [PATCH 14/31] Slicing ESResults now returns a new ESResults object
 Previously, slicing an ESResults object made it run a query against
 ElasticSearch and return the results This commit changes this by making slice
 return a new ESResults object with start and stop limits applied. To get
 results, you now have to iterate the ESResults object.

---
 .../wagtailsearch/backends/elasticsearch.py   | 62 ++++++++++++++++---
 1 file changed, 52 insertions(+), 10 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index eecbd2f1d..6bd48aafe 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -116,24 +116,47 @@ class ElasticSearchResults(object):
     def __init__(self, backend, query):
         self.backend = backend
         self.query = query
+        self.start = 0
+        self.stop = None
 
-    def _do_search(self, offset=0, limit=None):
+    def _set_limits(self, start=None, stop=None):
+        if stop is not None:
+            if self.stop is not None:
+                self.stop = min(self.stop, self.start + stop)
+            else:
+                self.stop = self.start + stop
+
+        if start is not None:
+            if self.stop is not None:
+                self.start = min(self.stop, self.start + start)
+            else:
+                self.start = self.start + start
+
+    def _clone(self):
+        klass = self.__class__
+        new = klass(self.backend, self.query)
+        new.start = self.start
+        new.stop = self.stop
+        return new
+
+    def _do_search(self):
         # Params for elasticsearch query
         params = dict(
             index=self.backend.es_index,
             body=dict(query=self.query.to_es()),
             _source=False,
             fields='pk',
-            from_=offset,
+            from_=self.start,
         )
 
-        # Add limit if set
-        if limit is not None:
-            params['size'] = limit
+        # Add size if set
+        if self.stop is not None:
+            params['size'] = self.stop - self.start
 
         # Send to ElasticSearch
         hits = self.backend.es.search(**params)
 
+        # Get pks from results
         pks = [hit['fields']['pk'] for hit in hits['hits']['hits']]
 
         # ElasticSearch 1.x likes to pack pks into lists, unpack them if this has happened
@@ -151,6 +174,7 @@ class ElasticSearchResults(object):
         return [results[str(pk)] for pk in pks if results[str(pk)]]
 
     def _do_count(self):
+        # Get query
         query = self.query.to_es()
 
         # Elasticsearch 1.x
@@ -166,15 +190,33 @@ class ElasticSearchResults(object):
                 body=query,
             )
 
-        return count['count']
+        # Get count
+        hit_count = count['count']
+
+        # Add limits
+        hit_count -= self.start
+        if self.stop is not None:
+            hit_count = min(hit_count, self.stop - self.start)
+
+        return max(hit_count, 0)
 
     def __getitem__(self, key):
+        new = self._clone()
+
         if isinstance(key, slice):
-            # Run query
-            return self._do_search(key.start, key.stop - key.start)
+            # Set limits
+            start = int(key.start) if key.start else None
+            stop = int(key.stop) if key.stop else None
+            new._set_limits(start, stop)
+
+            return new
         else:
-            # Return a single item
-            return self._do_search(key, key + 1)[0]
+            new.start = key
+            new.stop = key + 1
+            return list(new)[0]
+
+    def __iter__(self):
+        return iter(self._do_search())
 
     def __len__(self):
         return self._do_count()

From 1475d440e3b5a38eb9ff4a939b7bce4fcc1d0850 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:42:17 +0100
Subject: [PATCH 15/31] len(ESResults) now runs query and gets the length of
 the results. Added ESResults.count() which asks Elasticsearch for the count
 but doesn't run the query

---
 wagtail/wagtailsearch/backends/elasticsearch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 6bd48aafe..5f62b96a8 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -173,7 +173,7 @@ class ElasticSearchResults(object):
         # Return results in order given by ElasticSearch
         return [results[str(pk)] for pk in pks if results[str(pk)]]
 
-    def _do_count(self):
+    def count(self):
         # Get query
         query = self.query.to_es()
 
@@ -219,7 +219,7 @@ class ElasticSearchResults(object):
         return iter(self._do_search())
 
     def __len__(self):
-        return self._do_count()
+        return len(self._do_search())
 
 
 class ElasticSearch(BaseSearch):

From d4aa5d50905ecde6bf5199218cdc102b87b8ab1b Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:43:33 +0100
Subject: [PATCH 16/31] Added __repr__ method to ElasticSearchResults

---
 wagtail/wagtailsearch/backends/elasticsearch.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 5f62b96a8..e9db7fbf1 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -221,6 +221,12 @@ class ElasticSearchResults(object):
     def __len__(self):
         return len(self._do_search())
 
+    def __repr__(self):
+        data = list(self[:21])
+        if len(data) > 20:
+            data[-1] = "...(remaining elements truncated)..."
+        return repr(data)
+
 
 class ElasticSearch(BaseSearch):
     def __init__(self, params):

From a5834513c31b2260287c2411ec3a3dcb25e11d12 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 20 Jun 2014 11:51:38 +0100
Subject: [PATCH 17/31] Implemented results caching on ElasticSearchResults

---
 .../wagtailsearch/backends/elasticsearch.py   | 28 +++++++++++++++++--
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index e9db7fbf1..29141542a 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -118,6 +118,8 @@ class ElasticSearchResults(object):
         self.query = query
         self.start = 0
         self.stop = None
+        self._results_cache = None
+        self._count_cache = None
 
     def _set_limits(self, start=None, stop=None):
         if stop is not None:
@@ -173,7 +175,7 @@ class ElasticSearchResults(object):
         # Return results in order given by ElasticSearch
         return [results[str(pk)] for pk in pks if results[str(pk)]]
 
-    def count(self):
+    def _do_count(self):
         # Get query
         query = self.query.to_es()
 
@@ -200,6 +202,19 @@ class ElasticSearchResults(object):
 
         return max(hit_count, 0)
 
+    def results(self):
+        if self._results_cache is None:
+            self._results_cache = self._do_search()
+        return self._results_cache
+
+    def count(self):
+        if self._count_cache is None:
+            if self._results_cache is not None:
+                self._count_cache = len(self._results_cache)
+            else:
+                self._count_cache = self._do_count()
+        return self._count_cache
+
     def __getitem__(self, key):
         new = self._clone()
 
@@ -209,17 +224,24 @@ class ElasticSearchResults(object):
             stop = int(key.stop) if key.stop else None
             new._set_limits(start, stop)
 
+            # Copy results cache
+            if self._results_cache is not None:
+                new._results_cache = self._results_cache[key]
+
             return new
         else:
+            if self._results_cache is not None:
+                return self._results_cache[key]
+
             new.start = key
             new.stop = key + 1
             return list(new)[0]
 
     def __iter__(self):
-        return iter(self._do_search())
+        return iter(self.results())
 
     def __len__(self):
-        return len(self._do_search())
+        return len(self.results())
 
     def __repr__(self):
         data = list(self[:21])

From 8cb94f0b5c37df70624453ca0a7a1b8269f8973b Mon Sep 17 00:00:00 2001
From: Dave Cranwell <xcession@xcessionsites.com>
Date: Fri, 20 Jun 2014 13:46:43 +0100
Subject: [PATCH 18/31] Update frontenddevelopers.rst

Added notes about new image manipulation techniques
---
 .../building_your_site/frontenddevelopers.rst | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index a8cf6a25e..bb12c5922 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -186,7 +186,24 @@ The available resizing methods are:
 More control over the ``img`` tag
 ---------------------------------
 
-In some cases greater control over the ``img`` tag is required, for example to add a custom ``class``. Rather than generating the ``img`` element for you, Wagtail can assign the relevant data to another object using Django's ``as`` syntax:
+Greater control over the ``img`` tag is often required, for example to add a custom ``class``. This can be most easily achieved in two ways:
+
+.. versionadded:: 0.4
+
+Adding attributes to the  {% image %} tag
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Simply add extra attributes with the syntax ``attribute="value"`` to the tag:
+
+.. code-block:: django
+
+    {% image self.photo width-400 class="foo" id="bar" %}
+
+
+Generating the image "as"
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Wagtail can assign the image data to another object using Django's ``as`` syntax:
 
 .. code-block:: django
     
@@ -197,12 +214,17 @@ In some cases greater control over the ``img`` tag is required, for example to a
     <img src="{{ tmp_photo.src }}" width="{{ tmp_photo.width }}" 
         height="{{ tmp_photo.height }}" alt="{{ tmp_photo.alt }}" class="my-custom-class" />
 
+
+The ``attrs`` shortcut
+-----------------------
+
 You can also use the ``attrs`` property as a shorthand to output the ``src``, ``width``, ``height`` and ``alt`` attributes in one go:
 
 .. code-block:: django
 
     <img {{ tmp_photo.attrs }} class="my-custom-class" />
 
+
 .. _rich-text-filter:
 
 Rich text (filter)

From c1f2f745afcd99cefccc9d9ac0f7615a2a73c69d Mon Sep 17 00:00:00 2001
From: Dave Cranwell <xcession@xcessionsites.com>
Date: Fri, 20 Jun 2014 13:49:25 +0100
Subject: [PATCH 19/31] Update frontenddevelopers.rst

heading level fix
---
 docs/building_your_site/frontenddevelopers.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index bb12c5922..7b674bf0d 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -190,8 +190,7 @@ Greater control over the ``img`` tag is often required, for example to add a cus
 
 .. versionadded:: 0.4
 
-Adding attributes to the  {% image %} tag
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**Adding attributes to the  {% image %} tag**
 
 Simply add extra attributes with the syntax ``attribute="value"`` to the tag:
 
@@ -200,8 +199,8 @@ Simply add extra attributes with the syntax ``attribute="value"`` to the tag:
     {% image self.photo width-400 class="foo" id="bar" %}
 
 
-Generating the image "as"
-~~~~~~~~~~~~~~~~~~~~~~~~~
+**Generating the image "as"**
+
 
 Wagtail can assign the image data to another object using Django's ``as`` syntax:
 

From 135d4f20a622196caf33c4eb2ac8294b5e8b7f66 Mon Sep 17 00:00:00 2001
From: Dave Cranwell <xcession@xcessionsites.com>
Date: Fri, 20 Jun 2014 14:04:11 +0100
Subject: [PATCH 20/31] Update frontenddevelopers.rst

---
 docs/building_your_site/frontenddevelopers.rst | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index 7b674bf0d..602c53c7f 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -186,34 +186,32 @@ The available resizing methods are:
 More control over the ``img`` tag
 ---------------------------------
 
-Greater control over the ``img`` tag is often required, for example to add a custom ``class``. This can be most easily achieved in two ways:
+Wagtail provides two shorcuts to gain grater greater control over the ``img`` element:
 
 .. versionadded:: 0.4
-
 **Adding attributes to the  {% image %} tag**
 
-Simply add extra attributes with the syntax ``attribute="value"`` to the tag:
+Extra attributes can be specified with the syntax ``attribute="value"``:
 
 .. code-block:: django
 
     {% image self.photo width-400 class="foo" id="bar" %}
 
+No validation is performed on attributes add in this way by the developer. It's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself.
+
 
 **Generating the image "as"**
 
-
 Wagtail can assign the image data to another object using Django's ``as`` syntax:
 
 .. code-block:: django
     
-    {% load image %}
-    ...
     {% image self.photo width-400 as tmp_photo %}
 
     <img src="{{ tmp_photo.src }}" width="{{ tmp_photo.width }}" 
         height="{{ tmp_photo.height }}" alt="{{ tmp_photo.alt }}" class="my-custom-class" />
 
-
+.. versionadded:: 0.4
 The ``attrs`` shortcut
 -----------------------
 

From ba0805e521986329a7ec4f55bde934aec30e678e Mon Sep 17 00:00:00 2001
From: Dave Cranwell <xcession@xcessionsites.com>
Date: Fri, 20 Jun 2014 14:07:33 +0100
Subject: [PATCH 21/31] Update frontenddevelopers.rst

typo fix
---
 docs/building_your_site/frontenddevelopers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index 602c53c7f..8d0c9a3bb 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -186,7 +186,7 @@ The available resizing methods are:
 More control over the ``img`` tag
 ---------------------------------
 
-Wagtail provides two shorcuts to gain grater greater control over the ``img`` element:
+Wagtail provides two shorcuts to give greater control over the ``img`` element:
 
 .. versionadded:: 0.4
 **Adding attributes to the  {% image %} tag**

From 55cd5aad835f604583e30cfd2375358d08e08606 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Fri, 20 Jun 2014 14:08:49 +0100
Subject: [PATCH 22/31] Revert frame-writing stuff from #315, as it fails when
 the preview loads before the placeholder - see
 https://github.com/torchbox/wagtail/pull/315#issuecomment-46669595

---
 .../static/wagtailadmin/js/page-editor.js     | 19 +++++--------------
 .../templates/wagtailadmin/pages/preview.html |  1 -
 wagtail/wagtailadmin/urls.py                  |  1 -
 wagtail/wagtailadmin/views/pages.py           |  6 ------
 4 files changed, 5 insertions(+), 22 deletions(-)

diff --git a/wagtail/wagtailadmin/static/wagtailadmin/js/page-editor.js b/wagtail/wagtailadmin/static/wagtailadmin/js/page-editor.js
index eacb8f86d..03adb7aca 100644
--- a/wagtail/wagtailadmin/static/wagtailadmin/js/page-editor.js
+++ b/wagtail/wagtailadmin/static/wagtailadmin/js/page-editor.js
@@ -329,9 +329,9 @@ $(function() {
     });
 
     /* Set up behaviour of preview button */
-    $('.action-preview').click(function(e) {        
+    $('.action-preview').click(function(e) {
         e.preventDefault();
-        
+
         var previewWindow = window.open($(this).data('placeholder'), $(this).data('windowname'));
 
         $.ajax({
@@ -340,18 +340,9 @@ $(function() {
             data: $('#page-edit-form').serialize(),
             success: function(data, textStatus, request) {
                 if (request.getResponseHeader('X-Wagtail-Preview') == 'ok') {
-                    var pdoc = previewWindow.document;
-                    var frame = pdoc.getElementById('preview-frame');
-
-                    frame = frame.contentWindow || frame.contentDocument.document || frame.contentDocument;
-                    frame.document.open();
-                    frame.document.write(data);                 
-                    frame.document.close();
-
-                    var hideTimeout = setTimeout(function(){
-                        pdoc.getElementById('loading-spinner-wrapper').className += 'remove';
-                        clearTimeout(hideTimeout);
-                    }, 50) // just enough to give effect without adding discernible slowness
+                    previewWindow.document.open();
+                    previewWindow.document.write(data);
+                    previewWindow.document.close();
                 } else {
                     previewWindow.close();
                     document.open();
diff --git a/wagtail/wagtailadmin/templates/wagtailadmin/pages/preview.html b/wagtail/wagtailadmin/templates/wagtailadmin/pages/preview.html
index d5c4b7a51..46a4d2867 100644
--- a/wagtail/wagtailadmin/templates/wagtailadmin/pages/preview.html
+++ b/wagtail/wagtailadmin/templates/wagtailadmin/pages/preview.html
@@ -11,6 +11,5 @@
         <div id="loading-spinner-wrapper">
             <div id="loading-spinner"></div>
         </div>
-        <iframe id="preview-frame" src="{% url 'wagtailadmin_pages_preview_loading' %}"></iframe>
     </body>
 </html>
diff --git a/wagtail/wagtailadmin/urls.py b/wagtail/wagtailadmin/urls.py
index 21f127c1d..ee68f4655 100644
--- a/wagtail/wagtailadmin/urls.py
+++ b/wagtail/wagtailadmin/urls.py
@@ -50,7 +50,6 @@ urlpatterns += [
     url(r'^pages/(\d+)/edit/preview/$', pages.preview_on_edit, name='wagtailadmin_pages_preview_on_edit'),
 
     url(r'^pages/preview/$', pages.preview, name='wagtailadmin_pages_preview'),
-    url(r'^pages/preview_loading/$', pages.preview_loading, name='wagtailadmin_pages_preview_loading'),
 
     url(r'^pages/(\d+)/view_draft/$', pages.view_draft, name='wagtailadmin_pages_view_draft'),
     url(r'^pages/(\d+)/add_subpage/$', pages.add_subpage, name='wagtailadmin_pages_add_subpage'),
diff --git a/wagtail/wagtailadmin/views/pages.py b/wagtail/wagtailadmin/views/pages.py
index bfca64463..f215eb213 100644
--- a/wagtail/wagtailadmin/views/pages.py
+++ b/wagtail/wagtailadmin/views/pages.py
@@ -420,12 +420,6 @@ def preview(request):
     """
     return render(request, 'wagtailadmin/pages/preview.html')
 
-def preview_loading(request):
-    """
-    This page is blank, but must be real HTML so its DOM can be written to once the preview of the page has rendered
-    """
-    return HttpResponse("<html><head><title></title></head><body></body></html>")
-
 @permission_required('wagtailadmin.access_admin')
 def unpublish(request, page_id):
     page = get_object_or_404(Page, id=page_id)

From 2afb13667fa99150bc6cbeaa52061f97dfd2aae6 Mon Sep 17 00:00:00 2001
From: jmiguelv <jose.m.vieira@kcl.ac.uk>
Date: Thu, 19 Jun 2014 10:48:51 +0100
Subject: [PATCH 23/31] Added a new hook to add new element rules to the
 Whitelister.

---
 wagtail/wagtailcore/whitelist.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/wagtail/wagtailcore/whitelist.py b/wagtail/wagtailcore/whitelist.py
index a3d377bd6..0cfbbd2db 100644
--- a/wagtail/wagtailcore/whitelist.py
+++ b/wagtail/wagtailcore/whitelist.py
@@ -5,6 +5,8 @@ specific rules.
 from bs4 import BeautifulSoup, NavigableString, Tag
 from urlparse import urlparse
 
+from wagtail.wagtailadmin import hooks
+
 
 ALLOWED_URL_SCHEMES = ['', 'http', 'https', 'ftp', 'mailto', 'tel']
 
@@ -65,7 +67,8 @@ class Whitelister(object):
         'h6': allow_without_attributes,
         'hr': allow_without_attributes,
         'i': allow_without_attributes,
-        'img': attribute_rule({'src': check_url, 'width': True, 'height': True, 'alt': True}),
+        'img': attribute_rule({'src': check_url, 'width': True, 'height': True,
+                               'alt': True}),
         'li': allow_without_attributes,
         'ol': allow_without_attributes,
         'p': allow_without_attributes,
@@ -77,7 +80,12 @@ class Whitelister(object):
 
     @classmethod
     def clean(cls, html):
-        """Clean up an HTML string to contain just the allowed elements / attributes"""
+        """Clean up an HTML string to contain just the allowed elements /
+        attributes"""
+        for fn in hooks.get_hooks('construct_whitelister_element_rules'):
+            cls.element_rules = dict(
+                cls.element_rules.items() + fn().items())
+
         doc = BeautifulSoup(html, 'lxml')
         cls.clean_node(doc, doc)
         return unicode(doc)

From 9bc770d0a65386b81955d7660e14af00d9230ed3 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Fri, 20 Jun 2014 14:50:10 +0100
Subject: [PATCH 24/31] Add unit tests for DbWhitelister, including ability to
 override the whitelist via hooks

---
 wagtail/tests/wagtail_hooks.py                |  9 ++++
 .../wagtailcore/tests/test_dbwhitelister.py   | 43 +++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 wagtail/wagtailcore/tests/test_dbwhitelister.py

diff --git a/wagtail/tests/wagtail_hooks.py b/wagtail/tests/wagtail_hooks.py
index 91c76c9f4..5dccca666 100644
--- a/wagtail/tests/wagtail_hooks.py
+++ b/wagtail/tests/wagtail_hooks.py
@@ -1,4 +1,5 @@
 from wagtail.wagtailadmin import hooks
+from wagtail.wagtailcore.whitelist import attribute_rule, check_url, allow_without_attributes
 
 def editor_css():
     return """<link rel="stylesheet" href="/path/to/my/custom.css">"""
@@ -8,3 +9,11 @@ hooks.register('insert_editor_css', editor_css)
 def editor_js():
     return """<script src="/path/to/my/custom.js"></script>"""
 hooks.register('insert_editor_js', editor_js)
+
+
+def whitelister_element_rules():
+    return {
+        'blockquote': allow_without_attributes,
+        'a': attribute_rule({'href': check_url, 'target': True}),
+    }
+hooks.register('construct_whitelister_element_rules', whitelister_element_rules)
diff --git a/wagtail/wagtailcore/tests/test_dbwhitelister.py b/wagtail/wagtailcore/tests/test_dbwhitelister.py
new file mode 100644
index 000000000..db5b931ee
--- /dev/null
+++ b/wagtail/wagtailcore/tests/test_dbwhitelister.py
@@ -0,0 +1,43 @@
+from django.test import TestCase
+from wagtail.wagtailcore.rich_text import DbWhitelister
+
+from bs4 import BeautifulSoup
+
+class TestDbWhitelister(TestCase):
+    def assertHtmlEqual(self, str1, str2):
+        """
+        Assert that two HTML strings are equal at the DOM level
+        (necessary because we can't guarantee the order that attributes are output in)
+        """
+        self.assertEqual(BeautifulSoup(str1), BeautifulSoup(str2))
+
+    def test_page_link_is_rewritten(self):
+        input_html = '<p>Look at the <a data-linktype="page" data-id="2" href="/">lovely homepage</a> of my <a href="http://wagtail.io/">Wagtail</a> site</p>'
+        output_html = DbWhitelister.clean(input_html)
+        expected = '<p>Look at the <a linktype="page" id="2">lovely homepage</a> of my <a href="http://wagtail.io/">Wagtail</a> site</p>'
+        self.assertHtmlEqual(expected, output_html)
+
+    def test_document_link_is_rewritten(self):
+        input_html = '<p>Look at our <a data-linktype="document" data-id="1" href="/documents/1/brochure.pdf">horribly oversized brochure</a></p>'
+        output_html = DbWhitelister.clean(input_html)
+        expected = '<p>Look at our <a linktype="document" id="1">horribly oversized brochure</a></p>'
+        self.assertHtmlEqual(expected, output_html)
+
+    def test_image_embed_is_rewritten(self):
+        input_html = '<p>OMG look at this picture of a kitten: <figure data-embedtype="image" data-id="5" data-format="image-with-caption" data-alt="A cute kitten" class="fancy-image"><img src="/media/images/kitten.jpg" width="320" height="200" alt="A cute kitten" /><figcaption>A kitten, yesterday.</figcaption></figure></p>'
+        output_html = DbWhitelister.clean(input_html)
+        expected = '<p>OMG look at this picture of a kitten: <embed embedtype="image" id="5" format="image-with-caption" alt="A cute kitten" /></p>'
+        self.assertHtmlEqual(expected, output_html)
+
+    def test_media_embed_is_rewritten(self):
+        input_html = '<p>OMG look at this video of a kitten: <iframe data-embedtype="media" data-url="https://www.youtube.com/watch?v=dQw4w9WgXcQ" width="640" height="480" src="//www.youtube.com/embed/dQw4w9WgXcQ" frameborder="0" allowfullscreen></iframe></p>'
+        output_html = DbWhitelister.clean(input_html)
+        expected = '<p>OMG look at this video of a kitten: <embed embedtype="media" url="https://www.youtube.com/watch?v=dQw4w9WgXcQ" /></p>'
+        self.assertHtmlEqual(expected, output_html)
+
+    def test_whitelist_hooks(self):
+        # wagtail.tests.wagtail_hooks overrides the whitelist to permit <blockquote> and <a target="...">
+        input_html = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank" tea="darjeeling">stand in water</a>.</blockquote><p>- <character>Gumby</character>'
+        output_html = DbWhitelister.clean(input_html)
+        expected = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank">stand in water</a>.</blockquote><p>- Gumby'
+        self.assertHtmlEqual(expected, output_html)

From 505a1291a80a3a7fb088004b98b5192afa561d01 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Fri, 20 Jun 2014 15:07:58 +0100
Subject: [PATCH 25/31] Move construct_whitelister_element_rules hook logic
 into DbWhitelister, as the whitelist module should be Wagtail-agnostic

---
 wagtail/wagtailcore/rich_text.py                | 14 ++++++++++++++
 wagtail/wagtailcore/tests/test_dbwhitelister.py | 11 +++++++++--
 wagtail/wagtailcore/whitelist.py                |  6 ------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/wagtail/wagtailcore/rich_text.py b/wagtail/wagtailcore/rich_text.py
index aa8f25336..8004cb30d 100644
--- a/wagtail/wagtailcore/rich_text.py
+++ b/wagtail/wagtailcore/rich_text.py
@@ -13,6 +13,8 @@ from wagtail.wagtaildocs.models import Document
 from wagtail.wagtailimages.models import get_image_model
 from wagtail.wagtailimages.formats import get_image_format
 
+from wagtail.wagtailadmin import hooks
+
 
 # Define a set of 'embed handlers' and 'link handlers'. These handle the translation
 # of 'special' HTML elements in rich text - ones which we do not want to include
@@ -158,6 +160,18 @@ LINK_HANDLERS = {
 # Prepare a whitelisting engine with custom behaviour:
 # rewrite any elements with a data-embedtype or data-linktype attribute
 class DbWhitelister(Whitelister):
+    has_loaded_custom_whitelist_rules = False
+
+    @classmethod
+    def clean(cls, html):
+        if not cls.has_loaded_custom_whitelist_rules:
+            for fn in hooks.get_hooks('construct_whitelister_element_rules'):
+                cls.element_rules = dict(
+                    cls.element_rules.items() + fn().items())
+            cls.has_loaded_custom_whitelist_rules = True
+
+        return super(DbWhitelister, cls).clean(html)
+
     @classmethod
     def clean_tag_node(cls, doc, tag):
         if 'data-embedtype' in tag.attrs:
diff --git a/wagtail/wagtailcore/tests/test_dbwhitelister.py b/wagtail/wagtailcore/tests/test_dbwhitelister.py
index db5b931ee..795b9637c 100644
--- a/wagtail/wagtailcore/tests/test_dbwhitelister.py
+++ b/wagtail/wagtailcore/tests/test_dbwhitelister.py
@@ -1,5 +1,6 @@
 from django.test import TestCase
 from wagtail.wagtailcore.rich_text import DbWhitelister
+from wagtail.wagtailcore.whitelist import Whitelister
 
 from bs4 import BeautifulSoup
 
@@ -37,7 +38,13 @@ class TestDbWhitelister(TestCase):
 
     def test_whitelist_hooks(self):
         # wagtail.tests.wagtail_hooks overrides the whitelist to permit <blockquote> and <a target="...">
-        input_html = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank" tea="darjeeling">stand in water</a>.</blockquote><p>- <character>Gumby</character>'
+        input_html = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank" tea="darjeeling">stand in water</a>.</blockquote><p>- <character>Gumby</character></p>'
         output_html = DbWhitelister.clean(input_html)
-        expected = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank">stand in water</a>.</blockquote><p>- Gumby'
+        expected = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank">stand in water</a>.</blockquote><p>- Gumby</p>'
+        self.assertHtmlEqual(expected, output_html)
+
+        # check that the base Whitelister class is unaffected by these custom whitelist rules
+        input_html = '<blockquote>I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1" target="_blank" tea="darjeeling">stand in water</a>.</blockquote><p>- <character>Gumby</character></p>'
+        output_html = Whitelister.clean(input_html)
+        expected = 'I would put a tax on all people who <a href="https://twitter.com/DMReporter/status/432914941201223680/photo/1">stand in water</a>.<p>- Gumby</p>'
         self.assertHtmlEqual(expected, output_html)
diff --git a/wagtail/wagtailcore/whitelist.py b/wagtail/wagtailcore/whitelist.py
index 0cfbbd2db..9a5f67cf1 100644
--- a/wagtail/wagtailcore/whitelist.py
+++ b/wagtail/wagtailcore/whitelist.py
@@ -5,8 +5,6 @@ specific rules.
 from bs4 import BeautifulSoup, NavigableString, Tag
 from urlparse import urlparse
 
-from wagtail.wagtailadmin import hooks
-
 
 ALLOWED_URL_SCHEMES = ['', 'http', 'https', 'ftp', 'mailto', 'tel']
 
@@ -82,10 +80,6 @@ class Whitelister(object):
     def clean(cls, html):
         """Clean up an HTML string to contain just the allowed elements /
         attributes"""
-        for fn in hooks.get_hooks('construct_whitelister_element_rules'):
-            cls.element_rules = dict(
-                cls.element_rules.items() + fn().items())
-
         doc = BeautifulSoup(html, 'lxml')
         cls.clean_node(doc, doc)
         return unicode(doc)

From 1aab35ba24341bb497316ccd880f813a095c6792 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Fri, 20 Jun 2014 16:08:50 +0100
Subject: [PATCH 26/31] add documentation for the
 construct_whitelister_element_rules hook

---
 docs/editing_api.rst | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/docs/editing_api.rst b/docs/editing_api.rst
index 88fc9458f..9f7602b11 100644
--- a/docs/editing_api.rst
+++ b/docs/editing_api.rst
@@ -546,6 +546,28 @@ Where ``'hook'`` is one of the following hook strings and ``function`` is a func
       + 'demo/css/vendor/font-awesome/css/font-awesome.min.css">')
     hooks.register('insert_editor_css', editor_css)
 
+.. _construct_whitelister_element_rules:
+
+``construct_whitelister_element_rules``
+  .. versionadded:: 0.4
+  Customise the rules that define which HTML elements are allowed in rich text areas. By default only a limited set of HTML elements and attributes are whitelisted - all others are stripped out. The callables passed into this hook must return a dict, which maps element names to handler functions that will perform some kind of manipulation of the element. These handler functions receive the element as a `BeautifulSoup <http://www.crummy.com/software/BeautifulSoup/bs4/doc/>`_ Tag object.
+
+  The ``wagtail.wagtailcore.whitelist`` module provides a few helper functions to assist in defining these handlers: ``allow_without_attributes``, a handler which preserves the element but strips out all of its attributes, and ``attribute_rule`` which accepts a dict specifying how to handle each attribute, and returns a handler function. This dict will map attribute names to either True (indicating that the attribute should be kept), False (indicating that it should be dropped), or a callable (which takes the initial attribute value and returns either a final value for the attribute, or None to drop the attribute).
+
+  For example, the following hook function will add the ``<blockquote>`` element to the whitelist, and allow the ``target`` attribute on ``<a>`` elements:
+
+  .. code-block:: python
+
+    from wagtail.wagtailadmin import hooks
+    from wagtail.wagtailcore.whitelist import attribute_rule, check_url, allow_without_attributes
+
+    def whitelister_element_rules():
+        return {
+            'blockquote': allow_without_attributes,
+            'a': attribute_rule({'href': check_url, 'target': True}),
+        }
+    hooks.register('construct_whitelister_element_rules', whitelister_element_rules)
+
 
 Image Formats in the Rich Text Editor
 -------------------------------------

From f4c51d8205f4c3a9cc5146021761e7d9f1d20671 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Fri, 20 Jun 2014 16:10:03 +0100
Subject: [PATCH 27/31] add changelog entry for
 construct_whitelister_element_rules hook

---
 CHANGELOG.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 19eec21e7..e5cda8b44 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -14,6 +14,7 @@ Changelog
  * Aesthetic improvements to preview experience
  * 'image' tag now accepts extra keyword arguments to be output as attributes on the img tag
  * Added an 'attrs' property to image rendition objects to output src, width, height and alt attributes all in one go
+ * Added 'construct_whitelister_element_rules' hook for customising the HTML whitelist used when saving rich text fields
  * Fix: Animated GIFs are now coalesced before resizing
  * Fix: Wand backend clones images before modifying them
  * Fix: Admin breadcrumb now positioned correctly on mobile

From 2801c913dc3b64d41cfd5ed70ebab270e434f9df Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Mon, 23 Jun 2014 14:22:45 +0100
Subject: [PATCH 28/31] Docs update for #342 merge

---
 docs/wagtail_search.rst | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/docs/wagtail_search.rst b/docs/wagtail_search.rst
index 3d8ea26b2..1fa2c6525 100644
--- a/docs/wagtail_search.rst
+++ b/docs/wagtail_search.rst
@@ -220,17 +220,14 @@ The default DB search backend uses Django's ``__icontains`` filter.
 
 Elasticsearch Backend
 `````````````````````
-Prerequisites are the Elasticsearch service itself and, via pip, the `elasticutils`_ and `pyelasticsearch`_ packages:
+Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
 
 .. code-block:: guess
 
-  pip install elasticutils==0.8.2 pyelasticsearch
+  pip install elasticsearch
 
 .. note::
-    ElasticUtils 0.9+ is not supported.
-
-.. note::
-    The dependency on elasticutils and pyelasticsearch is scheduled to be replaced by a dependency on `elasticsearch-py`_.
+  If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5```
 
 The backend is configured in settings:
 
@@ -246,7 +243,7 @@ The backend is configured in settings:
       }
   }
 
-Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticutils. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
+Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
 
 If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly:
 
@@ -256,8 +253,6 @@ If you prefer not to run an Elasticsearch server in development or production, t
 -  Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS``
 -  Run ``./manage.py update_index``
 
-.. _elasticutils: http://elasticutils.readthedocs.org
-.. _pyelasticsearch: http://pyelasticsearch.readthedocs.org
 .. _elasticsearch-py: http://elasticsearch-py.readthedocs.org
 .. _Searchly: http://www.searchly.com/
 .. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up

From 765657329e529f828e0bbf0185c162570301a539 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Mon, 23 Jun 2014 14:28:36 +0100
Subject: [PATCH 29/31] Changelog entry for #342

---
 CHANGELOG.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index e5cda8b44..ca58949b4 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -3,6 +3,7 @@ Changelog
 
 0.4 (xx.xx.20xx)
 ~~~~~~~~~~~~~~~~
+ * ElasticUtils/pyelasticsearch swapped for elasticsearch-py
  * Added 'original' as a resizing rule supported by the 'image' tag
  * Hallo.js updated to version 1.0.4
  * Snippets are now ordered alphabetically

From d25b6fb6c05a60de7f474201d794fbfcf8fbe135 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Tue, 24 Jun 2014 09:12:08 +0100
Subject: [PATCH 30/31] Put prefetch_related back

---
 .../wagtailsearch/backends/elasticsearch.py   | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 29141542a..5561a3cee 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -13,7 +13,6 @@ from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed
 
 
-
 class ElasticSearchQuery(object):
     def __init__(self, model, query_string, fields=None, filters={}):
         self.model = model
@@ -113,9 +112,10 @@ class ElasticSearchQuery(object):
 
 
 class ElasticSearchResults(object):
-    def __init__(self, backend, query):
+    def __init__(self, backend, query, prefetch_related=None):
         self.backend = backend
         self.query = query
+        self.prefetch_related = prefetch_related
         self.start = 0
         self.stop = None
         self._results_cache = None
@@ -136,7 +136,7 @@ class ElasticSearchResults(object):
 
     def _clone(self):
         klass = self.__class__
-        new = klass(self.backend, self.query)
+        new = klass(self.backend, self.query, prefetch_related=self.prefetch_related)
         new.start = self.start
         new.stop = self.stop
         return new
@@ -167,8 +167,15 @@ class ElasticSearchResults(object):
         # Initialise results dictionary
         results = dict((str(pk), None) for pk in pks)
 
-        # Find objects in database and add them to dict
+        # Get queryset
         queryset = self.query.model.objects.filter(pk__in=pks)
+
+        # Add prefetch related
+        if self.prefetch_related:
+            for prefetch in self.prefetch_related:
+                queryset = queryset.prefetch_related(prefetch)
+
+        # Find objects in database and add them to dict
         for obj in queryset:
             results[str(obj.pk)] = obj
 
@@ -417,8 +424,5 @@ class ElasticSearch(BaseSearch):
         if not query_string:
             return []
 
-        # Give deprecation warning if prefetch_related was used
-        warnings.warn("prefetch_related on search queries is no longer implemented. ", DeprecationWarning)
-
         # Return search results
-        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters))
+        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters), prefetch_related=prefetch_related)

From 3c1883bca0d4e60e9b60213f01dc405b63948035 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Tue, 24 Jun 2014 09:42:31 +0100
Subject: [PATCH 31/31] Put prefetch related back in database backend

---
 wagtail/wagtailsearch/backends/db.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
index 0531bc3aa..08fdaf048 100644
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -66,7 +66,9 @@ class DBSearch(BaseSearch):
         # Distinct
         query = query.distinct()
 
-        # Give deprecation warning if prefetch_related was used
-        warnings.warn("prefetch_related on search queries is no longer implemented. ", DeprecationWarning)
+        # Prefetch related
+        if prefetch_related:
+            for prefetch in prefetch_related:
+                query = query.prefetch_related(prefetch)
 
         return query
\ No newline at end of file