From b15522ac1e08dd18af9a9988f619eed9e0e22f20 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 15:49:19 +0100
Subject: [PATCH 01/49] Implemented search on queryset

---
 .../wagtailsearch/backends/elasticsearch.py   | 204 ++++++++++++------
 wagtail/wagtailsearch/indexed.py              |   9 +-
 wagtail/wagtailsearch/models.py               |   2 +-
 3 files changed, 146 insertions(+), 69 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 8fc2950dd..a8c590de2 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import json
 
 from django.db import models
+from django.db.models.query import QuerySet
 
 from elasticsearch import Elasticsearch, NotFoundError, RequestError
 from elasticsearch.helpers import bulk
@@ -109,12 +110,123 @@ class ElasticSearchMapping(object):
         return '<ElasticSearchMapping: %s>' % (self.model.__name__, )
 
 
+class FilterError(Exception):
+    pass
+
+
+class FieldError(Exception):
+    pass
+
+
 class ElasticSearchQuery(object):
-    def __init__(self, model, query_string, fields=None, filters={}):
-        self.model = model
+    def __init__(self, queryset, query_string, fields=None):
+        self.queryset = queryset
         self.query_string = query_string
-        self.fields = fields or ['_all', '_partials']
-        self.filters = filters
+        self.fields = fields or ['_all', 'partials']
+
+    def _get_filters_from_where(self, where_node):
+        # Check if this is a leaf node
+        if isinstance(where_node, tuple):
+            field_name = where_node[0].col
+            lookup = where_node[1]
+            value = where_node[3]
+
+            # Get field
+            field = dict(
+                (field.get_attname(self.queryset.model), field)
+                for field in self.queryset.model.get_filterable_search_fields()
+            ).get(field_name, None)
+
+            # Give error if the field doesn't exist
+            if field is None:
+                raise FieldError('Cannot filter ElasticSearch results with field "' + field_name + '". Please add FilterField(\'' + field_name + '\') to ' + self.queryset.model.__name__ + '.search_fields.')
+
+            # Get the name of the field in the index
+            field_index_name = field.get_index_name(self.queryset.model)
+
+            # Find lookup
+            if lookup == 'exact':
+                if value is None:
+                    return {
+                        'missing': {
+                            'field': field_index_name,
+                        }
+                    }
+                else:
+                    return {
+                        'term': {
+                            field_index_name: value,
+                        }
+                    }
+
+            if lookup == 'isnull':
+                if value:
+                    return {
+                        'missing': {
+                            'field': field_index_name,
+                        }
+                    }
+                else:
+                    return {
+                        'not': {
+                            'missing': {
+                                'field': field_index_name,
+                            }
+                        }
+                    }
+
+            if lookup in ['startswith', 'prefix']:
+                return {
+                    'prefix': {
+                        field_index_name: value,
+                    }
+                }
+
+            if lookup in ['gt', 'gte', 'lt', 'lte']:
+                return {
+                    'range': {
+                        field_index_name: {
+                            lookup: value,
+                        }
+                    }
+                }
+
+            if lookup == 'range':
+                lower, upper = value
+
+                return {
+                    'range': {
+                        field_index_name: {
+                            'gte': lower,
+                            'lte': upper,
+                        }
+                    }
+                }
+
+            raise FilterError('Could not apply filter on ElasticSearch results "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.')
+
+        # Get child filters
+        connector = where_node.connector
+        child_filters = [self._get_filters_from_where(child) for child in where_node.children]
+        child_filters = [child_filter for child_filter in child_filters if child_filter]
+
+        # Connect them
+        if child_filters:
+            if len(child_filters) == 1:
+                filter_out = child_filters[0]
+            else:
+                filter_out = {
+                    connector.lower(): [
+                        fil for fil in child_filters if fil is not None
+                    ]
+                }
+
+            if where_node.negated:
+                filter_out = {
+                    'not': filter_out
+                }
+
+            return filter_out
 
     def _get_filters(self):
         # Filters
@@ -123,59 +235,14 @@ class ElasticSearchQuery(object):
         # Filter by content type
         filters.append({
             'prefix': {
-                'content_type': self.model.indexed_get_content_type()
+                'content_type': self.queryset.model.indexed_get_content_type()
             }
         })
 
-        # Extra filters
-        if self.filters:
-            for key, value in self.filters.items():
-                if '__' in key:
-                    field, lookup = key.split('__')
-                else:
-                    field = key
-                    lookup = None
-
-                if lookup is None:
-                    if value is None:
-                        filters.append({
-                            'missing': {
-                                'field': field,
-                            }
-                        })
-                    else:
-                        filters.append({
-                            'term': {
-                                field: value
-                            }
-                        })
-
-                if lookup in ['startswith', 'prefix']:
-                    filters.append({
-                        'prefix': {
-                            field: value
-                        }
-                    })
-
-                if lookup in ['gt', 'gte', 'lt', 'lte']:
-                    filters.append({
-                        'range': {
-                            field: {
-                                lookup: value,
-                            }
-                        }
-                    })
-
-                if lookup == 'range':
-                    lower, upper = value
-                    filters.append({
-                        'range': {
-                            field: {
-                                'gte': lower,
-                                'lte': upper,
-                            }
-                        }
-                    })
+        # Apply filters from queryset
+        queryset_filters = self._get_filters_from_where(self.queryset.query.where)
+        if queryset_filters:
+            filters.append(queryset_filters)
 
         return filters
 
@@ -263,15 +330,8 @@ class ElasticSearchResults(object):
         # Initialise results dictionary
         results = dict((str(pk), None) for pk in pks)
 
-        # Get queryset
-        queryset = self.query.model.objects.filter(pk__in=pks)
-
-        # Add prefetch related
-        if self.prefetch_related:
-            for prefetch in self.prefetch_related:
-                queryset = queryset.prefetch_related(prefetch)
-
         # Find objects in database and add them to dict
+        queryset = self.query.queryset.filter(pk__in=pks)
         for obj in queryset:
             results[str(obj.pk)] = obj
 
@@ -502,7 +562,15 @@ class ElasticSearch(BaseSearch):
         except NotFoundError:
             pass  # Document doesn't exist, ignore this exception
 
-    def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
+    def search(self, query_string, model_or_queryset, fields=None, filters={}, prefetch_related=[]):
+        # Find model/queryset
+        if isinstance(model_or_queryset, QuerySet):
+            model = model_or_queryset.model
+            queryset = model_or_queryset
+        else:
+            model = model_or_queryset
+            queryset = model_or_queryset.objects.all()
+
         # Model must be a descendant of Indexed and be a django model
         if not issubclass(model, Indexed) or not issubclass(model, models.Model):
             return []
@@ -514,5 +582,13 @@ class ElasticSearch(BaseSearch):
         if not query_string:
             return []
 
+        # Apply filters to queryset
+        if filters:
+            queryset = queryset.filter(**filters)
+
+        # Prefetch related
+        for prefetch in prefetch_related:
+            queryset = queryset.prefetch_related(prefetch)
+
         # Return search results
-        return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters), prefetch_related=prefetch_related)
+        return ElasticSearchResults(self, ElasticSearchQuery(queryset, query_string, fields=fields))
diff --git a/wagtail/wagtailsearch/indexed.py b/wagtail/wagtailsearch/indexed.py
index b290f440b..a2cb54d14 100644
--- a/wagtail/wagtailsearch/indexed.py
+++ b/wagtail/wagtailsearch/indexed.py
@@ -112,13 +112,16 @@ class Indexed(object):
 
     @classmethod
     def get_searchable_search_fields(cls):
-        return filter(lambda field: field.searchable, cls.get_search_fields())
+        return filter(lambda field: isinstance(field, SearchField), cls.get_search_fields())
+
+    @classmethod
+    def get_filterable_search_fields(cls):
+        return filter(lambda field: isinstance(field, FilterField), cls.get_search_fields())
 
     indexed_fields = ()
 
 
 class BaseField(object):
-    searchable = False
     suffix = ''
 
     def __init__(self, field_name, **kwargs):
@@ -163,8 +166,6 @@ class BaseField(object):
 
 
 class SearchField(BaseField):
-    searchable = True
-
     def __init__(self, field_name, boost=None, partial_match=False, **kwargs):
         super(SearchField, self).__init__(field_name, **kwargs)
         self.boost = boost
diff --git a/wagtail/wagtailsearch/models.py b/wagtail/wagtailsearch/models.py
index 509368463..7cb8546d7 100644
--- a/wagtail/wagtailsearch/models.py
+++ b/wagtail/wagtailsearch/models.py
@@ -91,7 +91,7 @@ class SearchTest(models.Model, indexed.Indexed):
         indexed.SearchField('title'),
         indexed.SearchField('content'),
         indexed.SearchField('callable_indexed_field'),
-        indexed.SearchField('live'),
+        indexed.FilterField('live'),
     )
 
     def callable_indexed_field(self):

From 41199dd576ee9a4ad3ed5cbf414e655499bb2088 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 15:51:29 +0100
Subject: [PATCH 02/49] Minor optimisation in ElasticSearchQuery

---
 .../wagtailsearch/backends/elasticsearch.py   | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index a8c590de2..649a7a7f0 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -260,15 +260,24 @@ class ElasticSearchQuery(object):
 
         # Filters
         filters = self._get_filters()
-
-        return {
-            'filtered': {
-                'query': query,
-                'filter': {
-                    'and': filters,
+        if len(filters) == 1:
+            query = {
+                'filtered': {
+                    'query': query,
+                    'filter': filters[0],
                 }
             }
-        }
+        elif len(filters) > 1:
+            query = {
+                'filtered': {
+                    'query': query,
+                    'filter': {
+                        'and': filters,
+                    }
+                }
+            }
+
+        return query
 
     def __repr__(self):
         return json.dumps(self.to_es())

From d872768baffe4f6aa2e66ad92a82f1a3f075d69e Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 16:01:04 +0100
Subject: [PATCH 03/49] Allow query_string to be set to None This allows
 match_all queries to be created

---
 wagtail/wagtailsearch/backends/db.py          | 47 ++++++++++---------
 .../wagtailsearch/backends/elasticsearch.py   | 24 ++++++----
 2 files changed, 39 insertions(+), 32 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
index 4c05b19df..c91363740 100644
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -28,42 +28,43 @@ class DBSearch(BaseSearch):
         pass # Not needed
 
     def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
-        # Normalise query string
-        query_string = normalise_query_string(query_string)
-
-        # Get terms
-        terms = query_string.split()
-        if not terms:
-            return model.objects.none()
-
         # Get fields
         if fields is None:
             fields = [field.field_name for field in model.get_searchable_search_fields()]
 
-        # Start will all objects
+        # Start with all objects
         query = model.objects.all()
 
         # Apply filters
         if filters:
             query = query.filter(**filters)
 
-        # Filter by terms
-        for term in terms:
-            term_query = models.Q()
-            for field_name in fields:
-                # Check if the field exists (this will filter out indexed callables)
-                try:
-                    model._meta.get_field_by_name(field_name)
-                except:
-                    continue
+        if query_string is not None:
+            # Normalise query string
+            query_string = normalise_query_string(query_string)
 
-                # Filter on this field
-                term_query |= models.Q(**{'%s__icontains' % field_name: term})
+            # Get terms
+            terms = query_string.split()
+            if not terms:
+                return model.objects.none()
 
-            query = query.filter(term_query)
+            # Filter by terms
+            for term in terms:
+                term_query = models.Q()
+                for field_name in fields:
+                    # Check if the field exists (this will filter out indexed callables)
+                    try:
+                        model._meta.get_field_by_name(field_name)
+                    except:
+                        continue
 
-        # Distinct
-        query = query.distinct()
+                    # Filter on this field
+                    term_query |= models.Q(**{'%s__icontains' % field_name: term})
+
+                query = query.filter(term_query)
+
+            # Distinct
+            query = query.distinct()
 
         # Prefetch related
         if prefetch_related:
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 649a7a7f0..af810b02b 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -248,15 +248,20 @@ class ElasticSearchQuery(object):
 
     def to_es(self):
         # Query
-        query = {
-            'query_string': {
-                'query': self.query_string,
+        if self.query_string is not None:
+            query = {
+                'query_string': {
+                    'query': self.query_string,
+                }
             }
-        }
 
-        # Fields
-        if self.fields:
-            query['query_string']['fields'] = self.fields
+            # Fields
+            if self.fields:
+                query['query_string']['fields'] = self.fields
+        else:
+            query = {
+                'match_all': {}
+            }
 
         # Filters
         filters = self._get_filters()
@@ -585,10 +590,11 @@ class ElasticSearch(BaseSearch):
             return []
 
         # Normalise query string
-        query_string = normalise_query_string(query_string)
+        if query_string is not None:
+            query_string = normalise_query_string(query_string)
 
         # Check that theres still a query string after the clean up
-        if not query_string:
+        if query_string == "":
             return []
 
         # Apply filters to queryset

From 156eb89850e1efdcdc2b6fc31fb74681441098dd Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 16:10:50 +0100
Subject: [PATCH 04/49] Changed default values of filters and prefetch_related
 to None

---
 wagtail/wagtailsearch/backends/db.py            | 2 +-
 wagtail/wagtailsearch/backends/elasticsearch.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
index c91363740..56536e343 100644
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -27,7 +27,7 @@ class DBSearch(BaseSearch):
     def delete(self, obj):
         pass # Not needed
 
-    def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
+    def search(self, query_string, model, fields=None, filters=None, prefetch_related=None):
         # Get fields
         if fields is None:
             fields = [field.field_name for field in model.get_searchable_search_fields()]
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index af810b02b..14b1a3bdb 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -576,7 +576,7 @@ class ElasticSearch(BaseSearch):
         except NotFoundError:
             pass  # Document doesn't exist, ignore this exception
 
-    def search(self, query_string, model_or_queryset, fields=None, filters={}, prefetch_related=[]):
+    def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
         # Find model/queryset
         if isinstance(model_or_queryset, QuerySet):
             model = model_or_queryset.model
@@ -602,8 +602,9 @@ class ElasticSearch(BaseSearch):
             queryset = queryset.filter(**filters)
 
         # Prefetch related
-        for prefetch in prefetch_related:
-            queryset = queryset.prefetch_related(prefetch)
+        if prefetch_related:
+            for prefetch in prefetch_related:
+                queryset = queryset.prefetch_related(prefetch)
 
         # Return search results
         return ElasticSearchResults(self, ElasticSearchQuery(queryset, query_string, fields=fields))

From 8a99a63cf1df057e126d6a4d841f8af05ad97644 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 17:37:18 +0100
Subject: [PATCH 05/49] Fixed typo

---
 wagtail/wagtailsearch/backends/elasticsearch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 14b1a3bdb..ea0413eed 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -122,7 +122,7 @@ class ElasticSearchQuery(object):
     def __init__(self, queryset, query_string, fields=None):
         self.queryset = queryset
         self.query_string = query_string
-        self.fields = fields or ['_all', 'partials']
+        self.fields = fields or ['_all', '_partials']
 
     def _get_filters_from_where(self, where_node):
         # Check if this is a leaf node

From 2100c6372d0761a920cf22c62dffb9ed7d58c7b6 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 16:45:01 +0100
Subject: [PATCH 06/49] Moved wagtailsearch test models into wagtail.tests
 models

---
 wagtail/tests/models.py                       |  25 ++++
 ...to__del_searchtestchild__del_searchtest.py | 112 ++++++++++++++++++
 wagtail/wagtailsearch/models.py               |  26 ----
 wagtail/wagtailsearch/tests/test_backends.py  |   4 +-
 4 files changed, 139 insertions(+), 28 deletions(-)
 create mode 100644 wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py

diff --git a/wagtail/tests/models.py b/wagtail/tests/models.py
index ff6dc2dcc..5787c19ff 100644
--- a/wagtail/tests/models.py
+++ b/wagtail/tests/models.py
@@ -11,6 +11,7 @@ from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
 from wagtail.wagtaildocs.edit_handlers import DocumentChooserPanel
 from wagtail.wagtailforms.models import AbstractEmailForm, AbstractFormField
 from wagtail.wagtailsnippets.models import register_snippet
+from wagtail.wagtailsearch import indexed
 
 
 EVENT_AUDIENCE_CHOICES = (
@@ -316,3 +317,27 @@ class BusinessSubIndex(Page):
 
 class BusinessChild(Page):
     subpage_types = []
+
+
+class SearchTest(models.Model, indexed.Indexed):
+    title = models.CharField(max_length=255)
+    content = models.TextField()
+    live = models.BooleanField(default=False)
+
+    search_fields = (
+        indexed.SearchField('title'),
+        indexed.SearchField('content'),
+        indexed.SearchField('callable_indexed_field'),
+        indexed.FilterField('live'),
+    )
+
+    def callable_indexed_field(self):
+        return "Callable"
+
+
+class SearchTestChild(SearchTest):
+    extra_content = models.TextField()
+
+    search_fields = SearchTest.search_fields + (
+        indexed.SearchField('extra_content'),
+    )
diff --git a/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py
new file mode 100644
index 000000000..560dbaae9
--- /dev/null
+++ b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Deleting model 'SearchTestChild'
+        db.delete_table(u'wagtailsearch_searchtestchild')
+
+        # Deleting model 'SearchTest'
+        db.delete_table(u'wagtailsearch_searchtest')
+
+
+    def backwards(self, orm):
+        # Adding model 'SearchTestChild'
+        db.create_table(u'wagtailsearch_searchtestchild', (
+            ('extra_content', self.gf('django.db.models.fields.TextField')()),
+            (u'searchtest_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['wagtailsearch.SearchTest'], unique=True, primary_key=True)),
+        ))
+        db.send_create_signal(u'wagtailsearch', ['SearchTestChild'])
+
+        # Adding model 'SearchTest'
+        db.create_table(u'wagtailsearch_searchtest', (
+            ('content', self.gf('django.db.models.fields.TextField')()),
+            ('live', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('title', self.gf('django.db.models.fields.CharField')(max_length=255)),
+        ))
+        db.send_create_signal(u'wagtailsearch', ['SearchTest'])
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'wagtailcore.page': {
+            'Meta': {'object_name': 'Page'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pages'", 'to': u"orm['contenttypes.ContentType']"}),
+            'depth': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'has_unpublished_changes': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'live': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'numchild': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'owned_pages'", 'null': 'True', 'to': u"orm['auth.User']"}),
+            'path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'search_description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'seo_title': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'show_in_menus': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'url_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'})
+        },
+        u'wagtailsearch.editorspick': {
+            'Meta': {'ordering': "('sort_order',)", 'object_name': 'EditorsPick'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'page': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['wagtailcore.Page']"}),
+            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editors_picks'", 'to': u"orm['wagtailsearch.Query']"}),
+            'sort_order': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        u'wagtailsearch.query': {
+            'Meta': {'object_name': 'Query'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'query_string': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        u'wagtailsearch.querydailyhits': {
+            'Meta': {'unique_together': "(('query', 'date'),)", 'object_name': 'QueryDailyHits'},
+            'date': ('django.db.models.fields.DateField', [], {}),
+            'hits': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'daily_hits'", 'to': u"orm['wagtailsearch.Query']"})
+        }
+    }
+
+    complete_apps = ['wagtailsearch']
\ No newline at end of file
diff --git a/wagtail/wagtailsearch/models.py b/wagtail/wagtailsearch/models.py
index 7cb8546d7..c63a076a7 100644
--- a/wagtail/wagtailsearch/models.py
+++ b/wagtail/wagtailsearch/models.py
@@ -78,29 +78,3 @@ class EditorsPick(models.Model):
 
     class Meta:
         ordering = ('sort_order', )
-
-
-# Used for tests
-
-class SearchTest(models.Model, indexed.Indexed):
-    title = models.CharField(max_length=255)
-    content = models.TextField()
-    live = models.BooleanField(default=False)
-
-    search_fields = (
-        indexed.SearchField('title'),
-        indexed.SearchField('content'),
-        indexed.SearchField('callable_indexed_field'),
-        indexed.FilterField('live'),
-    )
-
-    def callable_indexed_field(self):
-        return "Callable"
-
-
-class SearchTestChild(SearchTest):
-    extra_content = models.TextField()
-
-    search_fields = SearchTest.search_fields + (
-        indexed.SearchField('extra_content'),
-    )
diff --git a/wagtail/wagtailsearch/tests/test_backends.py b/wagtail/wagtailsearch/tests/test_backends.py
index 86a67f21a..04d28689a 100644
--- a/wagtail/wagtailsearch/tests/test_backends.py
+++ b/wagtail/wagtailsearch/tests/test_backends.py
@@ -6,9 +6,9 @@ from django.conf import settings
 from django.core import management
 
 from wagtail.tests.utils import unittest
-from wagtail.wagtailsearch import models, get_search_backend
+from wagtail.tests import models
+from wagtail.wagtailsearch.backends import get_search_backend, InvalidSearchBackendError
 from wagtail.wagtailsearch.backends.db import DBSearch
-from wagtail.wagtailsearch.backends import InvalidSearchBackendError
 
 
 class BackendTests(object):

From 2679dc2fe69fbf08423d727c19e16a19c068ffbb Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 16:51:06 +0100
Subject: [PATCH 07/49] Split up backend tests

---
 wagtail/wagtailsearch/tests/test_backends.py  | 21 -------------------
 .../wagtailsearch/tests/test_db_backend.py    | 13 ++++++++++++
 .../tests/test_elasticsearch_backend.py       | 18 ++++++++++++++++
 3 files changed, 31 insertions(+), 21 deletions(-)
 create mode 100644 wagtail/wagtailsearch/tests/test_db_backend.py
 create mode 100644 wagtail/wagtailsearch/tests/test_elasticsearch_backend.py

diff --git a/wagtail/wagtailsearch/tests/test_backends.py b/wagtail/wagtailsearch/tests/test_backends.py
index 04d28689a..e9b656298 100644
--- a/wagtail/wagtailsearch/tests/test_backends.py
+++ b/wagtail/wagtailsearch/tests/test_backends.py
@@ -151,27 +151,6 @@ class BackendTests(object):
         self.assertEqual(len(results), 3)
 
 
-class TestDBBackend(BackendTests, TestCase):
-    backend_path = 'wagtail.wagtailsearch.backends.db.DBSearch'
-
-    @unittest.expectedFailure
-    def test_callable_indexed_field(self):
-        super(TestDBBackend, self).test_callable_indexed_field()
-
-
-class TestElasticSearchBackend(BackendTests, TestCase):
-    backend_path = 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch'
-
-    def test_search_with_spaces_only(self):
-        # Search for some space characters and hope it doesn't crash
-        results = self.backend.search("   ", models.SearchTest)
-
-        # Queries are lazily evaluated, force it to run
-        list(results)
-
-        # Didn't crash, yay!
-
-
 @override_settings(WAGTAILSEARCH_BACKENDS={
     'default': {'BACKEND': 'wagtail.wagtailsearch.backends.db.DBSearch'}
 })
diff --git a/wagtail/wagtailsearch/tests/test_db_backend.py b/wagtail/wagtailsearch/tests/test_db_backend.py
new file mode 100644
index 000000000..f471a34d1
--- /dev/null
+++ b/wagtail/wagtailsearch/tests/test_db_backend.py
@@ -0,0 +1,13 @@
+from wagtail.tests.utils import unittest
+
+from django.test import TestCase
+
+from .test_backends import BackendTests
+
+
+class TestDBBackend(BackendTests, TestCase):
+    backend_path = 'wagtail.wagtailsearch.backends.db.DBSearch'
+
+    @unittest.expectedFailure
+    def test_callable_indexed_field(self):
+        super(TestDBBackend, self).test_callable_indexed_field()
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
new file mode 100644
index 000000000..c2623fd54
--- /dev/null
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -0,0 +1,18 @@
+from wagtail.tests.utils import unittest
+
+from django.test import TestCase
+
+from .test_backends import BackendTests
+
+
+class TestElasticSearchBackend(BackendTests, TestCase):
+    backend_path = 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch'
+
+    def test_search_with_spaces_only(self):
+        # Search for some space characters and hope it doesn't crash
+        results = self.backend.search("   ", models.SearchTest)
+
+        # Queries are lazily evaluated, force it to run
+        list(results)
+
+        # Didn't crash, yay!

From f02a6937d262dcfa68ceb8e0551663799c7f9b17 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 17:31:52 +0100
Subject: [PATCH 08/49] Added more tests for ElasticSearch backend

---
 wagtail/tests/models.py                       |  15 +-
 .../tests/test_elasticsearch_backend.py       | 347 ++++++++++++++++++
 2 files changed, 357 insertions(+), 5 deletions(-)

diff --git a/wagtail/tests/models.py b/wagtail/tests/models.py
index 5787c19ff..86c85e7f9 100644
--- a/wagtail/tests/models.py
+++ b/wagtail/tests/models.py
@@ -323,21 +323,26 @@ class SearchTest(models.Model, indexed.Indexed):
     title = models.CharField(max_length=255)
     content = models.TextField()
     live = models.BooleanField(default=False)
+    published_date = models.DateField(null=True)
 
-    search_fields = (
-        indexed.SearchField('title'),
+    search_fields = [
+        indexed.SearchField('title', partial_match=True),
         indexed.SearchField('content'),
         indexed.SearchField('callable_indexed_field'),
+        indexed.FilterField('title'),
         indexed.FilterField('live'),
-    )
+        indexed.FilterField('published_date'),
+    ]
 
     def callable_indexed_field(self):
         return "Callable"
 
 
 class SearchTestChild(SearchTest):
+    subtitle = models.CharField(max_length=255, null=True, blank=True)
     extra_content = models.TextField()
 
-    search_fields = SearchTest.search_fields + (
+    search_fields = SearchTest.search_fields + [
+        indexed.SearchField('subtitle', partial_match=True),
         indexed.SearchField('extra_content'),
-    )
+    ]
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index c2623fd54..88aa87f8b 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -1,7 +1,11 @@
 from wagtail.tests.utils import unittest
+import datetime
+import json
 
 from django.test import TestCase
+from django.db.models import Q
 
+from wagtail.tests import models
 from .test_backends import BackendTests
 
 
@@ -16,3 +20,346 @@ class TestElasticSearchBackend(BackendTests, TestCase):
         list(results)
 
         # Didn't crash, yay!
+
+    def test_partial_search(self):
+        # Reset the index
+        self.backend.reset_index()
+        self.backend.add_type(models.SearchTest)
+        self.backend.add_type(models.SearchTestChild)
+
+        # Add some test data
+        obj = models.SearchTest()
+        obj.title = "HelloWorld"
+        obj.live = True
+        obj.save()
+        self.backend.add(obj)
+
+        # Refresh the index
+        self.backend.refresh_index()
+
+        # Search and check
+        results = self.backend.search("HelloW", models.SearchTest.objects.all())
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].id, obj.id)
+
+    def test_child_partial_search(self):
+        # Reset the index
+        self.backend.reset_index()
+        self.backend.add_type(models.SearchTest)
+        self.backend.add_type(models.SearchTestChild)
+
+        obj = models.SearchTestChild()
+        obj.title = "WorldHello"
+        obj.subtitle = "HelloWorld"
+        obj.live = True
+        obj.save()
+        self.backend.add(obj)
+
+        # Refresh the index
+        self.backend.refresh_index()
+
+        # Search and check
+        results = self.backend.search("HelloW", models.SearchTest.objects.all())
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].id, obj.id)
+
+
+class TestElasticSearchQuery(TestCase):
+    def assertDictEqual(self, a, b):
+        default = self.JSONSerializer().default
+        self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default))
+
+    def setUp(self):
+        # Import using a try-catch block to prevent crashes if the elasticsearch-py
+        # module is not installed
+        try:
+            from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchQuery
+            from elasticsearch.serializer import JSONSerializer
+        except ImportError:
+            raise unittest.SkipTest("elasticsearch-py not installed")
+
+        self.ElasticSearchQuery = ElasticSearchQuery
+        self.JSONSerializer = JSONSerializer
+
+    def test_simple(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_none_query_string(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.all(), None)
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'match_all': {}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_filter(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test"), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_and_filter(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test", live=True), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_or_filter(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(Q(title="Test") | Q(live=True)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'title_filter': 'Test'}}, {'term': {'live_filter': True}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_negated_filter(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.exclude(live=True), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_fields(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello", fields=['title'])
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['title']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_exact_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__exact="Test"), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_none_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title=None), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_isnull_true_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=True), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_isnull_false_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=False), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_startswith_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__startswith="Test"), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_gt_lookup(self):
+        # This shares the same code path as gte, lt and lte so theres no need to test those
+        # This also tests conversion of python dates to strings
+
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gt=datetime.datetime(2014, 4, 29)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_range_lookup(self):
+        start_date = datetime.datetime(2014, 4, 29)
+        end_date = datetime.datetime(2014, 8, 19)
+
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__range=(start_date, end_date)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+
+class TestElasticSearchMapping(TestCase):
+    def assertDictEqual(self, a, b):
+        default = self.JSONSerializer().default
+        self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default))
+
+    def setUp(self):
+        # Import using a try-catch block to prevent crashes if the elasticsearch-py
+        # module is not installed
+        try:
+            from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping
+            from elasticsearch.serializer import JSONSerializer
+        except ImportError:
+            raise unittest.SkipTest("elasticsearch-py not installed")
+
+        self.JSONSerializer = JSONSerializer
+
+        # Create ES mapping
+        self.es_mapping = ElasticSearchMapping(models.SearchTest)
+
+        # Create ES document
+        self.obj = models.SearchTest(title="Hello")
+        self.obj.save()
+
+    def test_get_document_type(self):
+        self.assertEqual(self.es_mapping.get_document_type(), 'tests_searchtest')
+
+    def test_get_mapping(self):
+        # Build mapping
+        mapping = self.es_mapping.get_mapping()
+
+        # Check
+        expected_result = {
+            'tests_searchtest': {
+                'properties': {
+                    'pk': {'index': 'not_analyzed', 'type': 'string', 'store': 'yes', 'include_in_all': False},
+                    'content_type': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
+                    '_partials': {'analyzer': 'edgengram_analyzer', 'include_in_all': False, 'type': 'string'},
+                    'live_filter': {'index': 'not_analyzed', 'type': 'boolean', 'include_in_all': False},
+                    'published_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
+                    'title': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'},
+                    'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
+                    'content': {'type': 'string', 'include_in_all': True},
+                    'callable_indexed_field': {'type': 'string', 'include_in_all': True}
+                }
+            }
+        }
+
+        self.assertDictEqual(mapping, expected_result)
+
+    def test_get_document_id(self):
+        self.assertEqual(self.es_mapping.get_document_id(self.obj), 'tests_searchtest:' + str(self.obj.pk))
+
+    def test_get_document(self):
+        # Get document
+        document = self.es_mapping.get_document(self.obj)
+
+        # Check
+        expected_result = {
+            'pk': str(self.obj.pk),
+            'content_type': 'tests_searchtest',
+            '_partials': ['Hello'],
+            'live_filter': False,
+            'published_date_filter': None,
+            'title': 'Hello',
+            'title_filter': 'Hello',
+            'callable_indexed_field': 'Callable',
+            'content': '',
+        }
+
+        self.assertDictEqual(document, expected_result)
+
+
+class TestElasticSearchMappingInheritance(TestCase):
+    def assertDictEqual(self, a, b):
+        default = self.JSONSerializer().default
+        self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default))
+
+    def setUp(self):
+        # Import using a try-catch block to prevent crashes if the elasticsearch-py
+        # module is not installed
+        try:
+            from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping
+            from elasticsearch.serializer import JSONSerializer
+        except ImportError:
+            raise unittest.SkipTest("elasticsearch-py not installed")
+
+        self.JSONSerializer = JSONSerializer
+
+        # Create ES mapping
+        self.es_mapping = ElasticSearchMapping(models.SearchTestChild)
+
+        # Create ES document
+        self.obj = models.SearchTestChild(title="Hello", subtitle="World")
+        self.obj.save()
+
+    def test_get_document_type(self):
+        self.assertEqual(self.es_mapping.get_document_type(), 'tests_searchtest_tests_searchtestchild')
+
+    def test_get_mapping(self):
+        # Build mapping
+        mapping = self.es_mapping.get_mapping()
+
+        # Check
+        expected_result = {
+            'tests_searchtest_tests_searchtestchild': {
+                'properties': {
+                    # New
+                    'extra_content': {'type': 'string', 'include_in_all': True},
+                    'subtitle': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'},
+
+                    # Inherited
+                    'pk': {'index': 'not_analyzed', 'type': 'string', 'store': 'yes', 'include_in_all': False},
+                    'content_type': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
+                    '_partials': {'analyzer': 'edgengram_analyzer', 'include_in_all': False, 'type': 'string'},
+                    'live_filter': {'index': 'not_analyzed', 'type': 'boolean', 'include_in_all': False},
+                    'published_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False},
+                    'title': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'},
+                    'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
+                    'content': {'type': 'string', 'include_in_all': True},
+                    'callable_indexed_field': {'type': 'string', 'include_in_all': True}
+                }
+            }
+        }
+
+        self.assertDictEqual(mapping, expected_result)
+
+    def test_get_document_id(self):
+        # This must be tests_searchtest instead of 'tests_searchtest_tests_searchtestchild'
+        # as it uses the contents base content type name.
+        # This prevents the same object being accidentally indexed twice.
+        self.assertEqual(self.es_mapping.get_document_id(self.obj), 'tests_searchtest:' + str(self.obj.pk))
+
+    def test_get_document(self):
+        # Build document
+        document = self.es_mapping.get_document(self.obj)
+
+        # Sort partials
+        if '_partials' in document:
+            document['_partials'].sort()
+
+        # Check
+        expected_result = {
+            # New
+            'extra_content': '',
+            'subtitle': 'World',
+
+            # Changed
+            'content_type': 'tests_searchtest_tests_searchtestchild',
+
+            # Inherited
+            'pk': str(self.obj.pk),
+            '_partials': ['Hello', 'World'],
+            'live_filter': False,
+            'published_date_filter': None,
+            'title': 'Hello',
+            'title_filter': 'Hello',
+            'callable_indexed_field': 'Callable',
+            'content': '',
+        }
+
+        self.assertDictEqual(document, expected_result)

From 04c507e37be80b9ba5b88fdc6140acdad43f3ed3 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 18:25:42 +0100
Subject: [PATCH 09/49] Added tests for indexed class

---
 wagtail/tests/models.py                       | 28 +++++++++++
 .../wagtailsearch/tests/test_indexed_class.py | 49 +++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 wagtail/wagtailsearch/tests/test_indexed_class.py

diff --git a/wagtail/tests/models.py b/wagtail/tests/models.py
index 86c85e7f9..504648933 100644
--- a/wagtail/tests/models.py
+++ b/wagtail/tests/models.py
@@ -346,3 +346,31 @@ class SearchTestChild(SearchTest):
         indexed.SearchField('subtitle', partial_match=True),
         indexed.SearchField('extra_content'),
     ]
+
+
+class SearchTestOldConfig(models.Model, indexed.Indexed):
+    """
+    This tests that the Indexed class can correctly handle models that
+    use the old "indexed_fields" configuration format.
+    """
+    indexed_fields = {
+        # A search field with predictive search and boosting
+        'title': {
+            'type': 'string',
+            'analyzer': 'edgengram_analyzer',
+            'boost': 100,
+        },
+
+        # A filter field
+        'live': {
+            'type': 'boolean',
+            'index': 'not_analyzed',
+        },
+    }
+
+class SearchTestOldConfigList(models.Model, indexed.Indexed):
+    """
+    This tests that the Indexed class can correctly handle models that
+    use the old "indexed_fields" configuration format using a list.
+    """
+    indexed_fields = ['title', 'content']
diff --git a/wagtail/wagtailsearch/tests/test_indexed_class.py b/wagtail/wagtailsearch/tests/test_indexed_class.py
new file mode 100644
index 000000000..a54f135c3
--- /dev/null
+++ b/wagtail/wagtailsearch/tests/test_indexed_class.py
@@ -0,0 +1,49 @@
+from django.test import TestCase
+from wagtail.tests import models
+import json
+
+from wagtail.wagtailsearch import indexed
+
+
+class TestContentTypeNames(TestCase):
+    def test_base_content_type_name(self):
+        name = models.SearchTestChild.indexed_get_toplevel_content_type()
+        self.assertEqual(name, 'tests_searchtest')
+
+    def test_qualified_content_type_name(self):
+        name = models.SearchTestChild.indexed_get_content_type()
+        self.assertEqual(name, 'tests_searchtest_tests_searchtestchild')
+
+
+class TestIndexedFieldsBackwardsCompatibility(TestCase):
+    def test_indexed_fields_backwards_compatibility(self):
+        # Get search fields
+        search_fields = models.SearchTestOldConfig.get_search_fields()
+
+        search_fields_dict = dict(
+            ((field.field_name, type(field)), field)
+            for field in search_fields
+        )
+
+        # Check that the fields were found
+        self.assertEqual(len(search_fields_dict), 2)
+        self.assertIn(('title', indexed.SearchField), search_fields_dict.keys())
+        self.assertIn(('live', indexed.FilterField), search_fields_dict.keys())
+
+        # Check that the title field has the correct settings
+        self.assertTrue(search_fields_dict[('title', indexed.SearchField)].partial_match)
+        self.assertEqual(search_fields_dict[('title', indexed.SearchField)].boost, 100)
+
+    def test_indexed_fields_backwards_compatibility_list(self):
+        # Get search fields
+        search_fields = models.SearchTestOldConfigList.get_search_fields()
+
+        search_fields_dict = dict(
+            ((field.field_name, type(field)), field)
+            for field in search_fields
+        )
+
+        # Check that the fields were found
+        self.assertEqual(len(search_fields_dict), 2)
+        self.assertIn(('title', indexed.SearchField), search_fields_dict.keys())
+        self.assertIn(('content', indexed.SearchField), search_fields_dict.keys())

From 63e5ca34ce9ad6b9bd1229a207767a51a76328e9 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Thu, 3 Jul 2014 12:31:18 +0100
Subject: [PATCH 10/49] Make sure field filters are sorted before checking them

---
 .../tests/test_elasticsearch_backend.py         | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index 88aa87f8b..0c7906b39 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -113,15 +113,26 @@ class TestElasticSearchQuery(TestCase):
 
         # Check it
         expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
-        self.assertDictEqual(query.to_es(), expected_result)
+
+        # Make sure field filters are sorted (as they can be in any order which may cause false positives)
+        query = query.to_es()
+        field_filters = query['filtered']['filter']['and'][1]['and']
+        field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0])
+
+        self.assertDictEqual(query, expected_result)
 
     def test_or_filter(self):
         # Create a query
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(Q(title="Test") | Q(live=True)), "Hello")
 
+        # Make sure field filters are sorted (as they can be in any order which may cause false positives)
+        query = query.to_es()
+        field_filters = query['filtered']['filter']['and'][1]['or']
+        field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0])
+
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'title_filter': 'Test'}}, {'term': {'live_filter': True}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
-        self.assertDictEqual(query.to_es(), expected_result)
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query, expected_result)
 
     def test_negated_filter(self):
         # Create a query

From 001fcabb46e986f5be8bbe0c4b2e61cd14507d10 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Thu, 3 Jul 2014 12:44:53 +0100
Subject: [PATCH 11/49] Fixed python 3.2 incompatible migration

---
 ...to__del_searchtestchild__del_searchtest.py | 70 ++++++++++---------
 1 file changed, 36 insertions(+), 34 deletions(-)

diff --git a/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py
index 560dbaae9..6826ddaa1 100644
--- a/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py
+++ b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py
@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from south.utils import datetime_utils as datetime
 from south.db import db
 from south.v2 import SchemaMigration
@@ -9,76 +11,76 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
         # Deleting model 'SearchTestChild'
-        db.delete_table(u'wagtailsearch_searchtestchild')
+        db.delete_table('wagtailsearch_searchtestchild')
 
         # Deleting model 'SearchTest'
-        db.delete_table(u'wagtailsearch_searchtest')
+        db.delete_table('wagtailsearch_searchtest')
 
 
     def backwards(self, orm):
         # Adding model 'SearchTestChild'
-        db.create_table(u'wagtailsearch_searchtestchild', (
+        db.create_table('wagtailsearch_searchtestchild', (
             ('extra_content', self.gf('django.db.models.fields.TextField')()),
-            (u'searchtest_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['wagtailsearch.SearchTest'], unique=True, primary_key=True)),
+            ('searchtest_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['wagtailsearch.SearchTest'], unique=True, primary_key=True)),
         ))
-        db.send_create_signal(u'wagtailsearch', ['SearchTestChild'])
+        db.send_create_signal('wagtailsearch', ['SearchTestChild'])
 
         # Adding model 'SearchTest'
-        db.create_table(u'wagtailsearch_searchtest', (
+        db.create_table('wagtailsearch_searchtest', (
             ('content', self.gf('django.db.models.fields.TextField')()),
             ('live', self.gf('django.db.models.fields.BooleanField')(default=False)),
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('title', self.gf('django.db.models.fields.CharField')(max_length=255)),
         ))
-        db.send_create_signal(u'wagtailsearch', ['SearchTest'])
+        db.send_create_signal('wagtailsearch', ['SearchTest'])
 
 
     models = {
-        u'auth.group': {
+        'auth.group': {
             'Meta': {'object_name': 'Group'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
-            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
         },
-        u'auth.permission': {
-            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
             'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
         },
-        u'auth.user': {
+        'auth.user': {
             'Meta': {'object_name': 'User'},
             'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
             'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
-            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'user_set'", 'blank': 'True', 'to': "orm['auth.Group']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
             'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'user_set'", 'blank': 'True', 'to': "orm['auth.Permission']"}),
             'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
         },
-        u'contenttypes.contenttype': {
+        'contenttypes.contenttype': {
             'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
             'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
             'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
         },
-        u'wagtailcore.page': {
+        'wagtailcore.page': {
             'Meta': {'object_name': 'Page'},
-            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pages'", 'to': u"orm['contenttypes.ContentType']"}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pages'", 'to': "orm['contenttypes.ContentType']"}),
             'depth': ('django.db.models.fields.PositiveIntegerField', [], {}),
             'has_unpublished_changes': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'live': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'numchild': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}),
-            'owner': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'owned_pages'", 'null': 'True', 'to': u"orm['auth.User']"}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'owned_pages'", 'null': 'True', 'to': "orm['auth.User']"}),
             'path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
             'search_description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
             'seo_title': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
@@ -87,25 +89,25 @@ class Migration(SchemaMigration):
             'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
             'url_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'})
         },
-        u'wagtailsearch.editorspick': {
+        'wagtailsearch.editorspick': {
             'Meta': {'ordering': "('sort_order',)", 'object_name': 'EditorsPick'},
             'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'page': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['wagtailcore.Page']"}),
-            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editors_picks'", 'to': u"orm['wagtailsearch.Query']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'page': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['wagtailcore.Page']"}),
+            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editors_picks'", 'to': "orm['wagtailsearch.Query']"}),
             'sort_order': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
         },
-        u'wagtailsearch.query': {
+        'wagtailsearch.query': {
             'Meta': {'object_name': 'Query'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'query_string': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
         },
-        u'wagtailsearch.querydailyhits': {
+        'wagtailsearch.querydailyhits': {
             'Meta': {'unique_together': "(('query', 'date'),)", 'object_name': 'QueryDailyHits'},
             'date': ('django.db.models.fields.DateField', [], {}),
             'hits': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'daily_hits'", 'to': u"orm['wagtailsearch.Query']"})
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'daily_hits'", 'to': "orm['wagtailsearch.Query']"})
         }
     }
 

From 08ced063ff09d102922deaf279275e4c0fb5ff4d Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 18:56:08 +0100
Subject: [PATCH 12/49] Moved search method into base search backend

---
 wagtail/wagtailcore/models.py                 |  2 +-
 wagtail/wagtailsearch/backends/base.py        | 38 ++++++++++++++++++-
 wagtail/wagtailsearch/backends/db.py          | 34 +++++------------
 .../wagtailsearch/backends/elasticsearch.py   | 34 +----------------
 4 files changed, 48 insertions(+), 60 deletions(-)

diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index 4964966a3..b6a664548 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -514,7 +514,7 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index
 
         # Search
         s = get_search_backend()
-        return s.search(query_string, model=cls, fields=fields, filters=filters, prefetch_related=prefetch_related)
+        return s.search(query_string, cls, fields=fields, filters=filters, prefetch_related=prefetch_related)
 
     @classmethod
     def clean_subpage_types(cls):
diff --git a/wagtail/wagtailsearch/backends/base.py b/wagtail/wagtailsearch/backends/base.py
index 393ecae1b..82e2e8d56 100644
--- a/wagtail/wagtailsearch/backends/base.py
+++ b/wagtail/wagtailsearch/backends/base.py
@@ -1,6 +1,9 @@
 from django.db import models
+from django.db.models.query import QuerySet
+from django.core.exceptions import ImproperlyConfigured
 
 from wagtail.wagtailsearch.indexed import Indexed
+from wagtail.wagtailsearch.utils import normalise_query_string
 
 
 class BaseSearch(object):
@@ -32,5 +35,38 @@ class BaseSearch(object):
     def delete(self, obj):
         return NotImplemented
 
-    def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
+    def _search(self, queryset, query_string, fields=None):
         return NotImplemented
+
+    def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
+        # Find model/queryset
+        if isinstance(model_or_queryset, QuerySet):
+            model = model_or_queryset.model
+            queryset = model_or_queryset
+        else:
+            model = model_or_queryset
+            queryset = model_or_queryset.objects.all()
+
+        # Model must be a descendant of Indexed and be a django model
+        if not issubclass(model, Indexed) or not issubclass(model, models.Model):
+            return []
+
+        # Normalise query string
+        if query_string is not None:
+            query_string = normalise_query_string(query_string)
+
+        # Check that theres still a query string after the clean up
+        if query_string == "":
+            return []
+
+        # Apply filters to queryset
+        if filters:
+            queryset = queryset.filter(**filters)
+
+        # Prefetch related
+        if prefetch_related:
+            for prefetch in prefetch_related:
+                queryset = queryset.prefetch_related(prefetch)
+
+        # Search
+        return self._search(queryset, query_string, fields=fields)
diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py
index 56536e343..a94fe3c58 100644
--- a/wagtail/wagtailsearch/backends/db.py
+++ b/wagtail/wagtailsearch/backends/db.py
@@ -2,7 +2,6 @@ from django.db import models
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed
-from wagtail.wagtailsearch.utils import normalise_query_string
 
 
 class DBSearch(BaseSearch):
@@ -27,26 +26,16 @@ class DBSearch(BaseSearch):
     def delete(self, obj):
         pass # Not needed
 
-    def search(self, query_string, model, fields=None, filters=None, prefetch_related=None):
-        # Get fields
-        if fields is None:
-            fields = [field.field_name for field in model.get_searchable_search_fields()]
-
-        # Start with all objects
-        query = model.objects.all()
-
-        # Apply filters
-        if filters:
-            query = query.filter(**filters)
-
+    def _search(self, queryset, query_string, fields=None):
         if query_string is not None:
-            # Normalise query string
-            query_string = normalise_query_string(query_string)
+            # Get fields
+            if fields is None:
+                fields = [field.field_name for field in queryset.model.get_searchable_search_fields()]
 
             # Get terms
             terms = query_string.split()
             if not terms:
-                return model.objects.none()
+                return queryset.model.objects.none()
 
             # Filter by terms
             for term in terms:
@@ -54,21 +43,16 @@ class DBSearch(BaseSearch):
                 for field_name in fields:
                     # Check if the field exists (this will filter out indexed callables)
                     try:
-                        model._meta.get_field_by_name(field_name)
+                        queryset.model._meta.get_field_by_name(field_name)
                     except:
                         continue
 
                     # Filter on this field
                     term_query |= models.Q(**{'%s__icontains' % field_name: term})
 
-                query = query.filter(term_query)
+                queryset = queryset.filter(term_query)
 
             # Distinct
-            query = query.distinct()
+            queryset = queryset.distinct()
 
-        # Prefetch related
-        if prefetch_related:
-            for prefetch in prefetch_related:
-                query = query.prefetch_related(prefetch)
-
-        return query
+        return queryset
diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index ea0413eed..8f17ba6ce 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -3,14 +3,12 @@ from __future__ import absolute_import
 import json
 
 from django.db import models
-from django.db.models.query import QuerySet
 
 from elasticsearch import Elasticsearch, NotFoundError, RequestError
 from elasticsearch.helpers import bulk
 
 from wagtail.wagtailsearch.backends.base import BaseSearch
 from wagtail.wagtailsearch.indexed import Indexed, SearchField, FilterField
-from wagtail.wagtailsearch.utils import normalise_query_string
 
 
 class ElasticSearchMapping(object):
@@ -576,35 +574,5 @@ class ElasticSearch(BaseSearch):
         except NotFoundError:
             pass  # Document doesn't exist, ignore this exception
 
-    def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
-        # Find model/queryset
-        if isinstance(model_or_queryset, QuerySet):
-            model = model_or_queryset.model
-            queryset = model_or_queryset
-        else:
-            model = model_or_queryset
-            queryset = model_or_queryset.objects.all()
-
-        # Model must be a descendant of Indexed and be a django model
-        if not issubclass(model, Indexed) or not issubclass(model, models.Model):
-            return []
-
-        # Normalise query string
-        if query_string is not None:
-            query_string = normalise_query_string(query_string)
-
-        # Check that theres still a query string after the clean up
-        if query_string == "":
-            return []
-
-        # Apply filters to queryset
-        if filters:
-            queryset = queryset.filter(**filters)
-
-        # Prefetch related
-        if prefetch_related:
-            for prefetch in prefetch_related:
-                queryset = queryset.prefetch_related(prefetch)
-
-        # Return search results
+    def _search(self, queryset, query_string, fields=None):
         return ElasticSearchResults(self, ElasticSearchQuery(queryset, query_string, fields=fields))

From e35e20dd10e2be669addbde992b24b41d78d9f5a Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 19:14:46 +0100
Subject: [PATCH 13/49] Added search method to PageQuerySet

---
 wagtail/wagtailcore/models.py | 3 +++
 wagtail/wagtailcore/query.py  | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index b6a664548..daee31fb3 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -228,6 +228,9 @@ class PageManager(models.Manager):
     def not_type(self, model):
         return self.get_queryset().not_type(model)
 
+    def search(self, query_string, fields=None, backend='default'):
+        return self.get_queryset().search(query_string, fields=fields, backend=backend)
+
 
 class PageBase(models.base.ModelBase):
     """Metaclass for Page"""
diff --git a/wagtail/wagtailcore/query.py b/wagtail/wagtailcore/query.py
index 57e8ffff3..422e46f47 100644
--- a/wagtail/wagtailcore/query.py
+++ b/wagtail/wagtailcore/query.py
@@ -2,6 +2,8 @@ from django.db.models import Q
 from django.contrib.contenttypes.models import ContentType
 from treebeard.mp_tree import MP_NodeQuerySet
 
+from wagtail.wagtailsearch.backends import get_search_backend
+
 
 class PageQuerySet(MP_NodeQuerySet):
     """
@@ -107,3 +109,7 @@ class PageQuerySet(MP_NodeQuerySet):
 
     def not_type(self, model):
         return self.exclude(self.type_q(model))
+
+    def search(self, query_string, fields=None, backend='default'):
+        search_backend = get_search_backend(backend)
+        return search_backend.search(query_string, self, fields=None)

From b78b6486829f748daf42d6240426abb44d8329a5 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karlhobley10@gmail.com>
Date: Sun, 22 Jun 2014 21:07:09 +0100
Subject: [PATCH 14/49] Implemented __in lookup in ElasticSearch backend

Also added an error message if a user attempts to use a subquery with the __in lookup.
---
 wagtail/wagtailsearch/backends/elasticsearch.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 8f17ba6ce..289b0f4b2 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -3,6 +3,7 @@ from __future__ import absolute_import
 import json
 
 from django.db import models
+from django.db.models.sql.where import SubqueryConstraint
 
 from elasticsearch import Elasticsearch, NotFoundError, RequestError
 from elasticsearch.helpers import bulk
@@ -201,7 +202,16 @@ class ElasticSearchQuery(object):
                     }
                 }
 
-            raise FilterError('Could not apply filter on ElasticSearch results "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.')
+            if lookup == 'in':
+                return {
+                    'terms': {
+                        field_index_name: value,
+                    }
+                }
+
+            raise FilterError('Could not apply filter on ElasticSearch results: "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.')
+        elif isinstance(where_node, SubqueryConstraint):
+            raise FilterError('Could not apply filter on ElasticSearch results: Subqueries are not allowed.')
 
         # Get child filters
         connector = where_node.connector

From 9e1bd2d601ab540eb0d2d383ddeec369ef7c94c1 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Tue, 24 Jun 2014 14:27:40 +0100
Subject: [PATCH 15/49] Added get_indexed_objects method to Indexed class This
 gives developers control over the QuerySet used when the model is added to
 the indexed

---
 wagtail/wagtailsearch/indexed.py                          | 4 ++++
 wagtail/wagtailsearch/management/commands/update_index.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/wagtail/wagtailsearch/indexed.py b/wagtail/wagtailsearch/indexed.py
index a2cb54d14..2e506d690 100644
--- a/wagtail/wagtailsearch/indexed.py
+++ b/wagtail/wagtailsearch/indexed.py
@@ -118,6 +118,10 @@ class Indexed(object):
     def get_filterable_search_fields(cls):
         return filter(lambda field: isinstance(field, FilterField), cls.get_search_fields())
 
+    @classmethod
+    def get_indexed_objects(cls):
+        return cls.objects.all()
+
     indexed_fields = ()
 
 
diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py
index 2c1593635..c52b75639 100644
--- a/wagtail/wagtailsearch/management/commands/update_index.py
+++ b/wagtail/wagtailsearch/management/commands/update_index.py
@@ -24,7 +24,7 @@ class Command(BaseCommand):
             toplevel_content_type = model.indexed_get_toplevel_content_type()
 
             # Loop through objects
-            for obj in model.objects.all():
+            for obj in model.get_indexed_objects():
                 # Get key for this object
                 key = toplevel_content_type + ':' + str(obj.pk)
 

From b9d3e44ebf1b146987338c137c9bb867a81f52fd Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Tue, 24 Jun 2014 14:28:40 +0100
Subject: [PATCH 16/49] Use get_indexed_objects to speed up indexing of
 Images/Documents Previously, this created a query for every single image and
 document to get the tags. This was very slow on RCA which has over 15000
 images. This commit fixes this by adding a prefetch_related to the QuerySet
 used for indexing.

---
 wagtail/wagtailadmin/taggable.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/wagtail/wagtailadmin/taggable.py b/wagtail/wagtailadmin/taggable.py
index 91c36ca7f..3b7de479c 100644
--- a/wagtail/wagtailadmin/taggable.py
+++ b/wagtail/wagtailadmin/taggable.py
@@ -21,7 +21,11 @@ class TagSearchable(indexed.Indexed):
 
     @property
     def get_tags(self):
-        return ' '.join([tag.name for tag in self.tags.all()])
+        return ' '.join([tag.name for tag in self.prefetched_tags()])
+
+    @classmethod
+    def get_indexed_objects(cls):
+        return super(TagSearchable, cls).get_indexed_objects().prefetch_related('tagged_items__tag')
 
     @classmethod
     def search(cls, q, results_per_page=None, page=1, prefetch_tags=False, filters={}):

From 532bb6241e7cbced94fb044306e3eba7a447be82 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Thu, 3 Jul 2014 14:26:05 +0100
Subject: [PATCH 17/49] Added some more FilterFields to Page

---
 wagtail/wagtailcore/models.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index daee31fb3..75021d584 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -285,8 +285,12 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index
 
     search_fields = (
         indexed.SearchField('title', partial_match=True, boost=100),
+        indexed.FilterField('id'),
         indexed.FilterField('live'),
+        indexed.FilterField('owner'),
+        indexed.FilterField('content_type'),
         indexed.FilterField('path'),
+        indexed.FilterField('depth'),
     )
 
     def __init__(self, *args, **kwargs):

From dcca9bd515f5bfbaed35c912236573d85a41d473 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 4 Jul 2014 10:24:32 +0100
Subject: [PATCH 18/49] Use a multi_match query instead of a query_string query

---
 .../wagtailsearch/backends/elasticsearch.py   |  4 +--
 .../tests/test_elasticsearch_backend.py       | 26 +++++++++----------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index 289b0f4b2..c28b676ee 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -258,14 +258,14 @@ class ElasticSearchQuery(object):
         # Query
         if self.query_string is not None:
             query = {
-                'query_string': {
+                'multi_match': {
                     'query': self.query_string,
                 }
             }
 
             # Fields
             if self.fields:
-                query['query_string']['fields'] = self.fields
+                query['multi_match']['fields'] = self.fields
         else:
             query = {
                 'match_all': {}
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index 0c7906b39..6d2df3b4c 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -88,7 +88,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_none_query_string(self):
@@ -104,7 +104,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test"), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_and_filter(self):
@@ -112,7 +112,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test", live=True), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
 
         # Make sure field filters are sorted (as they can be in any order which may cause false positives)
         query = query.to_es()
@@ -131,7 +131,7 @@ class TestElasticSearchQuery(TestCase):
         field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0])
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query, expected_result)
 
     def test_negated_filter(self):
@@ -139,7 +139,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.exclude(live=True), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_fields(self):
@@ -147,7 +147,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello", fields=['title'])
 
         # Check it
-        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['title']}}}}
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['title']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_exact_lookup(self):
@@ -155,7 +155,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__exact="Test"), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_none_lookup(self):
@@ -163,7 +163,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title=None), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_isnull_true_lookup(self):
@@ -171,7 +171,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=True), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_isnull_false_lookup(self):
@@ -179,7 +179,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=False), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_startswith_lookup(self):
@@ -187,7 +187,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__startswith="Test"), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_gt_lookup(self):
@@ -198,7 +198,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gt=datetime.datetime(2014, 4, 29)), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_range_lookup(self):
@@ -209,7 +209,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__range=(start_date, end_date)), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
 

From 7fc10398ddc657cc1708be8ce01b5285118b8598 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Fri, 4 Jul 2014 10:31:50 +0100
Subject: [PATCH 19/49] Minor optimisation when building single field queries

---
 .../wagtailsearch/backends/elasticsearch.py   | 24 ++++++++++++-------
 .../tests/test_elasticsearch_backend.py       |  2 +-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py
index c28b676ee..55f1971a2 100644
--- a/wagtail/wagtailsearch/backends/elasticsearch.py
+++ b/wagtail/wagtailsearch/backends/elasticsearch.py
@@ -121,7 +121,7 @@ class ElasticSearchQuery(object):
     def __init__(self, queryset, query_string, fields=None):
         self.queryset = queryset
         self.query_string = query_string
-        self.fields = fields or ['_all', '_partials']
+        self.fields = fields
 
     def _get_filters_from_where(self, where_node):
         # Check if this is a leaf node
@@ -257,15 +257,21 @@ class ElasticSearchQuery(object):
     def to_es(self):
         # Query
         if self.query_string is not None:
-            query = {
-                'multi_match': {
-                    'query': self.query_string,
-                }
-            }
+            fields = self.fields or ['_all', '_partials']
 
-            # Fields
-            if self.fields:
-                query['multi_match']['fields'] = self.fields
+            if len(fields) == 1:
+                query = {
+                    'match': {
+                        fields[0]: self.query_string,
+                    }
+                }
+            else:
+                query = {
+                    'multi_match': {
+                        'query': self.query_string,
+                        'fields': fields,
+                    }
+                }
         else:
             query = {
                 'match_all': {}
diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index 6d2df3b4c..eeb23ff33 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -147,7 +147,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello", fields=['title'])
 
         # Check it
-        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['title']}}}}
+        expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'match': {'title': 'Hello'}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_exact_lookup(self):

From 95c76542163e816c19310953c61d8f59507ae78a Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Tue, 8 Jul 2014 20:25:54 +0100
Subject: [PATCH 20/49] Page.search should invoke backend.search with cls as a
 plain arg, not a 'model' kwarg

---
 wagtail/wagtailcore/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index 11b6fe7ec..c6a4f6854 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -521,7 +521,7 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index
 
         # Search
         s = get_search_backend()
-        return s.search(query_string, model=cls, fields=fields, filters=filters, prefetch_related=prefetch_related)
+        return s.search(query_string, cls, fields=fields, filters=filters, prefetch_related=prefetch_related)
 
     @classmethod
     def clean_subpage_types(cls):

From 354895bad2d1e5e40559d7e980a69e4a5895a315 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 09:28:05 +0100
Subject: [PATCH 21/49] Updated setup.py requirements

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 6e0bc6949..11c32d522 100644
--- a/setup.py
+++ b/setup.py
@@ -25,9 +25,9 @@ PY3 = sys.version_info[0] == 3
 install_requires = [
     "Django>=1.6.2,<1.7",
     "South>=0.8.4",
-    "django-compressor>=1.3",
-    "django-libsass>=0.1",
-    "django-modelcluster>=0.1",
+    "django-compressor>=1.4",
+    "django-libsass>=0.2",
+    "django-modelcluster>=0.3",
     "django-taggit==0.11.2",
     "django-treebeard==2.0",
     "Pillow>=2.3.0",

From 1d9f792415c1ebd10f36758e72efff8ee1442bb2 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 09:50:58 +0100
Subject: [PATCH 22/49] Added search changes to changelog and 0.4 release notes

---
 CHANGELOG.txt         |  9 +++++++++
 docs/releases/0.4.rst | 14 ++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 8f391a29a..e915ffb22 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -9,6 +9,7 @@ Changelog
  * Added frontend cache invalidator
  * Added sitemap generator
  * Added notification preferences
+ * Added a new way to configure searchable/filterable fields on models
  * Added 'original' as a resizing rule supported by the 'image' tag
  * Hallo.js updated to version 1.0.4
  * Snippets are now ordered alphabetically
@@ -29,6 +30,14 @@ Changelog
  * Added init_new_page signal
  * Added page_published signal
  * Added copy method to Page to allow copying of pages
+ * Added ``search`` method to ``PageQuerySet``
+ * Added ``get_indexed_objects`` allowing developers to customise which objects get added to the search index
+ * Major refactor of Elasticsearch backend
+ * Use ``match`` instead of ``query_string`` queries
+ * Fields are now indexed in Elasticsearch with their correct type
+ * Filter fields are no longer included in '_all' (in Elasticsearch)
+ * Fields with partial matching are now indexed together into '_partials'
+
  * Fix: Animated GIFs are now coalesced before resizing
  * Fix: Wand backend clones images before modifying them
  * Fix: Admin breadcrumb now positioned correctly on mobile
diff --git a/docs/releases/0.4.rst b/docs/releases/0.4.rst
index a51c3402d..461689e05 100644
--- a/docs/releases/0.4.rst
+++ b/docs/releases/0.4.rst
@@ -70,6 +70,7 @@ Core
 
  * Any extra arguments given to ``Page.serve`` are now passed through to ``get_context`` and ``get_template``
  * Added ``in_menu`` and ``not_in_menu`` methods to ``PageQuerySet``
+ * Added ``search`` method to ``PageQuerySet``
  * Added ``get_next_siblings`` and ``get_prev_siblings`` to ``Page``
  * Added ``page_published`` signal
  * Added ``copy`` method to ``Page`` to allow copying of pages
@@ -90,6 +91,18 @@ Admin
  * Added ``init_new_page`` signal
 
 
+Search
+------
+
+ * Added a new way to configure searchable/filterable fields on models
+ * Added ``get_indexed_objects`` allowing developers to customise which objects get added to the search index
+ * Major refactor of Elasticsearch backend
+ * Use ``match`` instead of ``query_string`` queries
+ * Fields are now indexed in Elasticsearch with their correct type
+ * Filter fields are no longer included in '_all' (in Elasticsearch)
+ * Fields with partial matching are now indexed together into '_partials'
+
+
 Images
 ------
 
@@ -100,6 +113,7 @@ Images
 
 Other
 -----
+
  * Added styleguide (mainly for wagtail developers)
 
 

From 8aaff51b05e3cc781b89f6258166b6a58ba0295a Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 09:52:33 +0100
Subject: [PATCH 23/49] Minor tweak to 0.4 release notes

---
 docs/releases/0.4.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/releases/0.4.rst b/docs/releases/0.4.rst
index 461689e05..a53ca5463 100644
--- a/docs/releases/0.4.rst
+++ b/docs/releases/0.4.rst
@@ -99,7 +99,7 @@ Search
  * Major refactor of Elasticsearch backend
  * Use ``match`` instead of ``query_string`` queries
  * Fields are now indexed in Elasticsearch with their correct type
- * Filter fields are no longer included in '_all' (in Elasticsearch)
+ * Filter fields are no longer included in '_all'
  * Fields with partial matching are now indexed together into '_partials'
 
 

From 599eebf022a5b04cd7fcd30bfeec7dfbc4261693 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 14:30:39 +0100
Subject: [PATCH 24/49] Splitup wagtailsearch docs

---
 docs/index.rst                                |   2 +-
 docs/wagtailsearch/configuration.rst          |  77 +++++++++
 docs/wagtailsearch/editors_picks.rst          |  41 +++++
 .../frontend_views.rst}                       | 152 ++++--------------
 docs/wagtailsearch/index.rst                  |  15 ++
 5 files changed, 162 insertions(+), 125 deletions(-)
 create mode 100644 docs/wagtailsearch/configuration.rst
 create mode 100644 docs/wagtailsearch/editors_picks.rst
 rename docs/{wagtail_search.rst => wagtailsearch/frontend_views.rst} (53%)
 create mode 100644 docs/wagtailsearch/index.rst

diff --git a/docs/index.rst b/docs/index.rst
index 171c0e45a..b2d9a534d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -13,7 +13,7 @@ It supports Django 1.6.2+ on Python 2.6, 2.7, 3.2, 3.3 and 3.4. Django 1.7 suppo
    building_your_site/index
    editing_api
    snippets
-   wagtail_search
+   wagtailsearch/index
    form_builder
    model_recipes
    advanced_topics
diff --git a/docs/wagtailsearch/configuration.rst b/docs/wagtailsearch/configuration.rst
new file mode 100644
index 000000000..765e81b84
--- /dev/null
+++ b/docs/wagtailsearch/configuration.rst
@@ -0,0 +1,77 @@
+
+.. _wagtailsearch_configuration:
+
+
+Wagtailsearch configuration
+============================
+
+
+Adding extra search fields
+--------------------------
+
+ - SearchField
+ - FilterField
+ - Callable search fields
+ - Indexing fields in child/related objects
+
+
+Indexing non-page models
+------------------------
+
+
+Search Backends
+---------------
+
+Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.
+
+.. _Elasticsearch: http://www.elasticsearch.org/
+
+
+Default DB Backend
+``````````````````
+The default DB search backend uses Django's ``__icontains`` filter.
+
+
+Elasticsearch Backend
+`````````````````````
+Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
+
+.. code-block:: guess
+
+  pip install elasticsearch
+
+.. note::
+  If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5```
+
+The backend is configured in settings:
+
+.. code-block:: python
+
+  WAGTAILSEARCH_BACKENDS = {
+      'default': {
+          'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
+          'URLS': ['http://localhost:9200'],
+          'INDEX': 'wagtail',
+          'TIMEOUT': 5,
+          'FORCE_NEW': False,
+      }
+  }
+
+Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
+
+If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly:
+
+-  Sign up for an account at `dashboard.searchly.com/users/sign\_up`_
+-  Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo'
+-  Note the connection URL from your Searchly dashboard
+-  Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS``
+-  Run ``./manage.py update_index``
+
+.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org
+.. _Searchly: http://www.searchly.com/
+.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up
+
+
+Rolling Your Own
+````````````````
+Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``.
\ No newline at end of file
diff --git a/docs/wagtailsearch/editors_picks.rst b/docs/wagtailsearch/editors_picks.rst
new file mode 100644
index 000000000..f899302b4
--- /dev/null
+++ b/docs/wagtailsearch/editors_picks.rst
@@ -0,0 +1,41 @@
+
+.. _wagtailsearch_editors_picks:
+
+
+Editors picks
+=============
+
+Editor's Picks are a way of explicitly linking relevant content to search terms, so results pages can contain curated content instead of being at the mercy of the search algorithm. In a template using the search results view, editor's picks can be accessed through the variable ``query.editors_picks``. To include editor's picks in your search results template, use the following properties.
+
+``query.editors_picks.all``
+  This gathers all of the editor's picks objects relating to the current query, in order according to their sort order in the Wagtail admin. You can then iterate through them using a ``{% for ... %}`` loop. Each editor's pick object provides these properties:
+
+  ``editors_pick.page``
+    The page object associated with the pick. Use ``{% pageurl editors_pick.page %}`` to generate a URL or provide other properties of the page object.
+
+  ``editors_pick.description``
+    The description entered when choosing the pick, perhaps explaining why the page is relevant to the search terms.
+
+Putting this all together, a block of your search results template displaying editor's Picks might look like this:
+
+.. code-block:: django
+
+  {% with query.editors_picks.all as editors_picks %}
+    {% if editors_picks %}
+      <div class="well">
+      <h3>Editors picks</h3>
+    <ul>
+      {% for editors_pick in editors_picks %}
+        <li>
+          <h4>
+        <a href="{% pageurl editors_pick.page %}">
+          {{ editors_pick.page.title }}
+        </a>
+          </h4>
+          <p>{{ editors_pick.description|safe }}</p>
+        </li>
+      {% endfor %}
+    </ul>
+      </div>
+    {% endif %}
+  {% endwith %}
\ No newline at end of file
diff --git a/docs/wagtail_search.rst b/docs/wagtailsearch/frontend_views.rst
similarity index 53%
rename from docs/wagtail_search.rst
rename to docs/wagtailsearch/frontend_views.rst
index 4c0a1ac74..e83bf52a5 100644
--- a/docs/wagtail_search.rst
+++ b/docs/wagtailsearch/frontend_views.rst
@@ -1,10 +1,10 @@
 
-.. _wagtail_search:
+.. _wagtailsearch_frontend_views:
 
-Search
-======
 
-Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface.
+Wagtailsearch frontend views
+============================
+
 
 Default Page Search
 -------------------
@@ -71,43 +71,6 @@ The search view provides a context with a few useful variables.
   ``query``
     A Wagtail ``Query`` object matching the terms. The ``Query`` model provides several class methods for viewing the statistics of all queries, but exposes only one property for single objects, ``query.hits``, which tracks the number of time the search string has been used over the lifetime of the site. ``Query`` also joins to the Editor's Picks functionality though ``query.editors_picks``. See :ref:`editors-picks`.
 
-Editor's Picks
---------------
-
-Editor's Picks are a way of explicitly linking relevant content to search terms, so results pages can contain curated content instead of being at the mercy of the search algorithm. In a template using the search results view, editor's picks can be accessed through the variable ``query.editors_picks``. To include editor's picks in your search results template, use the following properties.
-
-``query.editors_picks.all``
-  This gathers all of the editor's picks objects relating to the current query, in order according to their sort order in the Wagtail admin. You can then iterate through them using a ``{% for ... %}`` loop. Each editor's pick object provides these properties:
-
-  ``editors_pick.page``
-    The page object associated with the pick. Use ``{% pageurl editors_pick.page %}`` to generate a URL or provide other properties of the page object.
-
-  ``editors_pick.description``
-    The description entered when choosing the pick, perhaps explaining why the page is relevant to the search terms.
-
-Putting this all together, a block of your search results template displaying editor's Picks might look like this:
-
-.. code-block:: django
-
-  {% with query.editors_picks.all as editors_picks %}
-    {% if editors_picks %}
-      <div class="well">
-      <h3>Editors picks</h3>
-	<ul>
-	  {% for editors_pick in editors_picks %}
-	    <li>
-	      <h4>
-		<a href="{% pageurl editors_pick.page %}">
-		  {{ editors_pick.page.title }}
-		</a>
-	      </h4>
-	      <p>{{ editors_pick.description|safe }}</p>
-	    </li>
-	  {% endfor %}
-	</ul>
-      </div>
-    {% endif %}
-  {% endwith %}
 
 Asynchronous Search with JSON and AJAX
 --------------------------------------
@@ -142,30 +105,30 @@ Finally, we'll use JQuery to make the asynchronous requests and handle the inter
     // when there's something in the input box, make the query
     searchBox.on('input', function() {
       if( searchBox.val() == ''){
-	resultsBox.html('');
-	return;
+    resultsBox.html('');
+    return;
       }
       // make the request to the Wagtail JSON search view
       $.ajax({
-	url: wagtailJSONSearchURL + "?q=" +  searchBox.val(),
-	dataType: "json"
+    url: wagtailJSONSearchURL + "?q=" +  searchBox.val(),
+    dataType: "json"
       })
       .done(function(data) {
-	console.log(data);
-	if( data == undefined ){
-	  resultsBox.html('');
-	  return;
-	}
-	// we're in business!  let's format the results
-	var htmlOutput = '';
-	data.forEach(function(element, index, array){
-	  htmlOutput += '<p><a href="' + element.url + '">' + element.title + '</a></p>';
-	});
-	// and display them
-	resultsBox.html(htmlOutput);
+    console.log(data);
+    if( data == undefined ){
+      resultsBox.html('');
+      return;
+    }
+    // we're in business!  let's format the results
+    var htmlOutput = '';
+    data.forEach(function(element, index, array){
+      htmlOutput += '<p><a href="' + element.url + '">' + element.title + '</a></p>';
+    });
+    // and display them
+    resultsBox.html(htmlOutput);
       })
       .error(function(data){
-	console.log(data);
+    console.log(data);
       });
     });
 
@@ -178,12 +141,12 @@ Results are returned as a JSON object with this structure:
   {
     [
       {
-	title: "Lumpy Space Princess",
-	url: "/oh-my-glob/"
+    title: "Lumpy Space Princess",
+    url: "/oh-my-glob/"
       },
       {
-	title: "Lumpy Space",
-	url: "/no-smooth-posers/"
+    title: "Lumpy Space",
+    url: "/no-smooth-posers/"
       },
       ...
     ]
@@ -199,67 +162,8 @@ The AJAX interface uses the same view as the normal HTML search, ``wagtailsearch
 
 In this template, you'll have access to the same context variables provided to the HTML template. You could provide a template in JSON format with extra properties, such as ``query.hits`` and editor's picks, or render an HTML snippet that can go directly into your results ``<div>``. If you need more flexibility, such as multiple formats/templates based on differing requests, you can set up a custom search view.
 
-.. _editors-picks:
 
+Custom Search Views
+-------------------
 
-Indexing Custom Fields & Custom Search Views
---------------------------------------------
-
-This functionality is still under active development to provide a streamlined interface, but take a look at ``wagtail/wagtail/wagtailsearch/views/frontend.py`` if you are interested in coding custom search views.
-
-
-Search Backends
----------------
-
-Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.
-
-.. _Elasticsearch: http://www.elasticsearch.org/
-
-
-Default DB Backend
-``````````````````
-The default DB search backend uses Django's ``__icontains`` filter.
-
-
-Elasticsearch Backend
-`````````````````````
-Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
-
-.. code-block:: guess
-
-  pip install elasticsearch
-
-.. note::
-  If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5```
-
-The backend is configured in settings:
-
-.. code-block:: python
-
-  WAGTAILSEARCH_BACKENDS = {
-      'default': {
-          'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
-          'URLS': ['http://localhost:9200'],
-          'INDEX': 'wagtail',
-          'TIMEOUT': 5,
-          'FORCE_NEW': False,
-      }
-  }
-
-Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
-
-If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly:
-
--  Sign up for an account at `dashboard.searchly.com/users/sign\_up`_
--  Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo'
--  Note the connection URL from your Searchly dashboard
--  Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS``
--  Run ``./manage.py update_index``
-
-.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org
-.. _Searchly: http://www.searchly.com/
-.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up
-
-Rolling Your Own
-````````````````
-Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``.
+This functionality is still under active development to provide a streamlined interface, but take a look at ``wagtail/wagtail/wagtailsearch/views/frontend.py`` if you are interested in coding custom search views.
\ No newline at end of file
diff --git a/docs/wagtailsearch/index.rst b/docs/wagtailsearch/index.rst
new file mode 100644
index 000000000..f73ea787b
--- /dev/null
+++ b/docs/wagtailsearch/index.rst
@@ -0,0 +1,15 @@
+
+.. _wagtailsearch:
+
+
+Wagtailsearch
+=============
+
+Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface.
+
+.. toctree::
+    :maxdepth: 2
+
+    configuration
+    frontend_views
+    editors_picks

From d4c4c08d7c5d772de9b87c7b94f18c06ca8913f2 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Wed, 9 Jul 2014 15:18:22 +0100
Subject: [PATCH 25/49] add tests for lt, gte, lte lookups in
 TestElasticSearchQuery

---
 .../tests/test_elasticsearch_backend.py       | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index 0c7906b39..f97166fc4 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -191,7 +191,6 @@ class TestElasticSearchQuery(TestCase):
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_gt_lookup(self):
-        # This shares the same code path as gte, lt and lte so theres no need to test those
         # This also tests conversion of python dates to strings
 
         # Create a query
@@ -201,6 +200,30 @@ class TestElasticSearchQuery(TestCase):
         expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
+    def test_lt_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lt=datetime.datetime(2014, 4, 29)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_gte_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gte=datetime.datetime(2014, 4, 29)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
+    def test_lte_lookup(self):
+        # Create a query
+        query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lte=datetime.datetime(2014, 4, 29)), "Hello")
+
+        # Check it
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        self.assertDictEqual(query.to_es(), expected_result)
+
     def test_range_lookup(self):
         start_date = datetime.datetime(2014, 4, 29)
         end_date = datetime.datetime(2014, 8, 19)

From 70261780f62c85babfafff2e1d0f3b32fe6f6c2b Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:28:47 +0100
Subject: [PATCH 26/49] Added search fields configuration docs

---
 docs/wagtailsearch/configuration.rst | 142 ++++++++++++++++++++++++---
 1 file changed, 128 insertions(+), 14 deletions(-)

diff --git a/docs/wagtailsearch/configuration.rst b/docs/wagtailsearch/configuration.rst
index 765e81b84..6e8739d1c 100644
--- a/docs/wagtailsearch/configuration.rst
+++ b/docs/wagtailsearch/configuration.rst
@@ -1,39 +1,152 @@
 
 .. _wagtailsearch_configuration:
 
-
+===========================
 Wagtailsearch configuration
-============================
+===========================
 
 
-Adding extra search fields
---------------------------
+Indexing extra fields
+=====================
 
- - SearchField
- - FilterField
- - Callable search fields
- - Indexing fields in child/related objects
+Fields need to be explicitly added to the search configuration in order for you to be able to search/filter on them.
+
+You can add new fields to the search index by overriding the ``search_fields`` property and appending a list of extra ``SearchField``/``FilterField`` objects to it.
+
+``Page`` sets a default value to ``search_fields`` indexing the ``title`` field as a ``SearchField`` and some other generally useful fields as ``FilterField``s.
+
+
+Quick example
+-------------
+
+This creates an ``EventPage`` model with two fields ``description`` and ``date``. ``description`` is indexed as a ``SearchField`` and ``date`` is indexed as a ``FilterField``
+
+
+.. code-block:: python
+
+    from wagtail.wagtailsearch import indexed
+
+    class EventPage(Page):
+        description = models.TextField()
+        date = models.DateField()
+
+        search_fields = Page.search_fields + ( # Inherit search_fields from Page
+            indexed.SearchField('description'),
+            indexed.FilterField('date'),
+        )
+
+
+    # Get future events which contain the string "Christmas" in the title or description
+    >>> EventPage.objects.filter(date__gt=timezone.now()).search("Christmas")
+
+
+``indexed.SearchField``
+-----------------------
+
+These are added to the search index and are used for performing full-text searches on your models. These would usually be text fields.
+
+
+Options
+```````
+
+ - **partial_match** (boolean) - Setting this to true allows results to be matched on parts of words. For example, this is set on the title field by default so a page titled "Hello World!" will be found if the user only types "Hel" into the search box.
+ - **boost** (number) - This allows you to set fields as being more important than others. Setting this to a high number on a field will make pages with matches in that field to be ranked higher. By default, this is set to 100 on the title field and 1 on all other fields.
+ - **es_extra** (dict) - This field is to allow the developer to set or override any setting on the field in the ElasticSearch mapping. Use this if you want to make use of any ElasticSearch features that are not yet supported in Wagtail.
+
+
+``indexed.FilterField``
+-----------------------
+
+These are added to the search index but are not used for full-text searches. Instead, they allow you to run filters on your search results.
+
+
+Indexing callables and other attributes
+---------------------------------------
+
+ .. note::
+
+     This is not supported in the `Database Backend`_
+
+
+Search/filter fields do not need to be Django fields, they could be any method or attribute on your class.
+
+One use for this is indexing ``get_*_display`` methods Django creates automatically for fields with choices.
+
+
+.. code-block:: python
+
+    from wagtail.wagtailsearch import indexed
+
+    class EventPage(Page):
+        IS_PRIVATE_CHOICES = (
+            (False, "Public"),
+            (True, "Private"),
+        )
+
+        is_private = models.BooleanField(choices=IS_PRIVATE_CHOICES)
+
+        search_fields = Page.search_fields + (
+            # Index the human-readable string for searching
+            indexed.SearchField('get_is_private_display'),
+
+            # Index the boolean value for filtering
+            indexed.FilterField('is_private'),
+        )
 
 
 Indexing non-page models
-------------------------
+========================
+
+Any Django model can be indexed and searched.
+
+To do this, inherit from ``indexed.Indexed`` and add some ``search_fields`` to the model.
+
+.. code-block:: python
+
+    from wagtail.wagtailsearch import indexed
+
+    class Book(models.Model, indexed.Indexed):
+        title = models.CharField(max_length=255)
+        genre = models.CharField(max_length=255, choices=GENRE_CHOICES)
+        author = models.ForeignKey(Author)
+        published_date = models.DateTimeField()
+
+        search_fields = [
+            SearchField('title', partial_match=True, boost=10),
+            SearchField('get_genre_display'),
+
+            FilterField('genre'),
+            FilterField('author'),
+            FilterField('published_date'),
+        ]
+
+    # As this model doesn't have a search method in its QuerySet, we have to call search directly on the backend
+    >>> from wagtail.wagtailsearch.backends import get_search_backend
+    >>> s = get_search_backend()
+
+    # Run a search for a book by Roald Dahl
+    >>> roald_dahl = Author.objects.get(name="Roald Dahl")
+    >>> s.search("chocolate factory", Book.objects.filter(author=roald_dahl))
+    [<Book: Charlie and the chocolate factory>]
 
 
 Search Backends
----------------
+===============
 
 Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.
 
 .. _Elasticsearch: http://www.elasticsearch.org/
 
 
-Default DB Backend
-``````````````````
+Database Backend
+----------------
+
 The default DB search backend uses Django's ``__icontains`` filter.
 
 
 Elasticsearch Backend
-`````````````````````
+---------------------
+
 Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
 
 .. code-block:: guess
@@ -73,5 +186,6 @@ If you prefer not to run an Elasticsearch server in development or production, t
 
 
 Rolling Your Own
-````````````````
+----------------
+
 Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``.
\ No newline at end of file

From cb556c9ede449aca4ef898e782144108eb769db3 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:42:44 +0100
Subject: [PATCH 27/49] Splitup search configuration docs

---
 docs/wagtailsearch/backends.rst               | 64 +++++++++++++++++
 ...guration.rst => for_python_developers.rst} | 70 ++-----------------
 docs/wagtailsearch/index.rst                  |  4 +-
 3 files changed, 72 insertions(+), 66 deletions(-)
 create mode 100644 docs/wagtailsearch/backends.rst
 rename docs/wagtailsearch/{configuration.rst => for_python_developers.rst} (64%)

diff --git a/docs/wagtailsearch/backends.rst b/docs/wagtailsearch/backends.rst
new file mode 100644
index 000000000..f6bb2f296
--- /dev/null
+++ b/docs/wagtailsearch/backends.rst
@@ -0,0 +1,64 @@
+
+.. _wagtailsearch_backends:
+
+======================
+Wagtailsearch backends
+======================
+
+
+Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.
+
+.. _Elasticsearch: http://www.elasticsearch.org/
+
+
+Database Backend
+===============
+
+The default DB search backend uses Django's ``__icontains`` filter.
+
+
+Elasticsearch Backend
+=====================
+
+Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
+
+.. code-block:: guess
+
+  pip install elasticsearch
+
+.. note::
+  If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5```
+
+The backend is configured in settings:
+
+.. code-block:: python
+
+  WAGTAILSEARCH_BACKENDS = {
+      'default': {
+          'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
+          'URLS': ['http://localhost:9200'],
+          'INDEX': 'wagtail',
+          'TIMEOUT': 5,
+          'FORCE_NEW': False,
+      }
+  }
+
+Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
+
+If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly:
+
+-  Sign up for an account at `dashboard.searchly.com/users/sign\_up`_
+-  Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo'
+-  Note the connection URL from your Searchly dashboard
+-  Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS``
+-  Run ``./manage.py update_index``
+
+.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org
+.. _Searchly: http://www.searchly.com/
+.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up
+
+
+Rolling Your Own
+================
+
+Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``.
\ No newline at end of file
diff --git a/docs/wagtailsearch/configuration.rst b/docs/wagtailsearch/for_python_developers.rst
similarity index 64%
rename from docs/wagtailsearch/configuration.rst
rename to docs/wagtailsearch/for_python_developers.rst
index 6e8739d1c..d183bfe31 100644
--- a/docs/wagtailsearch/configuration.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -1,9 +1,10 @@
 
-.. _wagtailsearch_configuration:
+.. _wagtailsearch_for_python_developers:
 
-===========================
-Wagtailsearch configuration
-===========================
+
+====================================
+Wagtailsearch: For python developers
+====================================
 
 
 Indexing extra fields
@@ -128,64 +129,3 @@ To do this, inherit from ``indexed.Indexed`` and add some ``search_fields`` to t
     >>> roald_dahl = Author.objects.get(name="Roald Dahl")
     >>> s.search("chocolate factory", Book.objects.filter(author=roald_dahl))
     [<Book: Charlie and the chocolate factory>]
-
-
-Search Backends
-===============
-
-Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.
-
-.. _Elasticsearch: http://www.elasticsearch.org/
-
-
-Database Backend
-----------------
-
-The default DB search backend uses Django's ``__icontains`` filter.
-
-
-Elasticsearch Backend
----------------------
-
-Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package:
-
-.. code-block:: guess
-
-  pip install elasticsearch
-
-.. note::
-  If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5```
-
-The backend is configured in settings:
-
-.. code-block:: python
-
-  WAGTAILSEARCH_BACKENDS = {
-      'default': {
-          'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
-          'URLS': ['http://localhost:9200'],
-          'INDEX': 'wagtail',
-          'TIMEOUT': 5,
-          'FORCE_NEW': False,
-      }
-  }
-
-Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``).
-
-If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly:
-
--  Sign up for an account at `dashboard.searchly.com/users/sign\_up`_
--  Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo'
--  Note the connection URL from your Searchly dashboard
--  Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS``
--  Run ``./manage.py update_index``
-
-.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org
-.. _Searchly: http://www.searchly.com/
-.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up
-
-
-Rolling Your Own
-----------------
-
-Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``.
\ No newline at end of file
diff --git a/docs/wagtailsearch/index.rst b/docs/wagtailsearch/index.rst
index f73ea787b..c11d7f675 100644
--- a/docs/wagtailsearch/index.rst
+++ b/docs/wagtailsearch/index.rst
@@ -7,9 +7,11 @@ Wagtailsearch
 
 Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface.
 
+
 .. toctree::
     :maxdepth: 2
 
-    configuration
+    for_python_developers
     frontend_views
     editors_picks
+    backends

From 474868439090f647aa869ecf0b4753cfa8e60d5b Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:43:07 +0100
Subject: [PATCH 28/49] Added basic useage docs for python developers

---
 docs/wagtailsearch/for_python_developers.rst | 21 ++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index d183bfe31..548510f6f 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -7,6 +7,27 @@ Wagtailsearch: For python developers
 ====================================
 
 
+Basic useage
+============
+
+All searches are performed on Django QuerySets:
+
+.. code-block:: python
+
+    # Search future EventPages
+    >>> from wagtail.wagtailcore.models import EventPage
+    EventPage.objects.filter(date__gt=timezone.now()).search("Hello world!")
+
+
+All methods of ``PageQuerySet`` are supported by wagtailsearch:
+
+.. code-block:: python
+
+    # Search all live EventPages that are under the events index
+    >>> EventPage.objects.live().descendant_of(events_index).search("Hello")
+    [<EventPage: Event 1>, <EventPage: Event 2>]
+
+
 Indexing extra fields
 =====================
 

From fa97565b2a9d424c894dddf01e4624adf1092951 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:45:41 +0100
Subject: [PATCH 29/49] Fix spelling mistake

---
 docs/wagtailsearch/for_python_developers.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index 548510f6f..a49603d43 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -7,8 +7,8 @@ Wagtailsearch: For python developers
 ====================================
 
 
-Basic useage
-============
+Basic usage
+===========
 
 All searches are performed on Django QuerySets:
 

From 33e947703292792d7f6e453a3452788cd936e040 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:46:36 +0100
Subject: [PATCH 30/49] Doc tweak

---
 docs/wagtailsearch/for_python_developers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index a49603d43..c337e1275 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -10,7 +10,7 @@ Wagtailsearch: For python developers
 Basic usage
 ===========
 
-All searches are performed on Django QuerySets:
+All searches are performed on Django QuerySets. Wagtail provides a ``search`` method on the queryset for all page models:
 
 .. code-block:: python
 

From dbab06a88cc7b12ff728d9c1721d126ed0690202 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 15:47:00 +0100
Subject: [PATCH 31/49] Fixed typo

---
 docs/wagtailsearch/for_python_developers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index c337e1275..44332a819 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -16,7 +16,7 @@ All searches are performed on Django QuerySets. Wagtail provides a ``search`` me
 
     # Search future EventPages
     >>> from wagtail.wagtailcore.models import EventPage
-    EventPage.objects.filter(date__gt=timezone.now()).search("Hello world!")
+    >>> EventPage.objects.filter(date__gt=timezone.now()).search("Hello world!")
 
 
 All methods of ``PageQuerySet`` are supported by wagtailsearch:

From 072f5c04e02a5412a0238286b0d3644d3a2a9267 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 16:07:53 +0100
Subject: [PATCH 32/49] Docs fixes

---
 docs/wagtailsearch/backends.rst              |  2 ++
 docs/wagtailsearch/for_python_developers.rst | 12 ++++++------
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/docs/wagtailsearch/backends.rst b/docs/wagtailsearch/backends.rst
index f6bb2f296..bd4974b11 100644
--- a/docs/wagtailsearch/backends.rst
+++ b/docs/wagtailsearch/backends.rst
@@ -11,6 +11,8 @@ Wagtail can degrade to a database-backed text search, but we strongly recommend
 .. _Elasticsearch: http://www.elasticsearch.org/
 
 
+.. _wagtailsearch_backends_database:
+
 Database Backend
 ===============
 
diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index 44332a819..01098fdde 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -87,7 +87,7 @@ Indexing callables and other attributes
 
  .. note::
 
-     This is not supported in the `Database Backend`_
+     This is not supported in the :ref:`wagtailsearch_backends_database`
 
 
 Search/filter fields do not need to be Django fields, they could be any method or attribute on your class.
@@ -134,12 +134,12 @@ To do this, inherit from ``indexed.Indexed`` and add some ``search_fields`` to t
         published_date = models.DateTimeField()
 
         search_fields = [
-            SearchField('title', partial_match=True, boost=10),
-            SearchField('get_genre_display'),
+            indexed.SearchField('title', partial_match=True, boost=10),
+            indexed.SearchField('get_genre_display'),
 
-            FilterField('genre'),
-            FilterField('author'),
-            FilterField('published_date'),
+            indexed.FilterField('genre'),
+            indexed.FilterField('author'),
+            indexed.FilterField('published_date'),
         ]
 
     # As this model doesn't have a search method in its QuerySet, we have to call search directly on the backend

From 3a8bc7f3ed993422ce5cf00e673a4f699bd97616 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 16:09:07 +0100
Subject: [PATCH 33/49] Use tuples in search_fields configuration

Not a requirement, just a best practise
---
 docs/wagtailsearch/for_python_developers.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index 01098fdde..506649976 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -133,14 +133,14 @@ To do this, inherit from ``indexed.Indexed`` and add some ``search_fields`` to t
         author = models.ForeignKey(Author)
         published_date = models.DateTimeField()
 
-        search_fields = [
+        search_fields = (
             indexed.SearchField('title', partial_match=True, boost=10),
             indexed.SearchField('get_genre_display'),
 
             indexed.FilterField('genre'),
             indexed.FilterField('author'),
             indexed.FilterField('published_date'),
-        ]
+        )
 
     # As this model doesn't have a search method in its QuerySet, we have to call search directly on the backend
     >>> from wagtail.wagtailsearch.backends import get_search_backend

From 677d0333f4906fbfd590a2c32262cad14b995180 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Wed, 9 Jul 2014 16:26:54 +0100
Subject: [PATCH 34/49] suppress DeprecationWarnings thrown in tests by classes
 with old-style indexed_fields definitions

---
 wagtail/tests/utils.py                            | 14 ++++++++++++++
 wagtail/wagtailsearch/tests/test_backends.py      |  8 +++++---
 wagtail/wagtailsearch/tests/test_indexed_class.py | 14 +++++++++-----
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/wagtail/tests/utils.py b/wagtail/tests/utils.py
index 9360f2206..2e95d0210 100644
--- a/wagtail/tests/utils.py
+++ b/wagtail/tests/utils.py
@@ -1,3 +1,6 @@
+from contextlib import contextmanager
+import warnings
+
 from django.contrib.auth.models import User
 from django.utils import six
 
@@ -25,3 +28,14 @@ class WagtailTestUtils(object):
 
     def assertRegex(self, *args, **kwargs):
         six.assertRegex(self, *args, **kwargs)
+
+    @staticmethod
+    @contextmanager
+    def ignore_deprecation_warnings():
+        with warnings.catch_warnings(record=True) as warning_list:  # catch all warnings
+            yield
+
+        # rethrow all warnings that were not DeprecationWarnings
+        for w in warning_list:
+            if not issubclass(w.category, DeprecationWarning):
+                warnings.showwarning(message=w.message, category=w.category, filename=w.filename, lineno=w.lineno, file=w.file, line=w.line)
diff --git a/wagtail/wagtailsearch/tests/test_backends.py b/wagtail/wagtailsearch/tests/test_backends.py
index e9b656298..9baf64957 100644
--- a/wagtail/wagtailsearch/tests/test_backends.py
+++ b/wagtail/wagtailsearch/tests/test_backends.py
@@ -1,17 +1,18 @@
 from six import StringIO
+import warnings
 
 from django.test import TestCase
 from django.test.utils import override_settings
 from django.conf import settings
 from django.core import management
 
-from wagtail.tests.utils import unittest
+from wagtail.tests.utils import unittest, WagtailTestUtils
 from wagtail.tests import models
 from wagtail.wagtailsearch.backends import get_search_backend, InvalidSearchBackendError
 from wagtail.wagtailsearch.backends.db import DBSearch
 
 
-class BackendTests(object):
+class BackendTests(WagtailTestUtils):
     # To test a specific backend, subclass BackendTests and define self.backend_path.
 
     def setUp(self):
@@ -144,7 +145,8 @@ class BackendTests(object):
         self.backend.reset_index()
 
         # Run update_index command
-        management.call_command('update_index', backend=self.backend, interactive=False, stdout=StringIO())
+        with self.ignore_deprecation_warnings():  # ignore any DeprecationWarnings thrown by models with old-style indexed_fields definitions
+            management.call_command('update_index', backend=self.backend, interactive=False, stdout=StringIO())
 
         # Check that there are still 3 results
         results = self.backend.search("Hello", models.SearchTest)
diff --git a/wagtail/wagtailsearch/tests/test_indexed_class.py b/wagtail/wagtailsearch/tests/test_indexed_class.py
index a54f135c3..983d8e0ba 100644
--- a/wagtail/wagtailsearch/tests/test_indexed_class.py
+++ b/wagtail/wagtailsearch/tests/test_indexed_class.py
@@ -1,8 +1,10 @@
+import warnings
+
 from django.test import TestCase
-from wagtail.tests import models
-import json
 
 from wagtail.wagtailsearch import indexed
+from wagtail.tests import models
+from wagtail.tests.utils import WagtailTestUtils
 
 
 class TestContentTypeNames(TestCase):
@@ -15,10 +17,11 @@ class TestContentTypeNames(TestCase):
         self.assertEqual(name, 'tests_searchtest_tests_searchtestchild')
 
 
-class TestIndexedFieldsBackwardsCompatibility(TestCase):
+class TestIndexedFieldsBackwardsCompatibility(TestCase, WagtailTestUtils):
     def test_indexed_fields_backwards_compatibility(self):
         # Get search fields
-        search_fields = models.SearchTestOldConfig.get_search_fields()
+        with self.ignore_deprecation_warnings():
+            search_fields = models.SearchTestOldConfig.get_search_fields()
 
         search_fields_dict = dict(
             ((field.field_name, type(field)), field)
@@ -36,7 +39,8 @@ class TestIndexedFieldsBackwardsCompatibility(TestCase):
 
     def test_indexed_fields_backwards_compatibility_list(self):
         # Get search fields
-        search_fields = models.SearchTestOldConfigList.get_search_fields()
+        with self.ignore_deprecation_warnings():
+            search_fields = models.SearchTestOldConfigList.get_search_fields()
 
         search_fields_dict = dict(
             ((field.field_name, type(field)), field)

From 5766224808e6972f1fe85148ce2fe64608367690 Mon Sep 17 00:00:00 2001
From: Robert Clark <robert@bablmedia.com>
Date: Wed, 9 Jul 2014 12:09:19 -0400
Subject: [PATCH 35/49] sitemap should return the full url

---
 wagtail/wagtailcore/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py
index c6a4f6854..18dc018ad 100644
--- a/wagtail/wagtailcore/models.py
+++ b/wagtail/wagtailcore/models.py
@@ -774,7 +774,7 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index
 
         return [
             {
-                'location': self.url,
+                'location': self.full_url,
                 'lastmod': latest_revision.created_at if latest_revision else None
             }
         ]

From bc2556db280e09f98b9088a92bec7e0138f56571 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 17:16:20 +0100
Subject: [PATCH 36/49] Updated tests for 5766224

---
 wagtail/contrib/wagtailsitemaps/tests.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/wagtail/contrib/wagtailsitemaps/tests.py b/wagtail/contrib/wagtailsitemaps/tests.py
index 469e210ad..a556ce156 100644
--- a/wagtail/contrib/wagtailsitemaps/tests.py
+++ b/wagtail/contrib/wagtailsitemaps/tests.py
@@ -44,21 +44,21 @@ class TestSitemapGenerator(TestCase):
         sitemap = Sitemap(self.site)
         urls = [url['location'] for url in sitemap.get_urls()]
 
-        self.assertIn('/', urls) # Homepage
-        self.assertIn('/hello-world/', urls) # Child page
+        self.assertIn('http://localhost/', urls) # Homepage
+        self.assertIn('http://localhost/hello-world/', urls) # Child page
 
     def test_render(self):
         sitemap = Sitemap(self.site)
         xml = sitemap.render()
 
         # Check that a URL has made it into the xml
-        self.assertIn('/hello-world/', xml)
+        self.assertIn('http://localhost/hello-world/', xml)
 
         # Make sure the unpublished page didn't make it into the xml
-        self.assertNotIn('/unpublished/', xml)
+        self.assertNotIn('http://localhost/unpublished/', xml)
 
         # Make sure the protected page didn't make it into the xml
-        self.assertNotIn('/protected/', xml)
+        self.assertNotIn('http://localhost/protected/', xml)
 
 
 class TestSitemapView(TestCase):

From 5eaa60531460249799875ec913b6eb978212e11d Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 17:33:58 +0100
Subject: [PATCH 37/49] Fixed search example in docs

---
 docs/wagtailsearch/for_python_developers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/wagtailsearch/for_python_developers.rst
index 506649976..be389e500 100644
--- a/docs/wagtailsearch/for_python_developers.rst
+++ b/docs/wagtailsearch/for_python_developers.rst
@@ -24,7 +24,7 @@ All methods of ``PageQuerySet`` are supported by wagtailsearch:
 .. code-block:: python
 
     # Search all live EventPages that are under the events index
-    >>> EventPage.objects.live().descendant_of(events_index).search("Hello")
+    >>> EventPage.objects.live().descendant_of(events_index).search("Event")
     [<EventPage: Event 1>, <EventPage: Event 2>]
 
 

From 8e0752827450d31e2bc9cecff3b343d6083451e3 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 17:44:26 +0100
Subject: [PATCH 38/49] Renamed 'wagtailsearch' section of the docs to 'search'

---
 docs/index.rst                                           | 2 +-
 docs/{wagtailsearch => search}/backends.rst              | 0
 docs/{wagtailsearch => search}/editors_picks.rst         | 0
 docs/{wagtailsearch => search}/for_python_developers.rst | 0
 docs/{wagtailsearch => search}/frontend_views.rst        | 0
 docs/{wagtailsearch => search}/index.rst                 | 4 ++--
 6 files changed, 3 insertions(+), 3 deletions(-)
 rename docs/{wagtailsearch => search}/backends.rst (100%)
 rename docs/{wagtailsearch => search}/editors_picks.rst (100%)
 rename docs/{wagtailsearch => search}/for_python_developers.rst (100%)
 rename docs/{wagtailsearch => search}/frontend_views.rst (100%)
 rename docs/{wagtailsearch => search}/index.rst (92%)

diff --git a/docs/index.rst b/docs/index.rst
index b2d9a534d..a040fe767 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -13,7 +13,7 @@ It supports Django 1.6.2+ on Python 2.6, 2.7, 3.2, 3.3 and 3.4. Django 1.7 suppo
    building_your_site/index
    editing_api
    snippets
-   wagtailsearch/index
+   search/index
    form_builder
    model_recipes
    advanced_topics
diff --git a/docs/wagtailsearch/backends.rst b/docs/search/backends.rst
similarity index 100%
rename from docs/wagtailsearch/backends.rst
rename to docs/search/backends.rst
diff --git a/docs/wagtailsearch/editors_picks.rst b/docs/search/editors_picks.rst
similarity index 100%
rename from docs/wagtailsearch/editors_picks.rst
rename to docs/search/editors_picks.rst
diff --git a/docs/wagtailsearch/for_python_developers.rst b/docs/search/for_python_developers.rst
similarity index 100%
rename from docs/wagtailsearch/for_python_developers.rst
rename to docs/search/for_python_developers.rst
diff --git a/docs/wagtailsearch/frontend_views.rst b/docs/search/frontend_views.rst
similarity index 100%
rename from docs/wagtailsearch/frontend_views.rst
rename to docs/search/frontend_views.rst
diff --git a/docs/wagtailsearch/index.rst b/docs/search/index.rst
similarity index 92%
rename from docs/wagtailsearch/index.rst
rename to docs/search/index.rst
index c11d7f675..e343f76c8 100644
--- a/docs/wagtailsearch/index.rst
+++ b/docs/search/index.rst
@@ -2,8 +2,8 @@
 .. _wagtailsearch:
 
 
-Wagtailsearch
-=============
+Search
+======
 
 Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface.
 

From 1cc3456f580b397d0b2353641064e53d713ceb55 Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 17:46:34 +0100
Subject: [PATCH 39/49] Removed 'wagtailsearch' from titles of a couple of
 documents

---
 docs/search/for_python_developers.rst | 6 +++---
 docs/search/frontend_views.rst        | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/search/for_python_developers.rst b/docs/search/for_python_developers.rst
index be389e500..c73799a7a 100644
--- a/docs/search/for_python_developers.rst
+++ b/docs/search/for_python_developers.rst
@@ -2,9 +2,9 @@
 .. _wagtailsearch_for_python_developers:
 
 
-====================================
-Wagtailsearch: For python developers
-====================================
+=====================
+For python developers
+=====================
 
 
 Basic usage
diff --git a/docs/search/frontend_views.rst b/docs/search/frontend_views.rst
index e83bf52a5..1e1ad3fd3 100644
--- a/docs/search/frontend_views.rst
+++ b/docs/search/frontend_views.rst
@@ -2,8 +2,8 @@
 .. _wagtailsearch_frontend_views:
 
 
-Wagtailsearch frontend views
-============================
+Frontend views
+==============
 
 
 Default Page Search

From b067fad1f9b8f60d994327a48cf4cedac1575e1a Mon Sep 17 00:00:00 2001
From: Karl Hobley <karl@torchbox.com>
Date: Wed, 9 Jul 2014 17:47:33 +0100
Subject: [PATCH 40/49] Renamed 'Wagtailsearch backends' to 'Backends'

---
 docs/search/backends.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/search/backends.rst b/docs/search/backends.rst
index bd4974b11..eba6d8a1a 100644
--- a/docs/search/backends.rst
+++ b/docs/search/backends.rst
@@ -1,9 +1,9 @@
 
 .. _wagtailsearch_backends:
 
-======================
-Wagtailsearch backends
-======================
+========
+Backends
+========
 
 
 Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_.

From 9b8ee979d211be83bd73691df893e9c915b65df0 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Wed, 9 Jul 2014 23:29:37 +0100
Subject: [PATCH 41/49] fix tests for lt, gte, lte filters to expect a
 multi_match response

---
 wagtail/wagtailsearch/tests/test_elasticsearch_backend.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
index 18b3f7ce0..a95d442ff 100644
--- a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
+++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
@@ -205,7 +205,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lt=datetime.datetime(2014, 4, 29)), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_gte_lookup(self):
@@ -213,7 +213,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gte=datetime.datetime(2014, 4, 29)), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_lte_lookup(self):
@@ -221,7 +221,7 @@ class TestElasticSearchQuery(TestCase):
         query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lte=datetime.datetime(2014, 4, 29)), "Hello")
 
         # Check it
-        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
+        expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
         self.assertDictEqual(query.to_es(), expected_result)
 
     def test_range_lookup(self):

From bd1c72c0aab71743633c31b444a50d997b9fe0d7 Mon Sep 17 00:00:00 2001
From: Dave Cranwell <davecranwell@users.noreply.github.com>
Date: Thu, 10 Jul 2014 11:47:40 +0100
Subject: [PATCH 42/49] References to image.src corrected - it's image.url

---
 docs/building_your_site/frontenddevelopers.rst | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index a3ba8eaa2..8c3ffa479 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -202,26 +202,31 @@ Extra attributes can be specified with the syntax ``attribute="value"``:
 
     {% image self.photo width-400 class="foo" id="bar" %}
 
-No validation is performed on attributes add in this way by the developer. It's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself.
+No validation is performed on attributes added in this way so it's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself.
 
 
-**Generating the image "as"**
+**Generating the image "as foo" to access individual properties **
 
-Wagtail can assign the image data to another object using Django's ``as`` syntax:
+Wagtail can assign the image data to another variable using Django's ``as`` syntax:
 
 .. code-block:: django
 
     {% image self.photo width-400 as tmp_photo %}
 
-    <img src="{{ tmp_photo.src }}" width="{{ tmp_photo.width }}" 
+    <img src="{{ tmp_photo.url }}" width="{{ tmp_photo.width }}" 
         height="{{ tmp_photo.height }}" alt="{{ tmp_photo.alt }}" class="my-custom-class" />
+        
+
+.. Note::      
+    The image property used for the `src` attribute is actually `image.url`, not `image.src`.
+
 
 The ``attrs`` shortcut
 -----------------------
 
 .. versionadded:: 0.4
 
-You can also use the ``attrs`` property as a shorthand to output the ``src``, ``width``, ``height`` and ``alt`` attributes in one go:
+You can also use the ``attrs`` property as a shorthand to output the attributes ``src``, ``width``, ``height`` and ``alt`` in one go:
 
 .. code-block:: django
 

From 0987946786a41665cd7e664d8644591c27b4f2de Mon Sep 17 00:00:00 2001
From: Dave Cranwell <davecranwell@users.noreply.github.com>
Date: Thu, 10 Jul 2014 11:48:57 +0100
Subject: [PATCH 43/49] Update frontenddevelopers.rst

---
 docs/building_your_site/frontenddevelopers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index 8c3ffa479..e9c775d16 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -205,7 +205,7 @@ Extra attributes can be specified with the syntax ``attribute="value"``:
 No validation is performed on attributes added in this way so it's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself.
 
 
-**Generating the image "as foo" to access individual properties **
+**Generating the image "as foo" to access individual properties**
 
 Wagtail can assign the image data to another variable using Django's ``as`` syntax:
 

From f73c5416ba00c6a78050e7657f1dd93b6270805b Mon Sep 17 00:00:00 2001
From: Dave Cranwell <davecranwell@users.noreply.github.com>
Date: Thu, 10 Jul 2014 11:49:41 +0100
Subject: [PATCH 44/49] Update frontenddevelopers.rst

---
 docs/building_your_site/frontenddevelopers.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index e9c775d16..1061856bb 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -218,7 +218,7 @@ Wagtail can assign the image data to another variable using Django's ``as`` synt
         
 
 .. Note::      
-    The image property used for the `src` attribute is actually `image.url`, not `image.src`.
+    The image property used for the ``src`` attribute is actually ``image.url``, not ``image.src``.
 
 
 The ``attrs`` shortcut

From 5faa0652a24cbb6ee0d66c4d965d4ace7ad34d2a Mon Sep 17 00:00:00 2001
From: Dave Cranwell <davecranwell@users.noreply.github.com>
Date: Thu, 10 Jul 2014 11:51:06 +0100
Subject: [PATCH 45/49] Update frontenddevelopers.rst

---
 docs/building_your_site/frontenddevelopers.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst
index 1061856bb..a5120af19 100644
--- a/docs/building_your_site/frontenddevelopers.rst
+++ b/docs/building_your_site/frontenddevelopers.rst
@@ -192,7 +192,7 @@ More control over the ``img`` tag
 
 Wagtail provides two shorcuts to give greater control over the ``img`` element:
 
-**Adding attributes to the  {% image %} tag**
+**1. Adding attributes to the  {% image %} tag**
 
 .. versionadded:: 0.4
 
@@ -205,7 +205,7 @@ Extra attributes can be specified with the syntax ``attribute="value"``:
 No validation is performed on attributes added in this way so it's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself.
 
 
-**Generating the image "as foo" to access individual properties**
+**2. Generating the image "as foo" to access individual properties**
 
 Wagtail can assign the image data to another variable using Django's ``as`` syntax:
 

From 4f8cfa6346fd645af7a37cbaca3be9183d68b8b3 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Thu, 10 Jul 2014 12:57:56 +0100
Subject: [PATCH 46/49] version / date bump to 0.4 in changelog, setup.py and
 docs

---
 CHANGELOG.txt | 2 +-
 docs/conf.py  | 4 ++--
 setup.py      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index e915ffb22..6dc5a8744 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,7 +1,7 @@
 Changelog
 =========
 
-0.4 (xx.xx.20xx)
+0.4 (10.07.2014)
 ~~~~~~~~~~~~~~~~
  * ElasticUtils/pyelasticsearch swapped for elasticsearch-py
  * Python 3.2, 3.3 and 3.4 support
diff --git a/docs/conf.py b/docs/conf.py
index ba6525d1b..d54c30df7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -59,9 +59,9 @@ copyright = u'2014, Torchbox'
 # built documents.
 #
 # The short X.Y version.
-version = '0.3.1'
+version = '0.4'
 # The full version, including alpha/beta/rc tags.
-release = '0.3.1'
+release = '0.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/setup.py b/setup.py
index 11c32d522..129a48468 100644
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@ if not PY3:
 
 setup(
     name='wagtail',
-    version='0.3.1',
+    version='0.4',
     description='A Django content management system focused on flexibility and user experience',
     author='Matthew Westcott',
     author_email='matthew.westcott@torchbox.com',

From f939af74dc5cf49056cce1e1fbf829ed0d2d9fe0 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Thu, 10 Jul 2014 13:33:20 +0100
Subject: [PATCH 47/49] Fix more doc syntax errors

---
 docs/index.rst                        | 1 +
 docs/releases/0.4.rst                 | 4 ++--
 docs/releases/index.rst               | 7 +++++++
 docs/search/backends.rst              | 2 +-
 docs/search/editors_picks.rst         | 2 +-
 docs/search/for_python_developers.rst | 2 +-
 6 files changed, 13 insertions(+), 5 deletions(-)
 create mode 100644 docs/releases/index.rst

diff --git a/docs/index.rst b/docs/index.rst
index a040fe767..2ab429eb6 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -28,3 +28,4 @@ It supports Django 1.6.2+ on Python 2.6, 2.7, 3.2, 3.3 and 3.4. Django 1.7 suppo
    support
    roadmap
    editor_manual/index
+   releases/index
diff --git a/docs/releases/0.4.rst b/docs/releases/0.4.rst
index a53ca5463..3aa630603 100644
--- a/docs/releases/0.4.rst
+++ b/docs/releases/0.4.rst
@@ -32,7 +32,7 @@ A new management command has been added (:ref:`publish_scheduled_pages`) to publ
 Search on QuerySet with Elasticsearch
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Its now possible to perform searches with Elasticsearch on ``PageQuerySet``s:
+Its now possible to perform searches with Elasticsearch on ``PageQuerySet`` objects:
 
     >>> from wagtail.wagtailcore.models import Page
     >>> Page.objects.live().descendant_of(events_index).search("Hello")
@@ -174,7 +174,7 @@ Previously, the ``route`` method called ``serve`` and returned a ``HttpResponse`
 
 
 Wagtailadmins ``hooks`` module has moved to wagtailcore
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you use any ``wagtail_hooks.py`` files in your project, you may have an import like: ``from wagtail.wagtailadmin import hooks``
 
diff --git a/docs/releases/index.rst b/docs/releases/index.rst
new file mode 100644
index 000000000..07bc8e888
--- /dev/null
+++ b/docs/releases/index.rst
@@ -0,0 +1,7 @@
+Release notes
+=============
+
+.. toctree::
+   :maxdepth: 1
+
+   0.4
\ No newline at end of file
diff --git a/docs/search/backends.rst b/docs/search/backends.rst
index eba6d8a1a..f03f06358 100644
--- a/docs/search/backends.rst
+++ b/docs/search/backends.rst
@@ -14,7 +14,7 @@ Wagtail can degrade to a database-backed text search, but we strongly recommend
 .. _wagtailsearch_backends_database:
 
 Database Backend
-===============
+================
 
 The default DB search backend uses Django's ``__icontains`` filter.
 
diff --git a/docs/search/editors_picks.rst b/docs/search/editors_picks.rst
index f899302b4..9ed41f3c5 100644
--- a/docs/search/editors_picks.rst
+++ b/docs/search/editors_picks.rst
@@ -1,5 +1,5 @@
 
-.. _wagtailsearch_editors_picks:
+.. _editors-picks:
 
 
 Editors picks
diff --git a/docs/search/for_python_developers.rst b/docs/search/for_python_developers.rst
index c73799a7a..7ff9a18dc 100644
--- a/docs/search/for_python_developers.rst
+++ b/docs/search/for_python_developers.rst
@@ -35,7 +35,7 @@ Fields need to be explicitly added to the search configuration in order for you
 
 You can add new fields to the search index by overriding the ``search_fields`` property and appending a list of extra ``SearchField``/``FilterField`` objects to it.
 
-``Page`` sets a default value to ``search_fields`` indexing the ``title`` field as a ``SearchField`` and some other generally useful fields as ``FilterField``s.
+The default value of ``search_fields`` (as set in ``Page``) indexes the ``title`` field as a ``SearchField`` and some other generally useful fields as ``FilterField`` rules.
 
 
 Quick example

From 91d449de1ba39c990be25325976ec47972c70fdb Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Thu, 10 Jul 2014 13:39:33 +0100
Subject: [PATCH 48/49] add changelog entry for private pages

---
 CHANGELOG.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 6dc5a8744..a0abb58db 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -6,6 +6,7 @@ Changelog
  * ElasticUtils/pyelasticsearch swapped for elasticsearch-py
  * Python 3.2, 3.3 and 3.4 support
  * Added scheduled publishing
+ * Added support for private (password-protected) pages
  * Added frontend cache invalidator
  * Added sitemap generator
  * Added notification preferences

From a9b247b7810fad4479f18f7b31d20cc16360d2e0 Mon Sep 17 00:00:00 2001
From: Matt Westcott <matt@west.co.tt>
Date: Thu, 10 Jul 2014 15:47:14 +0100
Subject: [PATCH 49/49] more explicit instructions for migrating away from
 deprecated things

---
 docs/model_recipes.rst | 2 ++
 docs/releases/0.4.rst  | 8 +++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/model_recipes.rst b/docs/model_recipes.rst
index 7f9b5f7b6..b8f006f99 100644
--- a/docs/model_recipes.rst
+++ b/docs/model_recipes.rst
@@ -38,6 +38,8 @@ Consider this example from the Wagtail demo site's ``models.py``, which serves a
 With this strategy, you could use Django or Python utilities to render your model in JSON or XML or any other format you'd like.
 
 
+.. _overriding_route_method:
+
 Adding Endpoints with Custom route() Methods
 --------------------------------------------
 
diff --git a/docs/releases/0.4.rst b/docs/releases/0.4.rst
index 3aa630603..1e6dbd15d 100644
--- a/docs/releases/0.4.rst
+++ b/docs/releases/0.4.rst
@@ -158,19 +158,21 @@ The following template tag libraries have been renamed:
  * ``embed_filters`` => ``wagtailembeds_tags``
  * ``image_tags`` => ``wagtailimages_tags``
 
+The old names will continue to work, but output a ``DeprecationWarning`` - you are advised to update any ``{% load %}`` tags in your templates to refer to the new names.
+
 
 New search field configuration format
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``indexed_fields`` is now deprecated and has been replaced by a new search field configuration format called ``search_fields``.
+``indexed_fields`` is now deprecated and has been replaced by a new search field configuration format called ``search_fields``. See :ref:`wagtailsearch_for_python_developers` for how to define a ``search_fields`` property on your models.
 
 
 ``Page.route`` method should now return a ``RouteResult``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Previously, the ``route`` method called ``serve`` and returned a ``HttpResponse`` object. This has now been split up so ``serve`` is called separately and ``route`` must now return a RouteResult object.
+Previously, the ``route`` method called ``serve`` and returned an ``HttpResponse`` object. This has now been split up so ``serve`` is called separately and ``route`` must now return a RouteResult object.
 
-:ref:`anatomy_of_a_wagtail_request`
+If you are overriding ``Page.route`` on any of your page models, you will need to update the method to return a ``RouteResult`` object. The old method of returning an ``HttpResponse`` will continue to work, but this will throw a ``DeprecationWarning`` and bypass the ``before_serve_page`` hook, which means in particular that :ref:`private_pages` will not work on those page types. See :ref:`overriding_route_method`.
 
 
 Wagtailadmins ``hooks`` module has moved to wagtailcore