Merge branch 'master' into sass

CONTRIBUTORS.rst

@@ -1,4 +1,3 @@
-
 Original Authors
 ================
 
@@ -13,3 +12,5 @@ Contributors
 * Balazs Endresz balazs.endresz@torchbox.com
 * Neal Todd neal.todd@torchbox.com
 * Paul Hallett (twilio) hello@phalt.co
+* Tom Dyson
+* Serafeim Papastefanos

setup.py

@@ -42,7 +42,6 @@ setup(
         "Django>=1.6.1",
         "South>=0.8.4",
         "django-compressor>=1.3",
-        "django-celery>=3.1.1",
         "django-modelcluster>=0.1",
         "elasticutils>=0.8.2",
         "pyelasticsearch>=0.6.1",

wagtail/wagtailadmin/static/wagtailadmin/js/hallo-plugins/hallo-wagtaillink.coffee

@@ -41,9 +41,9 @@
 
                     if lastSelection.collapsed
                         # TODO: don't hard-code this, as it may be changed in urls.py
-                        url = '/admin/choose-page/?allow_external_link=true&allow_email_link=true&prompt_for_link_text=true'
+                        url = window.chooserUrls.pageChooser + '?allow_external_link=true&allow_email_link=true&prompt_for_link_text=true'
                     else
-                        url = '/admin/choose-page/?allow_external_link=true&allow_email_link=true'
+                        url = window.chooserUrls.pageChooser + '?allow_external_link=true&allow_email_link=true'
 
                     ModalWorkflow
                         url: url

wagtail/wagtailadmin/static/wagtailadmin/js/page-chooser.js

@@ -4,8 +4,7 @@ function createPageChooser(id, pageType, openAtParentId) {
     var input = $('#' + id);
 
     $('.action-choose', chooserElement).click(function() {
-        var initialUrl = '/admin/choose-page/';
-        /* TODO: don't hard-code this URL, as it may be changed in urls.py */
+        var initialUrl = window.chooserUrls.pageChooser;
         if (openAtParentId) {
             initialUrl += openAtParentId + '/';
         }

wagtail/wagtailadmin/static/wagtailadmin/js/page-editor.js

@@ -257,10 +257,12 @@ $(function() {
     /* Set up behaviour of preview button */
     $('#action-preview').click(function() {
         var previewWindow = window.open($(this).data('placeholder'), $(this).data('windowname'));
-        $.post(
-            $(this).data('action'),
-            $('#page-edit-form').serialize(),
-            function(data, textStatus, request) {
+
+        $.ajax({
+            type: "POST",
+            url: $(this).data('action'),
+            data: $('#page-edit-form').serialize(),
+            success: function(data, textStatus, request) {
                 if (request.getResponseHeader('X-Wagtail-Preview') == 'ok') {
                     previewWindow.document.open();
                     previewWindow.document.write(data);
@@ -271,7 +273,17 @@ $(function() {
                     document.write(data);
                     document.close();
                 }
+            },
+            error: function(xhr, textStatus, errorThrown) {
+                /* If an error occurs, display it in the preview window so that
+                we aren't just showing the spinner forever. We preserve the original
+                error output rather than giving a 'friendly' error message so that
+                developers can debug template errors. (On a production site, we'd
+                typically be serving a friendly custom 500 page anyhow.) */
+                previewWindow.document.open();
+                previewWindow.document.write(xhr.responseText);
+                previewWindow.document.close();
             }
-        );
+        });
     });
 });

wagtail/wagtailadmin/taggable.py

@@ -4,7 +4,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.db.models import Count
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 
-from wagtail.wagtailsearch import Indexed, Search
+from wagtail.wagtailsearch import Indexed, get_search_backend
 
 
 class TagSearchable(Indexed):
@@ -33,10 +33,11 @@ class TagSearchable(Indexed):
     @classmethod
     def search(cls, q, results_per_page=None, page=1, prefetch_tags=False, filters={}):
         # Run search query
+        search_backend = get_search_backend()
         if prefetch_tags:
-            results = Search().search(q, cls, prefetch_related=['tagged_items__tag'], filters=filters)
+            results = search_backend.search(q, cls, prefetch_related=['tagged_items__tag'], filters=filters)
         else:
-            results = Search().search(q, cls, filters=filters)
+            results = search_backend.search(q, cls, filters=filters)
 
         # If results_per_page is set, return a paginator
         if results_per_page is not None:

wagtail/wagtailadmin/tasks.py

@@ -1,5 +1,3 @@
-from celery.decorators import task
-
 from django.template.loader import render_to_string
 from django.core.mail import send_mail
 from django.conf import settings
@@ -8,6 +6,27 @@ from django.db.models import Q
 
 from wagtail.wagtailcore.models import PageRevision, GroupPagePermission
 
+# The following will check to see if we can import task from celery - 
+# if not then we definitely haven't installed it
+try:
+    from celery.decorators import task
+    NO_CELERY = False
+except:
+    NO_CELERY = True
+
+   
+# However, we could have installed celery for other projects. So we will also
+# check if we have defined the BROKER_URL setting. If not then definitely we
+# haven't configured it. 
+if NO_CELERY or not hasattr(settings, 'BROKER_URL'):
+    # So if we enter here we will define a different "task" decorator that
+    # just returns the original function and sets its delay attribute to
+    # point to the original function: This way, the send_notification
+    # function will be actually called instead of the the 
+    # send_notification.delay() 
+    def task(f):
+        f.delay=f
+        return f
 
 def users_with_page_permission(page, permission_type, include_superusers=True):
     # Get user model
@@ -25,11 +44,11 @@ def users_with_page_permission(page, permission_type, include_superusers=True):
     return User.objects.filter(q).distinct()
 
 
-@task()
+@task
 def send_notification(page_revision_id, notification, excluded_user_id):
     # Get revision
     revision = PageRevision.objects.get(id=page_revision_id)
-
+    
     # Get list of recipients
     if notification == 'submitted':
         # Get list of publishers

wagtail/wagtailadmin/templates/wagtailadmin/pages/_editor_js.html

@@ -31,6 +31,16 @@
     <script src="{{ STATIC_URL }}admin/js/urlify.js"></script>
 {% endcompress %}
 
+<script type='text/javascript'>
+    window.chooserUrls = {
+        'documentChooser': '{% url "wagtaildocs_chooser" %}',
+        'imageChooser': '{% url "wagtailimages_chooser" %}',
+        'embedsChooser': '{% url "wagtailembeds_chooser" %}',
+        'pageChooser': '{% url "wagtailadmin_choose_page" %}',
+        'snippetChooser': '{% url "wagtailsnippets_choose_generic" %}'
+    };
+</script>
+
 <script>
     (function() {
         function fixPrefix(str) {return str;}

wagtail/wagtailcore/models.py

@@ -248,11 +248,20 @@ class Page(MP_Node, ClusterableModel, Indexed):
 
     def _update_descendant_url_paths(self, old_url_path, new_url_path):
         cursor = connection.cursor()
-        cursor.execute("""
-            UPDATE wagtailcore_page
-            SET url_path = %s || substring(url_path from %s)
-            WHERE path LIKE %s AND id <> %s
-        """, [new_url_path, len(old_url_path) + 1, self.path + '%', self.id])
+        if connection.vendor == 'sqlite':
+            update_statement = """
+                UPDATE wagtailcore_page
+                SET url_path = %s || substr(url_path, %s)
+                WHERE path LIKE %s AND id <> %s
+            """
+        else:
+            update_statement = """
+                UPDATE wagtailcore_page
+                SET url_path = %s || substring(url_path from %s)
+                WHERE path LIKE %s AND id <> %s
+            """
+        cursor.execute(update_statement, 
+            [new_url_path, len(old_url_path) + 1, self.path + '%', self.id])
 
     def object_indexed(self):
         # Exclude root node from index

wagtail/wagtaildocs/static/wagtaildocs/js/document-chooser.js

@@ -5,7 +5,7 @@ function createDocumentChooser(id) {
 
     $('.action-choose', chooserElement).click(function() {
         ModalWorkflow({
-            'url': '/admin/documents/chooser/', /* TODO: don't hard-code this, as it may be changed in urls.py */
+            'url': window.chooserUrls.documentChooser,
             'responses': {
                 'documentChosen': function(docData) {
                     input.val(docData.id);

wagtail/wagtaildocs/static/wagtaildocs/js/hallo-plugins/hallo-wagtaildoclink.coffee

@@ -24,7 +24,7 @@
             button.on "click", (event) ->
                 lastSelection = widget.options.editable.getSelection()
                 ModalWorkflow
-                    url: '/admin/documents/chooser/' # TODO: don't hard-code this, as it may be changed in urls.py
+                    url: window.chooserUrls.documentChooser
                     responses:
                         documentChosen: (docData) ->
                             a = document.createElement('a')

wagtail/wagtailembeds/embeds.py

@@ -13,15 +13,18 @@ def get_embed(url, max_width=None):
     except Embed.DoesNotExist:
         pass
 
-    # Call embedly API
-    client = Embedly(key=settings.EMBEDLY_KEY)
+    try:
+        # Call embedly API
+        client = Embedly(key=settings.EMBEDLY_KEY)
+    except AttributeError:
+        return None
     if max_width is not None:
         oembed = client.oembed(url, maxwidth=max_width, better=False)
     else:
         oembed = client.oembed(url, better=False)
 
     # Check for error
-    if oembed.error:
+    if oembed.get('error'):
         return None
 
     # Save result to database
@@ -29,18 +32,18 @@ def get_embed(url, max_width=None):
         url=url,
         max_width=max_width,
         defaults={
-            'type': oembed.type,
-            'title': oembed.title,
-            'thumbnail_url': oembed.thumbnail_url,
-            'width': oembed.width,
-            'height': oembed.height
+            'type': oembed['type'],
+            'title': oembed['title'],
+            'thumbnail_url': oembed.get('thumbnail_url'),
+            'width': oembed.get('width'),
+            'height': oembed.get('height')
         }
     )
 
-    if oembed.type == 'photo':
-        html = '<img src="%s" />' % (oembed.url, )
+    if oembed['type'] == 'photo':
+        html = '<img src="%s" />' % (oembed['url'], )
     else:
-        html = oembed.html
+        html = oembed.get('html')
 
     if html:
         row.html = html

wagtail/wagtailembeds/static/wagtailembeds/js/hallo-plugins/hallo-wagtailembeds.coffee

@@ -25,7 +25,7 @@
                 lastSelection = widget.options.editable.getSelection()
                 insertionPoint = $(lastSelection.endContainer).parentsUntil('.richtext').last()
                 ModalWorkflow
-                    url: '/admin/embeds/chooser/' # TODO: don't hard-code this, as it may be changed in urls.py
+                    url: window.chooserUrls.embedsChooser
                     responses:
                         embedChosen: (embedData) ->
                             elem = $(embedData).get(0)

wagtail/wagtailembeds/views/chooser.py

@@ -1,7 +1,7 @@
 from django.forms.util import ErrorList
+from django.conf import settings
 
 from wagtail.wagtailadmin.modal_workflow import render_modal_workflow
-
 from wagtail.wagtailembeds.forms import EmbedForm
 from wagtail.wagtailembeds.format import embed_to_editor_html
 
@@ -27,7 +27,10 @@ def chooser_upload(request):
                 )
             else:
                 errors = form._errors.setdefault('url', ErrorList())
-                errors.append('This URL is not recognised')
+                if not hasattr(settings, 'EMBEDLY_KEY'):
+                    errors.append('Please set EMBEDLY_KEY in your settings')
+                else:
+                    errors.append('This URL is not recognised')
                 return render_modal_workflow(request, 'wagtailembeds/chooser/chooser.html', 'wagtailembeds/chooser/chooser.js', {
                     'form': form,
                 })

wagtail/wagtailimages/static/wagtailimages/js/hallo-plugins/hallo-wagtailimage.coffee

@@ -25,7 +25,7 @@
                 lastSelection = widget.options.editable.getSelection()
                 insertionPoint = $(lastSelection.endContainer).parentsUntil('.richtext').last()
                 ModalWorkflow
-                    url: '/admin/images/chooser/?select_format=true' # TODO: don't hard-code this, as it may be changed in urls.py
+                    url: window.chooserUrls.imageChooser + '?select_format=true'
                     responses:
                         imageChosen: (imageData) ->
                             elem = $(imageData.html).get(0)

wagtail/wagtailimages/static/wagtailimages/js/image-chooser.js

@@ -5,7 +5,7 @@ function createImageChooser(id) {
 
     $('.action-choose', chooserElement).click(function() {
         ModalWorkflow({
-            'url': '/admin/images/chooser/', /* TODO: don't hard-code this, as it may be changed in urls.py */
+            'url': window.chooserUrls.imageChooser,
             'responses': {
                 'imageChosen': function(imageData) {
                     input.val(imageData.id);

wagtail/wagtailsearch/__init__.py

@@ -1,4 +1,4 @@
 from indexed import Indexed
-from search import Search
 from searcher import Searcher
 from signal_handlers import register_signal_handlers
+from backends import get_search_backend

wagtail/wagtailsearch/backends/__init__.py

@@ -0,0 +1,71 @@
+# Backend loading
+# Based on the Django cache framework
+# https://github.com/django/django/blob/5d263dee304fdaf95e18d2f0619d6925984a7f02/django/core/cache/__init__.py
+
+from importlib import import_module
+from django.utils import six
+import sys
+from django.conf import settings
+from base import InvalidSearchBackendError
+
+
+# Pinched from django 1.7 source code.
+# TODO: Replace this with "from django.utils.module_loading import import_string" when django 1.7 is released
+def import_string(dotted_path):
+    """
+    Import a dotted module path and return the attribute/class designated by the
+    last name in the path. Raise ImportError if the import failed.
+    """
+    try:
+        module_path, class_name = dotted_path.rsplit('.', 1)
+    except ValueError:
+        msg = "%s doesn't look like a module path" % dotted_path
+        six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])
+
+    module = import_module(module_path)
+
+    try:
+        return getattr(module, class_name)
+    except AttributeError:
+        msg = 'Module "%s" does not define a "%s" attribute/class' % (
+            dotted_path, class_name)
+        six.reraise(ImportError, ImportError(msg), sys.exc_info()[2])
+
+
+
+def get_search_backend(backend='default', **kwargs):
+    # Get configuration
+    default_conf = {
+        'default': {
+            'BACKEND': 'wagtail.wagtailsearch.backends.db.DBSearch',
+        },
+    }
+    WAGTAILSEARCH_BACKENDS = getattr(settings, 'WAGTAILSEARCH_BACKENDS', default_conf)
+
+    # Try to find the backend
+    try:
+        # Try to get the WAGTAILSEARCH_BACKENDS entry for the given backend name first
+        conf = WAGTAILSEARCH_BACKENDS[backend]
+    except KeyError:
+        try:
+            # Trying to import the given backend, in case it's a dotted path
+            import_string(backend)
+        except ImportError as e:
+            raise InvalidSearchBackendError("Could not find backend '%s': %s" % (
+                backend, e))
+        params = kwargs
+    else:
+        # Backend is a conf entry
+        params = conf.copy()
+        params.update(kwargs)
+        backend = params.pop('BACKEND')
+
+    # Try to import the backend
+    try:
+        backend_cls = import_string(backend)
+    except ImportError as e:
+        raise InvalidSearchBackendError("Could not find backend '%s': %s" % (
+            backend, e))
+
+    # Create backend
+    return backend_cls(params)

wagtail/wagtailsearch/backends/base.py

@@ -0,0 +1,49 @@
+from django.db import models
+from django.core.exceptions import ImproperlyConfigured
+
+from wagtail.wagtailsearch.indexed import Indexed
+
+
+class InvalidSearchBackendError(ImproperlyConfigured):
+    pass
+
+
+class BaseSearch(object):
+    def __init__(self, params):
+        pass
+
+    def object_can_be_indexed(self, obj):
+        # Object must be a decendant of Indexed and be a django model
+        if not isinstance(obj, Indexed) or not isinstance(obj, models.Model):
+            return False
+
+        # Check if this objects model has opted out of indexing
+        if not obj.__class__.indexed:
+            return False
+
+        # Check if this object has an "object_indexed" function
+        if hasattr(obj, "object_indexed"):
+            if obj.object_indexed() is False:
+                return False
+        return True
+
+    def reset_index(self):
+        return NotImplemented
+
+    def add_type(self, model):
+        return NotImplemented
+
+    def refresh_index(self):
+        return NotImplemented
+
+    def add(self, obj):
+        return NotImplemented
+
+    def add_bulk(self, obj_list):
+        return NotImplemented
+
+    def delete(self, obj):
+        return NotImplemented
+
+    def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
+        return NotImplemented

wagtail/wagtailsearch/backends/db.py

@@ -0,0 +1,71 @@
+from django.db import models
+
+from wagtail.wagtailsearch.backends.base import BaseSearch
+from wagtail.wagtailsearch.indexed import Indexed
+
+
+class DBSearch(BaseSearch):
+    def __init__(self, params):
+        super(DBSearch, self).__init__(params)
+
+    def reset_index(self):
+        pass # Not needed
+
+    def add_type(self, model):
+        pass # Not needed
+
+    def refresh_index(self):
+        pass # Not needed
+
+    def add(self, obj):
+        pass # Not needed
+
+    def add_bulk(self, obj_list):
+        pass # Not needed
+
+    def delete(self, obj):
+        pass # Not needed
+
+    def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
+        # Get terms
+        terms = query_string.split()
+        if not terms:
+            return model.objects.none()
+
+        # Get fields
+        if fields is None:
+            fields = model.indexed_get_indexed_fields().keys()
+
+        # Start will all objects
+        query = model.objects.all()
+
+        # Apply filters
+        if filters:
+            query = query.filter(**filters)
+
+        # Filter by terms
+        for term in terms:
+            term_query = None
+            for field_name in fields:
+                # Check if the field exists (this will filter out indexed callables)
+                try:
+                    model._meta.get_field_by_name(field_name)
+                except:
+                    continue
+
+                # Filter on this field
+                field_filter = {'%s__icontains' % field_name: term}
+                if term_query is None:
+                    term_query = models.Q(**field_filter)
+                else:
+                    term_query |= models.Q(**field_filter)
+            query = query.filter(term_query)
+
+        # Distinct
+        query = query.distinct()
+
+        # Prefetch related
+        for prefetch in prefetch_related:
+            query = query.prefetch_related(prefetch)
+
+        return query

wagtail/wagtailsearch/backends/elasticsearch.py

@@ -1,15 +1,16 @@
-import string
+from django.db import models
+from django.conf import settings
 
 from pyelasticsearch.exceptions import ElasticHttpNotFoundError
 from elasticutils import get_es, S
 
-from django.db import models
-from django.conf import settings
+from wagtail.wagtailsearch.backends.base import BaseSearch
+from wagtail.wagtailsearch.indexed import Indexed
 
-from indexed import Indexed
+import string
 
 
-class SearchResults(object):
+class ElasticSearchResults(object):
     def __init__(self, model, query, prefetch_related=[]):
         self.model = model
         self.query = query
@@ -53,11 +54,13 @@ class SearchResults(object):
         return self.count
 
 
-class Search(object):
-    def __init__(self):
+class ElasticSearch(BaseSearch):
+    def __init__(self, params):
+        super(ElasticSearch, self).__init__(params)
+
         # Get settings
-        self.es_urls = getattr(settings, "WAGTAILSEARCH_ES_URLS", ["http://localhost:9200"])
-        self.es_index = getattr(settings, "WAGTAILSEARCH_ES_INDEX", "wagtail")
+        self.es_urls = params.get('URLS', ['http://localhost:9200'])
+        self.es_index = params.get('INDEX', 'wagtail')
 
         # Get ElasticSearch interface
         self.es = get_es(urls=self.es_urls)
@@ -145,24 +148,9 @@ class Search(object):
     def refresh_index(self):
         self.es.refresh(self.es_index)
 
-    def can_be_indexed(self, obj):
-        # Object must be a decendant of Indexed and be a django model
-        if not isinstance(obj, Indexed) or not isinstance(obj, models.Model):
-            return False
-
-        # Check if this objects model has opted out of indexing
-        if not obj.__class__.indexed:
-            return False
-
-        # Check if this object has an "object_indexed" function
-        if hasattr(obj, "object_indexed"):
-            if obj.object_indexed() is False:
-                return False
-        return True
-
     def add(self, obj):
         # Make sure the object can be indexed
-        if not self.can_be_indexed(obj):
+        if not self.object_can_be_indexed(obj):
             return
 
         # Build document
@@ -176,7 +164,7 @@ class Search(object):
         type_set = {}
         for obj in obj_list:
             # Object must be a decendant of Indexed and be a django model
-            if not self.can_be_indexed(obj):
+            if not self.object_can_be_indexed(obj):
                 continue
 
             # Get object type
@@ -190,9 +178,11 @@ class Search(object):
             type_set[obj_type].append(obj.indexed_build_document())
 
         # Loop through each type and bulk add them
+        results = []
         for type_name, type_objects in type_set.items():
-            print type_name, len(type_objects)
+            results.append((type_name, len(type_objects)))
             self.es.bulk_index(self.es_index, type_name, type_objects)
+        return results
 
     def delete(self, obj):
         # Object must be a decendant of Indexed and be a django model
@@ -243,4 +233,4 @@ class Search(object):
             query = query.filter(**filters)
 
         # Return search results
-        return SearchResults(model, query, prefetch_related=prefetch_related)
+        return ElasticSearchResults(model, query, prefetch_related=prefetch_related)

wagtail/wagtailsearch/management/commands/search_garbage_collect.py

@@ -6,11 +6,11 @@ from wagtail.wagtailsearch import models
 class Command(NoArgsCommand):
     def handle_noargs(self, **options):
         # Clean daily hits
-        print "Cleaning daily hits records... ",
+        self.stdout.write("Cleaning daily hits records... ")
         models.QueryDailyHits.garbage_collect()
-        print "Done"
+        self.stdout.write("Done")
 
         # Clean queries
-        print "Cleaning query records... ",
+        self.stdout.write("Cleaning query records... ")
         models.Query.garbage_collect()
-        print "Done"
+        self.stdout.write("Done")

wagtail/wagtailsearch/management/commands/update_index.py

@@ -1,14 +1,13 @@
-from django.core.management.base import NoArgsCommand
+from django.core.management.base import BaseCommand
 from django.db import models
 
-from wagtail.wagtailsearch.indexed import Indexed
-from wagtail.wagtailsearch.search import Search
+from wagtail.wagtailsearch import Indexed, get_search_backend
 
 
-class Command(NoArgsCommand):
-    def handle_noargs(self, **options):
+class Command(BaseCommand):
+    def handle(self, backend='default', **options):
         # Print info
-        print "Getting object list"
+        self.stdout.write("Getting object list")
 
         # Get list of indexed models
         indexed_models = [model for model in models.get_models() if issubclass(model, Indexed)]
@@ -46,22 +45,25 @@ class Command(NoArgsCommand):
                     # Space free, take it
                     object_set[key] = obj
 
-        # Search object
-        s = Search()
+        # Search backend
+        s = get_search_backend(backend=backend)
 
         # Reset the index
-        print "Reseting index"
+        self.stdout.write("Reseting index")
         s.reset_index()
 
         # Add types
-        print "Adding types"
+        self.stdout.write("Adding types")
         for model in indexed_models:
             s.add_type(model)
 
         # Add objects to index
-        print "Adding objects"
-        s.add_bulk(object_set.values())
+        self.stdout.write("Adding objects")
+        results = s.add_bulk(object_set.values())
+        if results:
+            for result in results:
+                self.stdout.write(result[0] + ' ' + str(result[1]))
 
         # Refresh index
-        print "Refreshing index"
+        self.stdout.write("Refreshing index")
         s.refresh_index()

wagtail/wagtailsearch/models.py

@@ -16,11 +16,16 @@ class Query(models.Model):
 
         super(Query, self).save(*args, **kwargs)
 
-    def add_hit(self):
-        daily_hits, created = QueryDailyHits.objects.get_or_create(query=self, date=timezone.now().date())
+    def add_hit(self, date=None):
+        if date is None:
+            date = timezone.now().date()
+        daily_hits, created = QueryDailyHits.objects.get_or_create(query=self, date=date)
         daily_hits.hits = models.F('hits') + 1
         daily_hits.save()
 
+    def __unicode__(self):
+        return self.query_string
+
     @property
     def hits(self):
         return self.daily_hits.aggregate(models.Sum('hits'))['hits__sum']
@@ -38,6 +43,7 @@ class Query(models.Model):
 
     @classmethod
     def get_most_popular(cls, date_since=None):
+        # TODO: Implement date_since
         return cls.objects.filter(daily_hits__isnull=False).annotate(_hits=models.Sum('daily_hits__hits')).distinct().order_by('-_hits')
 
     @staticmethod
@@ -49,7 +55,7 @@ class Query(models.Model):
         query_string = ''.join([c for c in query_string if c not in string.punctuation])
 
         # Remove double spaces
-        ' '.join(query_string.split())
+        query_string = ' '.join(query_string.split())
 
         return query_string
 
@@ -90,10 +96,18 @@ class SearchTest(models.Model, Indexed):
     title = models.CharField(max_length=255)
     content = models.TextField()
 
-    indexed_fields = ("title", "content")
+    indexed_fields = ("title", "content", "callable_indexed_field")
 
     title_search = Searcher(["title"])
 
+    def object_indexed(self):
+        if self.title == "Don't index me!":
+            return False
+        return True
+
+    def callable_indexed_field(self):
+        return "Callable"
+
 
 class SearchTestChild(SearchTest):
     extra_content = models.TextField()

wagtail/wagtailsearch/searcher.py

@@ -1,4 +1,4 @@
-from search import Search
+from wagtail.wagtailsearch.backends import get_search_backend
 
 
 class Searcher(object):
@@ -8,7 +8,17 @@ class Searcher(object):
 
     def __get__(self, instance, cls):
         def dosearch(query_string, **kwargs):
+            # Get backend
+            if 'backend' in kwargs:
+                backend = kwargs['backend']
+                del kwargs['backend']
+            else:
+                backend = 'default'
+
+            # Build search kwargs
             search_kwargs = dict(model=cls, fields=self.fields, filters=self.filters)
             search_kwargs.update(kwargs)
-            return Search().search(query_string, **search_kwargs)
+
+            # Run search
+            return get_search_backend(backend=backend).search(query_string, **search_kwargs)
         return dosearch

wagtail/wagtailsearch/signal_handlers.py

@@ -1,16 +1,16 @@
 from django.db.models.signals import post_save, post_delete
 from django.db import models
 
-from search import Search
-from indexed import Indexed
+from wagtail.wagtailsearch.indexed import Indexed
+from wagtail.wagtailsearch.backends import get_search_backend
 
 
 def post_save_signal_handler(instance, **kwargs):
-    Search().add(instance)
+    get_search_backend().add(instance)
 
 
 def post_delete_signal_handler(instance, **kwargs):
-    Search().delete(instance)
+    get_search_backend().delete(instance)
 
 
 def register_signal_handlers():

wagtail/wagtailsearch/tests.py

@@ -1,13 +1,67 @@
 from django.test import TestCase
+from django.test.client import Client
+from django.utils import timezone
+from django.core import management
+from django.conf import settings
 
-import models
-from search import Search
+import datetime
+import unittest
+from StringIO import StringIO
+
+from wagtail.wagtailcore import models as core_models
+from wagtail.wagtailsearch import models
+from wagtail.wagtailsearch.backends import get_search_backend
+
+from wagtail.wagtailsearch.backends.base import InvalidSearchBackendError
+from wagtail.wagtailsearch.backends.db import DBSearch
+from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearch
 
 
-class TestSearch(TestCase):
-    def test_search(self):
-        # Create search interface and reset the index
-        s = Search()
+def find_backend(cls):
+    if not hasattr(settings, 'WAGTAILSEARCH_BACKENDS'):
+        if cls == DBSearch:
+            return 'default'
+        else:
+            return
+
+    for backend in settings.WAGTAILSEARCH_BACKENDS.keys():
+        if isinstance(get_search_backend(backend), cls):
+            return backend
+
+
+class TestBackend(TestCase):
+    def __init__(self, *args, **kwargs):
+        super(TestBackend, self).__init__(*args, **kwargs)
+
+        self.backends_tested = []
+
+    def test_backend_loader(self):
+        # Test DB backend import
+        db = get_search_backend(backend='wagtail.wagtailsearch.backends.db.DBSearch')
+        self.assertIsInstance(db, DBSearch)
+
+        # Test Elastic search backend import
+        elasticsearch = get_search_backend(backend='wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch')
+        self.assertIsInstance(elasticsearch, ElasticSearch)
+
+        # Test loading a non existant backend
+        self.assertRaises(InvalidSearchBackendError, get_search_backend, backend='wagtail.wagtailsearch.backends.doesntexist.DoesntExist')
+
+        # Test something that isn't a backend
+        self.assertRaises(InvalidSearchBackendError, get_search_backend, backend="I'm not a backend!")
+
+    def test_search(self, backend=None):
+        # Don't run this test directly (this will be called from other tests)
+        if backend is None:
+            raise unittest.SkipTest()
+
+        # Don't test the same backend more than once!
+        if backend in self.backends_tested:
+            return
+        self.backends_tested.append(backend)
+
+        # Get search backend and reset the index
+        s = get_search_backend(backend=backend)
         s.reset_index()
 
         # Create a couple of objects and add them to the index
@@ -33,12 +87,29 @@ class TestSearch(TestCase):
         results = s.search("Hello", models.SearchTest)
         self.assertEqual(len(results), 3)
 
-          # Ordinary search on "World"
+        # Retrieve single result
+        self.assertIsInstance(results[0], models.SearchTest)
+
+        # Retrieve results through iteration
+        iterations = 0
+        for result in results:
+            self.assertIsInstance(result, models.SearchTest)
+            iterations += 1
+        self.assertEqual(iterations, 3)
+
+        # Retrieve results through slice
+        iterations = 0
+        for result in results[:]:
+            self.assertIsInstance(result, models.SearchTest)
+            iterations += 1
+        self.assertEqual(iterations, 3)
+
+        # Ordinary search on "World"
         results = s.search("World", models.SearchTest)
         self.assertEqual(len(results), 1)
 
         # Searcher search
-        results = models.SearchTest.title_search("Hello")
+        results = models.SearchTest.title_search("Hello", backend=backend)
         self.assertEqual(len(results), 3)
 
         # Ordinary search on child
@@ -46,5 +117,278 @@ class TestSearch(TestCase):
         self.assertEqual(len(results), 1)
 
         # Searcher search on child
-        results = models.SearchTestChild.title_search("Hello")
+        results = models.SearchTestChild.title_search("Hello", backend=backend)
         self.assertEqual(len(results), 1)
+
+        # Delete a record
+        testc.delete()
+        results = s.search("Hello", models.SearchTest)
+        # TODO: FIXME Deleting records doesn't seem to be deleting them from the index! (but they still get deleted on update_index)
+        #self.assertEqual(len(results), 2)
+
+        # Try to index something that shouldn't be indexed
+        # TODO: This currently fails on the DB backend
+        if not isinstance(s, DBSearch):
+            testd = models.SearchTest()
+            testd.title = "Don't index me!"
+            testd.save()
+            s.add(testd)
+            results = s.search("Don't", models.SearchTest)
+            self.assertEqual(len(results), 0)
+
+        # Reset the index, this should clear out the index (but doesn't have to!)
+        s.reset_index()
+
+        # Run update_index command
+        management.call_command('update_index', backend, interactive=False, stdout=StringIO())
+
+        # Should have results again now
+        results = s.search("Hello", models.SearchTest)
+        self.assertEqual(len(results), 2)
+
+    def test_db_backend(self):
+        self.test_search(backend='wagtail.wagtailsearch.backends.db.DBSearch')
+
+    def test_elastic_search_backend(self):
+        backend = find_backend(ElasticSearch)
+
+        if backend is not None:
+            self.test_search(backend)
+        else:
+            raise unittest.SkipTest("Cannot find an ElasticSearch search backend in configuration.")
+
+    def test_query_hit_counter(self):
+        # Add 10 hits to hello query
+        for i in range(10):
+            models.Query.get("Hello").add_hit()
+
+        # Check that each hit was registered
+        self.assertEqual(models.Query.get("Hello").hits, 10)
+
+    def test_query_string_normalisation(self):
+        # Get a query
+        query = models.Query.get("Hello World!")
+
+        # Check that it it stored correctly
+        self.assertEqual(str(query), "hello world")
+
+        # Check queries that should be the same
+        self.assertEqual(query, models.Query.get("Hello World"))
+        self.assertEqual(query, models.Query.get("Hello  World!!"))
+        self.assertEqual(query, models.Query.get("hello world"))
+        self.assertEqual(query, models.Query.get("Hello' world"))
+
+        # Check queries that should be different
+        self.assertNotEqual(query, models.Query.get("HelloWorld"))
+        self.assertNotEqual(query, models.Query.get("Hello orld!!"))
+        self.assertNotEqual(query, models.Query.get("Hello"))
+
+    def test_query_popularity(self):
+        # Add 3 hits to unpopular query
+        for i in range(3):
+            models.Query.get("unpopular query").add_hit()
+
+        # Add 10 hits to popular query
+        for i in range(10):
+            models.Query.get("popular query").add_hit()
+
+        # Get most popular queries
+        popular_queries = models.Query.get_most_popular()
+
+        # Check list
+        self.assertEqual(popular_queries.count(), 2)
+        self.assertEqual(popular_queries[0], models.Query.get("popular query"))
+        self.assertEqual(popular_queries[1], models.Query.get("unpopular query"))
+
+        # Add 5 hits to little popular query
+        for i in range(5):
+            models.Query.get("little popular query").add_hit()
+
+        # Check list again, little popular query should be in the middle
+        self.assertEqual(popular_queries.count(), 3)
+        self.assertEqual(popular_queries[0], models.Query.get("popular query"))
+        self.assertEqual(popular_queries[1], models.Query.get("little popular query"))
+        self.assertEqual(popular_queries[2], models.Query.get("unpopular query"))
+
+        # Unpopular query goes viral!
+        for i in range(20):
+            models.Query.get("unpopular query").add_hit()
+
+        # Unpopular query should be most popular now
+        self.assertEqual(popular_queries.count(), 3)
+        self.assertEqual(popular_queries[0], models.Query.get("unpopular query"))
+        self.assertEqual(popular_queries[1], models.Query.get("popular query"))
+        self.assertEqual(popular_queries[2], models.Query.get("little popular query"))
+
+    @unittest.expectedFailure # Time based popularity isn't implemented yet
+    def test_query_popularity_over_time(self):
+        today = timezone.now().date()
+        two_days_ago = today - datetime.timedelta(days=2)
+        a_week_ago = today - datetime.timedelta(days=7)
+        a_month_ago = today - datetime.timedelta(days=30)
+
+        # Add 10 hits to a query that was very popular query a month ago
+        for i in range(10):
+            models.Query.get("old popular query").add_hit(date=a_month_ago)
+
+        # Add 5 hits to a query that is was popular 2 days ago
+        for i in range(5):
+            models.Query.get("new popular query").add_hit(date=two_days_ago)
+
+        # Get most popular queries
+        popular_queries = models.Query.get_most_popular()
+
+        # Old popular query should be most popular
+        self.assertEqual(popular_queries.count(), 2)
+        self.assertEqual(popular_queries[0], models.Query.get("old popular query"))
+        self.assertEqual(popular_queries[1], models.Query.get("new popular query"))
+
+        # Get most popular queries for past week
+        past_week_popular_queries = models.Query.get_most_popular(date_since=a_week_ago)
+
+        # Only new popular query should be in this list
+        self.assertEqual(past_week_popular_queries.count(), 1)
+        self.assertEqual(past_week_popular_queries[0], models.Query.get("new popular query"))
+
+        # Old popular query gets a couple more hits!
+        for i in range(2):
+            models.Query.get("old popular query").add_hit()
+
+        # Old popular query should now be in the most popular queries
+        self.assertEqual(past_week_popular_queries.count(), 2)
+        self.assertEqual(past_week_popular_queries[0], models.Query.get("new popular query"))
+        self.assertEqual(past_week_popular_queries[1], models.Query.get("old popular query"))
+
+    def test_editors_picks(self):
+        # Get root page
+        root = core_models.Page.objects.first()
+
+        # Create an editors pick to the root page
+        models.EditorsPick.objects.create(
+            query=models.Query.get("root page"),
+            page=root,
+            sort_order=0,
+            description="First editors pick",
+        )
+
+        # Get editors pick
+        self.assertEqual(models.Query.get("root page").editors_picks.count(), 1)
+        self.assertEqual(models.Query.get("root page").editors_picks.first().page, root)
+
+        # Create a couple more editors picks to test the ordering
+        models.EditorsPick.objects.create(
+            query=models.Query.get("root page"),
+            page=root,
+            sort_order=2,
+            description="Last editors pick",
+        )
+        models.EditorsPick.objects.create(
+            query=models.Query.get("root page"),
+            page=root,
+            sort_order=1,
+            description="Middle editors pick",
+        )
+
+        # Check
+        self.assertEqual(models.Query.get("root page").editors_picks.count(), 3)
+        self.assertEqual(models.Query.get("root page").editors_picks.first().description, "First editors pick")
+        self.assertEqual(models.Query.get("root page").editors_picks.last().description, "Last editors pick")
+
+        # Add editors pick with different terms
+        models.EditorsPick.objects.create(
+            query=models.Query.get("root page 2"),
+            page=root,
+            sort_order=0,
+            description="Other terms",
+        )
+
+        # Check
+        self.assertEqual(models.Query.get("root page 2").editors_picks.count(), 1)
+        self.assertEqual(models.Query.get("root page").editors_picks.count(), 3)
+
+    def test_garbage_collect(self):
+        # Call garbage collector command
+        management.call_command('search_garbage_collect', interactive=False, stdout=StringIO())
+
+
+def get_default_host():
+    from wagtail.wagtailcore.models import Site
+    return Site.objects.filter(is_default_site=True).first().root_url.split('://')[1]
+
+
+def get_search_page_test_data():
+    params_list = []
+    params_list.append({})
+
+    for query in ['', 'Hello', "'", '%^W&*$']:
+        params_list.append({'q': query})
+
+    for page in ['-1', '0', '1', '99999', 'Not a number']:
+        params_list.append({'q': 'Hello', 'p': page})
+
+    return params_list
+
+
+class TestFrontend(TestCase):
+    def setUp(self):
+        s = get_search_backend()
+
+        # Stick some documents into the index
+        testa = models.SearchTest()
+        testa.title = "Hello World"
+        testa.save()
+        s.add(testa)
+
+        testb = models.SearchTest()
+        testb.title = "Hello"
+        testb.save()
+        s.add(testb)
+
+        testc = models.SearchTestChild()
+        testc.title = "Hello"
+        testc.save()
+        s.add(testc)
+
+    def test_views(self):
+        c = Client()
+
+        # Test urls
+        for url in ['/search/', '/search/suggest/']:
+            for params in get_search_page_test_data():
+                r = c.get(url, params, HTTP_HOST=get_default_host())
+                self.assertEqual(r.status_code, 200)
+
+            # Try an extra one with AJAX
+            r = c.get(url, HTTP_HOST=get_default_host(), HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+            self.assertEqual(r.status_code, 200)
+
+
+class TestAdmin(TestCase):
+    def setUp(self):
+        # Create a user
+        from django.contrib.auth.models import User
+        User.objects.create_superuser(username='test', email='test@email.com', password='password')
+
+        # Setup client
+        self.c = Client()
+        self.c.login(username='test', password='password')
+
+    def test_editors_picks(self):
+        # Test index
+        for params in get_search_page_test_data():
+            r = self.c.get('/admin/search/editorspicks/', params, HTTP_HOST=get_default_host())
+            self.assertEqual(r.status_code, 200)
+
+        # Try an extra one with AJAX
+        r = self.c.get('/admin/search/editorspicks/', HTTP_HOST=get_default_host(), HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+        self.assertEqual(r.status_code, 200)
+
+
+    def test_queries_chooser(self):
+        for params in get_search_page_test_data():
+            r = self.c.get('/admin/search/queries/chooser/', params, HTTP_HOST=get_default_host())
+            self.assertEqual(r.status_code, 200)
+
+        # Try an extra one with AJAX
+        r = self.c.get('/admin/search/queries/chooser/', HTTP_HOST=get_default_host(), HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+        self.assertEqual(r.status_code, 200)

wagtail/wagtailsearch/views/frontend.py

@@ -25,24 +25,23 @@ def search(request):
 
         # Pagination
         paginator = Paginator(search_results, 10)
-        if paginator is not None:
-            try:
-                search_results = paginator.page(page)
-            except PageNotAnInteger:
-                search_results = paginator.page(1)
-            except EmptyPage:
-                search_results = paginator.page(paginator.num_pages)
-        else:
-            search_results = None
+        try:
+            search_results = paginator.page(page)
+        except PageNotAnInteger:
+            search_results = paginator.page(1)
+        except EmptyPage:
+            search_results = paginator.page(paginator.num_pages)
     else:
         query = None
         search_results = None
 
     # Render
+    template_name = None
     if request.is_ajax():
-        template_name = getattr(settings, "WAGTAILSEARCH_RESULTS_TEMPLATE_AJAX", "wagtailsearch/search_results.html")
-    else:
-        template_name = getattr(settings, "WAGTAILSEARCH_RESULTS_TEMPLATE", "wagtailsearch/search_results.html")
+        template_name = getattr(settings, 'WAGTAILSEARCH_RESULTS_TEMPLATE_AJAX', None)
+    if template_name is None:
+        template_name = getattr(settings, 'WAGTAILSEARCH_RESULTS_TEMPLATE', 'wagtailsearch/search_results.html')
+
     return render(request, template_name, dict(query_string=query_string, search_results=search_results, is_ajax=request.is_ajax(), query=query))
 
 

wagtail/wagtailsnippets/static/wagtailsnippets/js/snippet-chooser.js

@@ -5,7 +5,7 @@ function createSnippetChooser(id, contentType) {
 
     $('.action-choose', chooserElement).click(function() {
         ModalWorkflow({
-            'url': '/admin/snippets/choose/' + contentType + '/', /* TODO: don't hard-code this, as it may be changed in urls.py */
+            'url': window.chooserUrls.snippetChooser + contentType + '/',
             'responses': {
                 'snippetChosen': function(snippetData) {
                     input.val(snippetData.id);

wagtail/wagtailsnippets/urls.py

@@ -5,6 +5,7 @@ urlpatterns = patterns(
     'wagtail.wagtailsnippets.views',
     url(r'^$', 'snippets.index', name='wagtailsnippets_index'),
 
+    url(r'^choose/$', 'chooser.choose', name='wagtailsnippets_choose_generic'),
     url(r'^choose/(\w+)/(\w+)/$', 'chooser.choose', name='wagtailsnippets_choose'),
     url(r'^choose/(\w+)/(\w+)/(\d+)/$', 'chooser.chosen', name='wagtailsnippets_chosen'),