diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8f391a29a..a0abb58db 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,14 +1,16 @@ Changelog ========= -0.4 (xx.xx.20xx) +0.4 (10.07.2014) ~~~~~~~~~~~~~~~~ * ElasticUtils/pyelasticsearch swapped for elasticsearch-py * Python 3.2, 3.3 and 3.4 support * Added scheduled publishing + * Added support for private (password-protected) pages * Added frontend cache invalidator * Added sitemap generator * Added notification preferences + * Added a new way to configure searchable/filterable fields on models * Added 'original' as a resizing rule supported by the 'image' tag * Hallo.js updated to version 1.0.4 * Snippets are now ordered alphabetically @@ -29,6 +31,14 @@ Changelog * Added init_new_page signal * Added page_published signal * Added copy method to Page to allow copying of pages + * Added ``search`` method to ``PageQuerySet`` + * Added ``get_indexed_objects`` allowing developers to customise which objects get added to the search index + * Major refactor of Elasticsearch backend + * Use ``match`` instead of ``query_string`` queries + * Fields are now indexed in Elasticsearch with their correct type + * Filter fields are no longer included in '_all' (in Elasticsearch) + * Fields with partial matching are now indexed together into '_partials' + * Fix: Animated GIFs are now coalesced before resizing * Fix: Wand backend clones images before modifying them * Fix: Admin breadcrumb now positioned correctly on mobile diff --git a/docs/building_your_site/frontenddevelopers.rst b/docs/building_your_site/frontenddevelopers.rst index a3ba8eaa2..a5120af19 100644 --- a/docs/building_your_site/frontenddevelopers.rst +++ b/docs/building_your_site/frontenddevelopers.rst @@ -192,7 +192,7 @@ More control over the ``img`` tag Wagtail provides two shorcuts to give greater control over the ``img`` element: -**Adding attributes to the {% image %} tag** +**1. Adding attributes to the {% image %} tag** .. versionadded:: 0.4 @@ -202,26 +202,31 @@ Extra attributes can be specified with the syntax ``attribute="value"``: {% image self.photo width-400 class="foo" id="bar" %} -No validation is performed on attributes add in this way by the developer. It's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself. +No validation is performed on attributes added in this way so it's possible to add `src`, `width`, `height` and `alt` of your own that might conflict with those generated by the tag itself. -**Generating the image "as"** +**2. Generating the image "as foo" to access individual properties** -Wagtail can assign the image data to another object using Django's ``as`` syntax: +Wagtail can assign the image data to another variable using Django's ``as`` syntax: .. code-block:: django {% image self.photo width-400 as tmp_photo %} - {{ tmp_photo.alt }} + + +.. Note:: + The image property used for the ``src`` attribute is actually ``image.url``, not ``image.src``. + The ``attrs`` shortcut ----------------------- .. versionadded:: 0.4 -You can also use the ``attrs`` property as a shorthand to output the ``src``, ``width``, ``height`` and ``alt`` attributes in one go: +You can also use the ``attrs`` property as a shorthand to output the attributes ``src``, ``width``, ``height`` and ``alt`` in one go: .. code-block:: django diff --git a/docs/conf.py b/docs/conf.py index ba6525d1b..d54c30df7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,9 +59,9 @@ copyright = u'2014, Torchbox' # built documents. # # The short X.Y version. -version = '0.3.1' +version = '0.4' # The full version, including alpha/beta/rc tags. -release = '0.3.1' +release = '0.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 171c0e45a..2ab429eb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ It supports Django 1.6.2+ on Python 2.6, 2.7, 3.2, 3.3 and 3.4. Django 1.7 suppo building_your_site/index editing_api snippets - wagtail_search + search/index form_builder model_recipes advanced_topics @@ -28,3 +28,4 @@ It supports Django 1.6.2+ on Python 2.6, 2.7, 3.2, 3.3 and 3.4. Django 1.7 suppo support roadmap editor_manual/index + releases/index diff --git a/docs/model_recipes.rst b/docs/model_recipes.rst index 7f9b5f7b6..b8f006f99 100644 --- a/docs/model_recipes.rst +++ b/docs/model_recipes.rst @@ -38,6 +38,8 @@ Consider this example from the Wagtail demo site's ``models.py``, which serves a With this strategy, you could use Django or Python utilities to render your model in JSON or XML or any other format you'd like. +.. _overriding_route_method: + Adding Endpoints with Custom route() Methods -------------------------------------------- diff --git a/docs/releases/0.4.rst b/docs/releases/0.4.rst index a51c3402d..1e6dbd15d 100644 --- a/docs/releases/0.4.rst +++ b/docs/releases/0.4.rst @@ -32,7 +32,7 @@ A new management command has been added (:ref:`publish_scheduled_pages`) to publ Search on QuerySet with Elasticsearch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Its now possible to perform searches with Elasticsearch on ``PageQuerySet``s: +Its now possible to perform searches with Elasticsearch on ``PageQuerySet`` objects: >>> from wagtail.wagtailcore.models import Page >>> Page.objects.live().descendant_of(events_index).search("Hello") @@ -70,6 +70,7 @@ Core * Any extra arguments given to ``Page.serve`` are now passed through to ``get_context`` and ``get_template`` * Added ``in_menu`` and ``not_in_menu`` methods to ``PageQuerySet`` + * Added ``search`` method to ``PageQuerySet`` * Added ``get_next_siblings`` and ``get_prev_siblings`` to ``Page`` * Added ``page_published`` signal * Added ``copy`` method to ``Page`` to allow copying of pages @@ -90,6 +91,18 @@ Admin * Added ``init_new_page`` signal +Search +------ + + * Added a new way to configure searchable/filterable fields on models + * Added ``get_indexed_objects`` allowing developers to customise which objects get added to the search index + * Major refactor of Elasticsearch backend + * Use ``match`` instead of ``query_string`` queries + * Fields are now indexed in Elasticsearch with their correct type + * Filter fields are no longer included in '_all' + * Fields with partial matching are now indexed together into '_partials' + + Images ------ @@ -100,6 +113,7 @@ Images Other ----- + * Added styleguide (mainly for wagtail developers) @@ -144,23 +158,25 @@ The following template tag libraries have been renamed: * ``embed_filters`` => ``wagtailembeds_tags`` * ``image_tags`` => ``wagtailimages_tags`` +The old names will continue to work, but output a ``DeprecationWarning`` - you are advised to update any ``{% load %}`` tags in your templates to refer to the new names. + New search field configuration format ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``indexed_fields`` is now deprecated and has been replaced by a new search field configuration format called ``search_fields``. +``indexed_fields`` is now deprecated and has been replaced by a new search field configuration format called ``search_fields``. See :ref:`wagtailsearch_for_python_developers` for how to define a ``search_fields`` property on your models. ``Page.route`` method should now return a ``RouteResult`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Previously, the ``route`` method called ``serve`` and returned a ``HttpResponse`` object. This has now been split up so ``serve`` is called separately and ``route`` must now return a RouteResult object. +Previously, the ``route`` method called ``serve`` and returned an ``HttpResponse`` object. This has now been split up so ``serve`` is called separately and ``route`` must now return a RouteResult object. -:ref:`anatomy_of_a_wagtail_request` +If you are overriding ``Page.route`` on any of your page models, you will need to update the method to return a ``RouteResult`` object. The old method of returning an ``HttpResponse`` will continue to work, but this will throw a ``DeprecationWarning`` and bypass the ``before_serve_page`` hook, which means in particular that :ref:`private_pages` will not work on those page types. See :ref:`overriding_route_method`. Wagtailadmins ``hooks`` module has moved to wagtailcore -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you use any ``wagtail_hooks.py`` files in your project, you may have an import like: ``from wagtail.wagtailadmin import hooks`` diff --git a/docs/releases/index.rst b/docs/releases/index.rst new file mode 100644 index 000000000..07bc8e888 --- /dev/null +++ b/docs/releases/index.rst @@ -0,0 +1,7 @@ +Release notes +============= + +.. toctree:: + :maxdepth: 1 + + 0.4 \ No newline at end of file diff --git a/docs/search/backends.rst b/docs/search/backends.rst new file mode 100644 index 000000000..f03f06358 --- /dev/null +++ b/docs/search/backends.rst @@ -0,0 +1,66 @@ + +.. _wagtailsearch_backends: + +======== +Backends +======== + + +Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_. + +.. _Elasticsearch: http://www.elasticsearch.org/ + + +.. _wagtailsearch_backends_database: + +Database Backend +================ + +The default DB search backend uses Django's ``__icontains`` filter. + + +Elasticsearch Backend +===================== + +Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package: + +.. code-block:: guess + + pip install elasticsearch + +.. note:: + If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5``` + +The backend is configured in settings: + +.. code-block:: python + + WAGTAILSEARCH_BACKENDS = { + 'default': { + 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch', + 'URLS': ['http://localhost:9200'], + 'INDEX': 'wagtail', + 'TIMEOUT': 5, + 'FORCE_NEW': False, + } + } + +Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``). + +If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly: + +- Sign up for an account at `dashboard.searchly.com/users/sign\_up`_ +- Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo' +- Note the connection URL from your Searchly dashboard +- Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS`` +- Run ``./manage.py update_index`` + +.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org +.. _Searchly: http://www.searchly.com/ +.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up + + +Rolling Your Own +================ + +Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``. \ No newline at end of file diff --git a/docs/search/editors_picks.rst b/docs/search/editors_picks.rst new file mode 100644 index 000000000..9ed41f3c5 --- /dev/null +++ b/docs/search/editors_picks.rst @@ -0,0 +1,41 @@ + +.. _editors-picks: + + +Editors picks +============= + +Editor's Picks are a way of explicitly linking relevant content to search terms, so results pages can contain curated content instead of being at the mercy of the search algorithm. In a template using the search results view, editor's picks can be accessed through the variable ``query.editors_picks``. To include editor's picks in your search results template, use the following properties. + +``query.editors_picks.all`` + This gathers all of the editor's picks objects relating to the current query, in order according to their sort order in the Wagtail admin. You can then iterate through them using a ``{% for ... %}`` loop. Each editor's pick object provides these properties: + + ``editors_pick.page`` + The page object associated with the pick. Use ``{% pageurl editors_pick.page %}`` to generate a URL or provide other properties of the page object. + + ``editors_pick.description`` + The description entered when choosing the pick, perhaps explaining why the page is relevant to the search terms. + +Putting this all together, a block of your search results template displaying editor's Picks might look like this: + +.. code-block:: django + + {% with query.editors_picks.all as editors_picks %} + {% if editors_picks %} +
+

Editors picks

+ +
+ {% endif %} + {% endwith %} \ No newline at end of file diff --git a/docs/search/for_python_developers.rst b/docs/search/for_python_developers.rst new file mode 100644 index 000000000..7ff9a18dc --- /dev/null +++ b/docs/search/for_python_developers.rst @@ -0,0 +1,152 @@ + +.. _wagtailsearch_for_python_developers: + + +===================== +For python developers +===================== + + +Basic usage +=========== + +All searches are performed on Django QuerySets. Wagtail provides a ``search`` method on the queryset for all page models: + +.. code-block:: python + + # Search future EventPages + >>> from wagtail.wagtailcore.models import EventPage + >>> EventPage.objects.filter(date__gt=timezone.now()).search("Hello world!") + + +All methods of ``PageQuerySet`` are supported by wagtailsearch: + +.. code-block:: python + + # Search all live EventPages that are under the events index + >>> EventPage.objects.live().descendant_of(events_index).search("Event") + [, ] + + +Indexing extra fields +===================== + +Fields need to be explicitly added to the search configuration in order for you to be able to search/filter on them. + +You can add new fields to the search index by overriding the ``search_fields`` property and appending a list of extra ``SearchField``/``FilterField`` objects to it. + +The default value of ``search_fields`` (as set in ``Page``) indexes the ``title`` field as a ``SearchField`` and some other generally useful fields as ``FilterField`` rules. + + +Quick example +------------- + +This creates an ``EventPage`` model with two fields ``description`` and ``date``. ``description`` is indexed as a ``SearchField`` and ``date`` is indexed as a ``FilterField`` + + +.. code-block:: python + + from wagtail.wagtailsearch import indexed + + class EventPage(Page): + description = models.TextField() + date = models.DateField() + + search_fields = Page.search_fields + ( # Inherit search_fields from Page + indexed.SearchField('description'), + indexed.FilterField('date'), + ) + + + # Get future events which contain the string "Christmas" in the title or description + >>> EventPage.objects.filter(date__gt=timezone.now()).search("Christmas") + + +``indexed.SearchField`` +----------------------- + +These are added to the search index and are used for performing full-text searches on your models. These would usually be text fields. + + +Options +``````` + + - **partial_match** (boolean) - Setting this to true allows results to be matched on parts of words. For example, this is set on the title field by default so a page titled "Hello World!" will be found if the user only types "Hel" into the search box. + - **boost** (number) - This allows you to set fields as being more important than others. Setting this to a high number on a field will make pages with matches in that field to be ranked higher. By default, this is set to 100 on the title field and 1 on all other fields. + - **es_extra** (dict) - This field is to allow the developer to set or override any setting on the field in the ElasticSearch mapping. Use this if you want to make use of any ElasticSearch features that are not yet supported in Wagtail. + + +``indexed.FilterField`` +----------------------- + +These are added to the search index but are not used for full-text searches. Instead, they allow you to run filters on your search results. + + +Indexing callables and other attributes +--------------------------------------- + + .. note:: + + This is not supported in the :ref:`wagtailsearch_backends_database` + + +Search/filter fields do not need to be Django fields, they could be any method or attribute on your class. + +One use for this is indexing ``get_*_display`` methods Django creates automatically for fields with choices. + + +.. code-block:: python + + from wagtail.wagtailsearch import indexed + + class EventPage(Page): + IS_PRIVATE_CHOICES = ( + (False, "Public"), + (True, "Private"), + ) + + is_private = models.BooleanField(choices=IS_PRIVATE_CHOICES) + + search_fields = Page.search_fields + ( + # Index the human-readable string for searching + indexed.SearchField('get_is_private_display'), + + # Index the boolean value for filtering + indexed.FilterField('is_private'), + ) + + +Indexing non-page models +======================== + +Any Django model can be indexed and searched. + +To do this, inherit from ``indexed.Indexed`` and add some ``search_fields`` to the model. + +.. code-block:: python + + from wagtail.wagtailsearch import indexed + + class Book(models.Model, indexed.Indexed): + title = models.CharField(max_length=255) + genre = models.CharField(max_length=255, choices=GENRE_CHOICES) + author = models.ForeignKey(Author) + published_date = models.DateTimeField() + + search_fields = ( + indexed.SearchField('title', partial_match=True, boost=10), + indexed.SearchField('get_genre_display'), + + indexed.FilterField('genre'), + indexed.FilterField('author'), + indexed.FilterField('published_date'), + ) + + # As this model doesn't have a search method in its QuerySet, we have to call search directly on the backend + >>> from wagtail.wagtailsearch.backends import get_search_backend + >>> s = get_search_backend() + + # Run a search for a book by Roald Dahl + >>> roald_dahl = Author.objects.get(name="Roald Dahl") + >>> s.search("chocolate factory", Book.objects.filter(author=roald_dahl)) + [] diff --git a/docs/wagtail_search.rst b/docs/search/frontend_views.rst similarity index 53% rename from docs/wagtail_search.rst rename to docs/search/frontend_views.rst index 4c0a1ac74..1e1ad3fd3 100644 --- a/docs/wagtail_search.rst +++ b/docs/search/frontend_views.rst @@ -1,10 +1,10 @@ -.. _wagtail_search: +.. _wagtailsearch_frontend_views: -Search -====== -Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface. +Frontend views +============== + Default Page Search ------------------- @@ -71,43 +71,6 @@ The search view provides a context with a few useful variables. ``query`` A Wagtail ``Query`` object matching the terms. The ``Query`` model provides several class methods for viewing the statistics of all queries, but exposes only one property for single objects, ``query.hits``, which tracks the number of time the search string has been used over the lifetime of the site. ``Query`` also joins to the Editor's Picks functionality though ``query.editors_picks``. See :ref:`editors-picks`. -Editor's Picks --------------- - -Editor's Picks are a way of explicitly linking relevant content to search terms, so results pages can contain curated content instead of being at the mercy of the search algorithm. In a template using the search results view, editor's picks can be accessed through the variable ``query.editors_picks``. To include editor's picks in your search results template, use the following properties. - -``query.editors_picks.all`` - This gathers all of the editor's picks objects relating to the current query, in order according to their sort order in the Wagtail admin. You can then iterate through them using a ``{% for ... %}`` loop. Each editor's pick object provides these properties: - - ``editors_pick.page`` - The page object associated with the pick. Use ``{% pageurl editors_pick.page %}`` to generate a URL or provide other properties of the page object. - - ``editors_pick.description`` - The description entered when choosing the pick, perhaps explaining why the page is relevant to the search terms. - -Putting this all together, a block of your search results template displaying editor's Picks might look like this: - -.. code-block:: django - - {% with query.editors_picks.all as editors_picks %} - {% if editors_picks %} -
-

Editors picks

- -
- {% endif %} - {% endwith %} Asynchronous Search with JSON and AJAX -------------------------------------- @@ -142,30 +105,30 @@ Finally, we'll use JQuery to make the asynchronous requests and handle the inter // when there's something in the input box, make the query searchBox.on('input', function() { if( searchBox.val() == ''){ - resultsBox.html(''); - return; + resultsBox.html(''); + return; } // make the request to the Wagtail JSON search view $.ajax({ - url: wagtailJSONSearchURL + "?q=" + searchBox.val(), - dataType: "json" + url: wagtailJSONSearchURL + "?q=" + searchBox.val(), + dataType: "json" }) .done(function(data) { - console.log(data); - if( data == undefined ){ - resultsBox.html(''); - return; - } - // we're in business! let's format the results - var htmlOutput = ''; - data.forEach(function(element, index, array){ - htmlOutput += '

' + element.title + '

'; - }); - // and display them - resultsBox.html(htmlOutput); + console.log(data); + if( data == undefined ){ + resultsBox.html(''); + return; + } + // we're in business! let's format the results + var htmlOutput = ''; + data.forEach(function(element, index, array){ + htmlOutput += '

' + element.title + '

'; + }); + // and display them + resultsBox.html(htmlOutput); }) .error(function(data){ - console.log(data); + console.log(data); }); }); @@ -178,12 +141,12 @@ Results are returned as a JSON object with this structure: { [ { - title: "Lumpy Space Princess", - url: "/oh-my-glob/" + title: "Lumpy Space Princess", + url: "/oh-my-glob/" }, { - title: "Lumpy Space", - url: "/no-smooth-posers/" + title: "Lumpy Space", + url: "/no-smooth-posers/" }, ... ] @@ -199,67 +162,8 @@ The AJAX interface uses the same view as the normal HTML search, ``wagtailsearch In this template, you'll have access to the same context variables provided to the HTML template. You could provide a template in JSON format with extra properties, such as ``query.hits`` and editor's picks, or render an HTML snippet that can go directly into your results ``
``. If you need more flexibility, such as multiple formats/templates based on differing requests, you can set up a custom search view. -.. _editors-picks: +Custom Search Views +------------------- -Indexing Custom Fields & Custom Search Views --------------------------------------------- - -This functionality is still under active development to provide a streamlined interface, but take a look at ``wagtail/wagtail/wagtailsearch/views/frontend.py`` if you are interested in coding custom search views. - - -Search Backends ---------------- - -Wagtail can degrade to a database-backed text search, but we strongly recommend `Elasticsearch`_. - -.. _Elasticsearch: http://www.elasticsearch.org/ - - -Default DB Backend -`````````````````` -The default DB search backend uses Django's ``__icontains`` filter. - - -Elasticsearch Backend -````````````````````` -Prerequisites are the Elasticsearch service itself and, via pip, the `elasticsearch-py`_ package: - -.. code-block:: guess - - pip install elasticsearch - -.. note:: - If you are using Elasticsearch < 1.0, install elasticsearch-py version 0.4.5: ```pip install elasticsearch==0.4.5``` - -The backend is configured in settings: - -.. code-block:: python - - WAGTAILSEARCH_BACKENDS = { - 'default': { - 'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch', - 'URLS': ['http://localhost:9200'], - 'INDEX': 'wagtail', - 'TIMEOUT': 5, - 'FORCE_NEW': False, - } - } - -Other than ``BACKEND`` the keys are optional and default to the values shown. ``FORCE_NEW`` is used by elasticsearch-py. In addition, any other keys are passed directly to the Elasticsearch constructor as case-sensitive keyword arguments (e.g. ``'max_retries': 1``). - -If you prefer not to run an Elasticsearch server in development or production, there are many hosted services available, including `Searchly`_, who offer a free account suitable for testing and development. To use Searchly: - -- Sign up for an account at `dashboard.searchly.com/users/sign\_up`_ -- Use your Searchly dashboard to create a new index, e.g. 'wagtaildemo' -- Note the connection URL from your Searchly dashboard -- Configure ``URLS`` and ``INDEX`` in the Elasticsearch entry in ``WAGTAILSEARCH_BACKENDS`` -- Run ``./manage.py update_index`` - -.. _elasticsearch-py: http://elasticsearch-py.readthedocs.org -.. _Searchly: http://www.searchly.com/ -.. _dashboard.searchly.com/users/sign\_up: https://dashboard.searchly.com/users/sign_up - -Rolling Your Own -```````````````` -Wagtail search backends implement the interface defined in ``wagtail/wagtail/wagtailsearch/backends/base.py``. At a minimum, the backend's ``search()`` method must return a collection of objects or ``model.objects.none()``. For a fully-featured search backend, examine the Elasticsearch backend code in ``elasticsearch.py``. +This functionality is still under active development to provide a streamlined interface, but take a look at ``wagtail/wagtail/wagtailsearch/views/frontend.py`` if you are interested in coding custom search views. \ No newline at end of file diff --git a/docs/search/index.rst b/docs/search/index.rst new file mode 100644 index 000000000..e343f76c8 --- /dev/null +++ b/docs/search/index.rst @@ -0,0 +1,17 @@ + +.. _wagtailsearch: + + +Search +====== + +Wagtail provides a comprehensive and extensible search interface. In addition, it provides ways to promote search results through "Editor's Picks." Wagtail also collects simple statistics on queries made through the search interface. + + +.. toctree:: + :maxdepth: 2 + + for_python_developers + frontend_views + editors_picks + backends diff --git a/setup.py b/setup.py index 6e0bc6949..129a48468 100644 --- a/setup.py +++ b/setup.py @@ -25,9 +25,9 @@ PY3 = sys.version_info[0] == 3 install_requires = [ "Django>=1.6.2,<1.7", "South>=0.8.4", - "django-compressor>=1.3", - "django-libsass>=0.1", - "django-modelcluster>=0.1", + "django-compressor>=1.4", + "django-libsass>=0.2", + "django-modelcluster>=0.3", "django-taggit==0.11.2", "django-treebeard==2.0", "Pillow>=2.3.0", @@ -47,7 +47,7 @@ if not PY3: setup( name='wagtail', - version='0.3.1', + version='0.4', description='A Django content management system focused on flexibility and user experience', author='Matthew Westcott', author_email='matthew.westcott@torchbox.com', diff --git a/wagtail/contrib/wagtailsitemaps/tests.py b/wagtail/contrib/wagtailsitemaps/tests.py index 469e210ad..a556ce156 100644 --- a/wagtail/contrib/wagtailsitemaps/tests.py +++ b/wagtail/contrib/wagtailsitemaps/tests.py @@ -44,21 +44,21 @@ class TestSitemapGenerator(TestCase): sitemap = Sitemap(self.site) urls = [url['location'] for url in sitemap.get_urls()] - self.assertIn('/', urls) # Homepage - self.assertIn('/hello-world/', urls) # Child page + self.assertIn('http://localhost/', urls) # Homepage + self.assertIn('http://localhost/hello-world/', urls) # Child page def test_render(self): sitemap = Sitemap(self.site) xml = sitemap.render() # Check that a URL has made it into the xml - self.assertIn('/hello-world/', xml) + self.assertIn('http://localhost/hello-world/', xml) # Make sure the unpublished page didn't make it into the xml - self.assertNotIn('/unpublished/', xml) + self.assertNotIn('http://localhost/unpublished/', xml) # Make sure the protected page didn't make it into the xml - self.assertNotIn('/protected/', xml) + self.assertNotIn('http://localhost/protected/', xml) class TestSitemapView(TestCase): diff --git a/wagtail/tests/models.py b/wagtail/tests/models.py index fbf78993e..5eeca6fc3 100644 --- a/wagtail/tests/models.py +++ b/wagtail/tests/models.py @@ -11,6 +11,7 @@ from wagtail.wagtailimages.edit_handlers import ImageChooserPanel from wagtail.wagtaildocs.edit_handlers import DocumentChooserPanel from wagtail.wagtailforms.models import AbstractEmailForm, AbstractFormField from wagtail.wagtailsnippets.models import register_snippet +from wagtail.wagtailsearch import indexed EVENT_AUDIENCE_CHOICES = ( @@ -333,3 +334,60 @@ class BusinessSubIndex(Page): class BusinessChild(Page): subpage_types = [] + + +class SearchTest(models.Model, indexed.Indexed): + title = models.CharField(max_length=255) + content = models.TextField() + live = models.BooleanField(default=False) + published_date = models.DateField(null=True) + + search_fields = [ + indexed.SearchField('title', partial_match=True), + indexed.SearchField('content'), + indexed.SearchField('callable_indexed_field'), + indexed.FilterField('title'), + indexed.FilterField('live'), + indexed.FilterField('published_date'), + ] + + def callable_indexed_field(self): + return "Callable" + + +class SearchTestChild(SearchTest): + subtitle = models.CharField(max_length=255, null=True, blank=True) + extra_content = models.TextField() + + search_fields = SearchTest.search_fields + [ + indexed.SearchField('subtitle', partial_match=True), + indexed.SearchField('extra_content'), + ] + + +class SearchTestOldConfig(models.Model, indexed.Indexed): + """ + This tests that the Indexed class can correctly handle models that + use the old "indexed_fields" configuration format. + """ + indexed_fields = { + # A search field with predictive search and boosting + 'title': { + 'type': 'string', + 'analyzer': 'edgengram_analyzer', + 'boost': 100, + }, + + # A filter field + 'live': { + 'type': 'boolean', + 'index': 'not_analyzed', + }, + } + +class SearchTestOldConfigList(models.Model, indexed.Indexed): + """ + This tests that the Indexed class can correctly handle models that + use the old "indexed_fields" configuration format using a list. + """ + indexed_fields = ['title', 'content'] diff --git a/wagtail/tests/utils.py b/wagtail/tests/utils.py index 9360f2206..2e95d0210 100644 --- a/wagtail/tests/utils.py +++ b/wagtail/tests/utils.py @@ -1,3 +1,6 @@ +from contextlib import contextmanager +import warnings + from django.contrib.auth.models import User from django.utils import six @@ -25,3 +28,14 @@ class WagtailTestUtils(object): def assertRegex(self, *args, **kwargs): six.assertRegex(self, *args, **kwargs) + + @staticmethod + @contextmanager + def ignore_deprecation_warnings(): + with warnings.catch_warnings(record=True) as warning_list: # catch all warnings + yield + + # rethrow all warnings that were not DeprecationWarnings + for w in warning_list: + if not issubclass(w.category, DeprecationWarning): + warnings.showwarning(message=w.message, category=w.category, filename=w.filename, lineno=w.lineno, file=w.file, line=w.line) diff --git a/wagtail/wagtailadmin/taggable.py b/wagtail/wagtailadmin/taggable.py index 91c36ca7f..3b7de479c 100644 --- a/wagtail/wagtailadmin/taggable.py +++ b/wagtail/wagtailadmin/taggable.py @@ -21,7 +21,11 @@ class TagSearchable(indexed.Indexed): @property def get_tags(self): - return ' '.join([tag.name for tag in self.tags.all()]) + return ' '.join([tag.name for tag in self.prefetched_tags()]) + + @classmethod + def get_indexed_objects(cls): + return super(TagSearchable, cls).get_indexed_objects().prefetch_related('tagged_items__tag') @classmethod def search(cls, q, results_per_page=None, page=1, prefetch_tags=False, filters={}): diff --git a/wagtail/wagtailcore/models.py b/wagtail/wagtailcore/models.py index 11b6fe7ec..2ea9836af 100644 --- a/wagtail/wagtailcore/models.py +++ b/wagtail/wagtailcore/models.py @@ -235,6 +235,9 @@ class PageManager(models.Manager): def not_public(self): return self.get_queryset().not_public() + def search(self, query_string, fields=None, backend='default'): + return self.get_queryset().search(query_string, fields=fields, backend=backend) + class PageBase(models.base.ModelBase): """Metaclass for Page""" @@ -289,8 +292,12 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index search_fields = ( indexed.SearchField('title', partial_match=True, boost=100), + indexed.FilterField('id'), indexed.FilterField('live'), + indexed.FilterField('owner'), + indexed.FilterField('content_type'), indexed.FilterField('path'), + indexed.FilterField('depth'), ) def __init__(self, *args, **kwargs): @@ -521,7 +528,7 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index # Search s = get_search_backend() - return s.search(query_string, model=cls, fields=fields, filters=filters, prefetch_related=prefetch_related) + return s.search(query_string, cls, fields=fields, filters=filters, prefetch_related=prefetch_related) @classmethod def clean_subpage_types(cls): @@ -774,7 +781,7 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index return [ { - 'location': self.url, + 'location': self.full_url, 'lastmod': latest_revision.created_at if latest_revision else None } ] diff --git a/wagtail/wagtailcore/query.py b/wagtail/wagtailcore/query.py index e0b6cc869..a9c1bd4a8 100644 --- a/wagtail/wagtailcore/query.py +++ b/wagtail/wagtailcore/query.py @@ -2,6 +2,8 @@ from django.db.models import Q from django.contrib.contenttypes.models import ContentType from treebeard.mp_tree import MP_NodeQuerySet +from wagtail.wagtailsearch.backends import get_search_backend + class PageQuerySet(MP_NodeQuerySet): """ @@ -121,3 +123,7 @@ class PageQuerySet(MP_NodeQuerySet): def not_public(self): return self.exclude(self.public_q()) + + def search(self, query_string, fields=None, backend='default'): + search_backend = get_search_backend(backend) + return search_backend.search(query_string, self, fields=None) diff --git a/wagtail/wagtailsearch/backends/base.py b/wagtail/wagtailsearch/backends/base.py index 393ecae1b..82e2e8d56 100644 --- a/wagtail/wagtailsearch/backends/base.py +++ b/wagtail/wagtailsearch/backends/base.py @@ -1,6 +1,9 @@ from django.db import models +from django.db.models.query import QuerySet +from django.core.exceptions import ImproperlyConfigured from wagtail.wagtailsearch.indexed import Indexed +from wagtail.wagtailsearch.utils import normalise_query_string class BaseSearch(object): @@ -32,5 +35,38 @@ class BaseSearch(object): def delete(self, obj): return NotImplemented - def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]): + def _search(self, queryset, query_string, fields=None): return NotImplemented + + def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None): + # Find model/queryset + if isinstance(model_or_queryset, QuerySet): + model = model_or_queryset.model + queryset = model_or_queryset + else: + model = model_or_queryset + queryset = model_or_queryset.objects.all() + + # Model must be a descendant of Indexed and be a django model + if not issubclass(model, Indexed) or not issubclass(model, models.Model): + return [] + + # Normalise query string + if query_string is not None: + query_string = normalise_query_string(query_string) + + # Check that theres still a query string after the clean up + if query_string == "": + return [] + + # Apply filters to queryset + if filters: + queryset = queryset.filter(**filters) + + # Prefetch related + if prefetch_related: + for prefetch in prefetch_related: + queryset = queryset.prefetch_related(prefetch) + + # Search + return self._search(queryset, query_string, fields=fields) diff --git a/wagtail/wagtailsearch/backends/db.py b/wagtail/wagtailsearch/backends/db.py index 4c05b19df..a94fe3c58 100644 --- a/wagtail/wagtailsearch/backends/db.py +++ b/wagtail/wagtailsearch/backends/db.py @@ -2,7 +2,6 @@ from django.db import models from wagtail.wagtailsearch.backends.base import BaseSearch from wagtail.wagtailsearch.indexed import Indexed -from wagtail.wagtailsearch.utils import normalise_query_string class DBSearch(BaseSearch): @@ -27,47 +26,33 @@ class DBSearch(BaseSearch): def delete(self, obj): pass # Not needed - def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]): - # Normalise query string - query_string = normalise_query_string(query_string) + def _search(self, queryset, query_string, fields=None): + if query_string is not None: + # Get fields + if fields is None: + fields = [field.field_name for field in queryset.model.get_searchable_search_fields()] - # Get terms - terms = query_string.split() - if not terms: - return model.objects.none() + # Get terms + terms = query_string.split() + if not terms: + return queryset.model.objects.none() - # Get fields - if fields is None: - fields = [field.field_name for field in model.get_searchable_search_fields()] + # Filter by terms + for term in terms: + term_query = models.Q() + for field_name in fields: + # Check if the field exists (this will filter out indexed callables) + try: + queryset.model._meta.get_field_by_name(field_name) + except: + continue - # Start will all objects - query = model.objects.all() + # Filter on this field + term_query |= models.Q(**{'%s__icontains' % field_name: term}) - # Apply filters - if filters: - query = query.filter(**filters) + queryset = queryset.filter(term_query) - # Filter by terms - for term in terms: - term_query = models.Q() - for field_name in fields: - # Check if the field exists (this will filter out indexed callables) - try: - model._meta.get_field_by_name(field_name) - except: - continue + # Distinct + queryset = queryset.distinct() - # Filter on this field - term_query |= models.Q(**{'%s__icontains' % field_name: term}) - - query = query.filter(term_query) - - # Distinct - query = query.distinct() - - # Prefetch related - if prefetch_related: - for prefetch in prefetch_related: - query = query.prefetch_related(prefetch) - - return query + return queryset diff --git a/wagtail/wagtailsearch/backends/elasticsearch.py b/wagtail/wagtailsearch/backends/elasticsearch.py index 8fc2950dd..55f1971a2 100644 --- a/wagtail/wagtailsearch/backends/elasticsearch.py +++ b/wagtail/wagtailsearch/backends/elasticsearch.py @@ -3,13 +3,13 @@ from __future__ import absolute_import import json from django.db import models +from django.db.models.sql.where import SubqueryConstraint from elasticsearch import Elasticsearch, NotFoundError, RequestError from elasticsearch.helpers import bulk from wagtail.wagtailsearch.backends.base import BaseSearch from wagtail.wagtailsearch.indexed import Indexed, SearchField, FilterField -from wagtail.wagtailsearch.utils import normalise_query_string class ElasticSearchMapping(object): @@ -109,12 +109,132 @@ class ElasticSearchMapping(object): return '' % (self.model.__name__, ) +class FilterError(Exception): + pass + + +class FieldError(Exception): + pass + + class ElasticSearchQuery(object): - def __init__(self, model, query_string, fields=None, filters={}): - self.model = model + def __init__(self, queryset, query_string, fields=None): + self.queryset = queryset self.query_string = query_string - self.fields = fields or ['_all', '_partials'] - self.filters = filters + self.fields = fields + + def _get_filters_from_where(self, where_node): + # Check if this is a leaf node + if isinstance(where_node, tuple): + field_name = where_node[0].col + lookup = where_node[1] + value = where_node[3] + + # Get field + field = dict( + (field.get_attname(self.queryset.model), field) + for field in self.queryset.model.get_filterable_search_fields() + ).get(field_name, None) + + # Give error if the field doesn't exist + if field is None: + raise FieldError('Cannot filter ElasticSearch results with field "' + field_name + '". Please add FilterField(\'' + field_name + '\') to ' + self.queryset.model.__name__ + '.search_fields.') + + # Get the name of the field in the index + field_index_name = field.get_index_name(self.queryset.model) + + # Find lookup + if lookup == 'exact': + if value is None: + return { + 'missing': { + 'field': field_index_name, + } + } + else: + return { + 'term': { + field_index_name: value, + } + } + + if lookup == 'isnull': + if value: + return { + 'missing': { + 'field': field_index_name, + } + } + else: + return { + 'not': { + 'missing': { + 'field': field_index_name, + } + } + } + + if lookup in ['startswith', 'prefix']: + return { + 'prefix': { + field_index_name: value, + } + } + + if lookup in ['gt', 'gte', 'lt', 'lte']: + return { + 'range': { + field_index_name: { + lookup: value, + } + } + } + + if lookup == 'range': + lower, upper = value + + return { + 'range': { + field_index_name: { + 'gte': lower, + 'lte': upper, + } + } + } + + if lookup == 'in': + return { + 'terms': { + field_index_name: value, + } + } + + raise FilterError('Could not apply filter on ElasticSearch results: "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.') + elif isinstance(where_node, SubqueryConstraint): + raise FilterError('Could not apply filter on ElasticSearch results: Subqueries are not allowed.') + + # Get child filters + connector = where_node.connector + child_filters = [self._get_filters_from_where(child) for child in where_node.children] + child_filters = [child_filter for child_filter in child_filters if child_filter] + + # Connect them + if child_filters: + if len(child_filters) == 1: + filter_out = child_filters[0] + else: + filter_out = { + connector.lower(): [ + fil for fil in child_filters if fil is not None + ] + } + + if where_node.negated: + filter_out = { + 'not': filter_out + } + + return filter_out def _get_filters(self): # Filters @@ -123,85 +243,60 @@ class ElasticSearchQuery(object): # Filter by content type filters.append({ 'prefix': { - 'content_type': self.model.indexed_get_content_type() + 'content_type': self.queryset.model.indexed_get_content_type() } }) - # Extra filters - if self.filters: - for key, value in self.filters.items(): - if '__' in key: - field, lookup = key.split('__') - else: - field = key - lookup = None - - if lookup is None: - if value is None: - filters.append({ - 'missing': { - 'field': field, - } - }) - else: - filters.append({ - 'term': { - field: value - } - }) - - if lookup in ['startswith', 'prefix']: - filters.append({ - 'prefix': { - field: value - } - }) - - if lookup in ['gt', 'gte', 'lt', 'lte']: - filters.append({ - 'range': { - field: { - lookup: value, - } - } - }) - - if lookup == 'range': - lower, upper = value - filters.append({ - 'range': { - field: { - 'gte': lower, - 'lte': upper, - } - } - }) + # Apply filters from queryset + queryset_filters = self._get_filters_from_where(self.queryset.query.where) + if queryset_filters: + filters.append(queryset_filters) return filters def to_es(self): # Query - query = { - 'query_string': { - 'query': self.query_string, - } - } + if self.query_string is not None: + fields = self.fields or ['_all', '_partials'] - # Fields - if self.fields: - query['query_string']['fields'] = self.fields + if len(fields) == 1: + query = { + 'match': { + fields[0]: self.query_string, + } + } + else: + query = { + 'multi_match': { + 'query': self.query_string, + 'fields': fields, + } + } + else: + query = { + 'match_all': {} + } # Filters filters = self._get_filters() - - return { - 'filtered': { - 'query': query, - 'filter': { - 'and': filters, + if len(filters) == 1: + query = { + 'filtered': { + 'query': query, + 'filter': filters[0], } } - } + elif len(filters) > 1: + query = { + 'filtered': { + 'query': query, + 'filter': { + 'and': filters, + } + } + } + + return query def __repr__(self): return json.dumps(self.to_es()) @@ -263,15 +358,8 @@ class ElasticSearchResults(object): # Initialise results dictionary results = dict((str(pk), None) for pk in pks) - # Get queryset - queryset = self.query.model.objects.filter(pk__in=pks) - - # Add prefetch related - if self.prefetch_related: - for prefetch in self.prefetch_related: - queryset = queryset.prefetch_related(prefetch) - # Find objects in database and add them to dict + queryset = self.query.queryset.filter(pk__in=pks) for obj in queryset: results[str(obj.pk)] = obj @@ -502,17 +590,5 @@ class ElasticSearch(BaseSearch): except NotFoundError: pass # Document doesn't exist, ignore this exception - def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]): - # Model must be a descendant of Indexed and be a django model - if not issubclass(model, Indexed) or not issubclass(model, models.Model): - return [] - - # Normalise query string - query_string = normalise_query_string(query_string) - - # Check that theres still a query string after the clean up - if not query_string: - return [] - - # Return search results - return ElasticSearchResults(self, ElasticSearchQuery(model, query_string, fields=fields, filters=filters), prefetch_related=prefetch_related) + def _search(self, queryset, query_string, fields=None): + return ElasticSearchResults(self, ElasticSearchQuery(queryset, query_string, fields=fields)) diff --git a/wagtail/wagtailsearch/indexed.py b/wagtail/wagtailsearch/indexed.py index b290f440b..2e506d690 100644 --- a/wagtail/wagtailsearch/indexed.py +++ b/wagtail/wagtailsearch/indexed.py @@ -112,13 +112,20 @@ class Indexed(object): @classmethod def get_searchable_search_fields(cls): - return filter(lambda field: field.searchable, cls.get_search_fields()) + return filter(lambda field: isinstance(field, SearchField), cls.get_search_fields()) + + @classmethod + def get_filterable_search_fields(cls): + return filter(lambda field: isinstance(field, FilterField), cls.get_search_fields()) + + @classmethod + def get_indexed_objects(cls): + return cls.objects.all() indexed_fields = () class BaseField(object): - searchable = False suffix = '' def __init__(self, field_name, **kwargs): @@ -163,8 +170,6 @@ class BaseField(object): class SearchField(BaseField): - searchable = True - def __init__(self, field_name, boost=None, partial_match=False, **kwargs): super(SearchField, self).__init__(field_name, **kwargs) self.boost = boost diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py index 2c1593635..c52b75639 100644 --- a/wagtail/wagtailsearch/management/commands/update_index.py +++ b/wagtail/wagtailsearch/management/commands/update_index.py @@ -24,7 +24,7 @@ class Command(BaseCommand): toplevel_content_type = model.indexed_get_toplevel_content_type() # Loop through objects - for obj in model.objects.all(): + for obj in model.get_indexed_objects(): # Get key for this object key = toplevel_content_type + ':' + str(obj.pk) diff --git a/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py new file mode 100644 index 000000000..6826ddaa1 --- /dev/null +++ b/wagtail/wagtailsearch/migrations/0003_auto__del_searchtestchild__del_searchtest.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Deleting model 'SearchTestChild' + db.delete_table('wagtailsearch_searchtestchild') + + # Deleting model 'SearchTest' + db.delete_table('wagtailsearch_searchtest') + + + def backwards(self, orm): + # Adding model 'SearchTestChild' + db.create_table('wagtailsearch_searchtestchild', ( + ('extra_content', self.gf('django.db.models.fields.TextField')()), + ('searchtest_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['wagtailsearch.SearchTest'], unique=True, primary_key=True)), + )) + db.send_create_signal('wagtailsearch', ['SearchTestChild']) + + # Adding model 'SearchTest' + db.create_table('wagtailsearch_searchtest', ( + ('content', self.gf('django.db.models.fields.TextField')()), + ('live', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=255)), + )) + db.send_create_signal('wagtailsearch', ['SearchTest']) + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'user_set'", 'blank': 'True', 'to': "orm['auth.Group']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'user_set'", 'blank': 'True', 'to': "orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'wagtailcore.page': { + 'Meta': {'object_name': 'Page'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'pages'", 'to': "orm['contenttypes.ContentType']"}), + 'depth': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'has_unpublished_changes': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'live': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'numchild': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'owned_pages'", 'null': 'True', 'to': "orm['auth.User']"}), + 'path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'search_description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'seo_title': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'show_in_menus': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'url_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}) + }, + 'wagtailsearch.editorspick': { + 'Meta': {'ordering': "('sort_order',)", 'object_name': 'EditorsPick'}, + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'page': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['wagtailcore.Page']"}), + 'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editors_picks'", 'to': "orm['wagtailsearch.Query']"}), + 'sort_order': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + }, + 'wagtailsearch.query': { + 'Meta': {'object_name': 'Query'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query_string': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'wagtailsearch.querydailyhits': { + 'Meta': {'unique_together': "(('query', 'date'),)", 'object_name': 'QueryDailyHits'}, + 'date': ('django.db.models.fields.DateField', [], {}), + 'hits': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'query': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'daily_hits'", 'to': "orm['wagtailsearch.Query']"}) + } + } + + complete_apps = ['wagtailsearch'] \ No newline at end of file diff --git a/wagtail/wagtailsearch/models.py b/wagtail/wagtailsearch/models.py index f6cfdf1dd..f82bb2462 100644 --- a/wagtail/wagtailsearch/models.py +++ b/wagtail/wagtailsearch/models.py @@ -81,29 +81,3 @@ class EditorsPick(models.Model): class Meta: ordering = ('sort_order', ) - - -# Used for tests - -class SearchTest(models.Model, indexed.Indexed): - title = models.CharField(max_length=255) - content = models.TextField() - live = models.BooleanField(default=False) - - search_fields = ( - indexed.SearchField('title'), - indexed.SearchField('content'), - indexed.SearchField('callable_indexed_field'), - indexed.SearchField('live'), - ) - - def callable_indexed_field(self): - return "Callable" - - -class SearchTestChild(SearchTest): - extra_content = models.TextField() - - search_fields = SearchTest.search_fields + ( - indexed.SearchField('extra_content'), - ) diff --git a/wagtail/wagtailsearch/tests/test_backends.py b/wagtail/wagtailsearch/tests/test_backends.py index 86a67f21a..9baf64957 100644 --- a/wagtail/wagtailsearch/tests/test_backends.py +++ b/wagtail/wagtailsearch/tests/test_backends.py @@ -1,17 +1,18 @@ from six import StringIO +import warnings from django.test import TestCase from django.test.utils import override_settings from django.conf import settings from django.core import management -from wagtail.tests.utils import unittest -from wagtail.wagtailsearch import models, get_search_backend +from wagtail.tests.utils import unittest, WagtailTestUtils +from wagtail.tests import models +from wagtail.wagtailsearch.backends import get_search_backend, InvalidSearchBackendError from wagtail.wagtailsearch.backends.db import DBSearch -from wagtail.wagtailsearch.backends import InvalidSearchBackendError -class BackendTests(object): +class BackendTests(WagtailTestUtils): # To test a specific backend, subclass BackendTests and define self.backend_path. def setUp(self): @@ -144,34 +145,14 @@ class BackendTests(object): self.backend.reset_index() # Run update_index command - management.call_command('update_index', backend=self.backend, interactive=False, stdout=StringIO()) + with self.ignore_deprecation_warnings(): # ignore any DeprecationWarnings thrown by models with old-style indexed_fields definitions + management.call_command('update_index', backend=self.backend, interactive=False, stdout=StringIO()) # Check that there are still 3 results results = self.backend.search("Hello", models.SearchTest) self.assertEqual(len(results), 3) -class TestDBBackend(BackendTests, TestCase): - backend_path = 'wagtail.wagtailsearch.backends.db.DBSearch' - - @unittest.expectedFailure - def test_callable_indexed_field(self): - super(TestDBBackend, self).test_callable_indexed_field() - - -class TestElasticSearchBackend(BackendTests, TestCase): - backend_path = 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch' - - def test_search_with_spaces_only(self): - # Search for some space characters and hope it doesn't crash - results = self.backend.search(" ", models.SearchTest) - - # Queries are lazily evaluated, force it to run - list(results) - - # Didn't crash, yay! - - @override_settings(WAGTAILSEARCH_BACKENDS={ 'default': {'BACKEND': 'wagtail.wagtailsearch.backends.db.DBSearch'} }) diff --git a/wagtail/wagtailsearch/tests/test_db_backend.py b/wagtail/wagtailsearch/tests/test_db_backend.py new file mode 100644 index 000000000..f471a34d1 --- /dev/null +++ b/wagtail/wagtailsearch/tests/test_db_backend.py @@ -0,0 +1,13 @@ +from wagtail.tests.utils import unittest + +from django.test import TestCase + +from .test_backends import BackendTests + + +class TestDBBackend(BackendTests, TestCase): + backend_path = 'wagtail.wagtailsearch.backends.db.DBSearch' + + @unittest.expectedFailure + def test_callable_indexed_field(self): + super(TestDBBackend, self).test_callable_indexed_field() diff --git a/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py new file mode 100644 index 000000000..a95d442ff --- /dev/null +++ b/wagtail/wagtailsearch/tests/test_elasticsearch_backend.py @@ -0,0 +1,399 @@ +from wagtail.tests.utils import unittest +import datetime +import json + +from django.test import TestCase +from django.db.models import Q + +from wagtail.tests import models +from .test_backends import BackendTests + + +class TestElasticSearchBackend(BackendTests, TestCase): + backend_path = 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch' + + def test_search_with_spaces_only(self): + # Search for some space characters and hope it doesn't crash + results = self.backend.search(" ", models.SearchTest) + + # Queries are lazily evaluated, force it to run + list(results) + + # Didn't crash, yay! + + def test_partial_search(self): + # Reset the index + self.backend.reset_index() + self.backend.add_type(models.SearchTest) + self.backend.add_type(models.SearchTestChild) + + # Add some test data + obj = models.SearchTest() + obj.title = "HelloWorld" + obj.live = True + obj.save() + self.backend.add(obj) + + # Refresh the index + self.backend.refresh_index() + + # Search and check + results = self.backend.search("HelloW", models.SearchTest.objects.all()) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].id, obj.id) + + def test_child_partial_search(self): + # Reset the index + self.backend.reset_index() + self.backend.add_type(models.SearchTest) + self.backend.add_type(models.SearchTestChild) + + obj = models.SearchTestChild() + obj.title = "WorldHello" + obj.subtitle = "HelloWorld" + obj.live = True + obj.save() + self.backend.add(obj) + + # Refresh the index + self.backend.refresh_index() + + # Search and check + results = self.backend.search("HelloW", models.SearchTest.objects.all()) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0].id, obj.id) + + +class TestElasticSearchQuery(TestCase): + def assertDictEqual(self, a, b): + default = self.JSONSerializer().default + self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default)) + + def setUp(self): + # Import using a try-catch block to prevent crashes if the elasticsearch-py + # module is not installed + try: + from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchQuery + from elasticsearch.serializer import JSONSerializer + except ImportError: + raise unittest.SkipTest("elasticsearch-py not installed") + + self.ElasticSearchQuery = ElasticSearchQuery + self.JSONSerializer = JSONSerializer + + def test_simple(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_none_query_string(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.all(), None) + + # Check it + expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'match_all': {}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_filter(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test"), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_and_filter(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test", live=True), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + + # Make sure field filters are sorted (as they can be in any order which may cause false positives) + query = query.to_es() + field_filters = query['filtered']['filter']['and'][1]['and'] + field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0]) + + self.assertDictEqual(query, expected_result) + + def test_or_filter(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(Q(title="Test") | Q(live=True)), "Hello") + + # Make sure field filters are sorted (as they can be in any order which may cause false positives) + query = query.to_es() + field_filters = query['filtered']['filter']['and'][1]['or'] + field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0]) + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query, expected_result) + + def test_negated_filter(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.exclude(live=True), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_fields(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello", fields=['title']) + + # Check it + expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'match': {'title': 'Hello'}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_exact_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__exact="Test"), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_none_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title=None), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_isnull_true_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=True), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_isnull_false_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=False), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_startswith_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__startswith="Test"), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_gt_lookup(self): + # This also tests conversion of python dates to strings + + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gt=datetime.datetime(2014, 4, 29)), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_lt_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lt=datetime.datetime(2014, 4, 29)), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_gte_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gte=datetime.datetime(2014, 4, 29)), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_lte_lookup(self): + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lte=datetime.datetime(2014, 4, 29)), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + def test_range_lookup(self): + start_date = datetime.datetime(2014, 4, 29) + end_date = datetime.datetime(2014, 8, 19) + + # Create a query + query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__range=(start_date, end_date)), "Hello") + + # Check it + expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}} + self.assertDictEqual(query.to_es(), expected_result) + + +class TestElasticSearchMapping(TestCase): + def assertDictEqual(self, a, b): + default = self.JSONSerializer().default + self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default)) + + def setUp(self): + # Import using a try-catch block to prevent crashes if the elasticsearch-py + # module is not installed + try: + from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping + from elasticsearch.serializer import JSONSerializer + except ImportError: + raise unittest.SkipTest("elasticsearch-py not installed") + + self.JSONSerializer = JSONSerializer + + # Create ES mapping + self.es_mapping = ElasticSearchMapping(models.SearchTest) + + # Create ES document + self.obj = models.SearchTest(title="Hello") + self.obj.save() + + def test_get_document_type(self): + self.assertEqual(self.es_mapping.get_document_type(), 'tests_searchtest') + + def test_get_mapping(self): + # Build mapping + mapping = self.es_mapping.get_mapping() + + # Check + expected_result = { + 'tests_searchtest': { + 'properties': { + 'pk': {'index': 'not_analyzed', 'type': 'string', 'store': 'yes', 'include_in_all': False}, + 'content_type': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + '_partials': {'analyzer': 'edgengram_analyzer', 'include_in_all': False, 'type': 'string'}, + 'live_filter': {'index': 'not_analyzed', 'type': 'boolean', 'include_in_all': False}, + 'published_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False}, + 'title': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'}, + 'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'content': {'type': 'string', 'include_in_all': True}, + 'callable_indexed_field': {'type': 'string', 'include_in_all': True} + } + } + } + + self.assertDictEqual(mapping, expected_result) + + def test_get_document_id(self): + self.assertEqual(self.es_mapping.get_document_id(self.obj), 'tests_searchtest:' + str(self.obj.pk)) + + def test_get_document(self): + # Get document + document = self.es_mapping.get_document(self.obj) + + # Check + expected_result = { + 'pk': str(self.obj.pk), + 'content_type': 'tests_searchtest', + '_partials': ['Hello'], + 'live_filter': False, + 'published_date_filter': None, + 'title': 'Hello', + 'title_filter': 'Hello', + 'callable_indexed_field': 'Callable', + 'content': '', + } + + self.assertDictEqual(document, expected_result) + + +class TestElasticSearchMappingInheritance(TestCase): + def assertDictEqual(self, a, b): + default = self.JSONSerializer().default + self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps(b, sort_keys=True, default=default)) + + def setUp(self): + # Import using a try-catch block to prevent crashes if the elasticsearch-py + # module is not installed + try: + from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping + from elasticsearch.serializer import JSONSerializer + except ImportError: + raise unittest.SkipTest("elasticsearch-py not installed") + + self.JSONSerializer = JSONSerializer + + # Create ES mapping + self.es_mapping = ElasticSearchMapping(models.SearchTestChild) + + # Create ES document + self.obj = models.SearchTestChild(title="Hello", subtitle="World") + self.obj.save() + + def test_get_document_type(self): + self.assertEqual(self.es_mapping.get_document_type(), 'tests_searchtest_tests_searchtestchild') + + def test_get_mapping(self): + # Build mapping + mapping = self.es_mapping.get_mapping() + + # Check + expected_result = { + 'tests_searchtest_tests_searchtestchild': { + 'properties': { + # New + 'extra_content': {'type': 'string', 'include_in_all': True}, + 'subtitle': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'}, + + # Inherited + 'pk': {'index': 'not_analyzed', 'type': 'string', 'store': 'yes', 'include_in_all': False}, + 'content_type': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + '_partials': {'analyzer': 'edgengram_analyzer', 'include_in_all': False, 'type': 'string'}, + 'live_filter': {'index': 'not_analyzed', 'type': 'boolean', 'include_in_all': False}, + 'published_date_filter': {'index': 'not_analyzed', 'type': 'date', 'include_in_all': False}, + 'title': {'type': 'string', 'include_in_all': True, 'analyzer': 'edgengram_analyzer'}, + 'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False}, + 'content': {'type': 'string', 'include_in_all': True}, + 'callable_indexed_field': {'type': 'string', 'include_in_all': True} + } + } + } + + self.assertDictEqual(mapping, expected_result) + + def test_get_document_id(self): + # This must be tests_searchtest instead of 'tests_searchtest_tests_searchtestchild' + # as it uses the contents base content type name. + # This prevents the same object being accidentally indexed twice. + self.assertEqual(self.es_mapping.get_document_id(self.obj), 'tests_searchtest:' + str(self.obj.pk)) + + def test_get_document(self): + # Build document + document = self.es_mapping.get_document(self.obj) + + # Sort partials + if '_partials' in document: + document['_partials'].sort() + + # Check + expected_result = { + # New + 'extra_content': '', + 'subtitle': 'World', + + # Changed + 'content_type': 'tests_searchtest_tests_searchtestchild', + + # Inherited + 'pk': str(self.obj.pk), + '_partials': ['Hello', 'World'], + 'live_filter': False, + 'published_date_filter': None, + 'title': 'Hello', + 'title_filter': 'Hello', + 'callable_indexed_field': 'Callable', + 'content': '', + } + + self.assertDictEqual(document, expected_result) diff --git a/wagtail/wagtailsearch/tests/test_indexed_class.py b/wagtail/wagtailsearch/tests/test_indexed_class.py new file mode 100644 index 000000000..983d8e0ba --- /dev/null +++ b/wagtail/wagtailsearch/tests/test_indexed_class.py @@ -0,0 +1,53 @@ +import warnings + +from django.test import TestCase + +from wagtail.wagtailsearch import indexed +from wagtail.tests import models +from wagtail.tests.utils import WagtailTestUtils + + +class TestContentTypeNames(TestCase): + def test_base_content_type_name(self): + name = models.SearchTestChild.indexed_get_toplevel_content_type() + self.assertEqual(name, 'tests_searchtest') + + def test_qualified_content_type_name(self): + name = models.SearchTestChild.indexed_get_content_type() + self.assertEqual(name, 'tests_searchtest_tests_searchtestchild') + + +class TestIndexedFieldsBackwardsCompatibility(TestCase, WagtailTestUtils): + def test_indexed_fields_backwards_compatibility(self): + # Get search fields + with self.ignore_deprecation_warnings(): + search_fields = models.SearchTestOldConfig.get_search_fields() + + search_fields_dict = dict( + ((field.field_name, type(field)), field) + for field in search_fields + ) + + # Check that the fields were found + self.assertEqual(len(search_fields_dict), 2) + self.assertIn(('title', indexed.SearchField), search_fields_dict.keys()) + self.assertIn(('live', indexed.FilterField), search_fields_dict.keys()) + + # Check that the title field has the correct settings + self.assertTrue(search_fields_dict[('title', indexed.SearchField)].partial_match) + self.assertEqual(search_fields_dict[('title', indexed.SearchField)].boost, 100) + + def test_indexed_fields_backwards_compatibility_list(self): + # Get search fields + with self.ignore_deprecation_warnings(): + search_fields = models.SearchTestOldConfigList.get_search_fields() + + search_fields_dict = dict( + ((field.field_name, type(field)), field) + for field in search_fields + ) + + # Check that the fields were found + self.assertEqual(len(search_fields_dict), 2) + self.assertIn(('title', indexed.SearchField), search_fields_dict.keys()) + self.assertIn(('content', indexed.SearchField), search_fields_dict.keys())