From 20636a7119ce8c2060152b98c5d621bac923c69b Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Tue, 19 Aug 2014 11:36:16 +0100 Subject: [PATCH 1/6] Added multiple backend support into update_index command. Fixes #552 --- .../management/commands/update_index.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py index c52b75639..05e2e8f9f 100644 --- a/wagtail/wagtailsearch/management/commands/update_index.py +++ b/wagtail/wagtailsearch/management/commands/update_index.py @@ -1,11 +1,20 @@ from django.core.management.base import BaseCommand from django.db import models +from django.conf import settings from wagtail.wagtailsearch import Indexed, get_search_backend +def get_search_backends(): + if hasattr(settings, 'WAGTAILSEARCH_BACKENDS'): + for backend in settings.WAGTAILSEARCH_BACKENDS.keys(): + yield backend, get_search_backend(backend) + else: + yield 'default', get_search_backend('default') + + class Command(BaseCommand): - def handle(self, **options): + def get_object_list(self): # Print info self.stdout.write("Getting object list") @@ -40,26 +49,41 @@ class Command(BaseCommand): # Space free, take it object_set[key] = obj - # Search backend - if 'backend' in options: - s = options['backend'] - else: - s = get_search_backend() + return indexed_models, object_set.values() + + def update_backend(self, backend, models, object_list, backend_name=''): + # Print info + self.stdout.write("Updating backend: " + backend_name) + + # Get backend + if backend is None: + backend = get_search_backend(backend_name) # Reset the index - self.stdout.write("Reseting index") - s.reset_index() + self.stdout.write(backend_name + ": Reseting index") + backend.reset_index() # Add types - self.stdout.write("Adding types") - for model in indexed_models: - s.add_type(model) + self.stdout.write(backend_name + ": Adding types") + for model in models: + backend.add_type(model) # Add objects to index - self.stdout.write("Adding objects") - for result in s.add_bulk(object_set.values()): + self.stdout.write(backend_name + ": Adding objects") + for result in backend.add_bulk(object_list): self.stdout.write(result[0] + ' ' + str(result[1])) # Refresh index - self.stdout.write("Refreshing index") - s.refresh_index() + self.stdout.write(backend_name + ": Refreshing index") + backend.refresh_index() + + def handle(self, **options): + # Get object list + models, object_list = self.get_object_list() + + # Update backends + if 'backend' in options: + self.update_backend(options['backend'], models, object_list) + else: + for backend_name, backend in get_search_backends(): + self.update_backend(backend, models, object_list, backend_name=backend_name) From 64245e9c1ad95f035e4a6b3c035f30ab9dc92488 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 28 Aug 2014 16:28:24 +0100 Subject: [PATCH 2/6] Added opton to update_index to allow user to select which backend to update --- .../management/commands/update_index.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py index 05e2e8f9f..f0880b886 100644 --- a/wagtail/wagtailsearch/management/commands/update_index.py +++ b/wagtail/wagtailsearch/management/commands/update_index.py @@ -1,3 +1,5 @@ +from optparse import make_option + from django.core.management.base import BaseCommand from django.db import models from django.conf import settings @@ -77,6 +79,15 @@ class Command(BaseCommand): self.stdout.write(backend_name + ": Refreshing index") backend.refresh_index() + option_list = BaseCommand.option_list + ( + make_option('--backend', + action='store', + dest='backend_name', + default=False, + help="Specify a backend to update", + ), + ) + def handle(self, **options): # Get object list models, object_list = self.get_object_list() @@ -84,6 +95,9 @@ class Command(BaseCommand): # Update backends if 'backend' in options: self.update_backend(options['backend'], models, object_list) + elif 'backend_name' in options: + backend = dict(get_search_backends())[options['backend_name']] + self.update_backend(backend, models, object_list, backend_name=options['backend_name']) else: for backend_name, backend in get_search_backends(): self.update_backend(backend, models, object_list, backend_name=backend_name) From 1bce98795e4306859abd60642c2f39c8d205fb01 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 18 Sep 2014 10:37:09 +0100 Subject: [PATCH 3/6] Docs for update_index --backend option --- docs/reference/management_commands.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/reference/management_commands.rst b/docs/reference/management_commands.rst index 18358a869..9daffacce 100644 --- a/docs/reference/management_commands.rst +++ b/docs/reference/management_commands.rst @@ -47,7 +47,7 @@ Options: update_index ------------ -:code:`./manage.py update_index` +:code:`./manage.py update_index [--backend ]` This command rebuilds the search index from scratch. It is only required when using Elasticsearch. @@ -59,6 +59,24 @@ It is recommended to run this command once a week and at the following times: The search may not return any results while this command is running, so avoid running it at peak times. +Specifying which backend to update +`````````````````````````````````` + +.. versionadded:: 0.7 + + +By default, ``update_index`` will rebuild all the search indexes listed in ``WAGTAILSEARCH_BACKENDS``. + +If you have multiple backends and would only like to update one of them, you can use the ``--backend`` option. + +For example, to update just the default backend: + +.. code-block:: + + python manage.py update_index --backend default + + + .. _search_garbage_collect: search_garbage_collect From 9a752a2210dd742790547d5a2217cce4d1169a55 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Sun, 5 Oct 2014 14:24:37 +0100 Subject: [PATCH 4/6] Pass backend name to update_index command in tests --- wagtail/wagtailsearch/management/commands/update_index.py | 4 +--- wagtail/wagtailsearch/tests/test_backends.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py index f0880b886..e209fd526 100644 --- a/wagtail/wagtailsearch/management/commands/update_index.py +++ b/wagtail/wagtailsearch/management/commands/update_index.py @@ -93,9 +93,7 @@ class Command(BaseCommand): models, object_list = self.get_object_list() # Update backends - if 'backend' in options: - self.update_backend(options['backend'], models, object_list) - elif 'backend_name' in options: + if 'backend_name' in options: backend = dict(get_search_backends())[options['backend_name']] self.update_backend(backend, models, object_list, backend_name=options['backend_name']) else: diff --git a/wagtail/wagtailsearch/tests/test_backends.py b/wagtail/wagtailsearch/tests/test_backends.py index 9baf64957..77733f0be 100644 --- a/wagtail/wagtailsearch/tests/test_backends.py +++ b/wagtail/wagtailsearch/tests/test_backends.py @@ -20,6 +20,7 @@ class BackendTests(WagtailTestUtils): for backend_name, backend_conf in settings.WAGTAILSEARCH_BACKENDS.items(): if backend_conf['BACKEND'] == self.backend_path: self.backend = get_search_backend(backend_name) + self.backend_name = backend_name break else: # no conf entry found - skip tests for this backend @@ -146,7 +147,7 @@ class BackendTests(WagtailTestUtils): # Run update_index command with self.ignore_deprecation_warnings(): # ignore any DeprecationWarnings thrown by models with old-style indexed_fields definitions - management.call_command('update_index', backend=self.backend, interactive=False, stdout=StringIO()) + management.call_command('update_index', backend_name=self.backend_name, interactive=False, stdout=StringIO()) # Check that there are still 3 results results = self.backend.search("Hello", models.SearchTest) From ebb6223f63229ced2f8bae93b148c88636736058 Mon Sep 17 00:00:00 2001 From: Matt Westcott Date: Mon, 6 Oct 2014 13:15:03 +0200 Subject: [PATCH 5/6] Fix option handling in update_index and refactor to avoid unnecessary backend/backend_name juggling --- .../management/commands/update_index.py | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/wagtail/wagtailsearch/management/commands/update_index.py b/wagtail/wagtailsearch/management/commands/update_index.py index 2e4f981c9..2c9002c4a 100644 --- a/wagtail/wagtailsearch/management/commands/update_index.py +++ b/wagtail/wagtailsearch/management/commands/update_index.py @@ -8,14 +8,6 @@ from wagtail.wagtailsearch.indexed import Indexed from wagtail.wagtailsearch.backends import get_search_backend -def get_search_backends(): - if hasattr(settings, 'WAGTAILSEARCH_BACKENDS'): - for backend in settings.WAGTAILSEARCH_BACKENDS.keys(): - yield backend, get_search_backend(backend) - else: - yield 'default', get_search_backend('default') - - class Command(BaseCommand): def get_object_list(self): # Print info @@ -54,13 +46,12 @@ class Command(BaseCommand): return indexed_models, object_set.values() - def update_backend(self, backend, models, object_list, backend_name=''): + def update_backend(self, backend_name, models, object_list): # Print info self.stdout.write("Updating backend: " + backend_name) # Get backend - if backend is None: - backend = get_search_backend(backend_name) + backend = get_search_backend(backend_name) # Reset the index self.stdout.write(backend_name + ": Reseting index") @@ -84,7 +75,7 @@ class Command(BaseCommand): make_option('--backend', action='store', dest='backend_name', - default=False, + default=None, help="Specify a backend to update", ), ) @@ -93,10 +84,17 @@ class Command(BaseCommand): # Get object list models, object_list = self.get_object_list() - # Update backends - if 'backend_name' in options: - backend = dict(get_search_backends())[options['backend_name']] - self.update_backend(backend, models, object_list, backend_name=options['backend_name']) + # Get list of backends to index + if options['backend_name']: + # index only the passed backend + backend_names = [options['backend_name']] + elif hasattr(settings, 'WAGTAILSEARCH_BACKENDS'): + # index all backends listed in settings + backend_names = settings.WAGTAILSEARCH_BACKENDS.keys() else: - for backend_name, backend in get_search_backends(): - self.update_backend(backend, models, object_list, backend_name=backend_name) + # index the 'default' backend only + backend_names = ['default'] + + # Update backends + for backend_name in backend_names: + self.update_backend(backend_name, models, object_list) From 39ce3ce5cad832dd0579eb9b114af4e00534eec2 Mon Sep 17 00:00:00 2001 From: Matt Westcott Date: Mon, 6 Oct 2014 13:23:18 +0200 Subject: [PATCH 6/6] Release note for #556 --- CHANGELOG.txt | 1 + docs/releases/0.7.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 2c1afab52..ddc6c4108 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -14,6 +14,7 @@ Changelog * Project template updated to Django 1.7 * 'boost' applied to the title field on searches reduced from 100 to 2 * The 'type' method of PageQuerySet (used to filter the queryset to a specific page type) now includes subclasses of the given page type. + * The 'update_index' management command now updates all backends listed in WAGTAILSEARCH_BACKENDS, or a specific one passed on the command line, rather than just the default backend * Fix: 'wagtail start' command now works on Windows * Fix: The external image URL generator no longer stores generated images in Django's cache * Fix: Elasticsearch backend can now search querysets that have been filtered with an 'in' clause of a non-list type (such as a ValuesListQuerySet) diff --git a/docs/releases/0.7.rst b/docs/releases/0.7.rst index 17bde9b59..3f9f704d3 100644 --- a/docs/releases/0.7.rst +++ b/docs/releases/0.7.rst @@ -32,6 +32,7 @@ Minor features * The project template (used when running ``wagtail start``) has been updated to Django 1.7. * The 'boost' applied to the title field on searches has been reduced from 100 to 2. * The ``type`` method of ``PageQuerySet`` (used to filter the queryset to a specific page type) now includes subclasses of the given page type. + * The ``update_index`` management command now updates all backends listed in ``WAGTAILSEARCH_BACKENDS``, or a specific one passed on the command line, rather than just the default backend. Bug fixes ~~~~~~~~~