Hopiu/xapian-haystack
1
0
Fork 0
mirror of https://github.com/Hopiu/xapian-haystack.git synced 2026-03-16 22:20:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updated to use new SQ objects

Browse source
This commit is contained in:
David Sauve 2009-10-21 10:22:29 -04:00
parent 817943bfc7
commit 204feae63e
3 changed files with 119 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
tests/xapian_tests/tests/xapian_backend.py
View file

@ -21,17 +21,43 @@ import shutil
import xapian
from django.conf import settings
from django.db import models
from django.utils.encoding import force_unicode
from django.test import TestCase
from haystack import indexes, sites
from haystack.backends.xapian_backend import SearchBackend, DEFAULT_MAX_RESULTS
from core.models import MockModel, AnotherMockModel
from core.models import MockTag, AnotherMockModel
class XapianMockModel(models.Model):
"""
Same as tests.core.MockModel with a few extra fields for testing various
sorting and ordering criteria.
"""
author = models.CharField(max_length=255)
foo = models.CharField(max_length=255, blank=True)
pub_date = models.DateTimeField(default=datetime.datetime.now)
tag = models.ForeignKey(MockTag)
value = models.IntegerField(default=0)
flag = models.BooleanField(default=True)
slug = models.SlugField()
popularity = models.FloatField(default=0.0)
def __unicode__(self):
return self.author
def hello(self):
return 'World!'
class XapianMockSearchIndex(indexes.SearchIndex):
text = indexes.CharField(document=True, use_template=True)
text = indexes.CharField(
document=True, use_template=True,
template_name='search/indexes/core/mockmodel_text.txt'
)
name = indexes.CharField(model_attr='author')
pub_date = indexes.DateField(model_attr='pub_date')
value = indexes.IntegerField(model_attr='value')
@ -58,13 +84,13 @@ class XapianSearchBackendTestCase(TestCase):
self.site = XapianSearchSite()
self.sb = SearchBackend(site=self.site)
self.msi = XapianMockSearchIndex(MockModel, backend=self.sb)
self.site.register(MockModel, XapianMockSearchIndex)
self.msi = XapianMockSearchIndex(XapianMockModel, backend=self.sb)
self.site.register(XapianMockModel, XapianMockSearchIndex)
self.sample_objs = []
for i in xrange(1, 4):
mock = MockModel()
mock = XapianMockModel()
mock.id = i
mock.author = 'david%s' % i
mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i)
@ -114,9 +140,9 @@ class XapianSearchBackendTestCase(TestCase):
self.assertEqual(len(self.xapian_search('')), 3)
self.assertEqual([dict(doc) for doc in self.xapian_search('')], [
{'flag': u't', 'name': u'david1', 'text': u'Indexed!\n1', 'sites': u"['1', '2', '3']", 'pub_date': u'20090224000000', 'value': '000000000005', 'id': u'core.mockmodel.1', 'slug': 'http://example.com/1', 'popularity': '\xca\x84'},
{'flag': u'f', 'name': u'david2', 'text': u'Indexed!\n2', 'sites': u"['2', '4', '6']", 'pub_date': u'20090223000000', 'value': '000000000010', 'id': u'core.mockmodel.2', 'slug': 'http://example.com/2', 'popularity': '\xb4`'},
{'flag': u't', 'name': u'david3', 'text': u'Indexed!\n3', 'sites': u"['3', '6', '9']", 'pub_date': u'20090222000000', 'value': '000000000015', 'id': u'core.mockmodel.3', 'slug': 'http://example.com/3', 'popularity': '\xcb\x98'}
{'flag': u't', 'name': u'david1', 'text': u'Indexed!\n1', 'sites': u"['1', '2', '3']", 'pub_date': u'20090224000000', 'value': u'000000000005', 'id': u'tests.xapianmockmodel.1', 'slug': u'http://example.com/1', 'popularity': '\xca\x84'},
{'flag': u'f', 'name': u'david2', 'text': u'Indexed!\n2', 'sites': u"['2', '4', '6']", 'pub_date': u'20090223000000', 'value': u'000000000010', 'id': u'tests.xapianmockmodel.2', 'slug': u'http://example.com/2', 'popularity': '\xb4`'},
{'flag': u't', 'name': u'david3', 'text': u'Indexed!\n3', 'sites': u"['3', '6', '9']", 'pub_date': u'20090222000000', 'value': u'000000000015', 'id': u'tests.xapianmockmodel.3', 'slug': u'http://example.com/3', 'popularity': '\xcb\x98'}
])
def test_remove(self):
@ -126,8 +152,8 @@ class XapianSearchBackendTestCase(TestCase):
self.sb.remove(self.sample_objs[0])
self.assertEqual(len(self.xapian_search('')), 2)
self.assertEqual([dict(doc) for doc in self.xapian_search('')], [
{'flag': u'f', 'name': u'david2', 'text': u'Indexed!\n2', 'sites': u"['2', '4', '6']", 'pub_date': u'20090223000000', 'value': '000000000010', 'id': u'core.mockmodel.2', 'slug': 'http://example.com/2', 'popularity': '\xb4`'},
{'flag': u't', 'name': u'david3', 'text': u'Indexed!\n3', 'sites': u"['3', '6', '9']", 'pub_date': u'20090222000000', 'value': '000000000015', 'id': u'core.mockmodel.3', 'slug': 'http://example.com/3', 'popularity': '\xcb\x98'}
{'flag': u'f', 'name': u'david2', 'text': u'Indexed!\n2', 'sites': u"['2', '4', '6']", 'pub_date': u'20090223000000', 'value': u'000000000010', 'id': u'tests.xapianmockmodel.2', 'slug': u'http://example.com/2', 'popularity': '\xb4`'},
{'flag': u't', 'name': u'david3', 'text': u'Indexed!\n3', 'sites': u"['3', '6', '9']", 'pub_date': u'20090222000000', 'value': u'000000000015', 'id': u'tests.xapianmockmodel.3', 'slug': u'http://example.com/3', 'popularity': '\xcb\x98'}
])
def test_clear(self):
@ -143,13 +169,13 @@ class XapianSearchBackendTestCase(TestCase):
self.sb.clear([AnotherMockModel])
self.assertEqual(len(self.xapian_search('')), 3)
self.sb.clear([MockModel])
self.sb.clear([XapianMockModel])
self.assertEqual(len(self.xapian_search('')), 0)
self.sb.update(self.msi, self.sample_objs)
self.assertEqual(len(self.xapian_search('')), 3)
self.sb.clear([AnotherMockModel, MockModel])
self.sb.clear([AnotherMockModel, XapianMockModel])
self.assertEqual(len(self.xapian_search('')), 0)
def test_search(self):

75
tests/xapian_tests/tests/xapian_query.py
View file

@ -21,6 +21,7 @@ from django.conf import settings
from django.test import TestCase
from haystack.backends.xapian_backend import SearchBackend, SearchQuery
from haystack.query import SQ
from core.models import MockModel, AnotherMockModel
@ -52,56 +53,56 @@ class XapianSearchQueryTestCase(TestCase):
self.assertEqual(self.sq.build_query(), '*')
def test_build_query_single_word(self):
self.sq.add_filter('content', 'hello')
self.sq.add_filter(SQ(content='hello'))
self.assertEqual(self.sq.build_query(), 'hello')
def test_build_query_multiple_words_and(self):
self.sq.add_filter('content', 'hello')
self.sq.add_filter('content', 'world')
self.assertEqual(self.sq.build_query(), 'hello AND world')
self.sq.add_filter(SQ(content='hello'))
self.sq.add_filter(SQ(content='world'))
self.assertEqual(self.sq.build_query(), '(hello AND world)')
def test_build_query_multiple_words_not(self):
self.sq.add_filter('content', 'hello', use_not=True)
self.sq.add_filter('content', 'world', use_not=True)
self.assertEqual(self.sq.build_query(), 'NOT hello NOT world')
self.sq.add_filter(~SQ(content='hello'))
self.sq.add_filter(~SQ(content='world'))
self.assertEqual(self.sq.build_query(), '(NOT (hello) AND NOT (world))')
def test_build_query_multiple_words_or(self):
self.sq.add_filter('content', 'hello', use_or=True)
self.sq.add_filter('content', 'world', use_or=True)
self.assertEqual(self.sq.build_query(), 'hello OR world')
self.sq.add_filter(SQ(content='hello'), use_or=True)
self.sq.add_filter(SQ(content='world'), use_or=True)
self.assertEqual(self.sq.build_query(), '(hello OR world)')
def test_build_query_multiple_words_mixed(self):
self.sq.add_filter('content', 'why')
self.sq.add_filter('content', 'hello', use_or=True)
self.sq.add_filter('content', 'world', use_not=True)
self.assertEqual(self.sq.build_query(), 'why OR hello NOT world')
self.sq.add_filter(SQ(content='why'))
self.sq.add_filter(SQ(content='hello'), use_or=True)
self.sq.add_filter(~SQ(content='world'))
self.assertEqual(self.sq.build_query(), '((why OR hello) AND NOT (world))')
def test_build_query_phrase(self):
self.sq.add_filter('content', 'hello world')
self.sq.add_filter(SQ(content='hello world'))
self.assertEqual(self.sq.build_query(), '"hello world"')
def test_build_query_multiple_filter_types(self):
self.sq.add_filter('content', 'why')
self.sq.add_filter('pub_date__lte', datetime.datetime(2009, 2, 10, 1, 59))
self.sq.add_filter('author__gt', 'david')
self.sq.add_filter('created__lt', datetime.datetime(2009, 2, 12, 12, 13))
self.sq.add_filter('title__gte', 'B')
self.sq.add_filter('id__in', [1, 2, 3])
self.assertEqual(self.sq.build_query(), 'why AND pub_date:..20090210015900 AND NOT author:..david AND NOT created:20090212121300..* AND title:B..* AND (id:1 OR id:2 OR id:3)')
self.sq.add_filter(SQ(content='why'))
self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59)))
self.sq.add_filter(SQ(author__gt='david'))
self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13)))
self.sq.add_filter(SQ(title__gte='B'))
self.sq.add_filter(SQ(id__in=[1, 2, 3]))
self.assertEqual(self.sq.build_query(), '(why AND pub_date:..20090210015900 AND NOT author:..david AND NOT created:20090212121300..* AND title:B..* AND (id:1 OR id:2 OR id:3))')
def test_build_query_multiple_exclude_types(self):
self.sq.add_filter('content', 'why', use_not=True)
self.sq.add_filter('pub_date__lte', datetime.datetime(2009, 2, 10, 1, 59), use_not=True)
self.sq.add_filter('author__gt', 'david', use_not=True)
self.sq.add_filter('created__lt', datetime.datetime(2009, 2, 12, 12, 13), use_not=True)
self.sq.add_filter('title__gte', 'B', use_not=True)
self.sq.add_filter('id__in', [1, 2, 3], use_not=True)
self.assertEqual(self.sq.build_query(), 'NOT why AND NOT pub_date:..20090210015900 AND author:..david AND created:20090212121300..* AND NOT title:B..* AND NOT id:1 NOT id:2 NOT id:3')
self.sq.add_filter(~SQ(content='why'))
self.sq.add_filter(~SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59)))
self.sq.add_filter(~SQ(author__gt='david'))
self.sq.add_filter(~SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13)))
self.sq.add_filter(~SQ(title__gte='B'))
self.sq.add_filter(~SQ(id__in=[1, 2, 3]))
self.assertEqual(self.sq.build_query(), '(NOT (why) AND NOT (pub_date:..20090210015900) AND NOT (NOT author:..david) AND NOT (NOT created:20090212121300..*) AND NOT (title:B..*) AND NOT ((id:1 OR id:2 OR id:3)))')
def test_build_query_wildcard_filter_types(self):
self.sq.add_filter('content', 'why')
self.sq.add_filter('title__startswith', 'haystack')
self.assertEqual(self.sq.build_query(), 'why AND title:haystack*')
self.sq.add_filter(SQ(content='why'))
self.sq.add_filter(SQ(title__startswith='haystack'))
self.assertEqual(self.sq.build_query(), '(why AND title:haystack*)')
def test_clean(self):
self.assertEqual(self.sq.clean('hello world'), 'hello world')
@ -110,17 +111,17 @@ class XapianSearchQueryTestCase(TestCase):
self.assertEqual(self.sq.clean('so please NOTe i am in a bAND and bORed'), 'so please NOTe i am in a bAND and bORed')
def test_build_query_with_models(self):
self.sq.add_filter('content', 'hello')
self.sq.add_filter(SQ(content='hello'))
self.sq.add_model(MockModel)
self.assertEqual(self.sq.build_query(), u'(hello) django_ct:core.mockmodel')
self.assertEqual(self.sq.build_query(), u'(hello) AND (django_ct:core.mockmodel)')
self.sq.add_model(AnotherMockModel)
self.assertEqual(self.sq.build_query(), u'(hello) django_ct:core.anothermockmodel django_ct:core.mockmodel')
self.assertEqual(self.sq.build_query(), u'(hello) AND (django_ct:core.anothermockmodel OR django_ct:core.mockmodel)')
def test_build_query_with_datetime(self):
self.sq.add_filter('pub_date', datetime.datetime(2009, 5, 9, 16, 20))
self.sq.add_filter(SQ(pub_date=datetime.datetime(2009, 5, 9, 16, 20)))
self.assertEqual(self.sq.build_query(), u'pub_date:20090509162000')
def test_build_query_with_sequence_and_filter_not_in(self):
self.sq.add_filter('id__exact', [1, 2, 3])
self.sq.add_filter(SQ(id__exact=[1, 2, 3]))
self.assertEqual(self.sq.build_query(), u'id:[1, 2, 3]')

130
xapian_backend.py
View file

@ -919,10 +919,9 @@ class SearchBackend(BaseSearchBackend):
class SearchQuery(BaseSearchQuery):
"""
`SearchQuery` is responsible for converting search queries into a format
that Xapian can understand.
Most of the work is done by the :method:`build_query`.
This class is the Xapian specific version of the SearchQuery class.
It acts as an intermediary between the ``SearchQuerySet`` and the
``SearchBackend`` itself.
"""
def __init__(self, backend=None):
"""
@ -930,103 +929,57 @@ class SearchQuery(BaseSearchQuery):
specified. If no backend is set, will use the Xapian `SearchBackend`.
Optional arguments:
`backend` -- The `SearchBackend` to use (default = None)
``backend`` -- The ``SearchBackend`` to use (default = None)
"""
super(SearchQuery, self).__init__(backend=backend)
self.backend = backend or SearchBackend()
def build_query(self):
def build_query_fragment(self, field, filter_type, value):
"""
Builds a search query from previously set values, returning a query
string in a format ready for use by the Xapian `SearchBackend`.
Builds a search query fragment from a field, filter type and value.
Returns:
A query string suitable for parsing by Xapian.
A query string fragment suitable for parsing by Xapian.
"""
query = ''
if not self.query_filters:
query = '*'
result = ''
if not isinstance(value, (list, tuple)):
# Convert whatever we find to what xapian wants.
value = self.backend._marshal_value(value)
# Check to see if it's a phrase for an exact match.
if ' ' in value:
value = '"%s"' % value
# 'content' is a special reserved word, much like 'pk' in
# Django's ORM layer. It indicates 'no special field'.
if field == 'content':
result = value
else:
query_chunks = []
for the_filter in self.query_filters:
if the_filter.is_and():
query_chunks.append('AND')
filter_types = {
'exact': '%s:%s',
'gte': '%s:%s..*',
'gt': 'NOT %s:..%s',
'lte': '%s:..%s',
'lt': 'NOT %s:%s..*',
'startswith': '%s:%s*',
}
if the_filter.is_or():
query_chunks.append('OR')
if filter_type != 'in':
result = filter_types[filter_type] % (field, value)
else:
in_options = []
for possible_value in value:
in_options.append('%s:%s' % (field, possible_value))
result = '(%s)' % ' OR '.join(in_options)
if the_filter.is_not() and the_filter.field == 'content':
query_chunks.append('NOT')
value = the_filter.value
if not isinstance(value, (list, tuple)):
# Convert whatever we find to what xapian wants.
value = self.backend._marshal_value(value)
# Check to see if it's a phrase for an exact match.
if ' ' in value:
value = '"%s"' % value
# 'content' is a special reserved word, much like 'pk' in
# Django's ORM layer. It indicates 'no special field'.
if the_filter.field == 'content':
query_chunks.append(value)
else:
if the_filter.is_not():
query_chunks.append('AND')
filter_types = {
'exact': 'NOT %s:%s',
'gte': 'NOT %s:%s..*',
'gt': '%s:..%s',
'lte': 'NOT %s:..%s',
'lt': '%s:%s..*',
'startswith': 'NOT %s:%s*',
}
else:
filter_types = {
'exact': '%s:%s',
'gte': '%s:%s..*',
'gt': 'NOT %s:..%s',
'lte': '%s:..%s',
'lt': 'NOT %s:%s..*',
'startswith': '%s:%s*',
}
if the_filter.filter_type != 'in':
query_chunks.append(filter_types[the_filter.filter_type] % (the_filter.field, value))
else:
in_options = []
if the_filter.is_not():
for possible_value in value:
in_options.append('%s:%s' % (the_filter.field, possible_value))
query_chunks.append('NOT %s' % ' NOT '.join(in_options))
else:
for possible_value in value:
in_options.append('%s:%s' % (the_filter.field, possible_value))
query_chunks.append('(%s)' % ' OR '.join(in_options))
if query_chunks[0] in ('AND', 'OR'):
# Pull off an undesirable leading "AND" or "OR".
del(query_chunks[0])
query = ' '.join(query_chunks)
return result
if len(self.models):
models = ['django_ct:%s.%s' % (model._meta.app_label, model._meta.module_name) for model in self.models]
models_clause = ' '.join(models)
final_query = '(%s) %s' % (query, models_clause)
else:
final_query = query
return final_query
def run(self, spelling_query=None):
"""
Builds and executes the query. Returns a list of search results.
Returns:
List of search results
"""
final_query = self.build_query()
kwargs = {
@ -1069,6 +1022,9 @@ class SearchQuery(BaseSearchQuery):
def run_mlt(self):
"""
Builds and executes the query. Returns a list of search results.
Returns:
List of search results
"""
if self._more_like_this is False or self._mlt_instance is None:
raise MoreLikeThisError("No instance was provided to determine 'More Like This' results.")