matthias
/
xapian-haystack
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merged 2.0.X branch

This commit is contained in:
David Sauve 2012-04-20 12:07:44 -07:00
parent 69ee097c1c 9fff70d652
commit dcdc9eb319
9 changed files with 439 additions and 457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.rst
View File

@ -13,7 +13,7 @@ Requirements
- Python 2.4 (May work with 2.3, but untested)
- Django 1.0.x
- Django-Haystack 1.1.X (If you wish to use django-haystack 1.0.X, please use xapian-haystack 1.0.X)
- Django-Haystack 2.0.X
- Xapian 1.0.13+ (May work with earlier versions, but untested)
Notes
@ -38,8 +38,14 @@ Installation
``easy_install xapian-haystack``
#. Add ``HAYSTACK_XAPIAN_PATH`` to ``settings.py``
#. Set ``HAYSTACK_SEARCH_ENGINE`` to ``xapian``
#. Set to something similar to:
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.xapian_backend.XapianEngine',
'PATH': os.path.join(os.path.dirname(__file__), 'xapian_index')
},
}
Configuration
-------------
@ -96,7 +102,7 @@ xapian-haystack is maintained by `David Sauve <mailto:david.sauve@bag-of-holding
License
-------
xapian-haystack is Copyright (c) 2009, 2010, 2011 David Sauve, 2009, 2010 Trapeze. It is free software, and may be redistributed under the terms specified in the LICENSE file.
xapian-haystack is Copyright (c) 2009, 2010, 2011, 2012 David Sauve, 2009, 2010 Trapeze. It is free software, and may be redistributed under the terms specified in the LICENSE file.
Questions, Comments, Concerns:
------------------------------

12
tests/xapian_settings.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze
import os
@ -8,6 +8,10 @@ INSTALLED_APPS += [
'xapian_tests',
]
HAYSTACK_SEARCH_ENGINE = 'xapian'
HAYSTACK_XAPIAN_PATH = os.path.join('tmp', 'test_xapian_query')
HAYSTACK_INCLUDE_SPELLING = True
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.xapian_backend.XapianEngine',
'PATH': os.path.join('tmp', 'test_xapian_query'),
'INCLUDE_SPELLING': True,
}
}

2
tests/xapian_tests/__init__.py
View File

@ -1,2 +1,2 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze

2
tests/xapian_tests/models.py
View File

@ -1,2 +1,2 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze

2
tests/xapian_tests/tests/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze
import warnings

145
tests/xapian_tests/tests/xapian_backend.py
View File

@ -1,9 +1,8 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze
# Based on original code by Daniel Lindsley as part of the Haystack test suite.
import cPickle as pickle
import datetime
import os
import shutil
@ -14,12 +13,12 @@ from django.conf import settings
from django.db import models
from django.test import TestCase
from haystack import indexes, sites, backends
from haystack.backends.xapian_backend import SearchBackend, SearchQuery, _marshal_value
from haystack.exceptions import HaystackError
from haystack import connections, reset_search_queries
from haystack import indexes
from haystack.backends.xapian_backend import _marshal_value
from haystack.models import SearchResult
from haystack.query import SearchQuerySet, SQ
from haystack.sites import SearchSite
from haystack.utils.loading import UnifiedIndex
from core.models import MockTag, MockModel, AnotherMockModel, AFourthMockModel
from core.tests.mocks import MockSearchResult
@ -71,6 +70,9 @@ class XapianMockSearchIndex(indexes.SearchIndex):
keys = indexes.MultiValueField()
titles = indexes.MultiValueField()
def get_model(self):
return XapianMockModel
def prepare_sites(self, obj):
return ['%d' % (i * obj.id) for i in xrange(1, 4)]
@ -92,7 +94,6 @@ class XapianMockSearchIndex(indexes.SearchIndex):
return ['object two title one', 'object two title two']
else:
return ['object three title one', 'object three title two']
pub_date = indexes.DateField(model_attr='pub_date')
def prepare_month(self, obj):
return '%02d' % obj.pub_date.month
@ -110,15 +111,20 @@ class XapianBoostMockSearchIndex(indexes.SearchIndex):
editor = indexes.CharField(model_attr='editor')
pub_date = indexes.DateField(model_attr='pub_date')
def get_model(self):
return AFourthMockModel
class XapianSearchBackendTestCase(TestCase):
def setUp(self):
super(XapianSearchBackendTestCase, self).setUp()
self.site = SearchSite()
self.backend = SearchBackend(site=self.site)
self.index = XapianMockSearchIndex(XapianMockModel, backend=self.backend)
self.site.register(XapianMockModel, XapianMockSearchIndex)
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.index = XapianMockSearchIndex()
self.ui.build(indexes=[self.index])
self.backend = connections['default'].get_backend()
connections['default']._index = self.ui
self.sample_objs = []
@ -139,9 +145,10 @@ class XapianSearchBackendTestCase(TestCase):
self.sample_objs[2].popularity = 972.0
def tearDown(self):
if os.path.exists(settings.HAYSTACK_XAPIAN_PATH):
shutil.rmtree(settings.HAYSTACK_XAPIAN_PATH)
if os.path.exists(settings.HAYSTACK_CONNECTIONS['default']['PATH']):
shutil.rmtree(settings.HAYSTACK_CONNECTIONS['default']['PATH'])
connections['default']._index = self.old_ui
super(XapianSearchBackendTestCase, self).tearDown()
def test_update(self):
@ -152,7 +159,7 @@ class XapianSearchBackendTestCase(TestCase):
def test_duplicate_update(self):
self.backend.update(self.index, self.sample_objs)
self.backend.update(self.index, self.sample_objs) # Duplicates should be updated, not appended -- http://github.com/notanumber/xapian-haystack/issues/#issue/6
self.backend.update(self.index, self.sample_objs) # Duplicates should be updated, not appended -- http://github.com/notanumber/xapian-haystack/issues/#issue/6
self.assertEqual(self.backend.document_count(), 3)
@ -315,36 +322,6 @@ class XapianSearchBackendTestCase(TestCase):
# Ensure that swapping the ``result_class`` works.
self.assertTrue(isinstance(self.backend.more_like_this(self.sample_objs[0], result_class=MockSearchResult)['results'][0], MockSearchResult))
def test_use_correct_site(self):
test_site = SearchSite()
test_site.register(XapianMockModel, XapianMockSearchIndex)
self.backend.update(self.index, self.sample_objs)
# Make sure that ``_process_results`` uses the right ``site``.
self.assertEqual(self.backend.search(xapian.Query('indexed'))['hits'], 3)
self.assertEqual([result.pk for result in self.backend.search(xapian.Query('indexed'))['results']], [1, 2, 3])
self.site.unregister(XapianMockModel)
self.assertEqual(len(self.site.get_indexed_models()), 0)
self.backend.site = test_site
self.assertTrue(len(self.backend.site.get_indexed_models()) > 0)
# Should still be there, despite the main ``site`` not having that model
# registered any longer.
self.assertEqual(self.backend.search(xapian.Query('indexed'))['hits'], 3)
self.assertEqual([result.pk for result in self.backend.search(xapian.Query('indexed'))['results']], [1, 2, 3])
# Unregister it on the backend & make sure it takes effect.
self.backend.site.unregister(XapianMockModel)
self.assertEqual(len(self.backend.site.get_indexed_models()), 0)
self.assertEqual(self.backend.search(xapian.Query('indexed'))['hits'], 0)
# Nuke it & fallback on the main ``site``.
self.backend.site = haystack.site
self.assertEqual(self.backend.search(xapian.Query('indexed'))['hits'], 0)
self.site.register(XapianMockModel, XapianMockSearchIndex)
self.assertEqual(self.backend.search(xapian.Query('indexed'))['hits'], 3)
def test_order_by(self):
self.backend.update(self.index, self.sample_objs)
self.assertEqual(self.backend.document_count(), 3)
@ -406,7 +383,7 @@ class XapianSearchBackendTestCase(TestCase):
self.assertEqual(_marshal_value(datetime.datetime(2009, 5, 18, 1, 16, 30, 250)), u'20090518011630000250')
def test_build_schema(self):
(content_field_name, fields) = self.backend.build_schema(self.site.all_searchfields())
(content_field_name, fields) = self.backend.build_schema(connections['default'].get_unified_index().all_searchfields())
self.assertEqual(content_field_name, 'text')
self.assertEqual(len(fields), 15)
self.assertEqual(fields, [
@ -451,6 +428,9 @@ class LiveXapianMockSearchIndex(indexes.SearchIndex):
created = indexes.DateField()
title = indexes.CharField()
def get_model(self):
return MockModel
class LiveXapianSearchQueryTestCase(TestCase):
"""
@ -461,13 +441,19 @@ class LiveXapianSearchQueryTestCase(TestCase):
def setUp(self):
super(LiveXapianSearchQueryTestCase, self).setUp()
site = SearchSite()
backend = SearchBackend(site=site)
index = LiveXapianMockSearchIndex(MockModel, backend=backend)
site.register(MockModel, LiveXapianMockSearchIndex)
self.old_ui = connections['default'].get_unified_index()
ui = UnifiedIndex()
index = LiveXapianMockSearchIndex()
ui.build(indexes=[index])
backend = connections['default'].get_backend()
connections['default']._index = ui
backend.update(index, MockModel.objects.all())
self.sq = SearchQuery(backend=backend)
self.sq = connections['default'].get_query()
def tearDown(self):
connections['default']._index = self.old_ui
super(LiveXapianSearchQueryTestCase, self).tearDown()
def test_get_spelling(self):
self.sq.add_filter(SQ(content='indxd'))
@ -504,32 +490,32 @@ class LiveXapianSearchQueryTestCase(TestCase):
self.assertEqual(str(self.sq.build_query()), u'Xapian::Query(((Zwhi OR why) AND VALUE_RANGE 3 00010101000000 20090210015900 AND (<alldocuments> AND_NOT VALUE_RANGE 2 a david) AND (<alldocuments> AND_NOT VALUE_RANGE 1 20090212121300 99990101000000) AND VALUE_RANGE 5 b zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz AND (Q1 OR Q2 OR Q3)))')
def test_log_query(self):
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Stow.
old_debug = settings.DEBUG
settings.DEBUG = False
len(self.sq.get_results())
self.assertEqual(len(backends.queries), 0)
self.assertEqual(len(connections['default'].queries), 0)
settings.DEBUG = True
# Redefine it to clear out the cached results.
self.sq = SearchQuery(backend=SearchBackend())
self.sq = connections['default'].get_query()
self.sq.add_filter(SQ(name='bar'))
len(self.sq.get_results())
self.assertEqual(len(backends.queries), 1)
self.assertEqual(str(backends.queries[0]['query_string']), u'Xapian::Query((ZXNAMEbar OR XNAMEbar))')
self.assertEqual(len(connections['default'].queries), 1)
self.assertEqual(str(connections['default'].queries[0]['query_string']), u'Xapian::Query((ZXNAMEbar OR XNAMEbar))')
# And again, for good measure.
self.sq = SearchQuery(backend=SearchBackend())
self.sq = connections['default'].get_query()
self.sq.add_filter(SQ(name='bar'))
self.sq.add_filter(SQ(text='moof'))
len(self.sq.get_results())
self.assertEqual(len(backends.queries), 2)
self.assertEqual(str(backends.queries[0]['query_string']), u'Xapian::Query((ZXNAMEbar OR XNAMEbar))')
self.assertEqual(str(backends.queries[1]['query_string']), u'Xapian::Query(((ZXNAMEbar OR XNAMEbar) AND (ZXTEXTmoof OR XTEXTmoof)))')
self.assertEqual(len(connections['default'].queries), 2)
self.assertEqual(str(connections['default'].queries[0]['query_string']), u'Xapian::Query((ZXNAMEbar OR XNAMEbar))')
self.assertEqual(str(connections['default'].queries[1]['query_string']), u'Xapian::Query(((ZXNAMEbar OR XNAMEbar) AND (ZXTEXTmoof OR XTEXTmoof)))')
# Restore.
settings.DEBUG = old_debug
@ -544,14 +530,20 @@ class LiveXapianSearchQuerySetTestCase(TestCase):
def setUp(self):
super(LiveXapianSearchQuerySetTestCase, self).setUp()
site = SearchSite()
backend = SearchBackend(site=site)
index = LiveXapianMockSearchIndex(MockModel, backend=backend)
site.register(MockModel, LiveXapianMockSearchIndex)
backend.update(index, MockModel.objects.all())
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.index = LiveXapianMockSearchIndex()
self.ui.build(indexes=[self.index])
self.backend = connections['default'].get_backend()
connections['default']._index = self.ui
self.backend.update(self.index, MockModel.objects.all())
self.sq = SearchQuery(backend=backend)
self.sqs = SearchQuerySet(query=self.sq)
self.sq = connections['default'].get_query()
self.sqs = SearchQuerySet()
def tearDown(self):
connections['default']._index = self.old_ui
super(LiveXapianSearchQuerySetTestCase, self).tearDown()
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
@ -571,15 +563,13 @@ class XapianBoostBackendTestCase(TestCase):
def setUp(self):
super(XapianBoostBackendTestCase, self).setUp()
self.site = SearchSite()
self.sb = SearchBackend(site=self.site)
self.smmi = XapianBoostMockSearchIndex(AFourthMockModel, backend=self.sb)
self.site.register(AFourthMockModel, XapianBoostMockSearchIndex)
# Stow.
import haystack
self.old_site = haystack.site
haystack.site = self.site
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.index = XapianBoostMockSearchIndex()
self.ui.build(indexes=[self.index])
self.sb = connections['default'].get_backend()
connections['default']._index = self.ui
self.sample_objs = []
@ -596,12 +586,11 @@ class XapianBoostBackendTestCase(TestCase):
self.sample_objs.append(mock)
def tearDown(self):
import haystack
haystack.site = self.old_site
connections['default']._index = self.old_ui
super(XapianBoostBackendTestCase, self).tearDown()
def test_boost(self):
self.sb.update(self.smmi, self.sample_objs)
self.sb.update(self.index, self.sample_objs)
sqs = SearchQuerySet()

10
tests/xapian_tests/tests/xapian_query.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze
import datetime
@ -8,7 +8,7 @@ import shutil
from django.conf import settings
from django.test import TestCase
from haystack.backends.xapian_backend import SearchBackend, SearchQuery
from haystack import connections
from haystack.query import SQ
from core.models import MockModel, AnotherMockModel
@ -17,11 +17,11 @@ from core.models import MockModel, AnotherMockModel
class XapianSearchQueryTestCase(TestCase):
def setUp(self):
super(XapianSearchQueryTestCase, self).setUp()
self.sq = SearchQuery(backend=SearchBackend())
self.sq = connections['default'].get_query()
def tearDown(self):
if os.path.exists(settings.HAYSTACK_XAPIAN_PATH):
shutil.rmtree(settings.HAYSTACK_XAPIAN_PATH)
if os.path.exists(settings.HAYSTACK_CONNECTIONS['default']['PATH']):
shutil.rmtree(settings.HAYSTACK_CONNECTIONS['default']['PATH'])
super(XapianSearchQueryTestCase, self).tearDown()

135
xapian_backend.py
View File

@ -1,8 +1,8 @@
# Copyright (C) 2009, 2010, 2011 David Sauve
# Copyright (C) 2009, 2010, 2011, 2012 David Sauve
# Copyright (C) 2009, 2010 Trapeze
__author__ = 'David Sauve'
__version__ = (1, 1, 6, 'beta')
__version__ = (2, 0, 0, 'beta')
import time
import datetime
@ -11,16 +11,15 @@ import os
import re
import shutil
import sys
import warnings
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.encoding import smart_unicode, force_unicode
from django.utils.encoding import force_unicode
from haystack.backends import BaseSearchBackend, BaseSearchQuery, SearchNode, log_query
from haystack.constants import ID, DJANGO_CT, DJANGO_ID
from haystack.exceptions import HaystackError, MissingDependency, MoreLikeThisError
from haystack.fields import DateField, DateTimeField, IntegerField, FloatField, BooleanField, MultiValueField
from haystack import connections
from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query
from haystack.constants import ID
from haystack.exceptions import HaystackError, MissingDependency
from haystack.models import SearchResult
from haystack.utils import get_identifier
@ -36,8 +35,6 @@ DOCUMENT_CT_TERM_PREFIX = DOCUMENT_CUSTOM_TERM_PREFIX + 'CONTENTTYPE'
MEMORY_DB_NAME = ':memory:'
BACKEND_NAME = 'xapian'
DEFAULT_XAPIAN_FLAGS = (
xapian.QueryParser.FLAG_PHRASE |
xapian.QueryParser.FLAG_BOOLEAN |
@ -54,7 +51,8 @@ class InvalidIndexError(HaystackError):
class XHValueRangeProcessor(xapian.ValueRangeProcessor):
def __init__(self, backend):
self.backend = backend or SearchBackend()
# FIXME: This needs to get smarter about pulling the right backend.
self.backend = backend or XapianSearchBackend()
xapian.ValueRangeProcessor.__init__(self)
def __call__(self, begin, end):
@ -73,7 +71,7 @@ class XHValueRangeProcessor(xapian.ValueRangeProcessor):
if field_dict['field_name'] == field_name:
if not begin:
if field_dict['type'] == 'text':
begin = u'a' # TODO: A better way of getting a min text value?
begin = u'a' # TODO: A better way of getting a min text value?
elif field_dict['type'] == 'long':
begin = -sys.maxint - 1
elif field_dict['type'] == 'float':
@ -82,7 +80,7 @@ class XHValueRangeProcessor(xapian.ValueRangeProcessor):
begin = u'00010101000000'
elif end == '*':
if field_dict['type'] == 'text':
end = u'z' * 100 # TODO: A better way of getting a max text value?
end = u'z' * 100 # TODO: A better way of getting a max text value?
elif field_dict['type'] == 'long':
end = sys.maxint
elif field_dict['type'] == 'float':
@ -111,7 +109,7 @@ class XHExpandDecider(xapian.ExpandDecider):
return True
class SearchBackend(BaseSearchBackend):
class XapianSearchBackend(BaseSearchBackend):
"""
`SearchBackend` defines the Xapian search backend for use with the Haystack
API for Django search.
@ -124,32 +122,35 @@ class SearchBackend(BaseSearchBackend):
`WSGIApplicationGroup to %{GLOBAL}` when using mod_wsgi, or
`PythonInterpreter main_interpreter` when using mod_python.
In order to use this backend, `HAYSTACK_XAPIAN_PATH` must be set in
your settings. This should point to a location where you would your
In order to use this backend, `PATH` must be included in the
`connection_options`. This should point to a location where you would your
indexes to reside.
"""
inmemory_db = None
def __init__(self, site=None, language='english'):
def __init__(self, connection_alias, language='english', **connection_options):
"""
Instantiates an instance of `SearchBackend`.
Optional arguments:
`site` -- The site to associate the backend with (default = None)
`stemming_language` -- The stemming language (default = 'english')
`connection_alias` -- The name of the connection
`language` -- The stemming language (default = 'english')
`**connection_options` -- The various options needed to setup
the backend.
Also sets the stemming language to be used to `stemming_language`.
Also sets the stemming language to be used to `language`.
"""
super(SearchBackend, self).__init__(site)
super(XapianSearchBackend, self).__init__(connection_alias, **connection_options)
if not hasattr(settings, 'HAYSTACK_XAPIAN_PATH'):
raise ImproperlyConfigured('You must specify a HAYSTACK_XAPIAN_PATH in your settings.')
if not 'PATH' in connection_options:
raise ImproperlyConfigured("You must specify a 'PATH' in your settings for connection '%s'." % connection_alias)
if settings.HAYSTACK_XAPIAN_PATH != MEMORY_DB_NAME and \
not os.path.exists(settings.HAYSTACK_XAPIAN_PATH):
os.makedirs(settings.HAYSTACK_XAPIAN_PATH)
self.path = connection_options.get('PATH')
if self.path != MEMORY_DB_NAME and not os.path.exists(self.path):
os.makedirs(self.path)
self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS)
self.language = language
self._schema = None
self._content_field_name = None
@ -157,13 +158,13 @@ class SearchBackend(BaseSearchBackend):
@property
def schema(self):
if not self._schema:
self._content_field_name, self._schema = self.build_schema(self.site.all_searchfields())
self._content_field_name, self._schema = self.build_schema(connections[self.connection_alias].get_unified_index().all_searchfields())
return self._schema
@property
def content_field_name(self):
if not self._content_field_name:
self._content_field_name, self._schema = self.build_schema(self.site.all_searchfields())
self._content_field_name, self._schema = self.build_schema(connections[self.connection_alias].get_unified_index().all_searchfields())
return self._content_field_name
def update(self, index, iterable):
@ -212,7 +213,7 @@ class SearchBackend(BaseSearchBackend):
term_generator = xapian.TermGenerator()
term_generator.set_database(database)
term_generator.set_stemmer(xapian.Stem(self.language))
if getattr(settings, 'HAYSTACK_INCLUDE_SPELLING', False) is True:
if self.include_spelling is True:
term_generator.set_flags(xapian.TermGenerator.FLAG_SPELLING)
term_generator.set_document(document)
@ -304,10 +305,10 @@ class SearchBackend(BaseSearchBackend):
database = self._database(writable=True)
if not models:
# Because there does not appear to be a "clear all" method,
# it's much quicker to remove the contents of the `HAYSTACK_XAPIAN_PATH`
# it's much quicker to remove the contents of the `self.path`
# folder than it is to remove each document one at a time.
if os.path.exists(settings.HAYSTACK_XAPIAN_PATH):
shutil.rmtree(settings.HAYSTACK_XAPIAN_PATH)
if os.path.exists(self.path):
shutil.rmtree(self.path)
else:
for model in models:
database.delete_document(
@ -357,16 +358,11 @@ class SearchBackend(BaseSearchBackend):
If `query` is None, returns no results.
If `HAYSTACK_INCLUDE_SPELLING` was enabled in `settings.py`, the
If `INCLUDE_SPELLING` was enabled in the connection options, the
extra flag `FLAG_SPELLING_CORRECTION` will be passed to the query parser
and any suggestions for spell correction will be returned as well as
the results.
"""
if not self.site:
from haystack import site
else:
site = self.site
if xapian.Query.empty(query):
return {
'results': [],
@ -378,7 +374,7 @@ class SearchBackend(BaseSearchBackend):
if result_class is None:
result_class = SearchResult
if getattr(settings, 'HAYSTACK_INCLUDE_SPELLING', False) is True:
if self.include_spelling is True:
spelling_suggestion = self._do_spelling_suggestion(database, query, spelling_query)
else:
spelling_suggestion = ''
@ -391,7 +387,7 @@ class SearchBackend(BaseSearchBackend):
)
if limit_to_registered_models:
registered_models = self.build_registered_models_list()
registered_models = self.build_models_list()
if len(registered_models) > 0:
query = xapian.Query(
@ -414,9 +410,9 @@ class SearchBackend(BaseSearchBackend):
for sort_field in sort_by:
if sort_field.startswith('-'):
reverse = True
sort_field = sort_field[1:] # Strip the '-'
sort_field = sort_field[1:] # Strip the '-'
else:
reverse = False # Reverse is inverted in Xapian -- http://trac.xapian.org/ticket/311
reverse = False # Reverse is inverted in Xapian -- http://trac.xapian.org/ticket/311
sorter.add(self._value_column(sort_field), reverse)
enquire.set_sort_by_key_then_relevance(sorter, True)
@ -442,7 +438,7 @@ class SearchBackend(BaseSearchBackend):
)
}
results.append(
result_class(app_label, module_name, pk, match.percent, searchsite=site, **model_data)
result_class(app_label, module_name, pk, match.percent, **model_data)
)
if facets:
@ -491,11 +487,6 @@ class SearchBackend(BaseSearchBackend):
Finally, processes the resulting matches and returns.
"""
if not self.site:
from haystack import site
else:
site = self.site
database = self._database()
if result_class is None:
@ -523,7 +514,7 @@ class SearchBackend(BaseSearchBackend):
xapian.Query.OP_AND_NOT, [query, DOCUMENT_ID_TERM_PREFIX + get_identifier(model_instance)]
)
if limit_to_registered_models:
registered_models = self.build_registered_models_list()
registered_models = self.build_models_list()
if len(registered_models) > 0:
query = xapian.Query(
@ -547,7 +538,7 @@ class SearchBackend(BaseSearchBackend):
for match in matches:
app_label, module_name, pk, model_data = pickle.loads(self._get_document_data(database, match.document))
results.append(
result_class(app_label, module_name, pk, match.percent, searchsite=site, **model_data)
result_class(app_label, module_name, pk, match.percent, **model_data)
)
return {
@ -571,11 +562,10 @@ class SearchBackend(BaseSearchBackend):
Returns a xapian.Query
"""
if query_string == '*':
return xapian.Query('') # Match everything
return xapian.Query('') # Match everything
elif query_string == '':
return xapian.Query() # Match nothing
return xapian.Query() # Match nothing
flags = getattr(settings, 'HAYSTACK_XAPIAN_FLAGS', DEFAULT_XAPIAN_FLAGS)
qp = xapian.QueryParser()
qp.set_database(self._database())
qp.set_stemmer(xapian.Stem(self.language))
@ -591,7 +581,7 @@ class SearchBackend(BaseSearchBackend):
vrp = XHValueRangeProcessor(self)
qp.add_valuerangeprocessor(vrp)
return qp.parse_query(query_string, flags)
return qp.parse_query(query_string, self.flags)
def build_schema(self, fields):
"""
@ -651,7 +641,7 @@ class SearchBackend(BaseSearchBackend):
`text` -- The text to be highlighted
"""
for term in query:
for match in re.findall('[^A-Z]+', term): # Ignore field identifiers
for match in re.findall('[^A-Z]+', term): # Ignore field identifiers
match_re = re.compile(match, re.I)
content = match_re.sub('<%s>%s</%s>' % (tag, term, tag), content)
@ -677,7 +667,7 @@ class SearchBackend(BaseSearchBackend):
for result in results:
field_value = getattr(result, field)
if self._multi_value_field(field):
for item in field_value: # Facet each item in a MultiValueField
for item in field_value: # Facet each item in a MultiValueField
facet_list[item] = facet_list.get(item, 0) + 1
else:
facet_list[field_value] = facet_list.get(field_value, 0) + 1
@ -746,7 +736,7 @@ class SearchBackend(BaseSearchBackend):
elif gap_type == 'second':
date_range += datetime.timedelta(seconds=int(gap_value))
facet_list = sorted(facet_list, key=lambda n:n[0], reverse=True)
facet_list = sorted(facet_list, key=lambda n: n[0], reverse=True)
for result in results:
result_date = getattr(result, date_facet)
@ -808,7 +798,7 @@ class SearchBackend(BaseSearchBackend):
term_set = set()
for term in query:
for match in re.findall('[^A-Z]+', term): # Ignore field identifiers
for match in re.findall('[^A-Z]+', term): # Ignore field identifiers
term_set.add(database.get_spelling_suggestion(match))
return ' '.join(term_set)
@ -827,12 +817,12 @@ class SearchBackend(BaseSearchBackend):
SearchBackend.inmemory_db = xapian.inmemory_open()
return SearchBackend.inmemory_db
if writable:
database = xapian.WritableDatabase(settings.HAYSTACK_XAPIAN_PATH, xapian.DB_CREATE_OR_OPEN)
database = xapian.WritableDatabase(self.path, xapian.DB_CREATE_OR_OPEN)
else:
try:
database = xapian.Database(settings.HAYSTACK_XAPIAN_PATH)
database = xapian.Database(self.path)
except xapian.DatabaseOpeningError:
raise InvalidIndexError(u'Unable to open index at %s' % settings.HAYSTACK_XAPIAN_PATH)
raise InvalidIndexError(u'Unable to open index at %s' % self.path)
return database
@ -916,24 +906,12 @@ class SearchBackend(BaseSearchBackend):
return False
class SearchQuery(BaseSearchQuery):
class XapianSearchQuery(BaseSearchQuery):
"""
This class is the Xapian specific version of the SearchQuery class.
It acts as an intermediary between the ``SearchQuerySet`` and the
``SearchBackend`` itself.
"""
def __init__(self, backend=None, site=None):
"""
Create a new instance of the SearchQuery setting the backend as
specified. If no backend is set, will use the Xapian `SearchBackend`.
Optional arguments:
``backend`` -- The ``SearchBackend`` to use (default = None)
``site`` -- The site to use (default = None)
"""
super(SearchQuery, self).__init__(backend=backend)
self.backend = backend or SearchBackend(site=site)
def build_params(self, *args, **kwargs):
kwargs = super(SearchQuery, self).build_params(*args, **kwargs)
@ -955,7 +933,7 @@ class SearchQuery(BaseSearchQuery):
DOCUMENT_CT_TERM_PREFIX,
model._meta.app_label, model._meta.module_name
)
), 0 # Pure boolean sub-query
), 0 # Pure boolean sub-query
) for model in self.models
]
query = xapian.Query(
@ -1274,3 +1252,8 @@ def _marshal_datetime(dt):
dt.year, dt.month, dt.day, dt.hour,
dt.minute, dt.second
)
class XapianEngine(BaseEngine):
backend = XapianSearchBackend
query = XapianSearchQuery