kallithea Changeset - 168cc92c1b53

Changeset - 168cc92c1b53

Parent rev.

Child rev.

[Not reviewed]

default

0 2 0

FUJIWARA Katsunori - 9 years ago 2017-01-22 18:17:38
foozy@lares.dti.ne.jp

search: prevent pathname related conditions from removing "stop words"

Before this revision, pathname related conditions below cause
unintentional ignorance of "stop words".

- path:,extension: (for "File contents" or "File names")
- added:, removed:, changed: (for "Commit messages")

Therefore, pathname related conditions with "this", "a", "you", and so
on are completely ignored, even if they are valid pathname components.

To prevent pathname related conditions from removing "stop words",
this revision explicitly specifies "analyzer" for pathname related
fields of SCHEMA and CHGSETS_SCHEMA.

Difference between PATHANALYZER and default analyzer of TEXT is
whether "stop words" are preserved or not. Tokenization is still
applied on pathnames.

This revision requires full re-building index tables, because indexing
schemas are changed.

2 files changed with 19 insertions and 11 deletions:

kallithea/lib/indexers/__init__.py

kallithea/tests/functional/test_search_indexing.py

0 comments (0 inline, 0 general)

kallithea/lib/indexers/__init__.py

➞

Show inline comments

@@ @@ -40,82 +40,90 @@ from whoosh.highlight import highlight a @@
 from kallithea.lib.utils2 import LazyProperty
 log = logging.getLogger(__name__)
 # CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 # CUSTOM ANALYZER raw-string + lowercase filter
+#
 # This is useful to:
 # - avoid tokenization
 # - avoid removing "stop words" from text
 # - search case-insensitively
+#
 ICASEIDANALYZER = IDTokenizer() | LowercaseFilter()
 # CUSTOM ANALYZER raw-string
+#
 # This is useful to:
 # - avoid tokenization
 # - avoid removing "stop words" from text
+#
 IDANALYZER = IDTokenizer()
 # CUSTOM ANALYZER wordsplit + lowercase filter, for pathname-like text
+#
 # This is useful to:
 # - avoid removing "stop words" from text
 # - search case-insensitively
+#
 PATHANALYZER = RegexTokenizer() | LowercaseFilter()
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(
     fileid=ID(unique=True),
     owner=TEXT(),
     # this field preserves case of repository name for exact matching
     repository_rawname=TEXT(analyzer=IDANALYZER),
     repository=TEXT(stored=True, analyzer=ICASEIDANALYZER),
     path=TEXT(stored=True),
+    path=TEXT(stored=True, analyzer=PATHANALYZER),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     extension=TEXT(stored=True)
+    extension=TEXT(stored=True, analyzer=PATHANALYZER)
+)
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 CHGSETS_SCHEMA = Schema(
     raw_id=ID(unique=True, stored=True),
     date=NUMERIC(stored=True),
     last=BOOLEAN(),
     owner=TEXT(),
     # this field preserves case of repository name for exact matching
     # and unique-ness in index table
     repository_rawname=ID(unique=True),
     repository=ID(stored=True, analyzer=ICASEIDANALYZER),
     author=TEXT(stored=True),
     message=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     parents=TEXT(),
     added=TEXT(),
     removed=TEXT(),
     changed=TEXT(),
     added=TEXT(analyzer=PATHANALYZER),
     removed=TEXT(analyzer=PATHANALYZER),
     changed=TEXT(analyzer=PATHANALYZER),
+)
 CHGSET_IDX_NAME = 'CHGSET_INDEX'
 # used only to generate queries in journal
 JOURNAL_SCHEMA = Schema(
     username=ID(),
     date=DATETIME(),
     action=TEXT(),
     repository=ID(),
     ip=TEXT(),
+)
 class WhooshResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items,
                  repo_location):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200
         self.repo_location = repo_location

kallithea/tests/functional/test_search_indexing.py

➞

Show inline comments

@@ @@ -135,61 +135,61 @@ class TestSearchControllerIndexing(TestC @@
         lname = u'indexing_test-foo'
         uname = u'indexing_test-FOO'
         # (1) "repository:REPONAME" condition should match against
         # repositories case-insensitively
         q = 'repository:%s %s' % (lname, query)
         response = self.app.get(url(controller='search', action='index'),
                                 {'q': q, 'type': searchtype})
         response.mustcontain('>%d results' % (hit * 2))
         # (2) on the other hand, searching under the specific
         # repository should return results only for that repository,
         # even if specified name matches against another repository
         # case-insensitively.
         response = self.app.get(url(controller='search', action='index',
                                     repo_name=uname),
                                 {'q': query, 'type': searchtype})
         response.mustcontain('>%d results' % hit)
         # confirm that there is no matching against lower name repository
         assert uname in response
-        #assert lname not in response
         assert lname not in response
     @parametrize('searchtype,query,hit', [
         ('content', 'path:this/is/it def test', 37),
         ('commit', 'added:this/is/it bother to ask where', 4),
         ('content', 'path:this/is/it def test', 1),
         ('commit', 'added:this/is/it bother to ask where', 1),
         # this condition matches against files below, because
         # "path:" condition is also applied on "repository path".
         # - "this/is/it" in "stopword_test" repo
         # - "this_should_be_unique_filename.txt" in "this-is-it" repo
-        ('path', 'this/is/it', 0),
+        ('path', 'this/is/it', 2),
         ('content', 'extension:us', 0),
         ('path', 'extension:us', 0),
         ('content', 'extension:us', 1),
         ('path', 'extension:us', 1),
     ])
     def test_filename_stopword(self, searchtype, query, hit):
         response = self.app.get(url(controller='search', action='index'),
                                 {'q': query, 'type': searchtype})
         response.mustcontain('>%d results' % hit)
     @parametrize('searchtype,query,hit', [
         # matching against both 2 files
         ('content', 'owner:"this is it"', 0),
         ('content', 'owner:this-is-it', 0),
         ('path', 'owner:"this is it"', 0),
         ('path', 'owner:this-is-it', 0),
         # matching against both 2 revisions
         ('commit', 'owner:"this is it"', 0),
         ('commit', 'owner:"this-is-it"', 0),
         # matching against only 1 revision
         ('commit', 'author:"this is it"', 0),
         ('commit', 'author:"this-is-it"', 0),
     ])
     def test_mailaddr_stopword(self, searchtype, query, hit):
         response = self.app.get(url(controller='search', action='index'),

0 comments (0 inline, 0 general)