schema_defn = {

            'content': SCHEMA,

            'commit': CHGSETS_SCHEMA,

            'path': SCHEMA

        }.get(c.cur_type, SCHEMA)

        log.debug('IDX: %s', index_name)

        log.debug('SCHEMA: %s', schema_defn)

        if c.cur_query:

            cur_query = c.cur_query.lower()

            log.debug(cur_query)

        if c.cur_query:

            p = safe_int(request.GET.get('page'), 1)

            highlight_items = set()

            try:

                idx = open_dir(config['app_conf']['index_dir'],

                               indexname=index_name)

                searcher = idx.searcher()

                qp = QueryParser(search_type, schema=schema_defn)

                if c.repo_name:

                try:

                    query = qp.parse(unicode(cur_query))

                    # extract words for highlight

                    if isinstance(query, Phrase):

                        highlight_items.update(query.words)

                    elif isinstance(query, Prefix):

                        highlight_items.add(query.text)

                    else:

                        for i in query.all_terms():

                            if i[0] in ['content', 'message']:

                                highlight_items.add(i[1])

                    matcher = query.matcher(searcher)

                    log.debug('query: %s', query)

                    log.debug('hl terms: %s', highlight_items)

                    results = searcher.search(query)

                    res_ln = len(results)

                    c.runtime = '%s results (%.3f seconds)' % (

                        res_ln, results.runtime

                    )

                    def url_generator(**kw):

                        q = urllib.quote(safe_str(c.cur_query))

# Add location of top level folder to sys.path

sys.path.append(dirname(dirname(dirname(os.path.realpath(__file__)))))

from whoosh.analysis import RegexTokenizer, LowercaseFilter, IDTokenizer

from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME

from whoosh.formats import Characters

from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter

from kallithea.lib.utils2 import LazyProperty

log = logging.getLogger(__name__)

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

# CUSTOM ANALYZER raw-string + lowercase filter

#

# This is useful to:

# - avoid tokenization

# - avoid removing "stop words" from text

# - search case-insensitively

#

ICASEIDANALYZER = IDTokenizer() | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True, analyzer=ICASEIDANALYZER),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

CHGSETS_SCHEMA = Schema(

    raw_id=ID(unique=True, stored=True),

    date=NUMERIC(stored=True),

    last=BOOLEAN(),

    owner=TEXT(),

    author=TEXT(stored=True),

    message=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    parents=TEXT(),

    added=TEXT(),

    removed=TEXT(),

    changed=TEXT(),

)

CHGSET_IDX_NAME = 'CHGSET_INDEX'

# used only to generate queries in journal

JOURNAL_SCHEMA = Schema(

    username=ID(),

    date=DATETIME(),

    action=TEXT(),

    repository=ID(),

    ip=TEXT(),

)

class WhooshResultWrapper(object):

    def __init__(self, search_type, searcher, matcher, highlight_items,

                 repo_location):

            log.debug("couldn't add doc - %s did not have %r at %s", repo, path, index_rev)

            return 0, 0

        indexed = indexed_w_content = 0

        if self.is_indexable_node(node):

            u_content = node.content

            if not isinstance(u_content, unicode):

                log.warning('  >> %s Could not get this content as unicode '

                            'replacing with empty content' % path)

                u_content = u''

            else:

                log.debug('    >> %s [WITH CONTENT]', path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s', path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        p = safe_unicode(path)

        writer.add_document(

            fileid=p,

            owner=unicode(repo.contact),

            repository=safe_unicode(repo_name),

            path=p,

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

    def index_changesets(self, writer, repo_name, repo, start_rev=None):

        """

        Add all changeset in the vcs repo starting at start_rev

        to the index writer

        :param writer: the whoosh index writer to add to

        :param repo_name: name of the repository from whence the

          changeset originates including the repository group

        :param repo: the vcs repository instance to index changesets for,

          the presumption is the repo has changesets to index

        :param start_rev=None: the full sha id to start indexing from

          if start_rev is None then index from the first changeset in

          the repo

        """

        if start_rev is None:

            start_rev = repo[0].raw_id

        log.debug('indexing changesets in %s starting at rev: %s',

                  repo_name, start_rev)

        indexed = 0

        cs_iter = repo.get_changesets(start=start_rev)

        total = len(cs_iter)

        for cs in cs_iter:

            log.debug('    >> %s/%s', cs, total)

            writer.add_document(

                raw_id=unicode(cs.raw_id),

                owner=unicode(repo.contact),

                date=cs._timestamp,

                repository=safe_unicode(repo_name),

                author=cs.author,

                message=cs.message,

                last=cs.last,

                added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),

                removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),

                changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),

                parents=u' '.join([cs.raw_id for cs in cs.parents]),

            )

            indexed += 1

        log.debug('indexed %d changesets for repo %s', indexed, repo_name)

        return indexed

    def index_files(self, file_idx_writer, repo_name, repo):

        """

        Index files for given repo_name

        :param file_idx_writer: the whoosh index writer to add to

        :param repo_name: name of the repository we're indexing

        :param repo: instance of vcs repo

        """

        i_cnt = iwc_cnt = 0

        log.debug('building index for %s @revision:%s', repo.path,

        rebuild_index(full_index=True) # rebuild fully for subsequent tests

    @parametrize('reponame', [

        (u'indexing_test'),

        (u'indexing_test-fork'),

        (u'group/indexing_test'),

        (u'this-is-it'),

        (u'*-fork'),

        (u'group/*'),

    ])

    @parametrize('searchtype,query,hit', [

        ('content', 'this_should_be_unique_content', 1),

        ('commit', 'this_should_be_unique_commit_log', 1),

        ('path', 'this_should_be_unique_filename.txt', 1),

    ])

    def test_repository_tokenization(self, reponame, searchtype, query, hit):

        self.log_user()

        q = 'repository:%s %s' % (reponame, query)

        response = self.app.get(url(controller='search', action='index'),

                                {'q': q, 'type': searchtype})

        response.mustcontain('>%d results' % hit)

    @parametrize('searchtype,query,hit', [

        ('commit', 'this_should_be_unique_commit_log', 1),

    ])

    def test_repository_case_sensitivity(self, searchtype, query, hit):

        self.log_user()

        lname = u'indexing_test-foo'

        uname = u'indexing_test-FOO'

        # (1) "repository:REPONAME" condition should match against

        # repositories case-insensitively

        q = 'repository:%s %s' % (lname, query)

        response = self.app.get(url(controller='search', action='index'),

                                {'q': q, 'type': searchtype})

        # (2) on the other hand, searching under the specific

        # repository should return results only for that repository,

        # even if specified name matches against another repository

        # case-insensitively.

        response = self.app.get(url(controller='search', action='index',

                                    repo_name=uname),

                                {'q': query, 'type': searchtype})

        response.mustcontain('>%d results' % hit)

        # confirm that there is no matching against lower name repository

        assert uname in response

        #assert lname not in response

    @parametrize('searchtype,query,hit', [

        ('content', 'path:this/is/it def test', 37),

        ('commit', 'added:this/is/it bother to ask where', 4),

        # this condition matches against files below, because

        # "path:" condition is also applied on "repository path".

        # - "this/is/it" in "stopword_test" repo

        # - "this_should_be_unique_filename.txt" in "this-is-it" repo

        ('path', 'this/is/it', 0),