class SearchController(BaseRepoController):

    def __before__(self):

        super(SearchController, self).__before__()

    @LoginRequired()

    def index(self, repo_name=None):

        c.repo_name = repo_name

        c.formated_results = []

        c.runtime = ''

        c.cur_query = request.GET.get('q', None)

        c.cur_type = request.GET.get('type', 'content')

        c.cur_search = search_type = {'content': 'content',

                                      'commit': 'message',

                                      'path': 'path',

                                      'repository': 'repository'

                                      }.get(c.cur_type, 'content')

        index_name = {

            'content': IDX_NAME,

            'commit': CHGSET_IDX_NAME,

            'path': IDX_NAME

        }.get(c.cur_type, IDX_NAME)

        schema_defn = {

            'content': SCHEMA,

            'commit': CHGSETS_SCHEMA,

            'path': SCHEMA

        }.get(c.cur_type, SCHEMA)

        log.debug('IDX: %s', index_name)

        log.debug('SCHEMA: %s', schema_defn)

        if c.cur_query:

            cur_query = c.cur_query.lower()

            log.debug(cur_query)

        if c.cur_query:

            p = safe_int(request.GET.get('page'), 1)

            highlight_items = set()

            try:

                idx = open_dir(config['app_conf']['index_dir'],

                               indexname=index_name)

                searcher = idx.searcher()

                qp = QueryParser(search_type, schema=schema_defn)

                if c.repo_name:

                try:

                    query = qp.parse(unicode(cur_query))

                    # extract words for highlight

                    if isinstance(query, Phrase):

                        highlight_items.update(query.words)

                    elif isinstance(query, Prefix):

                        highlight_items.add(query.text)

                    else:

                        for i in query.all_terms():

                            if i[0] in ['content', 'message']:

                                highlight_items.add(i[1])

                    matcher = query.matcher(searcher)

                    log.debug('query: %s', query)

                    log.debug('hl terms: %s', highlight_items)

                    results = searcher.search(query)

                    res_ln = len(results)

                    c.runtime = '%s results (%.3f seconds)' % (

                        res_ln, results.runtime

                    )

                    def url_generator(**kw):

                        q = urllib.quote(safe_str(c.cur_query))

                        return update_params("?q=%s&type=%s" \

                        % (q, safe_str(c.cur_type)), **kw)

                    repo_location = RepoModel().repos_path

                    c.formated_results = Page(

                        WhooshResultWrapper(search_type, searcher, matcher,

                                            highlight_items, repo_location),

                        page=p,

                        item_count=res_ln,

                        items_per_page=10,

                        url=url_generator

                    )

                except QueryParserError:

                    c.runtime = _('Invalid search query. Try quoting it.')

                searcher.close()

            except (EmptyIndexError, IOError):

                log.error(traceback.format_exc())

                log.error('Empty Index data')

                c.runtime = _('There is no index to search in. '

                              'Please run whoosh indexer')

            except Exception:

                log.error(traceback.format_exc())

                c.runtime = _('An error occurred during search operation.')

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers

~~~~~~~~~~~~~~~~~~~~~~

Whoosh indexing module for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import logging

from os.path import dirname

# Add location of top level folder to sys.path

sys.path.append(dirname(dirname(dirname(os.path.realpath(__file__)))))

from whoosh.analysis import RegexTokenizer, LowercaseFilter, IDTokenizer

from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME

from whoosh.formats import Characters

from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter

from kallithea.lib.utils2 import LazyProperty

log = logging.getLogger(__name__)

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

# CUSTOM ANALYZER raw-string + lowercase filter

#

# This is useful to:

# - avoid tokenization

# - avoid removing "stop words" from text

# - search case-insensitively

#

ICASEIDANALYZER = IDTokenizer() | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True, analyzer=ICASEIDANALYZER),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

CHGSETS_SCHEMA = Schema(

    raw_id=ID(unique=True, stored=True),

    date=NUMERIC(stored=True),

    last=BOOLEAN(),

    owner=TEXT(),

    author=TEXT(stored=True),

    message=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    parents=TEXT(),

    added=TEXT(),

    removed=TEXT(),

    changed=TEXT(),

)

CHGSET_IDX_NAME = 'CHGSET_INDEX'

# used only to generate queries in journal

JOURNAL_SCHEMA = Schema(

    username=ID(),

    date=DATETIME(),

    action=TEXT(),

    repository=ID(),

    ip=TEXT(),

)

class WhooshResultWrapper(object):

    def __init__(self, search_type, searcher, matcher, highlight_items,

                 repo_location):

        self.search_type = search_type

        self.searcher = searcher

        self.matcher = matcher

        self.highlight_items = highlight_items

        self.fragment_size = 200

        self.repo_location = repo_location

    @LazyProperty

    def doc_ids(self):

        docs_id = []

        while self.matcher.is_active():

            docnum = self.matcher.id()

            chunks = [offsets for offsets in self.get_chunks()]

            docs_id.append([docnum, chunks])

            self.matcher.next()

        return docs_id

    def __str__(self):

        return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))

    def __repr__(self):

        return self.__str__()

    def __len__(self):

        assert path[len(repo_path)] in (os.path.sep, os.path.altsep)

        node_path = path[len(repo_path) + 1:]

        cs = self._get_index_changeset(repo, index_rev=index_rev)

        node = cs.get_node(node_path)

        return node

    def is_indexable_node(self, node):

        """

        Just index the content of chosen files, skipping binary files

        """

        return (node.extension in INDEX_EXTENSIONS or node.name in INDEX_FILENAMES) and \

               not node.is_binary

    def get_node_mtime(self, node):

        return mktime(node.last_changeset.date.timetuple())

    def add_doc(self, writer, path, repo, repo_name, index_rev=None):

        """

        Adding doc to writer this function itself fetches data from

        the instance of vcs backend

        """

        try:

            node = self.get_node(repo, path, index_rev)

        except (ChangesetError, NodeDoesNotExistError):

            log.debug("couldn't add doc - %s did not have %r at %s", repo, path, index_rev)

            return 0, 0

        indexed = indexed_w_content = 0

        if self.is_indexable_node(node):

            u_content = node.content

            if not isinstance(u_content, unicode):

                log.warning('  >> %s Could not get this content as unicode '

                            'replacing with empty content' % path)

                u_content = u''

            else:

                log.debug('    >> %s [WITH CONTENT]', path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s', path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        p = safe_unicode(path)

        writer.add_document(

            fileid=p,

            owner=unicode(repo.contact),

            repository=safe_unicode(repo_name),

            path=p,

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

    def index_changesets(self, writer, repo_name, repo, start_rev=None):

        """

        Add all changeset in the vcs repo starting at start_rev

        to the index writer

        :param writer: the whoosh index writer to add to

        :param repo_name: name of the repository from whence the

          changeset originates including the repository group

        :param repo: the vcs repository instance to index changesets for,

          the presumption is the repo has changesets to index

        :param start_rev=None: the full sha id to start indexing from

          if start_rev is None then index from the first changeset in

          the repo

        """

        if start_rev is None:

            start_rev = repo[0].raw_id

        log.debug('indexing changesets in %s starting at rev: %s',

                  repo_name, start_rev)

        indexed = 0

        cs_iter = repo.get_changesets(start=start_rev)

        total = len(cs_iter)

        for cs in cs_iter:

            log.debug('    >> %s/%s', cs, total)

            writer.add_document(

                raw_id=unicode(cs.raw_id),

                owner=unicode(repo.contact),

                date=cs._timestamp,

                repository=safe_unicode(repo_name),

                author=cs.author,

                message=cs.message,

                last=cs.last,

                added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),

                removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),

                changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),

                parents=u' '.join([cs.raw_id for cs in cs.parents]),

            )

            indexed += 1

        log.debug('indexed %d changesets for repo %s', indexed, repo_name)

        return indexed

    def index_files(self, file_idx_writer, repo_name, repo):

        """

        Index files for given repo_name

        :param file_idx_writer: the whoosh index writer to add to

        :param repo_name: name of the repository we're indexing

        :param repo: instance of vcs repo

        """

        i_cnt = iwc_cnt = 0

        log.debug('building index for %s @revision:%s', repo.path,

                                                self._get_index_revision(repo))

        index_rev = self._get_index_revision(repo)

        for idx_path in self.get_paths(repo):

            i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name, index_rev)

            i_cnt += i

            iwc_cnt += iwc

        log.debug('added %s files %s with content for repo %s',

                  i_cnt + iwc_cnt, iwc_cnt, repo.path)

        return i_cnt, iwc_cnt

    def update_changeset_index(self):

        idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)

        with idx.searcher() as searcher:

            writer = idx.writer()

            writer_is_dirty = False

            try:

                indexed_total = 0

                repo_name = None

                for repo_name, repo in self.repo_paths.items():

                    # skip indexing if there aren't any revs in the repo

                    num_of_revs = len(repo)

                    if num_of_revs < 1:

                                           repo_group=groupname)

            repoids[reponame] = repo.repo_id

        # treat "it" as indexable filename

        filenames_mock = list(INDEX_FILENAMES)

        filenames_mock.append('it')

        with mock.patch('kallithea.lib.indexers.daemon.INDEX_FILENAMES',

                        filenames_mock):

            rebuild_index(full_index=False) # only for newly added repos

    @classmethod

    def teardown_class(cls):

        # delete in reversed order, to delete fork destination at first

        for reponame, init_or_fork, groupname in reversed(repos):

            RepoModel().delete(repoids[reponame])

        for reponame, init_or_fork, groupname in reversed(repos):

            if groupname in groupids:

                RepoGroupModel().delete(groupids.pop(groupname),

                                        force_delete=True)

        Session().commit()

        Session.remove()

        rebuild_index(full_index=True) # rebuild fully for subsequent tests

    @parametrize('reponame', [

        (u'indexing_test'),

        (u'indexing_test-fork'),

        (u'group/indexing_test'),

        (u'this-is-it'),

        (u'*-fork'),

        (u'group/*'),

    ])

    @parametrize('searchtype,query,hit', [

        ('content', 'this_should_be_unique_content', 1),

        ('commit', 'this_should_be_unique_commit_log', 1),

        ('path', 'this_should_be_unique_filename.txt', 1),

    ])

    def test_repository_tokenization(self, reponame, searchtype, query, hit):

        self.log_user()

        q = 'repository:%s %s' % (reponame, query)

        response = self.app.get(url(controller='search', action='index'),

                                {'q': q, 'type': searchtype})

        response.mustcontain('>%d results' % hit)

    @parametrize('searchtype,query,hit', [

        ('commit', 'this_should_be_unique_commit_log', 1),

    ])

    def test_repository_case_sensitivity(self, searchtype, query, hit):

        self.log_user()

        lname = u'indexing_test-foo'

        uname = u'indexing_test-FOO'

        # (1) "repository:REPONAME" condition should match against

        # repositories case-insensitively

        q = 'repository:%s %s' % (lname, query)

        response = self.app.get(url(controller='search', action='index'),

                                {'q': q, 'type': searchtype})

        # (2) on the other hand, searching under the specific

        # repository should return results only for that repository,

        # even if specified name matches against another repository

        # case-insensitively.

        response = self.app.get(url(controller='search', action='index',

                                    repo_name=uname),

                                {'q': query, 'type': searchtype})

        response.mustcontain('>%d results' % hit)

        # confirm that there is no matching against lower name repository

        assert uname in response

        #assert lname not in response

    @parametrize('searchtype,query,hit', [

        ('content', 'path:this/is/it def test', 37),

        ('commit', 'added:this/is/it bother to ask where', 4),

        # this condition matches against files below, because

        # "path:" condition is also applied on "repository path".

        # - "this/is/it" in "stopword_test" repo

        # - "this_should_be_unique_filename.txt" in "this-is-it" repo

        ('path', 'this/is/it', 0),

        ('content', 'extension:us', 0),

        ('path', 'extension:us', 0),

    ])

    def test_filename_stopword(self, searchtype, query, hit):

        response = self.app.get(url(controller='search', action='index'),

                                {'q': query, 'type': searchtype})

        response.mustcontain('>%d results' % hit)

    @parametrize('searchtype,query,hit', [

        # matching against both 2 files

        ('content', 'owner:"this is it"', 0),

        ('content', 'owner:this-is-it', 0),

        ('path', 'owner:"this is it"', 0),

        ('path', 'owner:this-is-it', 0),

        # matching against both 2 revisions

        ('commit', 'owner:"this is it"', 0),

        ('commit', 'owner:"this-is-it"', 0),

        # matching against only 1 revision

        ('commit', 'author:"this is it"', 0),

        ('commit', 'author:"this-is-it"', 0),

    ])