:author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

import traceback

from pylons.i18n.translation import _

from pylons import request, config, tmpl_context as c

from rhodecode.lib.auth import LoginRequired

from rhodecode.lib.base import BaseController, render

from webhelpers.paginate import Page

from webhelpers.util import update_params

from whoosh.index import open_dir, EmptyIndexError

from whoosh.qparser import QueryParser, QueryParserError

from whoosh.query import Phrase, Wildcard, Term, Prefix

from rhodecode.model.repo import RepoModel

log = logging.getLogger(__name__)

class SearchController(BaseController):

    @LoginRequired()

    def __before__(self):

        super(SearchController, self).__before__()

    def index(self, search_repo=None):

        c.repo_name = search_repo

        c.formated_results = []

        c.runtime = ''

        c.cur_query = request.GET.get('q', None)

        c.cur_search = search_type = {'content': 'content',

                                      'path': 'path',

                                      'repository': 'repository'}\

                                      .get(c.cur_type, 'content')

        if c.cur_query:

            cur_query = c.cur_query.lower()

        if c.cur_query:

            p = int(request.params.get('page', 1))

            highlight_items = set()

            try:

                idx = open_dir(config['app_conf']['index_dir'],

                searcher = idx.searcher()

                if c.repo_name:

                    cur_query = u'repository:%s %s' % (c.repo_name, cur_query)

                try:

                    query = qp.parse(unicode(cur_query))

                    # extract words for highlight

                    if isinstance(query, Phrase):

                        highlight_items.update(query.words)

                    elif isinstance(query, Prefix):

                        highlight_items.add(query.text)

                    else:

                        for i in query.all_terms():

                                highlight_items.add(i[1])

                    matcher = query.matcher(searcher)

                    results = searcher.search(query)

                    res_ln = len(results)

                    c.runtime = '%s results (%.3f seconds)' % (

                        res_ln, results.runtime

                    )

                    def url_generator(**kw):

                        return update_params("?q=%s&type=%s" \

                    repo_location = RepoModel().repos_path

                    c.formated_results = Page(

                        WhooshResultWrapper(search_type, searcher, matcher,

                                            highlight_items, repo_location),

                        page=p,

                        item_count=res_ln,

                        items_per_page=10,

                        url=url_generator

                    )

                except QueryParserError:

                    c.runtime = _('Invalid search query. Try quoting it.')

                searcher.close()

            except (EmptyIndexError, IOError):

                log.error(traceback.format_exc())

                log.error('Empty Index data')

                c.runtime = _('There is no index to search in. '

                              'Please run whoosh indexer')

            except (Exception):

                log.error(traceback.format_exc())

                c.runtime = _('An error occurred during this search operation')

        # Return a rendered template

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import sys

import traceback

import logging

from os.path import dirname as dn, join as jn

#to get the rhodecode import

sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))

from string import strip

from shutil import rmtree

from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter

from webhelpers.html.builder import escape, literal

from sqlalchemy import engine_from_config

from rhodecode.model import init_model

from rhodecode.model.scm import ScmModel

from rhodecode.model.repo import RepoModel

from rhodecode.config.environment import load_environment

from rhodecode.lib.utils2 import LazyProperty

from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\

    load_rcextensions

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

class MakeIndex(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    summary = "Creates index for full text search given configuration file"

    group_name = "RhodeCode"

    takes_config_file = -1

    parser = Command.standard_parser(verbose=True)

    def command(self):

        logging.config.fileConfig(self.path_to_ini_file)

        from pylons import config

        add_cache(config)

        engine = engine_from_config(config, 'sqlalchemy.db1.')

        init_model(engine)

        index_location = config['index_dir']

        repo_location = self.options.repo_location \

            if self.options.repo_location else RepoModel().repos_path

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        repo_update_list = map(strip, self.options.repo_update_list.split(',')) \

        return len(self.doc_ids)

    def __iter__(self):

        """

        Allows Iteration over results,and lazy generate content

        *Requires* implementation of ``__getitem__`` method.

        """

        for docid in self.doc_ids:

            yield self.get_full_content(docid)

    def __getitem__(self, key):

        """

        Slicing of resultWrapper

        """

        i, j = key.start, key.stop

        slices = []

        for docid in self.doc_ids[i:j]:

            slices.append(self.get_full_content(docid))

        return slices

    def get_full_content(self, docid):

        res = self.searcher.stored_fields(docid[0])

        full_repo_path = jn(self.repo_location, res['repository'])

        f_path = res['path'].split(full_repo_path)[-1]

        f_path = f_path.lstrip(os.sep)

        content_short = self.get_short_content(res, docid[1])

        res.update({'content_short': content_short,

        return res

    def get_short_content(self, res, chunks):

        return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])

    def get_chunks(self):

        """

        Smart function that implements chunking the content

        but not overlap chunks so it doesn't highlight the same

        close occurrences twice.

        :param matcher:

        :param size:

        """

        memory = [(0, 0)]

        for span in self.matcher.spans():

            start = span.startchar or 0

            end = span.endchar or 0

            start_offseted = max(0, start - self.fragment_size)

            end_offseted = end + self.fragment_size

            if start_offseted < memory[-1][1]:

                start_offseted = memory[-1][1]

            memory.append((start_offseted, end_offseted,))

            yield (start_offseted, end_offseted,)

    def highlight(self, content, top=5):

            return ''

        hl = highlight(

            text=content,

            terms=self.highlight_items,

            analyzer=ANALYZER,

            fragmenter=FRAGMENTER,

            formatter=FORMATTER,

            top=top

        )

        return hl

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import sys

import logging

import traceback

from shutil import rmtree

from time import mktime

from os.path import dirname as dn

from os.path import join as jn

#to get the rhodecode import

project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))

sys.path.append(project_path)

from rhodecode.config.conf import INDEX_EXTENSIONS

from rhodecode.model.scm import ScmModel

from rhodecode.lib.utils2 import safe_unicode

from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \

    NodeDoesNotExistError

log = logging.getLogger('whoosh_indexer')

class WhooshIndexingDaemon(object):

    """

    Daemon for atomic indexing jobs

    """

    def __init__(self, indexname=IDX_NAME, index_location=None,

                 repo_location=None, sa=None, repo_list=None,

                 repo_update_list=None):

        self.indexname = indexname

        self.index_location = index_location

        if not index_location:

            raise Exception('You have to provide index location')

        self.repo_location = repo_location

        if not repo_location:

            raise Exception('You have to provide repositories location')

        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)

        #filter repo list

        if repo_list:

            self.filtered_repo_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_list:

                    self.filtered_repo_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_paths

        #filter update repo list

        self.filtered_repo_update_paths = {}

        if repo_update_list:

            self.filtered_repo_update_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_update_list:

                    self.filtered_repo_update_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_update_paths

        if not os.path.isdir(self.index_location):

            os.makedirs(self.index_location)

            log.info('Cannot run incremental index since it does not'

                     ' yet exist running full build')

    def get_paths(self, repo):

        """

        recursive walk in root dir and return a set of all path in that dir

        based on repository walk function

        """

        index_paths_ = set()

        try:

            tip = repo.get_changeset('tip')

            for topnode, dirs, files in tip.walk('/'):

                for f in files:

                    index_paths_.add(jn(repo.path, f.path))

        except RepositoryError, e:

            log.debug(traceback.format_exc())

            pass

        return index_paths_

    def get_node(self, repo, path):

        n_path = path[len(repo.path) + 1:]

        node = repo.get_changeset().get_node(n_path)

        return node

    def get_node_mtime(self, node):

                            'replacing with empty content' % path)

                u_content = u''

            else:

                log.debug('    >> %s [WITH CONTENT]' % path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s' % path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        p = safe_unicode(path)

        writer.add_document(

            fileid=p,

            owner=unicode(repo.contact),

            repository=safe_unicode(repo_name),

            path=p,

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

            i_cnt = iwc_cnt = 0

            for idx_path in self.get_paths(repo):

                i_cnt += i

                iwc_cnt += iwc

        writer.commit(merge=True)

        log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '

                   'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))

        idx = open_dir(self.index_location, indexname=self.indexname)

        # The set of all paths in the index

        indexed_paths = set()

        # The set of all paths we need to re-index

        to_index = set()

        writer = idx.writer()

        # Loop over the stored fields in the index

        for fields in reader.all_stored_fields():

            indexed_path = fields['path']

            indexed_repo_path = fields['repository']

            indexed_paths.add(indexed_path)

            if not indexed_repo_path in self.filtered_repo_update_paths:

                continue

            repo = self.repo_paths[indexed_repo_path]

            try:

                node = self.get_node(repo, indexed_path)

                # Check if this file was changed since it was indexed

                indexed_time = fields['modtime']

                mtime = self.get_node_mtime(node)

                if mtime > indexed_time:

                    # The file has changed, delete it and add it to the list of

                    # files to reindex

                    log.debug('adding to reindex list %s mtime: %s vs %s' % (

                                    indexed_path, mtime, indexed_time)

                    )

                    writer.delete_by_term('fileid', indexed_path)

                    to_index.add(indexed_path)

            except (ChangesetError, NodeDoesNotExistError):

                # This file was deleted since it was indexed

                log.debug('removing from index %s' % indexed_path)

                writer.delete_by_term('path', indexed_path)

        # Loop over the files in the filesystem

        # Assume we have a function that gathers the filenames of the

        # documents to be indexed

        ri_cnt_total = 0  # indexed

        riwc_cnt_total = 0  # indexed with content

        for repo_name, repo in self.repo_paths.items():

            ri_cnt = 0   # indexed

            riwc_cnt = 0  # indexed with content

            for path in self.get_paths(repo):

                path = safe_unicode(path)

                if path in to_index or path not in indexed_paths:

                    # This is either a file that's changed, or a new file

                    # that wasn't indexed before. So index it!

                    i, iwc = self.add_doc(writer, path, repo, repo_name)

                    log.debug('re indexing %s' % path)

                    ri_cnt += i

                    ri_cnt_total += 1

                    riwc_cnt += iwc

                    riwc_cnt_total += iwc

            log.debug('added %s files %s with content for repo %s' % (

                         ri_cnt + riwc_cnt, riwc_cnt, repo.path)

            )

        log.debug('indexed %s files in total and %s with content' % (

                    ri_cnt_total, riwc_cnt_total)

        )

        log.debug('>> COMMITING CHANGES <<')

        writer.commit(merge=True)

        log.debug('>>> FINISHED REBUILDING INDEX <<<')

    def run(self, full_index=False):

        """Run daemon"""

        if full_index or self.initial:

        else:

		${h.form(h.url('search_repo',search_repo=c.repo_name),method='get')}

	%else:

		${h.form(h.url('search'),method='get')}

	%endif

	<div class="form">

		<div class="fields">

			<div class="field field-first field-noborder">

             <div class="label">

                 <label for="q">${_('Search term')}</label>

             </div>

				<div class="input">${h.text('q',c.cur_query,class_="small")}

					<div class="button highlight">

						<input type="submit" value="${_('Search')}" class="ui-button"/>

					</div>

				</div>

				<div style="font-weight: bold;clear:Both;margin-left:200px">${c.runtime}</div>

			</div>

			<div class="field">

	            <div class="label">

	                <label for="type">${_('Search in')}</label>

	            </div>

                <div class="select">

                    ${h.select('type',c.cur_type,[('content',_('File contents')),

                        ('path',_('File names')),

                        ##('repository',_('Repository names')),

                        ])}

                </div>

             </div>

		</div>

	</div>

	${h.end_form()}

    <div class="search">

        <%include file='search_content.html'/>

        <%include file='search_path.html'/>

        <%include file='search_commit.html'/>

        <%include file='search_repository.html'/>

    %endif

    </div>

</div>

</%def>

class TestSearchController(TestController):

    def test_index(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'))

        self.assertTrue('class="small" id="q" name="q" type="text"' in

                        response.body)

        # Test response...

    def test_empty_search(self):

        if os.path.isdir(self.index_location):

            raise SkipTest('skipped due to existing index')

        else:

            self.log_user()

            response = self.app.get(url(controller='search', action='index'),

                                    {'q': HG_REPO})

            self.assertTrue('There is no index to search in. '

                            'Please run whoosh indexer' in response.body)

    def test_normal_search(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'def repo'})

    def test_repo_search(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'repository:%s def test' % HG_REPO})

        response.mustcontain('4 results')