# -*- coding: utf-8 -*-

"""

    rhodecode.lib.indexers.__init__

    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Whoosh indexing module for RhodeCode

    :created_on: Aug 17, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import sys

import traceback

import logging

from os.path import dirname as dn, join as jn

#to get the rhodecode import

sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))

from string import strip

from shutil import rmtree

from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter

from webhelpers.html.builder import escape, literal

from sqlalchemy import engine_from_config

from rhodecode.model import init_model

from rhodecode.model.scm import ScmModel

from rhodecode.model.repo import RepoModel

from rhodecode.config.environment import load_environment

from rhodecode.lib.utils2 import LazyProperty

from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\

    load_rcextensions

log = logging.getLogger(__name__)

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

CHGSETS_SCHEMA = Schema(

    raw_id=ID(unique=True, stored=True),

    last=BOOLEAN(),

    owner=TEXT(),

    repository=ID(unique=True, stored=True),

    author=TEXT(stored=True),

    message=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    parents=TEXT(),

    added=TEXT(),

    removed=TEXT(),

    changed=TEXT(),

)

CHGSET_IDX_NAME = 'CHGSET_INDEX'

class MakeIndex(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    summary = "Creates index for full text search given configuration file"

    group_name = "RhodeCode"

    takes_config_file = -1

    parser = Command.standard_parser(verbose=True)

    def command(self):

        logging.config.fileConfig(self.path_to_ini_file)

        from pylons import config

        add_cache(config)

        engine = engine_from_config(config, 'sqlalchemy.db1.')

        init_model(engine)

        index_location = config['index_dir']

        repo_location = self.options.repo_location \

            if self.options.repo_location else RepoModel().repos_path

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        repo_update_list = map(strip, self.options.repo_update_list.split(',')) \

            if self.options.repo_update_list else None

        load_rcextensions(config['here'])

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from rhodecode.lib.pidlock import LockHeld, DaemonLock

        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

        try:

            l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock'))

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list,

                                 repo_update_list=repo_update_list)\

                .run(full_index=self.options.full_index)

            l.release()

        except LockHeld:

            sys.exit(1)

    def update_parser(self):

        self.parser.add_option('--repo-location',

                          action='store',

                          dest='repo_location',

                          help="Specifies repositories location to index OPTIONAL",

                          )

        self.parser.add_option('--index-only',

                          action='store',

                          dest='repo_list',

                          help="Specifies a comma separated list of repositores "

                                "to build index on. If not given all repositories "

                                "are scanned for indexing. OPTIONAL",

                          )

        self.parser.add_option('--update-only',

                          action='store',

                          dest='repo_update_list',

                          help="Specifies a comma separated list of repositores "

                                "to re-build index on. OPTIONAL",

                          )

        self.parser.add_option('-f',

                          action='store_true',

                          dest='full_index',

                          help="Specifies that index should be made full i.e"

                                " destroy old and build from scratch",

                          default=False)

class WhooshResultWrapper(object):

    def __init__(self, search_type, searcher, matcher, highlight_items,

                 repo_location):

        self.search_type = search_type

        self.searcher = searcher

        self.matcher = matcher

        self.highlight_items = highlight_items

        self.fragment_size = 200

        self.repo_location = repo_location

    @LazyProperty

    def doc_ids(self):

        docs_id = []

        while self.matcher.is_active():

        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)

        #filter repo list

        if repo_list:

            self.filtered_repo_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_list:

                    self.filtered_repo_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_paths

        #filter update repo list

        self.filtered_repo_update_paths = {}

        if repo_update_list:

            self.filtered_repo_update_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_update_list:

                    self.filtered_repo_update_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_update_paths

        self.initial = True

        if not os.path.isdir(self.index_location):

            os.makedirs(self.index_location)

            log.info('Cannot run incremental index since it does not'

                     ' yet exist running full build')

        elif not exists_in(self.index_location, IDX_NAME):

            log.info('Running full index build as the file content'

                     ' index does not exist')

        elif not exists_in(self.index_location, CHGSET_IDX_NAME):

            log.info('Running full index build as the changeset'

                     ' index does not exist')

        else:

            self.initial = False

    def get_paths(self, repo):

        """

        recursive walk in root dir and return a set of all path in that dir

        based on repository walk function

        """

        index_paths_ = set()

        try:

            tip = repo.get_changeset('tip')

            for topnode, dirs, files in tip.walk('/'):

                for f in files:

                    index_paths_.add(jn(repo.path, f.path))

        except RepositoryError, e:

            log.debug(traceback.format_exc())

            pass

        return index_paths_

    def get_node(self, repo, path):

        n_path = path[len(repo.path) + 1:]

        node = repo.get_changeset().get_node(n_path)

        return node

    def get_node_mtime(self, node):

        return mktime(node.last_changeset.date.timetuple())

    def add_doc(self, writer, path, repo, repo_name):

        """

        Adding doc to writer this function itself fetches data from

        the instance of vcs backend

        """

        node = self.get_node(repo, path)

        indexed = indexed_w_content = 0

        # we just index the content of chosen files, and skip binary files

        if node.extension in INDEX_EXTENSIONS and not node.is_binary:

            u_content = node.content

            if not isinstance(u_content, unicode):

                log.warning('  >> %s Could not get this content as unicode '

                            'replacing with empty content' % path)

                u_content = u''

            else:

                log.debug('    >> %s [WITH CONTENT]' % path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s' % path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        p = safe_unicode(path)

        writer.add_document(

            fileid=p,

            owner=unicode(repo.contact),

            repository=safe_unicode(repo_name),

            path=p,

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

        """

        Add all changeset in the vcs repo starting at start_rev

        to the index writer

        """

        indexed=0

            writer.add_document(

                raw_id=unicode(cs.raw_id),

                owner=unicode(repo.contact),

                repository=safe_unicode(repo_name),

                author=cs.author,

                message=cs.message,

                last=cs.last,

                added=u' '.join([node.path for node in cs.added]).lower(),

                removed=u' '.join([node.path for node in cs.removed]).lower(),

                changed=u' '.join([node.path for node in cs.changed]).lower(),

                parents=u' '.join([cs.raw_id for cs in cs.parents]),

            )

            indexed += 1

        log.debug('indexed %d changesets for repo %s' % (indexed, repo_name))

    def index_files(self, file_idx_writer, repo_name, repo):

        i_cnt = iwc_cnt = 0

        log.debug('building index for [%s]' % repo.path)

        for idx_path in self.get_paths(repo):

            i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name)

            i_cnt += i

            iwc_cnt += iwc

        log.debug('added %s files %s with content for repo %s' % (i_cnt + iwc_cnt, iwc_cnt, repo.path))

    def update_changeset_index(self):

        idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)

        with idx.searcher() as searcher:

            writer = idx.writer()

            writer_is_dirty = False

            try:

                for repo_name, repo in self.repo_paths.items():

                    # skip indexing if there aren't any revs in the repo

                        continue

                    qp = QueryParser('repository', schema=CHGSETS_SCHEMA)

                    q = qp.parse(u"last:t AND %s" % repo_name)

                    last_rev = 0

                    if len(results) > 0:

                    # there are new changesets to index or a new repo to index

                        # delete the docs in the index for the previous last changeset(s)

                        for hit in results:

                            q = qp.parse(u"last:t AND %s AND raw_id:%s" % 

                                            (repo_name, hit['raw_id']))

                            writer.delete_by_query(q)

                        # index from the previous last changeset + all new ones

                        writer_is_dirty = True

            finally:

                if writer_is_dirty:

                    log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')

                    writer.commit(merge=True)

                    log.debug('>> COMMITTED CHANGES TO CHANGESET INDEX<<')

                else:

                    writer.cancel

    def update_file_index(self):

        log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '

                   'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))

        idx = open_dir(self.index_location, indexname=self.indexname)

        # The set of all paths in the index

        indexed_paths = set()

        # The set of all paths we need to re-index

        to_index = set()

        writer = idx.writer()

        writer_is_dirty = False

        try:

            with idx.reader() as reader:

                # Loop over the stored fields in the index

                for fields in reader.all_stored_fields():

                    indexed_path = fields['path']

                    indexed_repo_path = fields['repository']

                    indexed_paths.add(indexed_path)

                    if not indexed_repo_path in self.filtered_repo_update_paths:

                        continue

                    repo = self.repo_paths[indexed_repo_path]

                    try:

                        node = self.get_node(repo, indexed_path)

                        # Check if this file was changed since it was indexed

                        indexed_time = fields['modtime']

                        mtime = self.get_node_mtime(node)

                        if mtime > indexed_time:

                            # The file has changed, delete it and add it to the list of

                            # files to reindex

                            log.debug('adding to reindex list %s mtime: %s vs %s' % (

                                            indexed_path, mtime, indexed_time)

                            )

                            writer.delete_by_term('fileid', indexed_path)

                            writer_is_dirty = True

                            to_index.add(indexed_path)

                    except (ChangesetError, NodeDoesNotExistError):

                        # This file was deleted since it was indexed

                        log.debug('removing from index %s' % indexed_path)

                        writer.delete_by_term('path', indexed_path)

                        writer_is_dirty = True

            # Loop over the files in the filesystem

            # Assume we have a function that gathers the filenames of the

            # documents to be indexed

            ri_cnt_total = 0  # indexed

            riwc_cnt_total = 0  # indexed with content

            for repo_name, repo in self.repo_paths.items():

                # skip indexing if there aren't any revisions

                if len(repo) < 1:

                    continue

                ri_cnt = 0   # indexed

                riwc_cnt = 0  # indexed with content

                for path in self.get_paths(repo):

                    path = safe_unicode(path)

                    if path in to_index or path not in indexed_paths:

                        # This is either a file that's changed, or a new file

                        # that wasn't indexed before. So index it!

                        i, iwc = self.add_doc(writer, path, repo, repo_name)

                        writer_is_dirty = True

                        log.debug('re indexing %s' % path)

                        ri_cnt += i

                        ri_cnt_total += 1

                        riwc_cnt += iwc

                        riwc_cnt_total += iwc

                log.debug('added %s files %s with content for repo %s' % (

                             ri_cnt + riwc_cnt, riwc_cnt, repo.path)

                )

            log.debug('indexed %s files in total and %s with content' % (

                        ri_cnt_total, riwc_cnt_total)

            )

        finally:

            if writer_is_dirty:

                log.debug('>> COMMITING CHANGES <<')

                writer.commit(merge=True)

                log.debug('>>> FINISHED REBUILDING INDEX <<<')

            else:

                writer.cancel()

    def build_indexes(self):

import os

from rhodecode.tests import *

from nose.plugins.skip import SkipTest

class TestSearchController(TestController):

    def test_index(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'))

        self.assertTrue('class="small" id="q" name="q" type="text"' in

                        response.body)

        # Test response...

    def test_empty_search(self):

        if os.path.isdir(self.index_location):

            raise SkipTest('skipped due to existing index')

        else:

            self.log_user()

            response = self.app.get(url(controller='search', action='index'),

                                    {'q': HG_REPO})

            self.assertTrue('There is no index to search in. '

                            'Please run whoosh indexer' in response.body)

    def test_normal_search(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'def repo'})

        response.mustcontain('10 results')

    def test_repo_search(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'repository:%s def test' % HG_REPO})

        response.mustcontain('4 results')

    def test_search_last(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'last:t', 'type': 'commit'})

        response.mustcontain('1 results')

    def test_search_commit_message(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'bother to ask where to fetch repo during tests',

                                 'type': 'commit'})

        response.mustcontain('1 results')

        response.mustcontain('a00c1b6f5d7a6ae678fd553a8b81d92367f7ecf1')

    def test_search_commit_changed_file(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'changed:tests/utils.py',

                                 'type': 'commit'})

        response.mustcontain('a00c1b6f5d7a6ae678fd553a8b81d92367f7ecf1')

    def test_search_commit_added_file(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                {'q': 'added:README.rst',

                                 'type': 'commit'})

        response.mustcontain('1 results')

        response.mustcontain('3803844fdbd3b711175fc3da9bdacfcd6d29a6fb')

    def test_search_author(self):

        self.log_user()

        response = self.app.get(url(controller='search', action='index'),

                                 'type': 'commit'})

        response.mustcontain('1 results')