from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.fields import TEXT, ID, STORED, Schema, FieldType

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter

from webhelpers.html.builder import escape

from sqlalchemy import engine_from_config

from vcs.utils.lazy import LazyProperty

from rhodecode.model import init_model

from rhodecode.model.scm import ScmModel

from rhodecode.config.environment import load_environment

from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP

from rhodecode.lib.utils import BasePasterCommand, Command, add_cache

#EXTENSIONS WE WANT TO INDEX CONTENT OFF

INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys()

#CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

    group_name = "RhodeCode"

    takes_config_file = -1

    parser = Command.standard_parser(verbose=True)

    def command(self):

        from pylons import config

        add_cache(config)

        engine = engine_from_config(config, 'sqlalchemy.db1.')

        init_model(engine)

        index_location = config['index_dir']

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from rhodecode.lib.pidlock import LockHeld, DaemonLock

        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

        try:

            l = DaemonLock(file=jn(dn(dn(index_location)), 'make_index.lock'))

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,