action="show", conditions=dict(method=["GET"]))

        m.connect("formatted_notification", "/notifications/{notification_id}.{format}",

                  action="show", conditions=dict(method=["GET"]))

    #ADMIN MAIN PAGES

    with rmap.submapper(path_prefix=ADMIN_PREFIX,

                        controller='admin/admin') as m:

        m.connect('admin_home', '', action='index')

        m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9\. _-]*}',

                  action='add_repo')

    #==========================================================================

    # API V2

    #==========================================================================

    with rmap.submapper(path_prefix=ADMIN_PREFIX,

                        controller='api/api') as m:

        m.connect('api', '/api')

    #USER JOURNAL

    rmap.connect('journal', '%s/journal' % ADMIN_PREFIX, controller='journal')

    rmap.connect('public_journal', '%s/public_journal' % ADMIN_PREFIX,

                 controller='journal', action="public_journal")

                 controller='journal', action="public_journal_rss")

    rmap.connect('public_journal_atom',

                 '%s/public_journal_atom' % ADMIN_PREFIX, controller='journal',

                 action="public_journal_atom")

    rmap.connect('toggle_following', '%s/toggle_following' % ADMIN_PREFIX,

                 controller='journal', action='toggle_following',

                 conditions=dict(method=["POST"]))

    #SEARCH

    rmap.connect('search', '%s/search' % ADMIN_PREFIX, controller='search',)

    rmap.connect('search_repo', '%s/search/{search_repo:.*}' % ADMIN_PREFIX,

                  controller='search')

    #LOGIN/LOGOUT/REGISTER/SIGN IN

    rmap.connect('login_home', '%s/login' % ADMIN_PREFIX, controller='login')

    rmap.connect('logout_home', '%s/logout' % ADMIN_PREFIX, controller='login',

                 action='logout')

    rmap.connect('register', '%s/register' % ADMIN_PREFIX, controller='login',

                 action='register')

    rmap.connect('reset_password', '%s/password_reset' % ADMIN_PREFIX,

                 controller='login', action='password_reset')

    rmap.connect('reset_password_confirmation',

        c.journal_data = render('journal/journal_data.html')

        if request.environ.get('HTTP_X_PARTIAL_XHR'):

            return c.journal_data

        return render('journal/public_journal.html')

    @LoginRequired(api_access=True)

    def public_journal_atom(self):

        """

        Produce an atom-1.0 feed via feedgenerator module

        """

        c.following = self.sa.query(UserFollowing)\

            .filter(UserFollowing.user_id == self.rhodecode_user.user_id)\

            .options(joinedload(UserFollowing.follows_repository))\

            .all()

        journal = self._get_journal_data(c.following)

        feed = Atom1Feed(title=self.title % 'atom',

                         link=url('public_journal_atom', qualified=True),

                         description=_('Public journal'),

                         language=self.language,

                         ttl=self.ttl)

        for entry in journal[:self.feed_nr]:

                                 entry.repository.repo_name)

            desc = action_extra()

            feed.add_item(title=title,

                          pubdate=entry.action_date,

                          link=url('', qualified=True),

                          author_email=entry.user.email,

                          author_name=entry.user.full_contact,

                          description=desc)

        response.content_type = feed.mime_type

        return feed.writeString('utf-8')

    @LoginRequired(api_access=True)

    def public_journal_rss(self):

        """

        Produce an rss2 feed via feedgenerator module

        """

        c.following = self.sa.query(UserFollowing)\

            .filter(UserFollowing.user_id == self.rhodecode_user.user_id)\

            .options(joinedload(UserFollowing.follows_repository))\

            .all()

        journal = self._get_journal_data(c.following)

        feed = Rss201rev2Feed(title=self.title % 'rss',

                         link=url('public_journal_rss', qualified=True),

                         description=_('Public journal'),

                         language=self.language,

                         ttl=self.ttl)

        for entry in journal[:self.feed_nr]:

                                 entry.repository.repo_name)

            desc = action_extra()

            feed.add_item(title=title,

                          pubdate=entry.action_date,

                          link=url('', qualified=True),

                          author_email=entry.user.email,

                          author_name=entry.user.full_contact,

                          description=desc)

        response.content_type = feed.mime_type

        return feed.writeString('utf-8')

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import sys

import traceback

import logging

from os.path import dirname as dn, join as jn

#to get the rhodecode import

sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))

from string import strip

from shutil import rmtree

from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.fields import TEXT, ID, STORED, Schema, FieldType

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter

from sqlalchemy import engine_from_config

from rhodecode.model import init_model

from rhodecode.model.scm import ScmModel

from rhodecode.model.repo import RepoModel

from rhodecode.config.environment import load_environment

from rhodecode.lib.utils2 import LazyProperty

from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\

    load_rcextensions

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    owner=TEXT(),

    repository=TEXT(stored=True),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

class MakeIndex(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    summary = "Creates index for full text search given configuration file"

    group_name = "RhodeCode"

    takes_config_file = -1

    parser = Command.standard_parser(verbose=True)

        """

        Smart function that implements chunking the content

        but not overlap chunks so it doesn't highlight the same

        close occurrences twice.

        :param matcher:

        :param size:

        """

        memory = [(0, 0)]

        for span in self.matcher.spans():

            start = span.startchar or 0

            end = span.endchar or 0

            start_offseted = max(0, start - self.fragment_size)

            end_offseted = end + self.fragment_size

            if start_offseted < memory[-1][1]:

                start_offseted = memory[-1][1]

            memory.append((start_offseted, end_offseted,))

            yield (start_offseted, end_offseted,)

    def highlight(self, content, top=5):

        if self.search_type != 'content':

            return ''

        hl = highlight(

            terms=self.highlight_items,

            analyzer=ANALYZER,

            fragmenter=FRAGMENTER,

            formatter=FORMATTER,

            top=top

        )

        return hl

        """

        Adding doc to writer this function itself fetches data from

        the instance of vcs backend

        """

        node = self.get_node(repo, path)

        indexed = indexed_w_content = 0

        # we just index the content of chosen files, and skip binary files

        if node.extension in INDEX_EXTENSIONS and not node.is_binary:

            u_content = node.content

            if not isinstance(u_content, unicode):

                log.warning('  >> %s Could not get this content as unicode '

                            'replacing with empty content' % path)

                u_content = u''

            else:

                log.debug('    >> %s [WITH CONTENT]' % path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s' % path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        writer.add_document(

            owner=unicode(repo.contact),

            repository=safe_unicode(repo_name),

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

    def build_index(self):

        if os.path.exists(self.index_location):

            log.debug('removing previous index')

            rmtree(self.index_location)

        if not os.path.exists(self.index_location):

            os.mkdir(self.index_location)

        idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)

        writer = idx.writer()

        log.debug('BUILDIN INDEX FOR EXTENSIONS %s' % INDEX_EXTENSIONS)

        for repo_name, repo in self.repo_paths.items():

            log.debug('building index @ %s' % repo.path)

            i_cnt = iwc_cnt = 0

            for idx_path in self.get_paths(repo):

                i, iwc = self.add_doc(writer, idx_path, repo, repo_name)

                i_cnt += i

                iwc_cnt += iwc

        to_index = set()

        reader = idx.reader()

        writer = idx.writer()

        # Loop over the stored fields in the index

        for fields in reader.all_stored_fields():

            indexed_path = fields['path']

            indexed_repo_path = fields['repository']

            indexed_paths.add(indexed_path)

            if not indexed_repo_path in self.filtered_repo_update_paths:

                continue

            repo = self.repo_paths[indexed_repo_path]

            try:

                node = self.get_node(repo, indexed_path)

                # Check if this file was changed since it was indexed

                indexed_time = fields['modtime']

                mtime = self.get_node_mtime(node)

                if mtime > indexed_time:

                    # The file has changed, delete it and add it to the list of

                    # files to reindex

                    to_index.add(indexed_path)

            except (ChangesetError, NodeDoesNotExistError):

                # This file was deleted since it was indexed

                log.debug('removing from index %s' % indexed_path)

                writer.delete_by_term('path', indexed_path)

        # Loop over the files in the filesystem

        # Assume we have a function that gathers the filenames of the

        # documents to be indexed

        ri_cnt = riwc_cnt = 0

        for repo_name, repo in self.repo_paths.items():

            for path in self.get_paths(repo):

                path = safe_unicode(path)

                if path in to_index or path not in indexed_paths:

                    # This is either a file that's changed, or a new file

                    # that wasn't indexed before. So index it!

                    i, iwc = self.add_doc(writer, path, repo, repo_name)

                    log.debug('re indexing %s' % path)

                    ri_cnt += i

                    riwc_cnt += iwc

        log.debug('added %s files %s with content for repo %s' % (

                     ri_cnt + riwc_cnt, riwc_cnt, repo.path)

        )

        log.debug('>> COMMITING CHANGES <<')

        writer.commit(merge=True)

        log.debug('>>> FINISHED REBUILDING INDEX <<<')

    def run(self, full_index=False):

        """Run daemon"""

        if full_index or self.initial:

            self.build_index()

        else:

            self.update_index()

				<div class="input">${h.text('q',c.cur_query,class_="small")}

					<div class="button highlight">

						<input type="submit" value="${_('Search')}" class="ui-button"/>

					</div>

				</div>

				<div style="font-weight: bold;clear:Both;margin-left:200px">${c.runtime}</div>

			</div>

			<div class="field">

	            <div class="label">

	                <label for="type">${_('Search in')}</label>

	            </div>

                <div class="select">

                    ${h.select('type',c.cur_type,[('content',_('File contents')),

                        ##('commit',_('Commit messages')),

                        ('path',_('File names')),

                        ##('repository',_('Repository names')),

                        ])}

                </div>

             </div>

		</div>

	</div>

	${h.end_form()}

    %if c.cur_search == 'content':

        <%include file='search_content.html'/>

    %elif c.cur_search == 'path':

        <%include file='search_path.html'/>

    %elif c.cur_search == 'commit':

        <%include file='search_commit.html'/>

    %elif c.cur_search == 'repository':

        <%include file='search_repository.html'/>

    %endif

</div>

</%def>