# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.controllers.search

~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Search controller for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 7, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import traceback

import urllib

from tg.i18n import ugettext as _

from tg import request, config, tmpl_context as c

from whoosh.index import open_dir, exists_in, EmptyIndexError

from whoosh.qparser import QueryParser, QueryParserError

from whoosh.query import Phrase, Prefix

from kallithea.lib.auth import LoginRequired

from kallithea.lib.base import BaseRepoController, render

from kallithea.lib.indexers import CHGSETS_SCHEMA, SCHEMA, CHGSET_IDX_NAME, \

    IDX_NAME, WhooshResultWrapper

from kallithea.lib.page import Page

from kallithea.lib.utils2 import safe_str, safe_int

from kallithea.model.repo import RepoModel

log = logging.getLogger(__name__)

class SearchController(BaseRepoController):

    @LoginRequired(allow_default_user=True)

    def index(self, repo_name=None):

        c.repo_name = repo_name

        c.formated_results = []

        c.runtime = ''

        c.cur_query = request.GET.get('q', None)

        c.cur_type = request.GET.get('type', 'content')

        c.cur_search = search_type = {'content': 'content',

                                      'commit': 'message',

                                      'path': 'path',

                                      'repository': 'repository'

                                      }.get(c.cur_type, 'content')

        index_name = {

            'content': IDX_NAME,

            'commit': CHGSET_IDX_NAME,

            'path': IDX_NAME

        }.get(c.cur_type, IDX_NAME)

        schema_defn = {

            'content': SCHEMA,

            'commit': CHGSETS_SCHEMA,

            'path': SCHEMA

        }.get(c.cur_type, SCHEMA)

        log.debug('IDX: %s', index_name)

        log.debug('SCHEMA: %s', schema_defn)

        if c.cur_query:

            cur_query = c.cur_query.lower()

            log.debug(cur_query)

        if c.cur_query:

            p = safe_int(request.GET.get('page'), 1)

            highlight_items = set()

            index_dir = config['app_conf']['index_dir']

            try:

                if not exists_in(index_dir, index_name):

                    raise EmptyIndexError

                idx = open_dir(index_dir, indexname=index_name)

                searcher = idx.searcher()

                qp = QueryParser(search_type, schema=schema_defn)

                if c.repo_name:

                    # use "repository_rawname:" instead of "repository:"

                    # for case-sensitive matching

                    cur_query = u'repository_rawname:%s %s' % (c.repo_name, cur_query)

                try:

                    query = qp.parse(unicode(cur_query))

                    # extract words for highlight

                    if isinstance(query, Phrase):

                        highlight_items.update(query.words)

                    elif isinstance(query, Prefix):

                        highlight_items.add(query.text)

                    else:

                        for i in query.all_terms():

                            if i[0] in ['content', 'message']:

                                highlight_items.add(i[1])

                    matcher = query.matcher(searcher)

                    log.debug('query: %s', query)

                    log.debug('hl terms: %s', highlight_items)

                    results = searcher.search(query)

                    res_ln = len(results)

                    c.runtime = '%s results (%.3f seconds)' % (

                        res_ln, results.runtime

                    )

                    def url_generator(**kw):

                        q = urllib.quote(safe_str(c.cur_query))

                    repo_location = RepoModel().repos_path

                    c.formated_results = Page(

                        WhooshResultWrapper(search_type, searcher, matcher,

                                            highlight_items, repo_location),

                        page=p,

                        item_count=res_ln,

                        items_per_page=10,

                        url=url_generator

                    )

                except QueryParserError:

                    c.runtime = _('Invalid search query. Try quoting it.')

                searcher.close()

            except EmptyIndexError:

                log.error("Empty search index - run 'kallithea-cli index-create' regularly")

                c.runtime = _('The server has no search index.')

            except Exception:

                log.error(traceback.format_exc())

                c.runtime = _('An error occurred during search operation.')

        # Return a rendered template

        return render('/search/search.html')

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

Helper functions

Consists of functions to typically be used within templates, but also

available to Controllers. This module is available to both as 'h'.

"""

import hashlib

import json

import StringIO

import logging

import re

import urlparse

import textwrap

import random

from beaker.cache import cache_region

from pygments.formatters.html import HtmlFormatter

from pygments import highlight as code_highlight

from tg.i18n import ugettext as _

from webhelpers.pylonslib import Flash as _Flash

from kallithea.config.routing import url

from kallithea.lib.annotate import annotate_highlight

from kallithea.lib.pygmentsutils import get_custom_lexer

from kallithea.lib.utils2 import str2bool, safe_unicode, safe_str, \

    time_to_datetime, AttributeDict, safe_int, MENTIONS_REGEX

from kallithea.lib.markup_renderer import url_re

from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError

from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset

log = logging.getLogger(__name__)

def canonical_url(*args, **kargs):

    '''Like url(x, qualified=True), but returns url that not only is qualified

    but also canonical, as configured in canonical_url'''

    from kallithea import CONFIG

    try:

        parts = CONFIG.get('canonical_url', '').split('://', 1)

        kargs['host'] = parts[1]

        kargs['protocol'] = parts[0]

    except IndexError:

        kargs['qualified'] = True

    return url(*args, **kargs)

def canonical_hostname():

    '''Return canonical hostname of system'''

    from kallithea import CONFIG

    try:

        parts = CONFIG.get('canonical_url', '').split('://', 1)

        return parts[1].split('/', 1)[0]

    except IndexError:

        parts = url('home', qualified=True).split('://', 1)

        return parts[1].split('/', 1)[0]

def html_escape(s):

    """Return string with all html escaped.

    This is also safe for javascript in html but not necessarily correct.

    """

    return (s

        .replace('&', '&')

        .replace(">", ">")

        .replace("<", "&lt;")

        .replace('"', """)

        .replace("'", "'") # Note: this is HTML5 not HTML4 and might not work in mails

        )

def js(value):

    """Convert Python value to the corresponding JavaScript representation.

    This is necessary to safely insert arbitrary values into HTML <script>

    sections e.g. using Mako template expression substitution.

    Note: Rather than using this function, it's preferable to avoid the

    insertion of values into HTML <script> sections altogether. Instead,

    data should (to the extent possible) be passed to JavaScript using

    data attributes or AJAX calls, eliminating the need for JS specific

    escaping.

    Note: This is not safe for use in attributes (e.g. onclick), because

    quotes are not escaped.

    Because the rules for parsing <script> varies between XHTML (where

    normal rules apply for any special characters) and HTML (where

    entities are not interpreted, but the literal string "</script>"

    is forbidden), the function ensures that the result never contains

    '&', '<' and '>', thus making it safe in both those contexts (but

    not in attributes).

    """

    return literal(

        ('(' + json.dumps(value) + ')')

        # In JSON, the following can only appear in string literals.

        .replace('&', r'\x26')

        .replace('<', r'\x3c')

        .replace('>', r'\x3e')

    )

def jshtml(val):

    """HTML escapes a string value, then converts the resulting string

    to its corresponding JavaScript representation (see `js`).

    This is used when a plain-text string (possibly containing special

    HTML characters) will be used by a script in an HTML context (e.g.

    element.innerHTML or jQuery's 'html' method).

    If in doubt, err on the side of using `jshtml` over `js`, since it's

    better to escape too much than too little.

    """

    return js(escape(val))

def shorter(s, size=20, firstline=False, postfix='...'):

    """Truncate s to size, including the postfix string if truncating.

    If firstline, truncate at newline.

    """

    if firstline:

        s = s.split('\n', 1)[0].rstrip()

    if len(s) > size:

        return s[:size - len(postfix)] + postfix

    return s

safeid = _make_safe_id_component

def FID(raw_id, path):

    """

    Creates a unique ID for filenode based on it's hash of path and revision

    it's safe to use in urls

    :param raw_id:

    :param path:

    """

    return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_str(path)).hexdigest()[:12])

class _FilesBreadCrumbs(object):

    def __call__(self, repo_name, rev, paths):

        if isinstance(paths, str):

            paths = safe_unicode(paths)

        url_l = [link_to(repo_name, url('files_home',

                                        repo_name=repo_name,

                                        revision=rev, f_path=''),

                         class_='ypjax-link')]

        paths_l = paths.split('/')

        for cnt, p in enumerate(paths_l):

            if p != '':

                url_l.append(link_to(p,

                                     url('files_home',

                                         repo_name=repo_name,

                                         revision=rev,

                                         f_path='/'.join(paths_l[:cnt + 1])

                                         ),

                                     class_='ypjax-link'

                                     )

                             )

        return literal('/'.join(url_l))

files_breadcrumbs = _FilesBreadCrumbs()

class CodeHtmlFormatter(HtmlFormatter):

    """

    My code Html Formatter for source codes

    """

    def wrap(self, source, outfile):

        return self._wrap_div(self._wrap_pre(self._wrap_code(source)))

    def _wrap_code(self, source):

        for cnt, it in enumerate(source):

            i, t = it

            t = '<span id="L%s">%s</span>' % (cnt + 1, t)

            yield i, t

    def _wrap_tablelinenos(self, inner):

        dummyoutfile = StringIO.StringIO()

        lncount = 0

        for t, line in inner:

            if t:

                lncount += 1

            dummyoutfile.write(line)

        fl = self.linenostart

        mw = len(str(lncount + fl - 1))

        sp = self.linenospecial

        st = self.linenostep

        la = self.lineanchors

        aln = self.anchorlinenos

        nocls = self.noclasses

        if sp:

            lines = []

            for i in range(fl, fl + lncount):

                if i % st == 0:

                    if i % sp == 0:

                        if aln:

                            lines.append('<a href="#%s%d" class="special">%*d</a>' %

                                         (la, i, mw, i))

                        else:

                            lines.append('<span class="special">%*d</span>' % (mw, i))

                    else:

                        if aln:

                            lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))

                        else:

                            lines.append('%*d' % (mw, i))

                else:

                    lines.append('')

            ls = '\n'.join(lines)

        else:

            lines = []

            for i in range(fl, fl + lncount):

                if i % st == 0:

                    if aln:

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

Custom paging classes

"""

import logging

import math

import re

from kallithea.config.routing import url

from webhelpers.paginate import Page as _Page

log = logging.getLogger(__name__)

class Page(_Page):

    """

    Custom pager emitting Bootstrap paginators

    """

    def __init__(self, *args, **kwargs):

        kwargs.setdefault('url', url.current)

        _Page.__init__(self, *args, **kwargs)

    def _get_pos(self, cur_page, max_page, items):

        edge = (items / 2) + 1

        if (cur_page <= edge):

            radius = max(items / 2, items - cur_page)

        elif (max_page - cur_page) < edge:

            radius = (items - 1) - (max_page - cur_page)

        else:

            radius = items / 2

        left = max(1, (cur_page - (radius)))

        right = min(max_page, cur_page + (radius))

        return left, cur_page, right

    def _range(self, regexp_match):

        """

        Return range of linked pages (e.g. '1 2 [3] 4 5 6 7 8').

        Arguments:

        regexp_match

            A "re" (regular expressions) match object containing the

            radius of linked pages around the current page in

            regexp_match.group(1) as a string

        This function is supposed to be called as a callable in

        re.sub.

        """

        radius = int(regexp_match.group(1))

        # Compute the first and last page number within the radius

        # e.g. '1 .. 5 6 [7] 8 9 .. 12'

        # -> leftmost_page  = 5

        # -> rightmost_page = 9

        leftmost_page, _cur, rightmost_page = self._get_pos(self.page,

                                                            self.last_page,

                                                            (radius * 2) + 1)

        nav_items = []

        # Create a link to the first page (unless we are on the first page

        # or there would be no need to insert '..' spacers)

        if self.page != self.first_page and self.first_page < leftmost_page:

            nav_items.append(HTML.li(self._pagerlink(self.first_page, self.first_page)))

        # Insert dots if there are pages between the first page

        # and the currently displayed page range

        if leftmost_page - self.first_page > 1:

            # Wrap in a SPAN tag if nolink_attr is set

            text_ = '..'

            if self.dotdot_attr:

                text_ = HTML.span(c=text_, **self.dotdot_attr)

            nav_items.append(HTML.li(text_))

        for thispage in xrange(leftmost_page, rightmost_page + 1):

            # Highlight the current page number and do not use a link

            text_ = str(thispage)

            if thispage == self.page:

                # Wrap in a SPAN tag if nolink_attr is set

                if self.curpage_attr:

                    text_ = HTML.li(HTML.span(c=text_), **self.curpage_attr)

                nav_items.append(text_)

            # Otherwise create just a link to that page

            else:

                nav_items.append(HTML.li(self._pagerlink(thispage, text_)))

        # Insert dots if there are pages between the displayed

        # page numbers and the end of the page range

        if self.last_page - rightmost_page > 1:

            text_ = '..'

            # Wrap in a SPAN tag if nolink_attr is set

            if self.dotdot_attr:

                text_ = HTML.span(c=text_, **self.dotdot_attr)

            nav_items.append(HTML.li(text_))

        # Create a link to the very last page (unless we are on the last

        # page or there would be no need to insert '..' spacers)

        if self.page != self.last_page and rightmost_page < self.last_page:

            nav_items.append(HTML.li(self._pagerlink(self.last_page, self.last_page)))

        #_page_link = url.current()

        #nav_items.append(literal('<link rel="prerender" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))

        #nav_items.append(literal('<link rel="prefetch" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.utils2

~~~~~~~~~~~~~~~~~~~~

Some simple helper functions.

Note: all these functions should be independent of Kallithea classes, i.e.

models, controllers, etc.  to prevent import cycles.

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jan 5, 2011

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import re

import sys

import time

import uuid

import datetime

import urllib

import binascii

import pwd

import webob

import urlobject

from tg.i18n import ugettext as _, ungettext

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.lib.compat import json

def str2bool(_str):

    """

    returns True/False value from given string, it tries to translate the

    string into boolean

    :param _str: string value to translate into boolean

    :rtype: boolean

    :returns: boolean from given string

    """

    if _str is None:

        return False

    if _str in (True, False):

        return _str

    _str = str(_str).strip().lower()

    return _str in ('t', 'true', 'y', 'yes', 'on', '1')

def aslist(obj, sep=None, strip=True):

    """

    Returns given string separated by sep as list

    :param obj:

    :param sep:

    :param strip:

    """

    if isinstance(obj, (basestring)):

        lst = obj.split(sep)

        if strip:

            lst = [v.strip() for v in lst]

        return lst

    elif isinstance(obj, (list, tuple)):

        return obj

    elif obj is None:

        return []

    else:

        return [obj]

def convert_line_endings(line, mode):

    """

    Converts a given line  "line end" according to given mode

    Available modes are::

        0 - Unix

        1 - Mac

        2 - DOS

    :param line: given line to convert

    :param mode: mode to convert to

    :rtype: str

    :return: converted line according to mode

    """

    from string import replace

    if mode == 0:

            line = replace(line, '\r\n', '\n')

            line = replace(line, '\r', '\n')

    elif mode == 1:

            line = replace(line, '\r\n', '\r')

            line = replace(line, '\n', '\r')

    elif mode == 2:

            line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)

    return line

def detect_mode(line, default):

    """

    Detects line break for given line, if line break couldn't be found

    given default value is returned

    :param line: str line

    :param default: default

    :rtype: int

    :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS

    """

    if line.endswith('\r\n'):

        return 2

    elif line.endswith('\n'):

        return 0

    elif line.endswith('\r'):

        return 1

    else:

        return default

def generate_api_key():

    """

    Generates a random (presumably unique) API key.

    This value is used in URLs and "Bearer" HTTP Authorization headers,

    def set_invalidate(self):

        """

        Mark caches of this repo as invalid.

        """

        CacheInvalidation.set_invalidate(self.repo_name)

    _scm_instance = None

    @property

    def scm_instance(self):

        if self._scm_instance is None:

            self._scm_instance = self.scm_instance_cached()

        return self._scm_instance

    def scm_instance_cached(self, valid_cache_keys=None):

        @cache_region('long_term', 'scm_instance_cached')

        def _c(repo_name): # repo_name is just for the cache key

            log.debug('Creating new %s scm_instance and populating cache', repo_name)

            return self.scm_instance_no_cache()

        rn = self.repo_name

        valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)

        if not valid:

            log.debug('Cache for %s invalidated, getting new object', rn)

            region_invalidate(_c, None, 'scm_instance_cached', rn)

        else:

            log.debug('Trying to get scm_instance of %s from cache', rn)

        return _c(rn)

    def scm_instance_no_cache(self):

        repo_full_path = safe_str(self.repo_full_path)

        alias = get_scm(repo_full_path)[0]

        log.debug('Creating instance of %s repository from %s',

                  alias, self.repo_full_path)

        backend = get_backend(alias)

        if alias == 'hg':

            repo = backend(repo_full_path, create=False,

                           baseui=self._ui)

        else:

            repo = backend(repo_full_path, create=False)

        return repo

    def __json__(self):

        return dict(

            repo_id=self.repo_id,

            repo_name=self.repo_name,

            landing_rev=self.landing_rev,

        )

class RepoGroup(Base, BaseDbModel):

    __tablename__ = 'groups'

    __table_args__ = (

        _table_args_default_dict,

    )

    SEP = ' » '

    group_id = Column(Integer(), primary_key=True)

    group_name = Column(Unicode(255), nullable=False, unique=True) # full path

    parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)

    group_description = Column(Unicode(10000), nullable=False)

    owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')

    users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')

    parent_group = relationship('RepoGroup', remote_side=group_id)

    owner = relationship('User')

    @classmethod

    def query(cls, sorted=False):

        """Add RepoGroup-specific helpers for common query constructs.

        sorted: if True, apply the default ordering (name, case insensitive).

        """

        q = super(RepoGroup, cls).query()

        if sorted:

            q = q.order_by(func.lower(RepoGroup.group_name))

        return q

    def __init__(self, group_name='', parent_group=None):

        self.group_name = group_name

        self.parent_group = parent_group

    def __unicode__(self):

        return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,

                                      self.group_name)

    @classmethod

    def _generate_choice(cls, repo_group):

        """Return tuple with group_id and name as html literal"""

        if repo_group is None:

            return (-1, u'-- %s --' % _('top level'))

        return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))

    @classmethod

    def groups_choices(cls, groups):

        """Return tuples with group_id and name as html literal."""

        return sorted((cls._generate_choice(g) for g in groups),

                      key=lambda c: c[1].split(cls.SEP))

    @classmethod

    def url_sep(cls):

        return URL_SEP

    @classmethod

    def guess_instance(cls, value):

        return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)

    @classmethod

    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):

        group_name = group_name.rstrip('/')

        if case_insensitive:

            gr = cls.query() \

                .filter(func.lower(cls.group_name) == func.lower(group_name))

        else:

            gr = cls.query() \

                .filter(cls.group_name == group_name)

        if cache:

            gr = gr.options(FromCache(

                            "sql_cache_short",

                            "get_group_%s" % _hash_key(group_name)

                            )

            )

        return gr.scalar()

    @property

    def parents(self):

        groups = []

        group = self.parent_group

        while group is not None:

            groups.append(group)

            group = group.parent_group

            assert group not in groups, group # avoid recursion on bad db content

        groups.reverse()

        return groups

    @property

    def children(self):

        return RepoGroup.query().filter(RepoGroup.parent_group == self)

    @property

    def name(self):

        return self.group_name.split(RepoGroup.url_sep())[-1]

    @property

    def full_path(self):

        return self.group_name

    @property

    def full_path_splitted(self):

        return self.group_name.split(RepoGroup.url_sep())

    @property

    def repositories(self):

        return Repository.query(sorted=True).filter_by(group=self)

    @property

    def repositories_recursive_count(self):

        cnt = self.repositories.count()

        def children_count(group):

            cnt = 0

            for child in group.children:

                cnt += child.repositories.count()