:created_on: Nov 21, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

from itertools import groupby

from sqlalchemy import or_

from sqlalchemy.orm import joinedload

from sqlalchemy.sql.expression import func

from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed

from webob.exc import HTTPBadRequest

from pylons import request, tmpl_context as c, response, url

from pylons.i18n.translation import _

from kallithea.controllers.admin.admin import _journal_filter

from kallithea.model.db import UserLog, UserFollowing, Repository, User

from kallithea.model.meta import Session

from kallithea.model.repo import RepoModel

import kallithea.lib.helpers as h

from kallithea.lib.helpers import Page

from kallithea.lib.auth import LoginRequired, NotAnonymous

from kallithea.lib.base import BaseController, render

from kallithea.lib.utils2 import safe_int, AttributeDict

from kallithea.lib.compat import json

log = logging.getLogger(__name__)

class JournalController(BaseController):

    def __before__(self):

        super(JournalController, self).__before__()

        self.language = 'en-us'

        self.ttl = "5"

        self.feed_nr = 20

        c.search_term = request.GET.get('filter')

    def _get_daily_aggregate(self, journal):

        groups = []

        for k, g in groupby(journal, lambda x: x.action_as_day):

            user_group = []

            #groupby username if it's a present value, else fallback to journal username

                l = list(g2)

                user_group.append((l[0].user, l))

            groups.append((k, user_group,))

        return groups

    def _get_journal_data(self, following_repos):

        repo_ids = [x.follows_repository.repo_id for x in following_repos

                    if x.follows_repository is not None]

        user_ids = [x.follows_user.user_id for x in following_repos

                    if x.follows_user is not None]

        filtering_criterion = None

        if repo_ids and user_ids:

            filtering_criterion = or_(UserLog.repository_id.in_(repo_ids),

                        UserLog.user_id.in_(user_ids))

        if repo_ids and not user_ids:

            filtering_criterion = UserLog.repository_id.in_(repo_ids)

        if not repo_ids and user_ids:

            filtering_criterion = UserLog.user_id.in_(user_ids)

        if filtering_criterion is not None:

            journal = self.sa.query(UserLog)\

                .options(joinedload(UserLog.user))\

                .options(joinedload(UserLog.repository))

            #filter

            try:

                journal = _journal_filter(journal, c.search_term)

            except Exception:

                # we want this to crash for now

                raise

            journal = journal.filter(filtering_criterion)\

                        .order_by(UserLog.action_date.desc())

        else:

            journal = []

        return journal

    def _atom_feed(self, repos, public=True):

        journal = self._get_journal_data(repos)

        if public:

            _link = url('public_journal_atom', qualified=True)

            _desc = '%s %s %s' % (c.site_name, _('public journal'),

                                  'atom feed')

        else:

            _link = url('journal_atom', qualified=True)

            _desc = '%s %s %s' % (c.site_name, _('journal'), 'atom feed')

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.celerylib.tasks

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Kallithea task modules, containing all task that suppose to be run

by celery daemon

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Oct 6, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

from celery.decorators import task

import os

import traceback

import logging

from os.path import join as jn

from time import mktime

from operator import itemgetter

from string import lower

from pylons import config

from kallithea import CELERY_ON

from kallithea.lib.celerylib import run_task, locked_task, dbsession, \

    str2bool, __get_lockkey, LockHeld, DaemonLock, get_session

from kallithea.lib.helpers import person

from kallithea.lib.rcmail.smtp_mailer import SmtpMailer

from kallithea.lib.utils import add_cache, action_logger

from kallithea.lib.compat import json, OrderedDict

from kallithea.lib.hooks import log_create_repository

from kallithea.model.db import Statistics, Repository, User

add_cache(config)  # pragma: no cover

def get_logger(cls):

    if CELERY_ON:

        try:

            log = cls.get_logger()

        except Exception:

            log = logging.getLogger(__name__)

    else:

        log = logging.getLogger(__name__)

    return log

@task(ignore_result=True)

@locked_task

@dbsession

def whoosh_index(repo_location, full_index):

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    DBS = get_session()

    index_location = config['index_dir']

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location, sa=DBS)\

                         .run(full_index=full_index)

@task(ignore_result=True)

@dbsession

def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):

    log = get_logger(get_commits_stats)

    DBS = get_session()

    lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,

                            ts_max_y)

    lockkey_path = config['app_conf']['cache_dir']

    log.info('running task with lockkey %s' % lockkey)

    try:

        lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))

        # for js data compatibility cleans the key for person from '

        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}

        commits_by_day_aggregate = {}

        repo = Repository.get_by_repo_name(repo_name)

        if repo is None:

log = logging.getLogger(__name__)

class Changeset(dict):

    """A collection of changes to be applied to a database.

    Changesets are bound to a repository and manage a set of

    scripts from that repository.

    Behaves like a dict, for the most part. Keys are ordered based on step value.

    """

    def __init__(self, start, *changes, **k):

        """

        Give a start version; step must be explicitly stated.

        """

        self.step = k.pop('step', 1)

        self.start = version.VerNum(start)

        self.end = self.start

        for change in changes:

            self.add(change)

    def __iter__(self):

        return iter(self.items())

    def keys(self):

        """

        In a series of upgrades x -> y, keys are version x. Sorted.

        """

        ret = super(Changeset, self).keys()

        # Reverse order if downgrading

        ret.sort(reverse=(self.step < 1))

        return ret

    def values(self):

        return [self[k] for k in self.keys()]

    def items(self):

        return zip(self.keys(), self.values())

    def add(self, change):

        """Add new change to changeset"""

        key = self.end

        self.end += self.step

        self[key] = change

    def run(self, *p, **k):

        """Run the changeset scripts"""

            script.run(*p, **k)

class Repository(pathed.Pathed):

    """A project's change script repository"""

    _config = 'migrate.cfg'

    _versions = 'versions'

    def __init__(self, path):

        log.debug('Loading repository %s...' % path)

        self.verify(path)

        super(Repository, self).__init__(path)

        self.config = cfgparse.Config(os.path.join(self.path, self._config))

        self.versions = version.Collection(os.path.join(self.path,

                                                      self._versions))

        log.debug('Repository %s loaded successfully' % path)

        log.debug('Config: %r' % self.config.to_dict())

    @classmethod

    def verify(cls, path):

        """

        Ensure the target path is a valid repository.

        :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>`

        """

        # Ensure the existence of required files

        try:

            cls.require_found(path)

            cls.require_found(os.path.join(path, cls._config))

            cls.require_found(os.path.join(path, cls._versions))

        except exceptions.PathNotFoundError, e:

            raise exceptions.InvalidRepositoryError(path)

    @classmethod

    def prepare_config(cls, tmpl_dir, name, options=None):

        """

        Prepare a project configuration file for a new project.

        :param tmpl_dir: Path to Repository template

        :param config_file: Name of the config file in Repository template

        :param name: Repository name

        :type tmpl_dir: string

        :type config_file: string

        :type name: string

        :returns: Populated config file

        """

        if options is None:

    :param filenode:

    """

    lexer = get_custom_lexer(filenode.extension) or filenode.lexer

    return literal(markup_whitespace(

        code_highlight(filenode.content, lexer, CodeHtmlFormatter(**kwargs))))

def pygmentize_annotation(repo_name, filenode, **kwargs):

    """

    pygmentize function for annotation

    :param filenode:

    """

    color_dict = {}

    def gen_color(n=10000):

        """generator for getting n of evenly distributed colors using

        hsv color and golden ratio. It always return same order of colors

        :returns: RGB tuple

        """

        def hsv_to_rgb(h, s, v):

            if s == 0.0:

                return v, v, v

            i = int(h * 6.0)  # XXX assume int() truncates!

            f = (h * 6.0) - i

            p = v * (1.0 - s)

            q = v * (1.0 - s * f)

            t = v * (1.0 - s * (1.0 - f))

            i = i % 6

            if i == 0:

                return v, t, p

            if i == 1:

                return q, v, p

            if i == 2:

                return p, v, t

            if i == 3:

                return p, q, v

            if i == 4:

                return t, p, v

            if i == 5:

                return v, p, q

        golden_ratio = 0.618033988749895

        h = 0.22717784590367374

            h += golden_ratio

            h %= 1

            HSV_tuple = [h, 0.95, 0.95]

            RGB_tuple = hsv_to_rgb(*HSV_tuple)

            yield map(lambda x: str(int(x * 256)), RGB_tuple)

    cgenerator = gen_color()

    def get_color_string(cs):

        if cs in color_dict:

            col = color_dict[cs]

        else:

            col = color_dict[cs] = cgenerator.next()

        return "color: rgb(%s)! important;" % (', '.join(col))

    def url_func(repo_name):

        def _url_func(changeset):

            author = changeset.author

            date = changeset.date

            message = tooltip(changeset.message)

            tooltip_html = ("<div style='font-size:0.8em'><b>Author:</b>"

                            " %s<br/><b>Date:</b> %s</b><br/><b>Message:"

                            "</b> %s<br/></div>")

            tooltip_html = tooltip_html % (author, date, message)

            lnk_format = show_id(changeset)

            uri = link_to(

                    lnk_format,

                    url('changeset_home', repo_name=repo_name,

                        revision=changeset.raw_id),

                    style=get_color_string(changeset.raw_id),

                    class_='tooltip',

                    title=tooltip_html

                  )

            uri += '\n'

            return uri

        return _url_func

    return literal(markup_whitespace(annotate_highlight(filenode, url_func(repo_name), **kwargs)))

def is_following_repo(repo_name, user_id):

    from kallithea.model.scm import ScmModel

    return ScmModel().is_following_repo(repo_name, user_id)

            for rev in revs_ids[:revs_top_limit]:

                _op, _name = _get_op(rev)

                # we want parsed changesets, or new log store format is bad

                if parse_cs:

                    try:

                        if repo is None:

                            repo = user_log.repository.scm_instance

                        _rev = repo.get_changeset(rev)

                        revs.append(_rev)

                    except ChangesetDoesNotExistError:

                        log.error('cannot find revision %s in this repo' % rev)

                        revs.append(rev)

                        continue

                else:

                    _rev = AttributeDict({

                        'short_id': rev[:12],

                        'raw_id': rev,

                        'message': '',

                        'op': _op,

                        'ref_name': _name

                    })

                    revs.append(_rev)

        cs_links = [" " + ', '.join(

            [lnk(rev, repo_name) for rev in revs[:revs_limit]]

        )]

        _op1, _name1 = _get_op(revs_ids[0])

        _op2, _name2 = _get_op(revs_ids[-1])

        _rev = '%s...%s' % (_name1, _name2)

        compare_view = (

            ' <div class="compare_view tooltip" title="%s">'

            '<a href="%s">%s</a> </div>' % (

                _('Show all combined changesets %s->%s') % (

                    revs_ids[0][:12], revs_ids[-1][:12]

                ),

                url('changeset_home', repo_name=repo_name,

                    revision=_rev

                ),

                _('compare view')

            )

        )

        # if we have exactly one more than normally displayed

        # just display it, takes less space than displaying

        # "and 1 more revisions"

        if len(revs_ids) == revs_limit + 1:

        # hidden-by-default ones

        if len(revs_ids) > revs_limit + 1:

            uniq_id = revs_ids[0]

            html_tmpl = (

                '<span> %s <a class="show_more" id="_%s" '

                'href="#more">%s</a> %s</span>'

            )

            if not feed:

                cs_links.append(html_tmpl % (

                      _('and'),

                      uniq_id, _('%s more') % (len(revs_ids) - revs_limit),

                      _('revisions')

                    )

                )

            if not feed:

                html_tmpl = '<span id="%s" style="display:none">, %s </span>'

            else:

                html_tmpl = '<span id="%s"> %s </span>'

            morelinks = ', '.join(

              [lnk(rev, repo_name) for rev in revs[revs_limit:]]

            )

            if len(revs_ids) > revs_top_limit:

                morelinks += ', ...'

            cs_links.append(html_tmpl % (uniq_id, morelinks))

        if len(revs) > 1:

            cs_links.append(compare_view)

        return ''.join(cs_links)

    def get_fork_name():

        repo_name = action_params

        _url = url('summary_home', repo_name=repo_name)

        return _('fork name %s') % link_to(action_params, _url)

    def get_user_name():

        user_name = action_params

        return user_name

    def get_users_group():

        group_name = action_params

        return group_name

    def get_pull_request():

        pull_request_id = action_params

        urls = re.compile(r'(\<a.*?\<\/a\>)',)

        links = []

        for e in urls.split(t):

            if not urls.match(e):

                links.append('<a class="message-link" href="%s">%s</a>' % (l, e))

            else:

                links.append(e)

        return ''.join(links)

    # urlify changesets - extract revisions and make link out of them

    newtext = urlify_changesets(escaper(text_), repository)

    # extract http/https links and make them real urls

    newtext = urlify_text(newtext, safe=False)

    try:

        from kallithea import CONFIG

        conf = CONFIG

        # allow multiple issue servers to be used

        valid_indices = [

            x.group(1)

            for x in map(lambda x: re.match(r'issue_pat(.*)', x), conf.keys())

            if x and 'issue_server_link%s' % x.group(1) in conf

            and 'issue_prefix%s' % x.group(1) in conf

        ]

        log.debug('found issue server suffixes `%s` during valuation of: %s'

                  % (','.join(valid_indices), newtext))

        for pattern_index in valid_indices:

            ISSUE_PATTERN = conf.get('issue_pat%s' % pattern_index)

            ISSUE_SERVER_LNK = conf.get('issue_server_link%s' % pattern_index)

            ISSUE_PREFIX = conf.get('issue_prefix%s' % pattern_index)

            log.debug('pattern suffix `%s` PAT:%s SERVER_LINK:%s PREFIX:%s'

                      % (pattern_index, ISSUE_PATTERN, ISSUE_SERVER_LNK,

                         ISSUE_PREFIX))

            URL_PAT = re.compile(r'%s' % ISSUE_PATTERN)

            def url_func(match_obj):

                pref = ''

                if match_obj.group().startswith(' '):

                    pref = ' '

                issue_id = ''.join(match_obj.groups())

                '%(pref)s<a class="%(cls)s" href="%(url)s">'

                '%(issue-prefix)s%(id-repr)s'

                '</a>'

                     'pref': pref,

                     'cls': 'issue-tracker-link',

                     'id-repr': issue_id,

                     'issue-prefix': ISSUE_PREFIX,

                     'serv': ISSUE_SERVER_LNK,

                }

            newtext = URL_PAT.sub(url_func, newtext)

            log.debug('processed prefix:`%s` => %s' % (pattern_index, newtext))

        # if we actually did something above

        if link_:

            # wrap not links into final link => link_

            newtext = linkify_others(newtext, link_)

    except Exception:

        log.error(traceback.format_exc())

        pass

    return literal(newtext)

def rst(source):

    return literal('<div class="rst-block">%s</div>' %

                   MarkupRenderer.rst(source))

def rst_w_mentions(source):

    """

    Wrapped rst renderer with @mention highlighting

    :param source:

    """

    return literal('<div class="rst-block">%s</div>' %

                   MarkupRenderer.rst_with_mentions(source))

def short_ref(ref_type, ref_name):

    if ref_type == 'rev':

        return short_id(ref_name)

    return ref_name

def link_to_ref(repo_name, ref_type, ref_name, rev=None):

    """

    Return full markup for a href to changeset_home for a changeset.

    If ref_type is branch it will link to changelog.

    ref_name is shortened if ref_type is 'rev'.

    if rev is specified show it too, explicitly linking to that revision.

    """

    txt = short_ref(ref_type, ref_name)

    if ref_type == 'branch':

        u = url('changelog_home', repo_name=repo_name, branch=ref_name)

    else:

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_index

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-index paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

from __future__ import with_statement

import os

import sys

import logging

from string import strip

from kallithea.model.repo import RepoModel

from kallithea.lib.utils import BasePasterCommand, load_rcextensions

# Add location of top level folder to sys.path

from os.path import dirname as dn

rc_path = dn(dn(dn(os.path.realpath(__file__))))

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Creates or updates full text search index"

    def command(self):

        logging.config.fileConfig(self.path_to_ini_file)

        #get SqlAlchemy session

        self._init_session()

        from pylons import config

        index_location = config['index_dir']

        load_rcextensions(config['here'])

        repo_location = self.options.repo_location \

            if self.options.repo_location else RepoModel().repos_path

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        repo_update_list = map(strip, self.options.repo_update_list.split(',')) \

            if self.options.repo_update_list else None

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from kallithea.lib.pidlock import LockHeld, DaemonLock

        from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

        try:

            l = DaemonLock(file_=os.path.join(dn(dn(index_location)),

                                              'make_index.lock'))

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list,