# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.celerylib.tasks

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Kallithea task modules, containing all task that suppose to be run

by celery daemon

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Oct 6, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import os

import rfc822

import traceback

from collections import OrderedDict

from operator import itemgetter

from time import mktime

from tg import config

from kallithea import CELERY_ON

from kallithea.lib import celerylib

from kallithea.lib.compat import json

from kallithea.lib.helpers import person

from kallithea.lib.hooks import log_create_repository

from kallithea.lib.rcmail.smtp_mailer import SmtpMailer

from kallithea.lib.utils import action_logger

from kallithea.lib.utils2 import str2bool

from kallithea.lib.vcs.utils import author_email

from kallithea.model.db import RepoGroup, Repository, Statistics, User

__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']

log = logging.getLogger(__name__)

@celerylib.task

@celerylib.locked_task

@celerylib.dbsession

def whoosh_index(repo_location, full_index):

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    celerylib.get_session() # initialize database connection

    index_location = config['index_dir']

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location) \

                         .run(full_index=full_index)

@celerylib.task

@celerylib.dbsession

def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):

    DBS = celerylib.get_session()

    lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,

                            ts_max_y)

    lockkey_path = config['tg.cache_dir']  # The configured cache_dir is only available under this name ...

    log.info('running task with lockkey %s', lockkey)

    try:

        lock = celerylib.DaemonLock(os.path.join(lockkey_path, lockkey))

        # for js data compatibility cleans the key for person from '

        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}

        commits_by_day_aggregate = {}

        repo = Repository.get_by_repo_name(repo_name)

        if repo is None:

            return True

        repo = repo.scm_instance

        repo_size = repo.count()

        # return if repo have no revisions

        if repo_size < 1:

            lock.release()

            return True

        skip_date_limit = True

        parse_limit = int(config.get('commit_parse_limit'))

        last_rev = None

        last_cs = None

        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository) \

            .filter(Repository.repo_name == repo_name).scalar()

        cur_stats = DBS.query(Statistics) \

            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:

            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:

            # pass silently without any work if we're not on first revision or

            # current state of parsing revision(from db marker) is the

            # last revision

            lock.release()

            return True

        if cur_stats:

            commits_by_day_aggregate = OrderedDict(json.loads(

                                        cur_stats.commit_activity_combined))

            co_day_auth_aggr = json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s', parse_limit)

        lmktime = mktime

        last_rev = last_rev + 1 if last_rev >= 0 else 0

        log.debug('Getting revisions from %s to %s',

             last_rev, last_rev + parse_limit

        )

        for cs in repo[last_rev:last_rev + parse_limit]:

            log.debug('parsing %s', cs)

            last_cs = cs  # remember last parsed changeset

            k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],

                          cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])

            if akc(cs.author) in co_day_auth_aggr:

                try:

                    l = [timegetter(x) for x in

                         co_day_auth_aggr[akc(cs.author)]['data']]

                    time_pos = l.index(k)

                except ValueError:

                    time_pos = None

                if time_pos >= 0 and time_pos is not None:

                    datadict = \

                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1

                    datadict["added"] += len(cs.added)

                    datadict["changed"] += len(cs.changed)

                    datadict["removed"] += len(cs.removed)

                else:

                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {"time": k,

                                    "commits": 1,

                                    "added": len(cs.added),

                                    "changed": len(cs.changed),

                                    "removed": len(cs.removed),

                                   }

                        co_day_auth_aggr[akc(cs.author)]['data'] \

                            .append(datadict)

            else:

                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                    co_day_auth_aggr[akc(cs.author)] = {

                                        "label": akc(cs.author),

                                                 }],

                                        "schema": ["commits"],

                                        }

            # gather all data by day

            if k in commits_by_day_aggregate:

                commits_by_day_aggregate[k] += 1

            else:

                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),

                               key=itemgetter(0))

        if not co_day_auth_aggr:

            co_day_auth_aggr[akc(repo.contact)] = {

                "label": akc(repo.contact),

                "data": [0, 1],

                "schema": ["commits"],

            }

        stats = cur_stats if cur_stats else Statistics()

        stats.commit_activity = json.dumps(co_day_auth_aggr)

        stats.commit_activity_combined = json.dumps(overview_data)

        log.debug('last revision %s', last_rev)

        leftovers = len(repo.revisions[last_rev:])

        log.debug('revisions to parse %s', leftovers)

        if last_rev == 0 or leftovers < parse_limit:

            log.debug('getting code trending stats')

            stats.languages = json.dumps(__get_codes_stats(repo_name))

        try:

            stats.repository = dbrepo

            stats.stat_on_revision = last_cs.revision if last_cs else 0

            DBS.add(stats)

            DBS.commit()

        except:

            log.error(traceback.format_exc())

            DBS.rollback()

            lock.release()

            return False

        # final release

        lock.release()

        # execute another task if celery is enabled

        if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:

            get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)

        elif recurse_limit <= 0:

            log.debug('Not recursing - limit has been reached')

        else:

            log.debug('Not recursing')

    except celerylib.LockHeld:

        log.info('Task with key %s already running', lockkey)

        return 'Task with key %s already running' % lockkey

@celerylib.task

@celerylib.dbsession

def send_email(recipients, subject, body='', html_body='', headers=None, author=None):

    """

    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, if this is None, the defined email

        address from field 'email_to' and all admins is used instead

    :param subject: subject of the mail

    :param body: body of the mail

    :param html_body: html version of body

    :param headers: dictionary of prepopulated e-mail headers

    :param author: User object of the author of this mail, if known and relevant

    """

    assert isinstance(recipients, list), recipients

    if headers is None:

        headers = {}

    else:

        # do not modify the original headers object passed by the caller

        headers = headers.copy()

    email_config = config

    email_prefix = email_config.get('email_prefix', '')

    if email_prefix:

        subject = "%s %s" % (email_prefix, subject)

    if not recipients:

        # if recipients are not defined we send to email_config + all admins

        recipients = [u.email for u in User.query()

                      .filter(User.admin == True).all()]

        if email_config.get('email_to') is not None:

            recipients += email_config.get('email_to').split(',')

        # If there are still no recipients, there are no admins and no address

        # configured in email_to, so return.

        if not recipients:

            log.error("No recipients specified and no fallback available.")

            return False

        log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))

    # SMTP sender

    envelope_from = email_config.get('app_email_from', 'Kallithea')

    # 'From' header

    if author is not None:

        # set From header based on author but with a generic e-mail address

        # In case app_email_from is in "Some Name <e-mail>" format, we first

        # extract the e-mail address.

        envelope_addr = author_email(envelope_from)

        headers['From'] = '"%s" <%s>' % (

            rfc822.quote('%s (no-reply)' % author.full_name_or_username),

            envelope_addr)

    user = email_config.get('smtp_username')

    passwd = email_config.get('smtp_password')

    mail_server = email_config.get('smtp_server')

    mail_port = email_config.get('smtp_port')

    tls = str2bool(email_config.get('smtp_use_tls'))

    ssl = str2bool(email_config.get('smtp_use_ssl'))

    debug = str2bool(email_config.get('debug'))

    smtp_auth = email_config.get('smtp_auth')

    logmsg = ("Mail details:\n"

              "recipients: %s\n"

              "headers: %s\n"

              "subject: %s\n"

              "body:\n%s\n"

              "html:\n%s\n"

              % (' '.join(recipients), headers, subject, body, html_body))

    if mail_server:

        log.debug("Sending e-mail. " + logmsg)

    else:

        log.error("SMTP mail server not configured - cannot send e-mail.")

        log.warning(logmsg)

        return False

    try:

        m = SmtpMailer(envelope_from, user, passwd, mail_server, smtp_auth,

                       mail_port, ssl, tls, debug=debug)

        m.send(recipients, subject, body, html_body, headers=headers)

    except:

        log.error('Mail sending failed')

        log.error(traceback.format_exc())

        return False

    return True

@celerylib.task

@celerylib.dbsession

def create_repo(form_data, cur_user):

    from kallithea.model.repo import RepoModel

    from kallithea.model.db import Setting

    DBS = celerylib.get_session()

    cur_user = User.guess_instance(cur_user)

    owner = cur_user

    repo_name = form_data['repo_name']

    repo_name_full = form_data['repo_name_full']

    repo_type = form_data['repo_type']

    description = form_data['repo_description']

    private = form_data['repo_private']

    clone_uri = form_data.get('clone_uri')

    repo_group = form_data['repo_group']

    landing_rev = form_data['repo_landing_rev']

    copy_fork_permissions = form_data.get('copy_permissions')

    copy_group_permissions = form_data.get('repo_copy_permissions')

    fork_of = form_data.get('fork_parent_id')

    state = form_data.get('repo_state', Repository.STATE_PENDING)

    # repo creation defaults, private and repo_type are filled in form

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_downloads = defs.get('repo_enable_downloads')

    try:

        repo = RepoModel()._create_repo(

            repo_name=repo_name_full,

            repo_type=repo_type,

            description=description,

            owner=owner,

            private=private,

            clone_uri=clone_uri,

            repo_group=repo_group,

            landing_rev=landing_rev,

            fork_of=fork_of,

            copy_fork_permissions=copy_fork_permissions,

            copy_group_permissions=copy_group_permissions,

            enable_statistics=enable_statistics,

            enable_downloads=enable_downloads,

            state=state

        )

        }

        def _validate_python(self, value, state):

            if value in ['default', 'new_user']:

                msg = self.message('system_invalid_username', state, username=value)

                raise formencode.Invalid(msg, value, state)

            # check if user is unique

            old_un = None

            if edit:

                old_un = User.get(old_data.get('user_id')).username

            if old_un != value or not edit:

                if User.get_by_username(value, case_insensitive=True):

                    msg = self.message('username_exists', state, username=value)

                    raise formencode.Invalid(msg, value, state)

            if re.match(r'^[a-zA-Z0-9\_]{1}[a-zA-Z0-9\-\_\.]*$', value) is None:

                msg = self.message('invalid_username', state)

                raise formencode.Invalid(msg, value, state)

    return _validator

def ValidRegex(msg=None):

    class _validator(formencode.validators.Regex):

        messages = dict(invalid=msg or _('The input is not valid'))

    return _validator

def ValidRepoUser():

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_username': _('Username %(username)s is not valid')

        }

        def _validate_python(self, value, state):

            try:

                User.query().filter(User.active == True) \

                    .filter(User.username == value).one()

            except sqlalchemy.exc.InvalidRequestError: # NoResultFound/MultipleResultsFound

                msg = self.message('invalid_username', state, username=value)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(username=msg)

                )

    return _validator

def ValidUserGroup(edit=False, old_data=None):

    old_data = old_data or {}

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_group': _('Invalid user group name'),

            'group_exist': _('User group "%(usergroup)s" already exists'),

            'invalid_usergroup_name':

                _('user group name may only contain alphanumeric '

                  'characters underscores, periods or dashes and must begin '

                  'with alphanumeric character')

        }

        def _validate_python(self, value, state):

            if value in ['default']:

                msg = self.message('invalid_group', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(users_group_name=msg)

                )

            # check if group is unique

            old_ugname = None

            if edit:

                old_id = old_data.get('users_group_id')

                old_ugname = UserGroup.get(old_id).users_group_name

            if old_ugname != value or not edit:

                is_existing_group = UserGroup.get_by_group_name(value,

                                                        case_insensitive=True)

                if is_existing_group:

                    msg = self.message('group_exist', state, usergroup=value)

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(users_group_name=msg)

                    )

            if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:

                msg = self.message('invalid_usergroup_name', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(users_group_name=msg)

                )

    return _validator

def ValidRepoGroup(edit=False, old_data=None):

    old_data = old_data or {}

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'parent_group_id': _('Cannot assign this group as parent'),

            'group_exists': _('Group "%(group_name)s" already exists'),

            'repo_exists':

                _('Repository with name "%(group_name)s" already exists')

        }

        def _validate_python(self, value, state):

            # TODO WRITE VALIDATIONS

            group_name = value.get('group_name')

            parent_group_id = value.get('parent_group_id')

            # slugify repo group just in case :)

            slug = repo_name_slug(group_name)

            # check for parent of self

            parent_of_self = lambda: (

                old_data['group_id'] == parent_group_id

                if parent_group_id else False

            )

            if edit and parent_of_self():

                msg = self.message('parent_group_id', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(parent_group_id=msg)

                )

            old_gname = None

            if edit:

                old_gname = RepoGroup.get(old_data.get('group_id')).group_name

            if old_gname != group_name or not edit:

                # check group

                gr = RepoGroup.query() \

                      .filter(func.lower(RepoGroup.group_name) == func.lower(slug)) \

                      .filter(RepoGroup.parent_group_id == parent_group_id) \

                      .scalar()

                if gr is not None:

                    msg = self.message('group_exists', state, group_name=slug)

                    raise formencode.Invalid(msg, value, state,

                            error_dict=dict(group_name=msg)

                    )

                # check for same repo

                repo = Repository.query() \

                      .filter(func.lower(Repository.repo_name) == func.lower(slug)) \

                      .scalar()

                if repo is not None:

                    msg = self.message('repo_exists', state, group_name=slug)

                    raise formencode.Invalid(msg, value, state,

                            error_dict=dict(group_name=msg)

                    )

    return _validator

def ValidPassword():

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_password':

                _('Invalid characters (non-ascii) in password')

        }

        def _validate_python(self, value, state):

            try:

                (value or '').decode('ascii')

            except UnicodeError:

                msg = self.message('invalid_password', state)

                raise formencode.Invalid(msg, value, state,)

    return _validator

def ValidOldPassword(username):

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_password': _('Invalid old password')

        }

        def _validate_python(self, value, state):

            from kallithea.lib import auth_modules

            if auth_modules.authenticate(username, value, '') is None:

                msg = self.message('invalid_password', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(current_password=msg)

                )

    return _validator

def ValidPasswordsMatch(password_field, password_confirmation_field):

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'password_mismatch': _('Passwords do not match'),

        }

        def _validate_python(self, value, state):

            if value.get(password_field) != value[password_confirmation_field]:

                msg = self.message('password_mismatch', state)

                raise formencode.Invalid(msg, value, state,

                )

    return _validator

def ValidAuth():

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_auth': _(u'Invalid username or password'),

        }

        def _validate_python(self, value, state):

            from kallithea.lib import auth_modules

            password = value['password']

            username = value['username']

            # authenticate returns unused dict but has called

            # plugin._authenticate which has create_or_update'ed the username user in db

            if auth_modules.authenticate(username, password) is None:

                user = User.get_by_username_or_email(username)

                if user and not user.active:

                    log.warning('user %s is disabled', username)

                    msg = self.message('invalid_auth', state)

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(username=' ', password=msg)

                    )

                else:

                    log.warning('user %s failed to authenticate', username)

                    msg = self.message('invalid_auth', state)

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(username=' ', password=msg)

                    )

    return _validator

def ValidRepoName(edit=False, old_data=None):

    old_data = old_data or {}

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_repo_name':

                _('Repository name %(repo)s is not allowed'),

            'repository_exists':

                _('Repository named %(repo)s already exists'),

            'repository_in_group_exists': _('Repository "%(repo)s" already '

                                            'exists in group "%(group)s"'),

            'same_group_exists': _('Repository group with name "%(repo)s" '

                                   'already exists')

        }

        def _convert_to_python(self, value, state):

            repo_name = repo_name_slug(value.get('repo_name', ''))

            repo_group = value.get('repo_group')

            if repo_group:

                gr = RepoGroup.get(repo_group)

                group_path = gr.full_path

                group_name = gr.group_name

                # value needs to be aware of group name in order to check

                # db key This is an actual just the name to store in the

                # database

                repo_name_full = group_path + RepoGroup.url_sep() + repo_name

            else:

                group_name = group_path = ''

                repo_name_full = repo_name

            value['repo_name'] = repo_name

            value['repo_name_full'] = repo_name_full

            value['group_path'] = group_path

            value['group_name'] = group_name

            return value

        def _validate_python(self, value, state):

            repo_name = value.get('repo_name')

            repo_name_full = value.get('repo_name_full')

            group_path = value.get('group_path')

            group_name = value.get('group_name')

            if repo_name in [ADMIN_PREFIX, '']:

                msg = self.message('invalid_repo_name', state, repo=repo_name)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(repo_name=msg)

                )

            rename = old_data.get('repo_name') != repo_name_full

            create = not edit

            if rename or create:

                repo = Repository.get_by_repo_name(repo_name_full, case_insensitive=True)

                repo_group = RepoGroup.get_by_group_name(repo_name_full, case_insensitive=True)

                if group_path != '':

                    if repo is not None:

                        msg = self.message('repository_in_group_exists', state,

                                repo=repo.repo_name, group=group_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

                elif repo_group is not None:

                        msg = self.message('same_group_exists', state,

                                repo=repo_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

                elif repo is not None:

                        msg = self.message('repository_exists', state,

                                repo=repo.repo_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

            return value

    return _validator

def ValidForkName(*args, **kwargs):

    return ValidRepoName(*args, **kwargs)

def SlugifyName():

    class _validator(formencode.validators.FancyValidator):

        def _convert_to_python(self, value, state):

            return repo_name_slug(value)

        def _validate_python(self, value, state):

            pass

    return _validator

def ValidCloneUri():

    from kallithea.lib.utils import make_ui

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'clone_uri': _('Invalid repository URL'),

            'invalid_clone_uri': _('Invalid repository URL. It must be a '

                                   'valid http, https, ssh, svn+http or svn+https URL'),

        }

        def _validate_python(self, value, state):

            repo_type = value.get('repo_type')

            url = value.get('clone_uri')

            if url and url != value.get('clone_uri_hidden'):

                try:

                    is_valid_repo_uri(repo_type, url, make_ui(clear_session=False))

                except Exception:

                    log.exception('URL validation failed')

                    msg = self.message('clone_uri', state)

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(clone_uri=msg)

                    )

    return _validator

def ValidForkType(old_data=None):

    old_data = old_data or {}

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_fork_type': _('Fork has to be the same type as parent')

        }

        def _validate_python(self, value, state):

            if old_data['repo_type'] != value:

                msg = self.message('invalid_fork_type', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(repo_type=msg)

                )

    return _validator

def CanWriteGroup(old_data=None):

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'permission_denied': _("You don't have permissions "

                                   "to create repository in this group"),

            'permission_denied_root': _("no permission to create repository "

                                        "in root location")

        }

        def _convert_to_python(self, value, state):

            # root location

            if value == -1:

                return None

            return value

        def _validate_python(self, value, state):

            gr = RepoGroup.get(value)

            gr_name = gr.group_name if gr is not None else None # None means ROOT location

            # create repositories with write permission on group is set to true

            create_on_write = HasPermissionAny('hg.create.write_on_repogroup.true')()

            group_admin = HasRepoGroupPermissionLevel('admin')(gr_name,