lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,

                            ts_max_y)

    lockkey_path = config['app_conf']['cache_dir']

    log.info('running task with lockkey %s', lockkey)

    try:

        lock = celerylib.DaemonLock(os.path.join(lockkey_path, lockkey))

        # for js data compatibility cleans the key for person from '

        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}

        commits_by_day_aggregate = {}

        repo = Repository.get_by_repo_name(repo_name)

        if repo is None:

            return True

        repo = repo.scm_instance

        repo_size = repo.count()

        # return if repo have no revisions

        if repo_size < 1:

            lock.release()

            return True

        skip_date_limit = True

        parse_limit = int(config['app_conf'].get('commit_parse_limit'))

        last_rev = None

        last_cs = None

        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository) \

            .filter(Repository.repo_name == repo_name).scalar()

        cur_stats = DBS.query(Statistics) \

            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:

            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:

            # pass silently without any work if we're not on first revision or

            # current state of parsing revision(from db marker) is the

            # last revision

            lock.release()

            return True

        if cur_stats:

            commits_by_day_aggregate = OrderedDict(json.loads(

                                        cur_stats.commit_activity_combined))

            co_day_auth_aggr = json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s', parse_limit)

        lmktime = mktime

        last_rev = last_rev + 1 if last_rev >= 0 else 0

        log.debug('Getting revisions from %s to %s',

             last_rev, last_rev + parse_limit

        )

        for cs in repo[last_rev:last_rev + parse_limit]:

            log.debug('parsing %s', cs)

            last_cs = cs  # remember last parsed changeset

            k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],

                          cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])

            if akc(cs.author) in co_day_auth_aggr:

                try:

                    l = [timegetter(x) for x in

                         co_day_auth_aggr[akc(cs.author)]['data']]

                    time_pos = l.index(k)

                except ValueError:

                    time_pos = None

                if time_pos >= 0 and time_pos is not None:

                    datadict = \

                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1

                    datadict["added"] += len(cs.added)

                    datadict["changed"] += len(cs.changed)

                    datadict["removed"] += len(cs.removed)

                else:

                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {"time": k,

                                    "commits": 1,

                                    "added": len(cs.added),

                                    "changed": len(cs.changed),

                                    "removed": len(cs.removed),

                                   }

                        co_day_auth_aggr[akc(cs.author)]['data'] \

                            .append(datadict)

            else:

                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                    co_day_auth_aggr[akc(cs.author)] = {

                                        "label": akc(cs.author),

                                        "data": [{"time":k,

                                                 "commits":1,

                                                 "added":len(cs.added),

                                                 "changed":len(cs.changed),

                                                 "removed":len(cs.removed),

                                                 }],

                                        "schema": ["commits"],

                                        }

            # gather all data by day

            if k in commits_by_day_aggregate:

                commits_by_day_aggregate[k] += 1

            else:

                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),

                               key=itemgetter(0))

        if not co_day_auth_aggr:

            co_day_auth_aggr[akc(repo.contact)] = {

                "label": akc(repo.contact),

                "data": [0, 1],

                "schema": ["commits"],

            }

        stats = cur_stats if cur_stats else Statistics()

        stats.commit_activity = json.dumps(co_day_auth_aggr)

        stats.commit_activity_combined = json.dumps(overview_data)

        log.debug('last revision %s', last_rev)

        leftovers = len(repo.revisions[last_rev:])

        log.debug('revisions to parse %s', leftovers)

        if last_rev == 0 or leftovers < parse_limit:

            log.debug('getting code trending stats')

            stats.languages = json.dumps(__get_codes_stats(repo_name))

        try:

            stats.repository = dbrepo

            stats.stat_on_revision = last_cs.revision if last_cs else 0

            DBS.add(stats)

            DBS.commit()

        except:

            log.error(traceback.format_exc())

            DBS.rollback()

            lock.release()

            return False

        # final release

        lock.release()

        # execute another task if celery is enabled

        if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:

            get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)

        elif recurse_limit <= 0:

            log.debug('Not recursing - limit has been reached')

        else:

            log.debug('Not recursing')

    except celerylib.LockHeld:

        log.info('Task with key %s already running', lockkey)

        return 'Task with key %s already running' % lockkey

@celerylib.task

@celerylib.dbsession

def send_email(recipients, subject, body='', html_body='', headers=None, author=None):

    """

    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, if this is None, the defined email

        address from field 'email_to' and all admins is used instead

    :param subject: subject of the mail

    :param body: body of the mail

    :param html_body: html version of body

    :param headers: dictionary of prepopulated e-mail headers

    :param author: User object of the author of this mail, if known and relevant

    """

    assert isinstance(recipients, list), recipients

    if headers is None:

        headers = {}

    else:

        # do not modify the original headers object passed by the caller

        headers = headers.copy()

    email_config = config

    email_prefix = email_config.get('email_prefix', '')

    if email_prefix:

        subject = "%s %s" % (email_prefix, subject)

    if not recipients:

        # if recipients are not defined we send to email_config + all admins

        recipients = [u.email for u in User.query()

                      .filter(User.admin == True).all()]

        if email_config.get('email_to') is not None:

        # If there are still no recipients, there are no admins and no address

        # configured in email_to, so return.

        if not recipients:

            log.error("No recipients specified and no fallback available.")

            return False

        log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))

    # SMTP sender

    envelope_from = email_config.get('app_email_from', 'Kallithea')

    # 'From' header

    if author is not None:

        # set From header based on author but with a generic e-mail address

        # In case app_email_from is in "Some Name <e-mail>" format, we first

        # extract the e-mail address.

        envelope_addr = author_email(envelope_from)

        headers['From'] = '"%s" <%s>' % (

            rfc822.quote('%s (no-reply)' % author.full_name_or_username),

            envelope_addr)

    user = email_config.get('smtp_username')

    passwd = email_config.get('smtp_password')

    mail_server = email_config.get('smtp_server')

    mail_port = email_config.get('smtp_port')

    tls = str2bool(email_config.get('smtp_use_tls'))

    ssl = str2bool(email_config.get('smtp_use_ssl'))

    debug = str2bool(email_config.get('debug'))

    smtp_auth = email_config.get('smtp_auth')

    logmsg = ("Mail details:\n"

              "recipients: %s\n"

              "headers: %s\n"

              "subject: %s\n"

              "body:\n%s\n"

              "html:\n%s\n"

              % (' '.join(recipients), headers, subject, body, html_body))

    if mail_server:

        log.debug("Sending e-mail. " + logmsg)

    else:

        log.error("SMTP mail server not configured - cannot send e-mail.")

        log.warning(logmsg)

        return False

    try:

        m = SmtpMailer(envelope_from, user, passwd, mail_server, smtp_auth,

                       mail_port, ssl, tls, debug=debug)

        m.send(recipients, subject, body, html_body, headers=headers)

    except:

        log.error('Mail sending failed')

        log.error(traceback.format_exc())

        return False

    return True

@celerylib.task

@celerylib.dbsession

def create_repo(form_data, cur_user):

    from kallithea.model.repo import RepoModel

    from kallithea.model.db import Setting

    DBS = celerylib.get_session()

    cur_user = User.guess_instance(cur_user)

    owner = cur_user

    repo_name = form_data['repo_name']

    repo_name_full = form_data['repo_name_full']

    repo_type = form_data['repo_type']

    description = form_data['repo_description']

    private = form_data['repo_private']

    clone_uri = form_data.get('clone_uri')

    repo_group = form_data['repo_group']

    landing_rev = form_data['repo_landing_rev']

    copy_fork_permissions = form_data.get('copy_permissions')

    copy_group_permissions = form_data.get('repo_copy_permissions')

    fork_of = form_data.get('fork_parent_id')

    state = form_data.get('repo_state', Repository.STATE_PENDING)

    # repo creation defaults, private and repo_type are filled in form

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_locking = defs.get('repo_enable_locking')

    enable_downloads = defs.get('repo_enable_downloads')

    try:

        repo = RepoModel()._create_repo(

            repo_name=repo_name_full,

            repo_type=repo_type,

            description=description,

            owner=owner,

            private=private,

            clone_uri=clone_uri,

            repo_group=repo_group,

            landing_rev=landing_rev,

            fork_of=fork_of,

            copy_fork_permissions=copy_fork_permissions,

            copy_group_permissions=copy_group_permissions,

            enable_statistics=enable_statistics,

            enable_locking=enable_locking,

            enable_downloads=enable_downloads,

            state=state

        )

        action_logger(cur_user, 'user_created_repo',

                      form_data['repo_name_full'], '')

        DBS.commit()

        # now create this repo on Filesystem

        RepoModel()._create_filesystem_repo(

            repo_name=repo_name,

            repo_type=repo_type,

            repo_group=RepoGroup.guess_instance(repo_group),

            clone_uri=clone_uri,

        )

        repo = Repository.get_by_repo_name(repo_name_full)

        log_create_repository(repo.get_dict(), created_by=owner.username)

        # update repo changeset caches initially

        repo.update_changeset_cache()

        # set new created state

        repo.set_state(Repository.STATE_CREATED)

        DBS.commit()

    except Exception as e:

        log.warning('Exception %s occurred when forking repository, '

                    'doing cleanup...' % e)

        # rollback things manually !

        repo = Repository.get_by_repo_name(repo_name_full)

        if repo:

            Repository.delete(repo.repo_id)

            DBS.commit()

            RepoModel()._delete_filesystem_repo(repo)

        raise

    return True

@celerylib.task

@celerylib.dbsession

def create_repo_fork(form_data, cur_user):

    """

    Creates a fork of repository using interval VCS methods

    :param form_data:

    :param cur_user:

    """

    from kallithea.model.repo import RepoModel

    DBS = celerylib.get_session()

    base_path = Repository.base_path()

    cur_user = User.guess_instance(cur_user)

    repo_name = form_data['repo_name']  # fork in this case

    repo_name_full = form_data['repo_name_full']

    repo_type = form_data['repo_type']

    owner = cur_user

    private = form_data['private']

    clone_uri = form_data.get('clone_uri')

    repo_group = form_data['repo_group']

    landing_rev = form_data['landing_rev']

    copy_fork_permissions = form_data.get('copy_permissions')

    try:

        fork_of = Repository.guess_instance(form_data.get('fork_parent_id'))

        RepoModel()._create_repo(

            repo_name=repo_name_full,

            repo_type=repo_type,

            description=form_data['description'],

            owner=owner,

            private=private,

            clone_uri=clone_uri,

            repo_group=repo_group,

            landing_rev=landing_rev,

            fork_of=fork_of,

            copy_fork_permissions=copy_fork_permissions

        )

        action_logger(cur_user, 'user_forked_repo:%s' % repo_name_full,

                      fork_of.repo_name, '')

        DBS.commit()

        source_repo_path = os.path.join(base_path, fork_of.repo_name)

        # now create this repo on Filesystem

        RepoModel()._create_filesystem_repo(

            repo_name=repo_name,

            repo_type=repo_type,

            repo_group=RepoGroup.guess_instance(repo_group),

import mock

import kallithea

from kallithea.tests.base import *

from kallithea.model.db import User

class smtplib_mock(object):

    @classmethod

    def SMTP(cls, server, port, local_hostname):

        return smtplib_mock()

    def ehlo(self):

        pass

    def quit(self):

        pass

    def sendmail(self, sender, dest, msg):

        smtplib_mock.lastsender = sender

        smtplib_mock.lastdest = dest

        smtplib_mock.lastmsg = msg

        pass

@mock.patch('kallithea.lib.rcmail.smtp_mailer.smtplib', smtplib_mock)

class TestMail(TestController):

    def test_send_mail_trivial(self):

        mailserver = 'smtp.mailserver.org'

        recipients = ['rcpt1', 'rcpt2']

        envelope_from = 'noreply@mailserver.org'

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)

        assert smtplib_mock.lastdest == set(recipients)

        assert smtplib_mock.lastsender == envelope_from

        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

    def test_send_mail_no_recipients(self):

        mailserver = 'smtp.mailserver.org'

        recipients = []

        envelope_from = 'noreply@mailserver.org'

        email_to = 'admin@mailserver.org'

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

            'email_to': email_to,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)

        assert smtplib_mock.lastdest == set([TEST_USER_ADMIN_EMAIL, email_to])

        assert smtplib_mock.lastsender == envelope_from

        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

    def test_send_mail_no_recipients_no_email_to(self):

        mailserver = 'smtp.mailserver.org'

        recipients = []

        envelope_from = 'noreply@mailserver.org'

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)

        assert smtplib_mock.lastdest == set([TEST_USER_ADMIN_EMAIL])

        assert smtplib_mock.lastsender == envelope_from

        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

    def test_send_mail_with_author(self):

        mailserver = 'smtp.mailserver.org'

        recipients = ['rcpt1', 'rcpt2']

        envelope_from = 'noreply@mailserver.org'

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        author = User.get_by_username(TEST_USER_REGULAR_LOGIN)

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body, author=author)

        assert smtplib_mock.lastdest == set(recipients)

        assert smtplib_mock.lastsender == envelope_from

        assert 'From: "Kallithea Admin (no-reply)" <%s>' % envelope_from in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

    def test_send_mail_with_author_full_mail_from(self):

        mailserver = 'smtp.mailserver.org'

        recipients = ['rcpt1', 'rcpt2']

        envelope_addr = 'noreply@mailserver.org'

        envelope_from = 'Some Name <%s>' % envelope_addr

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        author = User.get_by_username(TEST_USER_REGULAR_LOGIN)

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body, author=author)

        assert smtplib_mock.lastdest == set(recipients)

        assert smtplib_mock.lastsender == envelope_from

        assert 'From: "Kallithea Admin (no-reply)" <%s>' % envelope_addr in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

    def test_send_mail_extra_headers(self):

        mailserver = 'smtp.mailserver.org'

        recipients = ['rcpt1', 'rcpt2']

        envelope_from = 'noreply@mailserver.org'

        subject = 'subject'

        body = 'body'

        html_body = 'html_body'

        author = User(name='foo', lastname=u'(fubar) "baz"')

        headers = {'extra': 'yes'}

        config_mock = {

            'smtp_server': mailserver,

            'app_email_from': envelope_from,

        }

        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):

            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body,

                                                     author=author, headers=headers)

        assert smtplib_mock.lastdest == set(recipients)

        assert smtplib_mock.lastsender == envelope_from

        assert r'From: "foo (fubar) \"baz\" (no-reply)" <%s>' % envelope_from in smtplib_mock.lastmsg

        assert 'Subject: %s' % subject in smtplib_mock.lastmsg

        assert body in smtplib_mock.lastmsg

        assert html_body in smtplib_mock.lastmsg

        assert 'Extra: yes' in smtplib_mock.lastmsg

        # verify that headers dict hasn't mutated by send_email

        assert headers == {'extra': 'yes'}