~~~~~~~~~~~~~~~~~~~~~~~

celery libs for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Nov 27, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import logging

from tg import config

from hashlib import md5

from decorator import decorator

from kallithea import CELERY_ON, CELERY_EAGER

from kallithea.lib.utils2 import safe_str

from kallithea.lib.pidlock import DaemonLock, LockHeld

from kallithea.model.base import init_model

from kallithea.model import meta

from sqlalchemy import engine_from_config

log = logging.getLogger(__name__)

class FakeTask(object):

    """Fake a sync result to make it look like a finished task"""

    def __init__(self, result):

        self.result = result

    def failed(self):

        return False

    traceback = None # if failed

    task_id = None

def task(f_org):

    """Wrapper of celery.task.task, running async if CELERY_ON

    """

    if CELERY_ON:

        def f_async(*args, **kwargs):

            log.info('executing %s task', f_org.__name__)

            try:

                f_org(*args, **kwargs)

            finally:

                log.info('executed %s task', f_org.__name__)

        f_async.__name__ = f_org.__name__

        from kallithea.lib import celerypylons

        runner = celerypylons.task(ignore_result=True)(f_async)

        def f_wrapped(*args, **kwargs):

            t = runner.apply_async(args=args, kwargs=kwargs)

            log.info('executing task %s in async mode - id %s', f_org, t.task_id)

            return t

    else:

        def f_wrapped(*args, **kwargs):

            log.info('executing task %s in sync', f_org.__name__)

            try:

                result = f_org(*args, **kwargs)

            except Exception as e:

                log.error('exception executing sync task %s in sync: %r', f_org.__name__, e)

                raise # TODO: return this in FakeTask as with async tasks?

            return FakeTask(result)

    return f_wrapped

def __get_lockkey(func, *fargs, **fkwargs):

    params = list(fargs)

    params.extend(['%s-%s' % ar for ar in fkwargs.items()])

    func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)

    lockkey = 'task_%s.lock' % \

        md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest()

    return lockkey

def locked_task(func):

    def __wrapper(func, *fargs, **fkwargs):

        lockkey = __get_lockkey(func, *fargs, **fkwargs)

        lockkey_path = config['app_conf']['cache_dir']

        log.info('running task with lockkey %s', lockkey)

        try:

            ret = func(*fargs, **fkwargs)

            l.release()

            return ret

        except LockHeld:

            log.info('LockHeld')

            return 'Task with key %s already running' % lockkey

    return decorator(__wrapper, func)

def get_session():

    if CELERY_ON:

        engine = engine_from_config(config, 'sqlalchemy.')

        init_model(engine)

    sa = meta.Session()

    return sa

def dbsession(func):

    def __wrapper(func, *fargs, **fkwargs):

        try:

            ret = func(*fargs, **fkwargs)

            return ret

        finally:

            if CELERY_ON and not CELERY_EAGER:

                meta.Session.remove()

    return decorator(__wrapper, func)

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.celerylib.tasks

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Kallithea task modules, containing all task that suppose to be run

by celery daemon

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Oct 6, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import traceback

import logging

import rfc822

from time import mktime

from operator import itemgetter

from string import lower

from tg import config

from kallithea import CELERY_ON

from kallithea.lib import celerylib

from kallithea.lib.helpers import person

from kallithea.lib.rcmail.smtp_mailer import SmtpMailer

from kallithea.lib.utils import setup_cache_regions, action_logger

from kallithea.lib.utils2 import str2bool

from kallithea.lib.vcs.utils import author_email

from kallithea.lib.compat import json, OrderedDict

from kallithea.lib.hooks import log_create_repository

from kallithea.model.db import Statistics, RepoGroup, Repository, User

setup_cache_regions(config)  # pragma: no cover

__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']

log = logging.getLogger(__name__)

@celerylib.task

@celerylib.locked_task

@celerylib.dbsession

def whoosh_index(repo_location, full_index):

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    celerylib.get_session() # initialize database connection

    index_location = config['index_dir']

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location) \

                         .run(full_index=full_index)

@celerylib.task

@celerylib.dbsession

def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):

    DBS = celerylib.get_session()

    lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,

                            ts_max_y)

    lockkey_path = config['app_conf']['cache_dir']

    log.info('running task with lockkey %s', lockkey)

    try:

        # for js data compatibility cleans the key for person from '

        akc = lambda k: person(k).replace('"', "")

        co_day_auth_aggr = {}

        commits_by_day_aggregate = {}

        repo = Repository.get_by_repo_name(repo_name)

        if repo is None:

            return True

        repo = repo.scm_instance

        repo_size = repo.count()

        # return if repo have no revisions

        if repo_size < 1:

            lock.release()

            return True

        skip_date_limit = True

        parse_limit = int(config['app_conf'].get('commit_parse_limit'))

        last_rev = None

        last_cs = None

        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository) \

            .filter(Repository.repo_name == repo_name).scalar()

        cur_stats = DBS.query(Statistics) \

            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:

            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:

            # pass silently without any work if we're not on first revision or

            # current state of parsing revision(from db marker) is the

            # last revision

            lock.release()

            return True

        if cur_stats:

            commits_by_day_aggregate = OrderedDict(json.loads(

                                        cur_stats.commit_activity_combined))

            co_day_auth_aggr = json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s', parse_limit)

        lmktime = mktime

        last_rev = last_rev + 1 if last_rev >= 0 else 0

        log.debug('Getting revisions from %s to %s',

             last_rev, last_rev + parse_limit

        )

        for cs in repo[last_rev:last_rev + parse_limit]:

            log.debug('parsing %s', cs)

            last_cs = cs  # remember last parsed changeset

            k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],

                          cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])

            if akc(cs.author) in co_day_auth_aggr:

                try:

                    l = [timegetter(x) for x in

                         co_day_auth_aggr[akc(cs.author)]['data']]

                    time_pos = l.index(k)

                except ValueError:

                    time_pos = None

                if time_pos >= 0 and time_pos is not None:

                    datadict = \

                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1

                    datadict["added"] += len(cs.added)

                    datadict["changed"] += len(cs.changed)

                    datadict["removed"] += len(cs.removed)

                else:

                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {"time": k,

                                    "commits": 1,

                                    "added": len(cs.added),

                                    "changed": len(cs.changed),

                                    "removed": len(cs.removed),

                                   }

                        co_day_auth_aggr[akc(cs.author)]['data'] \

                            .append(datadict)

            else:

                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                    co_day_auth_aggr[akc(cs.author)] = {

                                        "label": akc(cs.author),

                                        "data": [{"time":k,

                                                 "commits":1,

                                                 "added":len(cs.added),

                                                 "changed":len(cs.changed),

                                                 "removed":len(cs.removed),

                                                 }],

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_index

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-index gearbox command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from os.path import dirname

from string import strip

from kallithea.model.repo import RepoModel

from kallithea.lib.paster_commands.common import BasePasterCommand

from kallithea.lib.utils import load_rcextensions

class Command(BasePasterCommand):

    "Kallithea: Create or update full text search index"

    def take_action(self, args):

        index_location = self.config['index_dir']

        load_rcextensions(self.config['here'])

        repo_location = args.repo_location \

            if args.repo_location else RepoModel().repos_path

        repo_list = map(strip, args.repo_list.split(',')) \

            if args.repo_list else None

        repo_update_list = map(strip, args.repo_update_list.split(',')) \

            if args.repo_update_list else None

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from kallithea.lib.pidlock import LockHeld, DaemonLock

        from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

        try:

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list,

                                 repo_update_list=repo_update_list) \

                .run(full_index=args.full_index)

            l.release()

        except LockHeld:

            sys.exit(1)

    def get_parser(self, prog_name):

        parser = super(Command, self).get_parser(prog_name)

        parser.add_argument('--repo-location',

                          action='store',

                          dest='repo_location',

                          help="Specifies repositories location to index OPTIONAL",

                          )

        parser.add_argument('--index-only',

                          action='store',

                          dest='repo_list',

                          help="Specifies a comma separated list of repositories "

                                "to build index on. If not given all repositories "

                                "are scanned for indexing. OPTIONAL",

                          )

        parser.add_argument('--update-only',

                          action='store',

                          dest='repo_update_list',

                          help="Specifies a comma separated list of repositories "

                                "to re-build index on. OPTIONAL",

                          )

        parser.add_argument('-f',

                          action='store_true',

                          dest='full_index',

                          help="Specifies that index should be made full i.e"

                                " destroy old and build from scratch",

                          default=False)

        return parser

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import errno

from multiprocessing.util import Finalize

from kallithea.lib.compat import kill

class LockHeld(Exception):

    pass

class DaemonLock(object):

    """daemon locking

    USAGE:

    try:

        main()

        l.release()

    except LockHeld:

        sys.exit(1)

    """

                 desc='daemon lock', debug=False):

        self.callbackfn = callbackfn

        self.desc = desc

        self.debug = debug

        self.held = False

        #run the lock automatically !

        self.lock()

        self._finalize = Finalize(self, DaemonLock._on_finalize,

                                  args=(self, debug), exitpriority=10)

    @staticmethod

    def _on_finalize(lock, debug):

        if lock.held:

            if debug:

                print 'lock held finalizing and running lock.release()'

            lock.release()

    def lock(self):

        """

        locking function, if lock is present it

        will raise LockHeld exception

        """

        lockname = str(os.getpid())

        if self.debug:

            print 'running lock'

        self.trylock()

        self.makelock(lockname, self.pidfile)

        return True

    def trylock(self):

        running_pid = False

        if self.debug:

            print 'checking for already running process'

        try:

            with open(self.pidfile, 'r') as f:

                try:

                    running_pid = int(f.readline())

                except ValueError:

                    running_pid = -1

            if self.debug:

                print ('lock file present running_pid: %s, '

                       'checking for execution' % (running_pid,))

            # Now we check the PID from lock file matches to the current

            # process PID

            if running_pid:

                try:

                    kill(running_pid, 0)

                except OSError as exc:

                    if exc.errno in (errno.ESRCH, errno.EPERM):

                        print ("Lock File is there but"

                               " the program is not running")

                        print "Removing lock file for the: %s" % running_pid

                        self.release()

                    else:

                        raise

                else:

                    print "You already have an instance of the program running"

                    print "It is running as process %s" % running_pid

                    raise LockHeld()

        except IOError as e:

            if e.errno != 2:

                raise

    def release(self):

        """releases the pid by removing the pidfile

        """

        if self.debug:

            print 'trying to release the pidlock'

        if self.callbackfn:

            #execute callback function on release

            if self.debug:

                print 'executing callback function %s' % self.callbackfn

            self.callbackfn()

        try:

            if self.debug:

                print 'removing pidfile %s' % self.pidfile

            os.remove(self.pidfile)

            self.held = False

        except OSError as e:

            if self.debug:

                print 'removing pidfile failed %s' % e

            pass

    def makelock(self, lockname, pidfile):

        """

        this function will make an actual lock

        :param lockname: actual pid of file

        :param pidfile: the file to write the pid in

        """

        if self.debug:

            print 'creating a file %s and pid: %s' % (pidfile, lockname)

        dir_, file_ = os.path.split(pidfile)

                f_path=filename

            )

        return cs

    def review_changeset(self, repo, revision, status, author=TEST_USER_ADMIN_LOGIN):

        comment = ChangesetCommentsModel().create(u"review comment", repo, author, revision=revision, send_email=False)

        csm = ChangesetStatusModel().set_status(repo, ChangesetStatus.STATUS_APPROVED, author, comment, revision=revision)

        Session().commit()

        return csm

    def create_pullrequest(self, testcontroller, repo_name, pr_src_rev, pr_dst_rev, title=u'title'):

        org_ref = 'branch:stable:%s' % pr_src_rev

        other_ref = 'branch:default:%s' % pr_dst_rev

        with test_context(testcontroller.app): # needed to be able to mock request user

            org_repo = other_repo = Repository.get_by_repo_name(repo_name)

            owner_user = User.get_by_username(TEST_USER_ADMIN_LOGIN)

            reviewers = [User.get_by_username(TEST_USER_REGULAR_LOGIN)]

            request.authuser = request.user = AuthUser(dbuser=owner_user)

            # creating a PR sends a message with an absolute URL - without routing that requires mocking

            with mock.patch.object(helpers, 'url', (lambda arg, qualified=False, **kwargs: ('https://localhost' if qualified else '') + '/fake/' + arg)):

                cmd = CreatePullRequestAction(org_repo, other_repo, org_ref, other_ref, title, u'No description', owner_user, reviewers)

                pull_request = cmd.execute()

            Session().commit()

        return pull_request.pull_request_id

#==============================================================================

# Global test environment setup

#==============================================================================

def create_test_env(repos_test_path, config):

    """

    Makes a fresh database and

    install test repository into tmp dir

    """

    # PART ONE create db

    dbconf = config['sqlalchemy.url']

    log.debug('making test db %s', dbconf)

    # create test dir if it doesn't exist

    if not os.path.isdir(repos_test_path):

        log.debug('Creating testdir %s', repos_test_path)

        os.makedirs(repos_test_path)

    dbmanage = DbManage(dbconf=dbconf, root=config['here'],

                        tests=True)

    dbmanage.create_tables(override=True)

    # for tests dynamically set new root paths based on generated content

    dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))

    dbmanage.create_default_user()

    dbmanage.admin_prompt()

    dbmanage.create_permissions()

    dbmanage.populate_default_permissions()

    Session().commit()

    # PART TWO make test repo

    log.debug('making test vcs repositories')

    idx_path = config['index_dir']

    data_path = config['cache_dir']

    #clean index and data

    if idx_path and os.path.exists(idx_path):

        log.debug('remove %s', idx_path)

        shutil.rmtree(idx_path)

    if data_path and os.path.exists(data_path):

        log.debug('remove %s', data_path)

        shutil.rmtree(data_path)

    #CREATE DEFAULT TEST REPOS

    tar = tarfile.open(os.path.join(FIXTURES, 'vcs_test_hg.tar.gz'))

    tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))

    tar.close()

    tar = tarfile.open(os.path.join(FIXTURES, 'vcs_test_git.tar.gz'))

    tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))

    tar.close()

    #LOAD VCS test stuff

    from kallithea.tests.vcs import setup_package

    setup_package()

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    """

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    from kallithea.lib.pidlock import DaemonLock, LockHeld

    index_location = os.path.join(config['index_dir'])

    if not os.path.exists(index_location):

        os.makedirs(index_location)

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location) \

        .run(full_index=full_index)

    l.release()