Changeset - 71113f64b2d8
[Not reviewed]
beta
0 2 0
Marcin Kuzminski - 15 years ago 2010-11-27 20:32:34
marcin@python-works.com
fidex corrent variables passed to dnconfig,
fixed celerylib import problem on tests
2 files changed with 62 insertions and 24 deletions:
0 comments (0 inline, 0 general)
rhodecode/lib/celerylib/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
"""
 
    package.rhodecode.lib.celerylib.__init__
 
    ~~~~~~~~~~~~~~
 

	
 
    celery libs for RhodeCode
 
    
 
    :created_on: Nov 27, 2010
 
    :author: marcink
 
    :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>    
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 

	
 
import os
 
import sys
 
import socket
 
import traceback
 
import logging
 

	
 
from rhodecode.lib.pidlock import DaemonLock, LockHeld
 
from hashlib import md5
 
from decorator import decorator
 
from vcs.utils.lazy import LazyProperty
 
from decorator import decorator
 
from hashlib import md5
 

	
 
from rhodecode.lib.pidlock import DaemonLock, LockHeld
 

	
 
from pylons import  config
 

	
 
log = logging.getLogger(__name__)
 

	
 
def str2bool(v):
 
    return v.lower() in ["yes", "true", "t", "1"] if v else None
 

	
 
try:
 
CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
except KeyError:
 
    CELERY_ON = False
 

	
 
class ResultWrapper(object):
 
    def __init__(self, task):
 
        self.task = task
 

	
 
    @LazyProperty
 
    def result(self):
 
        return self.task
 

	
 
def run_task(task, *args, **kwargs):
 
    if CELERY_ON:
 
        try:
 
            t = task.delay(*args, **kwargs)
 
            log.info('running task %s:%s', t.task_id, task)
 
            return t
 
        except socket.error, e:
 
            if  e.errno == 111:
 
                log.debug('Unable to connect to celeryd. Sync execution')
 
            else:
 
                log.error(traceback.format_exc())
 
        except KeyError, e:
 
                log.debug('Unable to connect to celeryd. Sync execution')
 
        except Exception, e:
 
            log.error(traceback.format_exc())
 

	
 
    log.debug('executing task %s in sync mode', task)
 
    return ResultWrapper(task(*args, **kwargs))
 

	
 

	
 
def locked_task(func):
 
    def __wrapper(func, *fargs, **fkwargs):
 
        params = list(fargs)
 
        params.extend(['%s-%s' % ar for ar in fkwargs.items()])
 

	
 
        lockkey = 'task_%s' % \
 
            md5(str(func.__name__) + '-' + \
 
                '-'.join(map(str, params))).hexdigest()
 
        log.info('running task with lockkey %s', lockkey)
 
        try:
 
            l = DaemonLock(lockkey)
 
            ret = func(*fargs, **fkwargs)
 
            l.release()
 
            return ret
 
        except LockHeld:
 
            log.info('LockHeld')
 
            return 'Task with key %s already running' % lockkey
 

	
 
    return decorator(__wrapper, func)
 

	
 

	
 

	
 

	
 

	
 

	
 

	
 

	
rhodecode/lib/utils.py
Show inline comments
 
#!/usr/bin/env python
 
# encoding: utf-8
 
# Utilities for RhodeCode
 
# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
 
# -*- coding: utf-8 -*-
 
"""
 
    package.rhodecode.lib.utils
 
    ~~~~~~~~~~~~~~
 

	
 
    Utilities library for RhodeCode
 
    
 
    :created_on: Apr 18, 2010
 
    :author: marcink
 
    :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>    
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 18, 2010
 
Utilities for RhodeCode
 
@author: marcink
 
"""
 

	
 
import os
 
import logging
 
import datetime
 
import traceback
 
import ConfigParser
 

	
 
from UserDict import DictMixin
 

	
 
from mercurial import ui, config, hg
 
from mercurial.error import RepoError
 

	
 
from paste.script import command
 
from vcs.backends.base import BaseChangeset
 
from vcs.utils.lazy import LazyProperty
 

	
 
from rhodecode.model import meta
 
from rhodecode.model.caching_query import FromCache
 
from rhodecode.model.db import Repository, User, RhodeCodeUi, UserLog
 
from rhodecode.model.repo import RepoModel
 
from rhodecode.model.user import UserModel
 

	
 
from vcs.backends.base import BaseChangeset
 
from paste.script import command
 
import ConfigParser
 
from vcs.utils.lazy import LazyProperty
 
import traceback
 
import datetime
 
import logging
 
import os
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def get_repo_slug(request):
 
    return request.environ['pylons.routes_dict'].get('repo_name')
 

	
 
def action_logger(user, action, repo, ipaddr='', sa=None):
 
    """
 
    Action logger for various actions made by users
 
    
 
    :param user: user that made this action, can be a unique username string or
 
        object containing user_id attribute
 
    :param action: action to log, should be on of predefined unique actions for
 
        easy translations
 
    :param repo: string name of repository or object containing repo_id,
 
        that action was made on
 
    :param ipaddr: optional ip address from what the action was made
 
    :param sa: optional sqlalchemy session
 
    
 
    """
 

	
 
    if not sa:
 
        sa = meta.Session()
 

	
 
    try:
 
        um = UserModel()
 
        if hasattr(user, 'user_id'):
 
            user_obj = user
 
        elif isinstance(user, basestring):
 
            user_obj = um.get_by_username(user, cache=False)
 
        else:
 
            raise Exception('You have to provide user object or username')
 

	
 

	
 
        rm = RepoModel()
 
        if hasattr(repo, 'repo_id'):
 
            repo_obj = rm.get(repo.repo_id, cache=False)
 
            repo_name = repo_obj.repo_name
 
        elif  isinstance(repo, basestring):
 
            repo_name = repo.lstrip('/')
 
            repo_obj = rm.get_by_repo_name(repo_name, cache=False)
 
        else:
 
            raise Exception('You have to provide repository to action logger')
 

	
 

	
 
        user_log = UserLog()
 
        user_log.user_id = user_obj.user_id
 
        user_log.action = action
 

	
 
        user_log.repository_id = repo_obj.repo_id
 
        user_log.repository_name = repo_name
 

	
 
        user_log.action_date = datetime.datetime.now()
 
        user_log.user_ip = ipaddr
 
        sa.add(user_log)
 
        sa.commit()
 

	
 
        log.info('Adding user %s, action %s on %s', user_obj, action, repo)
 
    except:
 
        log.error(traceback.format_exc())
 
        sa.rollback()
 

	
 
def get_repos(path, recursive=False, initial=False):
 
    """
 
    Scans given path for repos and return (name,(type,path)) tuple 
 
    :param prefix:
 
    :param path:
 
    :param recursive:
 
    :param initial:
 
    """
 
    from vcs.utils.helpers import get_scm
 
    from vcs.exceptions import VCSError
 

	
 
    try:
 
        scm = get_scm(path)
 
    except:
 
        pass
 
    else:
 
        raise Exception('The given path %s should not be a repository got %s',
 
                        path, scm)
 

	
 
    for dirpath in os.listdir(path):
 
        try:
 
            yield dirpath, get_scm(os.path.join(path, dirpath))
 
        except VCSError:
 
            pass
 

	
 
def check_repo_fast(repo_name, base_path):
 
    """
 
    Check given path for existance of directory
 
    :param repo_name:
 
    :param base_path:
 
    
 
    :return False: if this directory is present
 
    """
 
    if os.path.isdir(os.path.join(base_path, repo_name)):return False
 
@@ -371,148 +377,148 @@ class OrderedDict(dict, DictMixin):
 
        inst_dict = vars(self).copy()
 
        self.__map, self.__end = tmp
 
        if inst_dict:
 
            return (self.__class__, (items,), inst_dict)
 
        return self.__class__, (items,)
 

	
 
    def keys(self):
 
        return list(self)
 

	
 
    setdefault = DictMixin.setdefault
 
    update = DictMixin.update
 
    pop = DictMixin.pop
 
    values = DictMixin.values
 
    items = DictMixin.items
 
    iterkeys = DictMixin.iterkeys
 
    itervalues = DictMixin.itervalues
 
    iteritems = DictMixin.iteritems
 

	
 
    def __repr__(self):
 
        if not self:
 
            return '%s()' % (self.__class__.__name__,)
 
        return '%s(%r)' % (self.__class__.__name__, self.items())
 

	
 
    def copy(self):
 
        return self.__class__(self)
 

	
 
    @classmethod
 
    def fromkeys(cls, iterable, value=None):
 
        d = cls()
 
        for key in iterable:
 
            d[key] = value
 
        return d
 

	
 
    def __eq__(self, other):
 
        if isinstance(other, OrderedDict):
 
            return len(self) == len(other) and self.items() == other.items()
 
        return dict.__eq__(self, other)
 

	
 
    def __ne__(self, other):
 
        return not self == other
 

	
 

	
 
#===============================================================================
 
# TEST FUNCTIONS AND CREATORS
 
#===============================================================================
 
def create_test_index(repo_location, full_index):
 
    """Makes default test index
 
    :param repo_location:
 
    :param full_index:
 
    """
 
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
 
    from rhodecode.lib.pidlock import DaemonLock, LockHeld
 
    import shutil
 

	
 
    index_location = os.path.join(repo_location, 'index')
 
    if os.path.exists(index_location):
 
        shutil.rmtree(index_location)
 

	
 
    try:
 
        l = DaemonLock()
 
        WhooshIndexingDaemon(index_location=index_location,
 
                             repo_location=repo_location)\
 
            .run(full_index=full_index)
 
        l.release()
 
    except LockHeld:
 
        pass
 

	
 
def create_test_env(repos_test_path, config):
 
    """Makes a fresh database and 
 
    install test repository into tmp dir
 
    """
 
    from rhodecode.lib.db_manage import DbManage
 
    from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \
 
        HG_FORK, GIT_FORK, TESTS_TMP_PATH
 
    import tarfile
 
    import shutil
 
    from os.path import dirname as dn, join as jn, abspath
 

	
 
    log = logging.getLogger('TestEnvCreator')
 
    # create logger
 
    log.setLevel(logging.DEBUG)
 
    log.propagate = True
 
    # create console handler and set level to debug
 
    ch = logging.StreamHandler()
 
    ch.setLevel(logging.DEBUG)
 

	
 
    # create formatter
 
    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 

	
 
    # add formatter to ch
 
    ch.setFormatter(formatter)
 

	
 
    # add ch to logger
 
    log.addHandler(ch)
 

	
 
    #PART ONE create db
 
    dbname = config['sqlalchemy.db1.url'].split('/')[-1]
 
    log.debug('making test db %s', dbname)
 
    dbconf = config['sqlalchemy.db1.url']
 
    log.debug('making test db %s', dbconf)
 

	
 
    dbmanage = DbManage(log_sql=True, dbname=dbname, root=config['here'],
 
    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
 
                        tests=True)
 
    dbmanage.create_tables(override=True)
 
    dbmanage.config_prompt(repos_test_path)
 
    dbmanage.create_default_user()
 
    dbmanage.admin_prompt()
 
    dbmanage.create_permissions()
 
    dbmanage.populate_default_permissions()
 

	
 
    #PART TWO make test repo
 
    log.debug('making test vcs repositories')
 

	
 
    #remove old one from previos tests
 
    for r in [HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, HG_FORK, GIT_FORK]:
 

	
 
        if os.path.isdir(jn(TESTS_TMP_PATH, r)):
 
            log.debug('removing %s', r)
 
            shutil.rmtree(jn(TESTS_TMP_PATH, r))
 

	
 
    #CREATE DEFAULT HG REPOSITORY
 
    cur_dir = dn(dn(abspath(__file__)))
 
    tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
 
    tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
 
    tar.close()
 

	
 
class UpgradeDb(command.Command):
 
    """Command used for paster to upgrade our database to newer version
 
    """
 

	
 
    max_args = 1
 
    min_args = 1
 

	
 
    usage = "CONFIG_FILE"
 
    summary = "Upgrades current db to newer version given configuration file"
 
    group_name = "RhodeCode"
 

	
 
    parser = command.Command.standard_parser(verbose=True)
 

	
 
    parser.add_option('--sql',
 
                      action='store_true',
 
                      dest='just_sql',
 
                      help="Prints upgrade sql for further investigation",
 
                      default=False)
 
    def command(self):
 
        config_name = self.args[0]
 
        p = config_name.split('/')
 
        root = '.' if len(p) == 1 else '/'.join(p[:-1])
 
        config = ConfigParser.ConfigParser({'here':root})
 
        config.read(config_name)
0 comments (0 inline, 0 general)