group.user = owner

            sa.add(group)

            perm_obj = rgm._create_default_perms(group)

            sa.add(perm_obj)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hooks=False, user=None, overwrite_git_hooks=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hooks: if this is True, also check and install git hook

        for a repo if missing

    :param overwrite_git_hooks: if this is True, overwrite any existing git hooks

        that may be encountered (even if user-deployed)

    """

    from kallithea.model.repo import RepoModel

    from kallithea.model.scm import ScmModel

    sa = meta.Session()

    repo_model = RepoModel()

    if user is None:

        user = User.get_first_admin()

    added = []

    ##creation defaults

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_locking = defs.get('repo_enable_locking')

    enable_downloads = defs.get('repo_enable_downloads')

    private = defs.get('repo_private')

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        unicode_name = safe_unicode(name)

        db_repo = repo_model.get_by_repo_name(unicode_name)

        # found repo that is on filesystem not in Kallithea database

        if not db_repo:

            log.info('repository %s not found, creating now', name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = repo_model._create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repo_group=getattr(group, 'group_id', None),

                owner=user,

                enable_locking=enable_locking,

                enable_downloads=enable_downloads,

                enable_statistics=enable_statistics,

                private=private,

                state=Repository.STATE_CREATED

            )

            sa.commit()

            # we added that repo just now, and make sure it has githook

            # installed, and updated server info

            if new_repo.repo_type == 'git':

                git_repo = new_repo.scm_instance

                ScmModel().install_git_hooks(git_repo)

                # update repository server-info

                log.debug('Running update server info')

                git_repo._update_server_info()

            new_repo.update_changeset_cache()

        elif install_git_hooks:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)

    removed = []

    # remove from database those repositories that are not in the filesystem

    unicode_initial_repo_list = set(safe_unicode(name) for name in initial_repo_list)

    for repo in sa.query(Repository).all():

        if repo.repo_name not in unicode_initial_repo_list:

            if remove_obsolete:

                log.debug("Removing non-existing repository found in db `%s`",

                          repo.repo_name)

                try:

                    RepoModel(sa).delete(repo, forks='detach', fs_remove=False)

                    sa.commit()

                except Exception:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

            removed.append(repo.repo_name)

    return added, removed

def load_rcextensions(root_path):

    import kallithea

    from kallithea.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = kallithea.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...', rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []):

            log.debug('settings custom INDEX_EXTENSIONS')

            conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])

        #ADDITIONAL MAPPINGS

        log.debug('adding extra into INDEX_EXTENSIONS')

        conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))

        # auto check if the module is not missing any data, set to default if is

        # this will help autoupdate new feature of rcext module

        #from kallithea.config import rcextensions

        #for k in dir(rcextensions):

        #    if not k.startswith('_') and not hasattr(EXT, k):

        #        setattr(EXT, k, getattr(rcextensions, k))

def get_custom_lexer(extension):

    """

    returns a custom lexer if it's defined in rcextensions module, or None

    if there's no custom lexer defined

    """

    import kallithea

    from pygments import lexers

    #check if we didn't define this extension as other lexer

    if kallithea.EXTENSIONS and extension in kallithea.EXTENSIONS.EXTRA_LEXERS:

        _lexer_name = kallithea.EXTENSIONS.EXTRA_LEXERS[extension]

        return lexers.get_lexer_by_name(_lexer_name)

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    :param config: test config

    :param full_index:

    """

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    from kallithea.lib.pidlock import DaemonLock, LockHeld

    repo_location = repo_location

    index_location = os.path.join(config['app_conf']['index_dir'])

    if not os.path.exists(index_location):

        os.makedirs(index_location)

    try:

        l = DaemonLock(file_=os.path.join(dirname(index_location), 'make_index.lock'))

        WhooshIndexingDaemon(index_location=index_location,

                             repo_location=repo_location) \

            .run(full_index=full_index)

        l.release()

    except LockHeld:

        pass

def create_test_env(repos_test_path, config):

    """

    Makes a fresh database and

    install test repository into tmp dir

    """

    from kallithea.lib.db_manage import DbManage

    from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH

    # PART ONE create db

    dbconf = config['sqlalchemy.db1.url']

    log.debug('making test db %s', dbconf)

    # create test dir if it doesn't exist

    if not os.path.isdir(repos_test_path):

        log.debug('Creating testdir %s', repos_test_path)

        os.makedirs(repos_test_path)

    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],

                        tests=True)

    dbmanage.create_tables(override=True)

    # for tests dynamically set new root paths based on generated content

    dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))

    dbmanage.create_default_user()

    dbmanage.admin_prompt()

    dbmanage.create_permissions()

    dbmanage.populate_default_permissions()

    Session().commit()

    # PART TWO make test repo

    log.debug('making test vcs repositories')

    idx_path = config['app_conf']['index_dir']

    data_path = config['app_conf']['cache_dir']

    #clean index and data

    if idx_path and os.path.exists(idx_path):

        log.debug('remove %s', idx_path)

        shutil.rmtree(idx_path)

    if data_path and os.path.exists(data_path):

        log.debug('remove %s', data_path)

        shutil.rmtree(data_path)

    #CREATE DEFAULT TEST REPOS

    cur_dir = dirname(dirname(abspath(__file__)))

    tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))

    tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))

    tar.close()

    cur_dir = dirname(dirname(abspath(__file__)))

    tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))

    tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))

    tar.close()

    #LOAD VCS test stuff

    from kallithea.tests.vcs import setup_package

    setup_package()

def check_git_version():

    """

    Checks what version of git is installed in system, and issues a warning

    if it's too old for Kallithea to work properly.

    """

    from kallithea import BACKENDS

    from kallithea.lib.vcs.backends.git.repository import GitRepository

    from kallithea.lib.vcs.conf import settings

    from distutils.version import StrictVersion

    if 'git' not in BACKENDS:

        return None

    stdout, stderr = GitRepository._run_git_command(['--version'], _bare=True,

                                                    _safe=True)

    m = re.search("\d+.\d+.\d+", stdout)

    if m:

        ver = StrictVersion(m.group(0))

    else:

        ver = StrictVersion('0.0.0')

    req_ver = StrictVersion('1.7.4')

    log.debug('Git executable: "%s" version %s detected: %s',

              settings.GIT_EXECUTABLE_PATH, ver, stdout)

    if stderr:

        log.warning('Error detecting git version: %r', stderr)

    elif ver < req_ver:

        log.warning('Kallithea detected git version %s, which is too old '

                    'for the system to function properly. '

                    'Please upgrade to version %s or later.' % (ver, req_ver))

    return ver

@decorator.decorator

def jsonify(func, *args, **kwargs):

    """Action decorator that formats output for JSON

    Given a function that will return content, this decorator will turn

    the result into JSON, with a content-type of 'application/json' and

    output it.

    """

    from pylons.decorators.util import get_pylons

    from kallithea.lib.compat import json

    pylons = get_pylons(args)

    pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'

    data = func(*args, **kwargs)

    if isinstance(data, (list, tuple)):

        msg = "JSON responses with Array envelopes are susceptible to " \

              "cross-site data leak attacks, see " \

              "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"

        warnings.warn(msg, Warning, 2)

        log.warning(msg)

    log.debug("Returning JSON wrapped action output")

    return json.dumps(data, encoding='utf-8')

def conditional_cache(region, prefix, condition, func):

    """

    Conditional caching function use like::

        def _c(arg):

            #heavy computation function

            return data

        # depending from condition the compute is wrapped in cache or not

        compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)

        return compute(arg)

    :param region: name of cache region

    :param prefix: cache region prefix

    :param condition: condition for cache to be triggered, and return data cached

    :param func: wrapped heavy function to compute

    """

    wrapped = func

    if condition:

        log.debug('conditional_cache: True, wrapping call of '

                  'func: %s into %s region cache' % (region, func))

        wrapped = _cache_decorate((prefix,), None, None, region)(func)

    return wrapped