parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s' % (lvl,

                                                                   group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hook=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hook: if this is True, also check and install githook

        for a repo if missing

    """

    from rhodecode.model.repo import RepoModel

    from rhodecode.model.scm import ScmModel

    sa = meta.Session()

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        db_repo = rm.get_by_repo_name(name)

        # found repo that is on filesystem not in RhodeCode database

        if not db_repo:

            log.info('repository %s not found creating now' % name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = rm.create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repos_group=getattr(group, 'group_id', None),

                owner=user,

                just_db=True

            )

            # we added that repo just now, and make sure it has githook

            # installed

            if new_repo.repo_type == 'git':

                ScmModel().install_git_hook(new_repo.scm_instance)

        elif install_git_hook:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hook(db_repo.scm_instance)

    sa.commit()

    removed = []

    if remove_obsolete:

        # remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            if repo.repo_name not in initial_repo_list.keys():

                log.debug("Removing non existing repository found in db `%s`" %

                          repo.repo_name)

                try:

                    sa.delete(repo)

                    sa.commit()

                    removed.append(repo.repo_name)

                except:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

    import rhodecode

    from rhodecode.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = rhodecode.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...' % rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:

            log.debug('settings custom INDEX_EXTENSIONS')

        return cls.query().filter(cls.follows_repo_id == repo_id)

class CacheInvalidation(Base, BaseModel):

    __tablename__ = 'cache_invalidation'

    __table_args__ = (

        UniqueConstraint('cache_key'),

        Index('key_idx', 'cache_key'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8'},

    )

    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    cache_key = Column("cache_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_args = Column("cache_args", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)

    def __init__(self, cache_key, cache_args=''):

        self.cache_key = cache_key

        self.cache_args = cache_args

        self.cache_active = False

    def __unicode__(self):

        return u"<%s('%s:%s')>" % (self.__class__.__name__,

                                  self.cache_id, self.cache_key)

    @classmethod

    def clear_cache(cls):

        cls.query().delete()

    @classmethod

    def _get_key(cls, key):

        """

        Wrapper for generating a key, together with a prefix

        :param key:

        """

        import rhodecode

        prefix = ''

        iid = rhodecode.CONFIG.get('instance_id')

        if iid:

            prefix = iid

        return "%s%s" % (prefix, key), prefix, key.rstrip('_README')

    @classmethod

    def get_by_key(cls, key):

        return cls.query().filter(cls.cache_key == key).scalar()

    @classmethod

        inv_obj = Session().query(cls).filter(cls.cache_key == key).scalar()

        if not inv_obj:

            try:

                inv_obj = CacheInvalidation(key, org_key)

                Session().add(inv_obj)

            except Exception:

                log.error(traceback.format_exc())

                Session().rollback()

        return inv_obj

    @classmethod

    def invalidate(cls, key):

        """

        Returns Invalidation object if this given key should be invalidated

        None otherwise. `cache_active = False` means that this cache

        state is not valid and needs to be invalidated

        :param key:

        """

        key, _prefix, _org_key = cls._get_key(key)

        inv = cls._get_or_create_key(key, _prefix, _org_key)

        if inv and inv.cache_active is False:

            return inv

    @classmethod

    def set_invalidate(cls, key):

        """

        Mark this Cache key for invalidation

        :param key:

        """

        key, _prefix, _org_key = cls._get_key(key)

        inv_objs = Session().query(cls).filter(cls.cache_args == _org_key).all()

        log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),

                                                             _org_key))

        try:

            for inv_obj in inv_objs:

                if inv_obj:

                    inv_obj.cache_active = False

                Session().add(inv_obj)

            Session().commit()

        except Exception:

            log.error(traceback.format_exc())

            Session().rollback()

    @classmethod

    def set_valid(cls, key):

        """

        Mark this cache key as active and currently cached