if cache_key.startswith('get_repo_cached_'):

        name = cache_key.split('get_repo_cached_')[-1]

        ScmModel().mark_for_invalidation(name)

def map_groups(path):

    """

    Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s' % (lvl,

                                                                   group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hook=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hook: if this is True, also check and install githook

        for a repo if missing

    """

    from rhodecode.model.repo import RepoModel

    from rhodecode.model.scm import ScmModel

    sa = meta.Session()

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

#    # clear cache keys

#    log.debug("Clearing cache keys now...")

#    CacheInvalidation.clear_cache()

#    sa.commit()

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        db_repo = rm.get_by_repo_name(name)

        # found repo that is on filesystem not in RhodeCode database

        if not db_repo:

            log.info('repository %s not found creating now' % name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = rm.create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repos_group=getattr(group, 'group_id', None),

                owner=user,

                just_db=True

            )

            # we added that repo just now, and make sure it has githook

            # installed

            if new_repo.repo_type == 'git':

                ScmModel().install_git_hook(new_repo.scm_instance)

        elif install_git_hook:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hook(db_repo.scm_instance)

        # during starting install all cache keys for all repositories in the

        # system, this will register all repos and multiple instances

        key, _prefix, _org_key = CacheInvalidation._get_key(name)

        CacheInvalidation._get_or_create_key(key, _prefix, _org_key, commit=False)

    sa.commit()

    removed = []

    if remove_obsolete:

        # remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            if repo.repo_name not in initial_repo_list.keys():

                log.debug("Removing non existing repository found in db `%s`" %

                          repo.repo_name)

                try:

                    sa.delete(repo)

                    sa.commit()

                    removed.append(repo.repo_name)

                except:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

    import rhodecode

    from rhodecode.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = rhodecode.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...' % rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:

            log.debug('settings custom INDEX_EXTENSIONS')

            conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])

        #ADDITIONAL MAPPINGS

        log.debug('adding extra into INDEX_EXTENSIONS')

        conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    :param config: test config

    :param full_index:

    """

    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

    from rhodecode.lib.pidlock import DaemonLock, LockHeld

    repo_location = repo_location

    index_location = os.path.join(config['app_conf']['index_dir'])

    if not os.path.exists(index_location):

        os.makedirs(index_location)

    try:

        l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))

        WhooshIndexingDaemon(index_location=index_location,

            self._locked = ':'.join(map(str, val))

        else:

            self._locked = None

    @classmethod

    def url_sep(cls):

        return URL_SEP

    @classmethod

    def get_by_repo_name(cls, repo_name):

        q = Session().query(cls).filter(cls.repo_name == repo_name)

        q = q.options(joinedload(Repository.fork))\

                .options(joinedload(Repository.user))\

                .options(joinedload(Repository.group))

        return q.scalar()

    @classmethod

    def get_by_full_path(cls, repo_full_path):

        repo_name = repo_full_path.split(cls.base_path(), 1)[-1]

        return cls.get_by_repo_name(repo_name.strip(URL_SEP))

    @classmethod

    def get_repo_forks(cls, repo_id):

        return cls.query().filter(Repository.fork_id == repo_id)

    @classmethod

    def base_path(cls):

        """

        Returns base path when all repos are stored

        :param cls:

        """

        q = Session().query(RhodeCodeUi)\

            .filter(RhodeCodeUi.ui_key == cls.url_sep())

        q = q.options(FromCache("sql_cache_short", "repository_repo_path"))

        return q.one().ui_value

    @property

    def forks(self):

        """

        Return forks of this repo

        """

        return Repository.get_repo_forks(self.repo_id)

    @property

    def parent(self):

        """

        Returns fork parent

        """

        return self.fork

    @property

    def just_name(self):

        return self.repo_name.split(Repository.url_sep())[-1]

    @property

    def groups_with_parents(self):

        groups = []

        if self.group is None:

            return groups

        cur_gr = self.group

        groups.insert(0, cur_gr)

        while 1:

            gr = getattr(cur_gr, 'parent_group', None)

            cur_gr = cur_gr.parent_group

            if gr is None:

                break

            groups.insert(0, gr)

        return groups

    @property

    def groups_and_repo(self):

        return self.groups_with_parents, self.just_name

    @LazyProperty

    def repo_path(self):

        """

        Returns base full path for that repository means where it actually

        exists on a filesystem

        """

        q = Session().query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==

                                              Repository.url_sep())

        q = q.options(FromCache("sql_cache_short", "repository_repo_path"))

        return q.one().ui_value

    @property

    def repo_full_path(self):

        p = [self.repo_path]

        # we need to split the name by / since this is how we store the

        # names in the database, but that eventually needs to be converted

        # into a valid system path

        p += self.repo_name.split(Repository.url_sep())

        return os.path.join(*p)

    def get_new_name(self, repo_name):

        """

        returns new full repository name based on assigned group and new new

        :param group_name:

        """

        path_prefix = self.group.full_path_splitted if self.group else []

        return Repository.url_sep().join(path_prefix + [repo_name])

    @property

    def _ui(self):

        """

        Creates an db based ui object for this repository

        """

        from mercurial import ui

        from mercurial import config

        baseui = ui.ui()

        #clean the baseui object

        baseui._ocfg = config.config()

        baseui._ucfg = config.config()

        baseui._tcfg = config.config()

        ret = RhodeCodeUi.query()\

            .options(FromCache("sql_cache_short", "repository_repo_ui")).all()

        hg_ui = ret

        for ui_ in hg_ui:

            if ui_.ui_active:

                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,

                          ui_.ui_key, ui_.ui_value)

                baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)

            if ui_.ui_key == 'push_ssl':

                # force set push_ssl requirement to False, rhodecode

                # handles that

                baseui.setconfig(ui_.ui_section, ui_.ui_key, False)

        return baseui

    @classmethod

    def inject_ui(cls, repo, extras={}):

        from rhodecode.lib.vcs.backends.hg import MercurialRepository

        from rhodecode.lib.vcs.backends.git import GitRepository

        required = (MercurialRepository, GitRepository)

        if not isinstance(repo, required):

            raise Exception('repo must be instance of %s' % required)

        # inject ui extra param to log this action via push logger

        for k, v in extras.items():

            repo._repo.ui.setconfig('rhodecode_extras', k, v)

    @classmethod

    def is_valid(cls, repo_name):

        """

        returns True if given repo name is a valid filesystem repository

        :param cls:

        :param repo_name:

        """

        from rhodecode.lib.utils import is_valid_repo

        return is_valid_repo(repo_name, cls.base_path())

    def get_api_data(self):

        """

        Common function for generating repo api data

        """

        repo = self

        data = dict(

            repo_id=repo.repo_id,

            repo_name=repo.repo_name,

            repo_type=repo.repo_type,

            clone_uri=repo.clone_uri,

            private=repo.private,

            created_on=repo.created_on,

            description=repo.description,

            landing_rev=repo.landing_rev,

            owner=repo.user.username,

            fork_of=repo.fork.repo_name if repo.fork else None

        )

        return data

    @classmethod

    def lock(cls, repo, user_id):

        repo.locked = [user_id, time.time()]

        Session().add(repo)

        Session().commit()

    @classmethod

    def unlock(cls, repo):

        repo.locked = None

        Session().add(repo)

        Session().commit()

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8'}

    )

    group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)

    group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)

    permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)

    user = relationship('User')

    group = relationship('RepoGroup')

    permission = relationship('Permission')

class UsersGroupRepoGroupToPerm(Base, BaseModel):

    __tablename__ = 'users_group_repo_group_to_perm'

    __table_args__ = (

        UniqueConstraint('users_group_id', 'group_id'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8'}

    )

    users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)

    group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)

    permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)

    users_group = relationship('UsersGroup')

    permission = relationship('Permission')

    group = relationship('RepoGroup')

class Statistics(Base, BaseModel):

    __tablename__ = 'statistics'

    __table_args__ = (

         UniqueConstraint('repository_id'),

         {'extend_existing': True, 'mysql_engine': 'InnoDB',

          'mysql_charset': 'utf8'}

    )

    stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)

    stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)

    commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data

    commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data

    languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data

    repository = relationship('Repository', single_parent=True)

class UserFollowing(Base, BaseModel):

    __tablename__ = 'user_followings'

    __table_args__ = (

        UniqueConstraint('user_id', 'follows_repository_id'),

        UniqueConstraint('user_id', 'follows_user_id'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8'}

    )

    user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)

    follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)

    follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)

    follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)

    user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')

    follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')

    follows_repository = relationship('Repository', order_by='Repository.repo_name')

    @classmethod

    def get_repo_followers(cls, repo_id):

        return cls.query().filter(cls.follows_repo_id == repo_id)

class CacheInvalidation(Base, BaseModel):

    __tablename__ = 'cache_invalidation'

    __table_args__ = (

        UniqueConstraint('cache_key'),

        Index('key_idx', 'cache_key'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8'},

    )

    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    cache_key = Column("cache_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_args = Column("cache_args", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)

    def __init__(self, cache_key, cache_args=''):

        self.cache_key = cache_key

        self.cache_args = cache_args

        self.cache_active = False

    def __unicode__(self):

        return u"<%s('%s:%s')>" % (self.__class__.__name__,

                                  self.cache_id, self.cache_key)

    @classmethod

    def clear_cache(cls):

        cls.query().delete()

    @classmethod

    def _get_key(cls, key):

        """

        Wrapper for generating a key, together with a prefix

        :param key:

        """

        import rhodecode

        prefix = ''

        iid = rhodecode.CONFIG.get('instance_id')

        if iid:

            prefix = iid

        return "%s%s" % (prefix, key), prefix, key.rstrip('_README')

    @classmethod

    def get_by_key(cls, key):

        return cls.query().filter(cls.cache_key == key).scalar()

    @classmethod

    def _get_or_create_key(cls, key, prefix, org_key, commit=True):

        inv_obj = Session().query(cls).filter(cls.cache_key == key).scalar()

        if not inv_obj:

            try:

                inv_obj = CacheInvalidation(key, org_key)

                Session().add(inv_obj)

                if commit:

                    Session().commit()

            except Exception:

                log.error(traceback.format_exc())

                Session().rollback()

        return inv_obj

    @classmethod

    def invalidate(cls, key):

        """

        Returns Invalidation object if this given key should be invalidated

        None otherwise. `cache_active = False` means that this cache

        state is not valid and needs to be invalidated

        :param key:

        """

        key, _prefix, _org_key = cls._get_key(key)

        inv = cls._get_or_create_key(key, _prefix, _org_key)

        if inv and inv.cache_active is False:

            return inv

    @classmethod

    def set_invalidate(cls, key):

        """

        Mark this Cache key for invalidation

        :param key:

        """

        key, _prefix, _org_key = cls._get_key(key)

        inv_objs = Session().query(cls).filter(cls.cache_args == _org_key).all()

        log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),

                                                             _org_key))

        try:

            for inv_obj in inv_objs:

                if inv_obj:

                    inv_obj.cache_active = False

                Session().add(inv_obj)

            Session().commit()

        except Exception:

            log.error(traceback.format_exc())

            Session().rollback()

    @classmethod

    def set_valid(cls, key):

        """

        Mark this cache key as active and currently cached

        :param key:

        """

        inv_obj = cls.get_by_key(key)

        inv_obj.cache_active = True

        Session().add(inv_obj)

        Session().commit()

    @classmethod

    def get_cache_map(cls):

        class cachemapdict(dict):

            def __init__(self, *args, **kwargs):

                fixkey = kwargs.get('fixkey')

                if fixkey:

                    del kwargs['fixkey']

                </div>

                <div class="checkboxes">

                    ${h.checkbox('enable_statistics',value="True")}

                    <span class="help-block">${_('Enable statistics window on summary page.')}</span>

                </div>

            </div>

            <div class="field">

                <div class="label label-checkbox">

                    <label for="enable_downloads">${_('Enable downloads')}:</label>

                </div>

                <div class="checkboxes">

                    ${h.checkbox('enable_downloads',value="True")}

                    <span class="help-block">${_('Enable download menu on summary page.')}</span>

                </div>

            </div>

            <div class="field">

                <div class="label label-checkbox">

                    <label for="enable_locking">${_('Enable locking')}:</label>

                </div>

                <div class="checkboxes">

                    ${h.checkbox('enable_locking',value="True")}

                    <span class="help-block">${_('Enable lock-by-pulling on repository.')}</span>

                </div>

            </div>            

            <div class="field">

                <div class="label">

                    <label for="user">${_('Owner')}:</label>

                </div>

                <div class="input input-medium ac">

                    <div class="perm_ac">

                       ${h.text('user',class_='yui-ac-input')}

                       <span class="help-block">${_('Change owner of this repository.')}</span>

                       <div id="owner_container"></div>

                    </div>

                </div>

             </div>

            <div class="field">

                <div class="label">

                    <label for="input">${_('Permissions')}:</label>

                </div>

                <div class="input">

                    <%include file="repo_edit_perms.html"/>

                </div>

                <div class="buttons">

                  ${h.submit('save',_('Save'),class_="ui-btn large")}

                  ${h.reset('reset',_('Reset'),class_="ui-btn large")}

                </div>

            </div>

    </div>

    </div>

    ${h.end_form()}

</div>

<div class="box box-right">

    <div class="title">

        <h5>${_('Administration')}</h5>

    </div>

        <h3>${_('Statistics')}</h3>

        ${h.form(url('repo_stats', repo_name=c.repo_info.repo_name),method='delete')}

        <div class="form">

           <div class="fields">

               ${h.submit('reset_stats_%s' % c.repo_info.repo_name,_('Reset current statistics'),class_="ui-btn",onclick="return confirm('"+_('Confirm to remove current statistics')+"');")}

               <div class="field" style="border:none;color:#888">

               <ul>

                    <li>${_('Fetched to rev')}: ${c.stats_revision}/${c.repo_last_rev}</li>

                    <li>${_('Stats gathered')}: ${c.stats_percentage}%</li>

               </ul>

               </div>

           </div>

        </div>

        ${h.end_form()}

        %if c.repo_info.clone_uri:

        <h3>${_('Remote')}</h3>

        ${h.form(url('repo_pull', repo_name=c.repo_info.repo_name),method='put')}

        <div class="form">

           <div class="fields">

               ${h.submit('remote_pull_%s' % c.repo_info.repo_name,_('Pull changes from remote location'),class_="ui-btn",onclick="return confirm('"+_('Confirm to pull changes from remote side')+"');")}

               <div class="field" style="border:none">

               <ul>

                    <li><a href="${c.repo_info.clone_uri}">${c.repo_info.clone_uri}</a></li>

               </ul>

               </div>

           </div>

        </div>

        ${h.end_form()}

        %endif

        <h3>${_('Cache')}</h3>

        ${h.form(url('repo_cache', repo_name=c.repo_info.repo_name),method='delete')}

        <div class="form">

           <div class="fields">

               ${h.submit('reset_cache_%s' % c.repo_info.repo_name,_('Invalidate repository cache'),class_="ui-btn",onclick="return confirm('"+_('Confirm to invalidate repository cache')+"');")}

           </div>

        </div>

        ${h.end_form()}

        <h3>${_('Public journal')}</h3>

        ${h.form(url('repo_public_journal', repo_name=c.repo_info.repo_name),method='put')}

        <div class="form">

        </div>

        ${h.end_form()}

        <h3>${_('Locking')}</h3>

        ${h.form(url('repo_locking', repo_name=c.repo_info.repo_name),method='put')}

        <div class="form">

           <div class="fields">

              %if c.repo_info.locked[0]:

               ${h.submit('set_unlock' ,_('Unlock locked repo'),class_="ui-btn",onclick="return confirm('"+_('Confirm to unlock repository')+"');")}

               ${'Locked by %s on %s' % (h.person_by_id(c.repo_info.locked[0]),h.fmt_date(h.time_to_datetime(c.repo_info.locked[1])))}

              %else:

                ${h.submit('set_lock',_('lock repo'),class_="ui-btn",onclick="return confirm('"+_('Confirm to lock repository')+"');")}

                ${_('Repository is not locked')}

              %endif

           </div>

           <div class="field" style="border:none;color:#888">

           <ul>

                <li>${_('Force locking on repository. Works only when anonymous access is disabled')}

                </li>

           </ul>

           </div>           

        </div>

        ${h.end_form()}

        <h3>${_('Set as fork of')}</h3>

        ${h.form(url('repo_as_fork', repo_name=c.repo_info.repo_name),method='put')}

        <div class="form">

           <div class="fields">

               ${h.select('id_fork_of','',c.repos_list,class_="medium")}

               ${h.submit('set_as_fork_%s' % c.repo_info.repo_name,_('set'),class_="ui-btn",)}

           </div>

               <div class="field" style="border:none;color:#888">

               <ul>

                    <li>${_('''Manually set this repository as a fork of another from the list''')}</li>

               </ul>

               </div>

        </div>        

        ${h.end_form()}

        <h3>${_('Delete')}</h3>

        ${h.form(url('repo', repo_name=c.repo_info.repo_name),method='delete')}

        <div class="form">

           <div class="fields">

               ${h.submit('remove_%s' % c.repo_info.repo_name,_('Remove this repository'),class_="ui-btn red",onclick="return confirm('"+_('Confirm to delete this repository')+"');")}

           </div>

           <div class="field" style="border:none;color:#888">

           <ul>

                <li>${_('''This repository will be renamed in a special way in order to be unaccesible for RhodeCode and VCS systems.

                         If you need fully delete it from filesystem please do it manually''')}

                </li>

           </ul>

           </div>

        </div>

        ${h.end_form()}        

</div>

</%def>