"""

        Update cache of last changeset for repository, keys should be::

            short_id

            raw_id

            revision

            message

            date

            author

        :param cs_cache:

        """

        from kallithea.lib.vcs.backends.base import BaseChangeset

        if cs_cache is None:

            cs_cache = EmptyChangeset()

            # use no-cache version here

            scm_repo = self.scm_instance_no_cache()

            if scm_repo:

                cs_cache = scm_repo.get_changeset()

        if isinstance(cs_cache, BaseChangeset):

            cs_cache = cs_cache.__json__()

        if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):

            _default = datetime.datetime.fromtimestamp(0)

            last_change = cs_cache.get('date') or _default

            log.debug('updated repo %s with new cs cache %s'

                      % (self.repo_name, cs_cache))

            self.updated_on = last_change

            self.changeset_cache = cs_cache

            Session().add(self)

            Session().commit()

        else:

            log.debug('changeset_cache for %s already up to date with %s'

                      % (self.repo_name, cs_cache['raw_id']))

    @property

    def tip(self):

        return self.get_changeset('tip')

    @property

    def author(self):

        return self.tip.author

    @property

    def last_change(self):

        return self.scm_instance.last_change

    def get_comments(self, revisions=None):

        """

        Returns comments for this repository grouped by revisions

        :param revisions: filter query by revisions only

        """

        cmts = ChangesetComment.query()\

            .filter(ChangesetComment.repo == self)

        if revisions is not None:

            if not revisions:

                return [] # don't use sql 'in' on empty set

            cmts = cmts.filter(ChangesetComment.revision.in_(revisions))

        grouped = collections.defaultdict(list)

        for cmt in cmts.all():

            grouped[cmt.revision].append(cmt)

        return grouped

    def statuses(self, revisions):

        """

        Returns statuses for this repository.

        PRs without any votes do _not_ show up as unreviewed.

        :param revisions: list of revisions to get statuses for

        """

        if not revisions:

            return {}

        statuses = ChangesetStatus.query()\

            .filter(ChangesetStatus.repo == self)\

            .filter(ChangesetStatus.version == 0)\

            .filter(ChangesetStatus.revision.in_(revisions))

        grouped = {}

        for stat in statuses.all():

            pr_id = pr_nice_id = pr_repo = None

            if stat.pull_request:

                pr_id = stat.pull_request.pull_request_id

                pr_nice_id = PullRequest.make_nice_id(pr_id)

                pr_repo = stat.pull_request.other_repo.repo_name

            grouped[stat.revision] = [str(stat.status), stat.status_lbl,

                                      pr_id, pr_repo, pr_nice_id]

        return grouped

    def _repo_size(self):

        from kallithea.lib import helpers as h

        log.debug('calculating repository size...')

        return h.format_byte_size(self.scm_instance.size)

    #==========================================================================

    # SCM CACHE INSTANCE

    #==========================================================================

    def set_invalidate(self):

        """

        Mark caches of this repo as invalid.

        """

        CacheInvalidation.set_invalidate(self.repo_name)

    def scm_instance_no_cache(self):

        return self.__get_instance()

    @property

    def scm_instance(self):

        import kallithea

        full_cache = str2bool(kallithea.CONFIG.get('vcs_full_cache'))

        if full_cache:

            return self.scm_instance_cached()

        return self.__get_instance()

    def scm_instance_cached(self, valid_cache_keys=None):

        @cache_region('long_term')

        def _c(repo_name):

            return self.__get_instance()

        rn = self.repo_name

        valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)

        if not valid:

            log.debug('Cache for %s invalidated, getting new object' % (rn))

            region_invalidate(_c, None, rn)

        else:

            log.debug('Getting scm_instance of %s from cache' % (rn))

        return _c(rn)

    def __get_instance(self):

        repo_full_path = self.repo_full_path

        alias = get_scm(repo_full_path)[0]

        log.debug('Creating instance of %s repository from %s'

                  % (alias, repo_full_path))

        backend = get_backend(alias)

        if alias == 'hg':

            repo = backend(safe_str(repo_full_path), create=False,

                           baseui=self._ui)

        else:

            repo = backend(repo_full_path, create=False)

        return repo

    def __json__(self):

        return dict(landing_rev = self.landing_rev)

class RepoGroup(Base, BaseModel):

    __tablename__ = 'groups'

    __table_args__ = (

        UniqueConstraint('group_name', 'group_parent_id'),

        CheckConstraint('group_id != group_parent_id'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8', 'sqlite_autoincrement': True},

    )

    __mapper_args__ = {'order_by': 'group_name'}

    SEP = ' » '

    group_id = Column(Integer(), nullable=False, unique=True, primary_key=True)

    group_name = Column(String(255, convert_unicode=False), nullable=False, unique=True, default=None)

    group_parent_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)

    group_description = Column(String(10000, convert_unicode=False), nullable=True, unique=None, default=None)

    enable_locking = Column(Boolean(), nullable=False, unique=None, default=False)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')

    users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')

    parent_group = relationship('RepoGroup', remote_side=group_id)

    user = relationship('User')

    def __init__(self, group_name='', parent_group=None):

        self.group_name = group_name

        self.parent_group = parent_group

    def __unicode__(self):

        return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,

                                      self.group_name)

    @classmethod

    def _generate_choice(cls, repo_group):

        """Return tuple with group_id and name as html literal"""

        from webhelpers.html import literal

        if repo_group is None:

            return (-1, u'-- %s --' % _('top level'))

        return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))

    @classmethod

        """Return tuples with group_id and name as html literal."""

    @classmethod

    def url_sep(cls):

        return URL_SEP

    @classmethod

    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):

        if case_insensitive:

            gr = cls.query()\

                .filter(cls.group_name.ilike(group_name))

        else:

            gr = cls.query()\

                .filter(cls.group_name == group_name)

        if cache:

            gr = gr.options(FromCache(

                            "sql_cache_short",

                            "get_group_%s" % _hash_key(group_name)

                            )

            )

        return gr.scalar()

    @property

    def parents(self):

        parents_recursion_limit = 10

        groups = []

        if self.parent_group is None:

            return groups

        cur_gr = self.parent_group

        groups.insert(0, cur_gr)

        cnt = 0

        while 1:

            cnt += 1

            gr = getattr(cur_gr, 'parent_group', None)

            cur_gr = cur_gr.parent_group

            if gr is None:

                break

            if cnt == parents_recursion_limit:

                # this will prevent accidental infinite loops

                log.error(('more than %s parents found for group %s, stopping '

                           'recursive parent fetching' % (parents_recursion_limit, self)))

                break

            groups.insert(0, gr)

        return groups

    @property

    def children(self):

        return RepoGroup.query().filter(RepoGroup.parent_group == self)

    @property

    def name(self):

        return self.group_name.split(RepoGroup.url_sep())[-1]

    @property

    def full_path(self):

        return self.group_name

    @property

    def full_path_splitted(self):

        return self.group_name.split(RepoGroup.url_sep())

    @property

    def repositories(self):

        return Repository.query()\

                .filter(Repository.group == self)\

                .order_by(Repository.repo_name)

    @property

    def repositories_recursive_count(self):

        cnt = self.repositories.count()

        def children_count(group):

            cnt = 0

            for child in group.children:

                cnt += child.repositories.count()

                cnt += children_count(child)

            return cnt

        return cnt + children_count(self)

    def _recursive_objects(self, include_repos=True):

        all_ = []

        def _get_members(root_gr):

            if include_repos:

                for r in root_gr.repositories:

                    all_.append(r)

            childs = root_gr.children.all()

            if childs:

                for gr in childs:

                    all_.append(gr)

                    _get_members(gr)

        _get_members(self)

        return [self] + all_

    def recursive_groups_and_repos(self):

        """

        Recursive return all groups, with repositories in those groups

        """

        return self._recursive_objects()

    def recursive_groups(self):

        """

        Returns all children groups for this group including children of children

        """

        return self._recursive_objects(include_repos=False)

    def get_new_name(self, group_name):

        """

        returns new full group name based on parent and new name

        :param group_name:

        """

        path_prefix = (self.parent_group.full_path_splitted if

                       self.parent_group else [])

        return RepoGroup.url_sep().join(path_prefix + [group_name])

    def get_api_data(self):

        """

        Common function for generating api data

        """

        group = self

        data = dict(

            group_id=group.group_id,

            group_name=group.group_name,

            group_description=group.group_description,

            parent_group=group.parent_group.group_name if group.parent_group else None,

            repositories=[x.repo_name for x in group.repositories],

            owner=group.user.username

        )

        return data

class Permission(Base, BaseModel):

    __tablename__ = 'permissions'

    __table_args__ = (

        Index('p_perm_name_idx', 'permission_name'),

        {'extend_existing': True, 'mysql_engine': 'InnoDB',

         'mysql_charset': 'utf8', 'sqlite_autoincrement': True},

    )

    PERMS = [

        ('hg.admin', _('Kallithea Administrator')),

        ('repository.none', _('Default user has no access to new Repositories')),

        ('repository.read', _('Default user has read access to new Repositories')),

        ('repository.write', _('Default user has write access to new Repositories')),

        ('repository.admin', _('Default user has admin access to new Repositories')),

        ('group.none', _('Default user has no access to new Repository Groups')),

        ('group.read', _('Default user has read access to new Repository Groups')),

        ('group.write', _('Default user has write access to new Repository Groups')),

        ('group.admin', _('Default user has admin access to new Repository Groups')),

        ('usergroup.none', _('Default user has no access to new User Groups')),

        ('usergroup.read', _('Default user has read access to new User Groups')),

        ('usergroup.write', _('Default user has write access to new User Groups')),

        ('usergroup.admin', _('Default user has admin access to new User Groups')),

        ('hg.repogroup.create.false', _('Only admins can create Repository Groups')),

        ('hg.repogroup.create.true', _('Non-admins can create Repository Groups')),

        ('hg.usergroup.create.false', _('Only admins can create User Groups')),

        ('hg.usergroup.create.true', _('Non-admins can create User Groups')),

        ('hg.create.none', _('Only admins can create top level Repositories')),

        ('hg.create.repository', _('Non-admins can create top level Repositories')),

        ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')),

        ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')),

        ('hg.fork.none', _('Only admins can fork repositories')),

        ('hg.fork.repository', _('Non-admins can can fork repositories')),

        ('hg.register.none', _('Registration disabled')),

        ('hg.register.manual_activate', _('User Registration with manual account activation')),

        ('hg.register.auto_activate', _('User Registration with automatic account activation')),

        ('hg.extern_activate.manual', _('Manual activation of external account')),

        ('hg.extern_activate.auto', _('Automatic activation of external account')),

    ]

    #definition of system default permissions for DEFAULT user

    DEFAULT_USER_PERMISSIONS = [

        'repository.read',

        'group.read',

        'usergroup.read',

        'hg.create.repository',

        'hg.create.write_on_repogroup.true',

        'hg.fork.repository',

        'hg.register.manual_activate',

            elif op == 'mod':

                if filename != old_filename:

                    #TODO: handle renames, needs vcs lib changes

                    imc.remove(filenode)

                    imc.add(FileNode(filename, content=content))

                else:

                    imc.change(filenode)

        # commit changes

        tip = imc.commit(message=message,

                         author=author,

                         parents=parents,

                         branch=parent_cs.branch)

        self.mark_for_invalidation(repo.repo_name)

        if trigger_push_hook:

            self._handle_push(scm_instance,

                              username=user.username,

                              action='push_local',

                              repo_name=repo.repo_name,

                              revisions=[tip.raw_id])

    def delete_nodes(self, user, repo, message, nodes, parent_cs=None,

                     author=None, trigger_push_hook=True):

        """

        Deletes specified nodes from repo.

        :param user: Kallithea User object or user_id, the committer

        :param repo: Kallithea Repository object

        :param message: commit message

        :param nodes: mapping {filename:{'content':content},...}

        :param parent_cs: parent changeset, can be empty than it's initial commit

        :param author: author of commit, cna be different that committer only for git

        :param trigger_push_hook: trigger push hooks

        :returns: new committed changeset after deletion

        """

        user = self._get_user(user)

        scm_instance = repo.scm_instance_no_cache()

        processed_nodes = []

        for f_path in nodes:

            f_path = self._sanitize_path(f_path)

            # content can be empty but for compatibility it allows same dicts

            # structure as add_nodes

            content = nodes[f_path].get('content')

            processed_nodes.append((f_path, content))

        message = safe_unicode(message)

        committer = user.full_contact

        author = safe_unicode(author) if author else committer

        IMC = self._get_IMC_module(scm_instance.alias)

        imc = IMC(scm_instance)

        if not parent_cs:

            parent_cs = EmptyChangeset(alias=scm_instance.alias)

        if isinstance(parent_cs, EmptyChangeset):

            # EmptyChangeset means we we're editing empty repository

            parents = None

        else:

            parents = [parent_cs]

        # add multiple nodes

        for path, content in processed_nodes:

            imc.remove(FileNode(path, content=content))

        tip = imc.commit(message=message,

                         author=author,

                         parents=parents,

                         branch=parent_cs.branch)

        self.mark_for_invalidation(repo.repo_name)

        if trigger_push_hook:

            self._handle_push(scm_instance,

                              username=user.username,

                              action='push_local',

                              repo_name=repo.repo_name,

                              revisions=[tip.raw_id])

        return tip

    def get_unread_journal(self):

        return self.sa.query(UserLog).count()

    def get_repo_landing_revs(self, repo=None):

        """

        Generates select option with tags branches and bookmarks (for hg only)

        grouped by type

        :param repo:

        """

        hist_l = []

        choices = []

        repo = self.__get_repo(repo)

        hist_l.append(['rev:tip', _('latest tip')])

        choices.append('rev:tip')

        if not repo:

            return choices, hist_l

        repo = repo.scm_instance

        branches_group = ([(u'branch:%s' % k, k) for k, v in

                           repo.branches.iteritems()], _("Branches"))

        hist_l.append(branches_group)

        choices.extend([x[0] for x in branches_group[0]])

        if repo.alias == 'hg':

            bookmarks_group = ([(u'book:%s' % k, k) for k, v in

                                repo.bookmarks.iteritems()], _("Bookmarks"))

            hist_l.append(bookmarks_group)

            choices.extend([x[0] for x in bookmarks_group[0]])

        tags_group = ([(u'tag:%s' % k, k) for k, v in

                       repo.tags.iteritems()], _("Tags"))

        hist_l.append(tags_group)

        choices.extend([x[0] for x in tags_group[0]])

        return choices, hist_l

    def install_git_hook(self, repo, force_create=False):

        """

        Creates a kallithea hook inside a git repository

        :param repo: Instance of VCS repo

        :param force_create: Create even if same name hook exists

        """

        loc = jn(repo.path, 'hooks')

        if not repo.bare:

            loc = jn(repo.path, '.git', 'hooks')

        if not os.path.isdir(loc):

            os.makedirs(loc)

        tmpl_post = pkg_resources.resource_string(

            'kallithea', jn('config', 'post_receive_tmpl.py')

        )

        tmpl_pre = pkg_resources.resource_string(

            'kallithea', jn('config', 'pre_receive_tmpl.py')

        )

        for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:

            _hook_file = jn(loc, '%s-receive' % h_type)

            has_hook = False

            log.debug('Installing git hook in repo %s' % repo)

            if os.path.exists(_hook_file):

                # let's take a look at this hook, maybe it's kallithea ?

                log.debug('hook exists, checking if it is from kallithea')

                with open(_hook_file, 'rb') as f:

                    data = f.read()

                    matches = re.compile(r'(?:%s)\s*=\s*(.*)'

                                         % 'KALLITHEA_HOOK_VER').search(data)

                    if matches:

                        try:

                            ver = matches.groups()[0]

                            log.debug('got %s it is kallithea' % (ver))

                            has_hook = True

                        except Exception:

                            log.error(traceback.format_exc())

            else:

                # there is no hook in this dir, so we want to create one

                has_hook = True

            if has_hook or force_create:

                log.debug('writing %s hook file !' % (h_type,))

                try:

                    with open(_hook_file, 'wb') as f:

                        tmpl = tmpl.replace('_TMPL_', kallithea.__version__)

                        f.write(tmpl)

                    os.chmod(_hook_file, 0755)

                except IOError, e:

                    log.error('error writing %s: %s' % (_hook_file, e))

            else:

                log.debug('skipping writing hook file')

def AvailableRepoGroupChoices(top_perms, repo_group_perms, extras=()):

    """Return group_id,string tuples with choices for all the repo groups where

    the user has the necessary permissions.

    Top level is -1.

    """

    groups = RepoGroup.query().all()

    if HasPermissionAll('hg.admin')('available repo groups'):

        groups.append(None)

    else:

        groups = list(RepoGroupList(groups, perm_set=repo_group_perms))

        if top_perms and HasPermissionAny(*top_perms)('available repo groups'):

            groups.append(None)

        for extra in extras:

            if not any(rg == extra for rg in groups):

                groups.append(extra)