kallithea Changeset - b03233c4a438

Changeset - b03233c4a438

Parent rev.

Child rev.

[Not reviewed]

default

0 2 0

Mads Kiilerich - 10 years ago 2016-03-14 16:17:46
madski@unity3d.com

scm: simplify get_repos and get rid of RepoList overloads

The RepoList variants might have made sense before any repo info was stored in
the database. Now it has no value and doesn't help at all.

2 files changed with 25 insertions and 121 deletions:

kallithea/controllers/home.py

kallithea/model/scm.py

114

0 comments (0 inline, 0 general)

kallithea/controllers/home.py

➞

Show inline comments

@@ @@ -31,115 +31,124 @@ import logging @@
 from pylons import tmpl_context as c, request
 from pylons.i18n.translation import _
 from webob.exc import HTTPBadRequest
 from sqlalchemy.sql.expression import func
 from kallithea.lib.utils import jsonify, conditional_cache
 from kallithea.lib.compat import json
 from kallithea.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from kallithea.lib.base import BaseController, render
 from kallithea.model.db import Repository, RepoGroup
 from kallithea.model.repo import RepoModel
 log = logging.getLogger(__name__)
 class HomeController(BaseController):
     def __before__(self):
         super(HomeController, self).__before__()
     def about(self):
         return render('/about.html')
     @LoginRequired()
     def index(self):
         c.groups = self.scm_model.get_repo_groups()
         c.group = None
         c.repos_list = Repository.query() \
                         .filter(Repository.group_id == None) \
                         .order_by(func.lower(Repository.repo_name)) \
                         .all()
         repos_data = RepoModel().get_repos_as_dict(repos_list=c.repos_list,
                                                    admin=False)
         #json used to render the grid
         c.data = json.dumps(repos_data)
         return render('/index.html')
     @LoginRequired()
     @jsonify
     def repo_switcher_data(self):
         #wrapper for conditional cache
         def _c():
             log.debug('generating switcher repo/groups list')
             all_repos = Repository.query().order_by(Repository.repo_name).all()
-            repo_iter = self.scm_model.get_repos(all_repos, simple=True)
             repo_iter = self.scm_model.get_repos(all_repos)
             all_groups = RepoGroup.query().order_by(RepoGroup.group_name).all()
             repo_groups_iter = self.scm_model.get_repo_groups(all_groups)
             res = [{
                     'text': _('Groups'),
                     'children': [
                        {'id': obj.group_name, 'text': obj.group_name,
                         'type': 'group', 'obj': {}} for obj in repo_groups_iter]
                    }, {
                        {'id': obj.group_name,
                         'text': obj.group_name,
                         'type': 'group',
                         'obj': {}}
                        for obj in repo_groups_iter
                     ],
                    },
+                   {
                     'text': _('Repositories'),
                     'children': [
                        {'id': obj['name'], 'text': obj['name'],
                         'type': 'repo', 'obj': obj['dbrepo']} for obj in repo_iter]
                        {'id': obj.repo_name,
                         'text': obj.repo_name,
                         'type': 'repo',
                         'obj': obj.get_dict()}
                        for obj in repo_iter
                     ],
                    }]
             data = {
                 'more': False,
                 'results': res
+                'results': res,
+            }
             return data
         if request.is_xhr:
             condition = False
             compute = conditional_cache('short_term', 'cache_desc',
                                         condition=condition, func=_c)
             return compute()
         else:
             raise HTTPBadRequest()
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def branch_tag_switcher(self, repo_name):
         if request.is_xhr:
             c.db_repo = Repository.get_by_repo_name(repo_name)
             if c.db_repo:
                 c.db_repo_scm_instance = c.db_repo.scm_instance
                 return render('/switch_to_list.html')
         raise HTTPBadRequest()
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def repo_refs_data(self, repo_name):
         repo = Repository.get_by_repo_name(repo_name).scm_instance
         res = []
         _branches = repo.branches.items()
         if _branches:
             res.append({
                 'text': _('Branch'),
                 'children': [{'id': rev, 'text': name, 'type': 'branch'} for name, rev in _branches]
             })
         _closed_branches = repo.closed_branches.items()
         if _closed_branches:
             res.append({
                 'text': _('Closed Branches'),
                 'children': [{'id': rev, 'text': name, 'type': 'closed-branch'} for name, rev in _closed_branches]
             })
         _tags = repo.tags.items()
         if _tags:
             res.append({
                 'text': _('Tag'),
                 'children': [{'id': rev, 'text': name, 'type': 'tag'} for name, rev in _tags]
             })
         _bookmarks = repo.bookmarks.items()

kallithea/model/scm.py

➞

Show inline comments

@@ @@ -33,182 +33,96 @@ import traceback @@
 import logging
 import cStringIO
 import pkg_resources
 from os.path import join as jn
 from sqlalchemy import func
 from pylons.i18n.translation import _
 import kallithea
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea import BACKENDS
 from kallithea.lib import helpers as h
 from kallithea.lib.utils2 import safe_str, safe_unicode, get_server_url, \
     _set_extras
 from kallithea.lib.auth import HasRepoPermissionAny, HasRepoGroupPermissionAny, \
     HasUserGroupPermissionAny, HasPermissionAny, HasPermissionAll
 from kallithea.lib.utils import get_filesystem_repos, make_ui, \
     action_logger
 from kallithea.model import BaseModel
 from kallithea.model.db import Repository, Ui, CacheInvalidation, \
     UserFollowing, UserLog, User, RepoGroup, PullRequest
 from kallithea.lib.hooks import log_push_action
 from kallithea.lib.exceptions import NonRelativePathError, IMCCommitError
 log = logging.getLogger(__name__)
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 class CachedRepoList(object):
     """
     Cached repo list. Uses super-fast in-memory cache after initialization.
     """
     def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
         if not perm_set:
             perm_set = ['repository.read', 'repository.write',
                         'repository.admin']
         self.perm_set = perm_set
     def __len__(self):
         return len(self.db_repo_list)
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
     def __iter__(self):
         # pre-propagated valid_cache_keys to save executing select statements
         # for each repo
         valid_cache_keys = CacheInvalidation.get_valid_cache_keys()
         for dbr in self.db_repo_list:
             scmr = dbr.scm_instance_cached(valid_cache_keys)
             # check permission at this level
             if not HasRepoPermissionAny(
                 *self.perm_set)(dbr.repo_name, 'get repo check'):
                 continue
             try:
                 last_change = scmr.last_change
                 tip = h.get_changeset_safe(scmr, 'tip')
             except Exception:
                 log.error(
                     '%s this repository is present in database but it '
                     'cannot be created as an scm instance, org_exc:%s'
                     % (dbr.repo_name, traceback.format_exc())
+                )
                 continue
             tmp_d = {}
             tmp_d['name'] = dbr.repo_name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['raw_name'] = tmp_d['name'].lower()
             tmp_d['description'] = dbr.description
             tmp_d['description_sort'] = tmp_d['description'].lower()
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision
             tmp_d['rev'] = tip.revision
             tmp_d['contact'] = dbr.user.full_contact
             tmp_d['contact_sort'] = tmp_d['contact']
             tmp_d['owner_sort'] = tmp_d['contact']
             tmp_d['repo_archives'] = list(scmr._get_archives())
             tmp_d['last_msg'] = tip.message
             tmp_d['author'] = tip.author
             tmp_d['dbrepo'] = dbr.get_dict()
             tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
             yield tmp_d
 class SimpleCachedRepoList(CachedRepoList):
     """
     Lighter version of CachedRepoList without the scm initialisation
     """
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             if not HasRepoPermissionAny(
                 *self.perm_set)(dbr.repo_name, 'get repo check'):
                 continue
             tmp_d = {
                 'name': dbr.repo_name,
                 'dbrepo': dbr.get_dict(),
                 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
+            }
             yield tmp_d
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
     def __len__(self):
         return len(self.obj_list)
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(
                     name, self.__class__.__name__, **self.extra_kwargs):
                 continue
             yield db_obj
 class RepoList(_PermCheckIterator):
     def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['repository.read', 'repository.write', 'repository.admin']
         super(RepoList, self).__init__(obj_list=db_repo_list,
                     obj_attr='repo_name', perm_set=perm_set,
                     perm_checker=HasRepoPermissionAny,
                     extra_kwargs=extra_kwargs)
 class RepoGroupList(_PermCheckIterator):
@@ @@ -255,127 +169,108 @@ class ScmModel(BaseModel): @@
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = self.sa.query(Ui).filter(Ui.ui_key == '/').one()
         return q.ui_value
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
         :param repos_path: path to directory containing repositories
         """
         if repos_path is None:
             repos_path = self.repos_path
         log.info('scanning for repositories in %s', repos_path)
         baseui = make_ui('db')
         repos = {}
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = Repository.normalize_repo_name(name)
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos[name] = klass(safe_str(path[1]), baseui=baseui)
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos[name] = klass(path[1])
             except OSError:
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
     def get_repos(self, all_repos=None, sort_key=None, simple=False):
         """
         Get all repos from db and for each repo create its
         backend instance and fill that backed with information from database
     def get_repos(self, repos):
         """Return the repos the user has access to"""
         return RepoList(repos)
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
         :param sort_key: initial sorting of repos
         :param simple: use SimpleCachedList - one without the SCM info
     def get_repo_groups(self, groups=None):
         """Return the repo groups the user has access to
         If no groups are specified, use top level groups.
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository) \
                         .filter(Repository.group_id == None) \
                         .order_by(func.lower(Repository.repo_name)).all()
         if simple:
             repo_iter = SimpleCachedRepoList(all_repos,
                                              repos_path=self.repos_path,
                                              order_by=sort_key)
         else:
             repo_iter = CachedRepoList(all_repos,
                                        repos_path=self.repos_path,
                                        order_by=sort_key)
         return repo_iter
     def get_repo_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query() \
         if groups is None:
             groups = RepoGroup.query() \
                 .filter(RepoGroup.group_parent_id == None).all()
-        return [x for x in RepoGroupList(all_groups)]
+        return RepoGroupList(groups)
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
         :param repo_name: the repo for which caches should be marked invalid
         """
         CacheInvalidation.set_invalidate(repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         if repo is not None:
             repo.update_changeset_cache()
     def toggle_following_repo(self, follow_repo_id, user_id):
         f = self.sa.query(UserFollowing) \
             .filter(UserFollowing.follows_repo_id == follow_repo_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing) \
             .filter(UserFollowing.follows_user_id == follow_user_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:

0 comments (0 inline, 0 general)