kallithea Changeset - 070f32743632

Changeset - 070f32743632

Parent rev.

Child rev.

[Not reviewed]

beta

0 11 0

Marcin Kuzminski - 15 years ago 2010-11-07 15:02:56
marcin@python-works.com

Moved out reposcan into hg Model.
Rewrote repo scann and caching of repositories, all is in hgModel.
Changed invalidate cache calls.
mergeds main repo list and repo switcher list into one new based on hgModel.

11 files changed with 122 insertions and 151 deletions:

rhodecode/config/environment.py

rhodecode/controllers/admin/repos.py

rhodecode/controllers/admin/settings.py

rhodecode/controllers/settings.py

rhodecode/lib/base.py

rhodecode/lib/middleware/simplegit.py

rhodecode/lib/middleware/simplehg.py

rhodecode/lib/utils.py

rhodecode/model/hg.py

rhodecode/templates/admin/users/user_edit_my_account.html

rhodecode/templates/base/base.html

0 comments (0 inline, 0 general)

rhodecode/config/environment.py

➞

Show inline comments

@@ @@ -6,7 +6,7 @@ from rhodecode.config.routing import mak @@
 from rhodecode.lib.auth import set_available_permissions, set_base_path
 from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config
 from rhodecode.model import init_model
-from rhodecode.model.hg import _get_repos_cached_initial
+from rhodecode.model.hg import HgModel
 from sqlalchemy import engine_from_config
 import logging
 import os
@@ @@ -69,7 +69,8 @@ def load_environment(global_conf, app_co @@
     #init baseui
     config['pylons.app_globals'].baseui = make_ui('db')
     repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial))
     g = config['pylons.app_globals']
     repo2db_mapper(HgModel().repo_scan(g.paths[0][1], g.baseui, initial))
     set_available_permissions(config)
     set_base_path(config)
     set_rhodecode_config(config)

rhodecode/controllers/admin/repos.py

➞

Show inline comments

@@ @@ -74,7 +74,6 @@ class ReposController(BaseController): @@
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.create(form_result, c.rhodecode_user)
             invalidate_cache('cached_repo_list')
             h.flash(_('created repository %s') % form_result['repo_name'],
                     category='success')
@@ @@ -133,7 +132,7 @@ class ReposController(BaseController): @@
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('Repository %s updated succesfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']
@@ @@ -182,7 +181,7 @@ class ReposController(BaseController): @@
             action_logger(self.rhodecode_user, 'admin_deleted_repo',
                               repo_name, '', self.sa)
             repo_model.delete(repo)
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
         except Exception, e:

rhodecode/controllers/admin/settings.py

➞

Show inline comments

@@ @@ -33,12 +33,13 @@ from rhodecode.lib.auth import LoginRequ @@
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \
     set_rhodecode_config, get_hg_settings, get_hg_ui_settings
 from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi
+from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi, Repository
 from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \
     ApplicationUiSettingsForm
 from rhodecode.model.hg import HgModel
 from rhodecode.model.user import UserModel
 from rhodecode.lib.celerylib import tasks, run_task
 from sqlalchemy import func
 import formencode
 import logging
 import traceback
@@ @@ -98,9 +99,12 @@ class SettingsController(BaseController) @@
             rm_obsolete = request.POST.get('destroy', False)
             log.debug('Rescanning directories with destroy=%s', rm_obsolete)
             initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
             initial = HgModel().repo_scan(g.paths[0][1], g.baseui)
             for repo_name in initial.keys():
                 invalidate_cache('get_repo_cached_%s' % repo_name)
             repo2db_mapper(initial, rm_obsolete)
-            invalidate_cache('cached_repo_list')
             h.flash(_('Repositories successfully rescanned'), category='success')
         if setting_id == 'whoosh':
@@ @@ -238,12 +242,14 @@ class SettingsController(BaseController) @@
         """
         GET /_admin/my_account Displays info about my account
         """
         # url('admin_settings_my_account')
         c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
         c.user_repos = []
         for repo in c.cached_repo_list.values():
             if repo.dbrepo.user.username == c.user.username:
                 c.user_repos.append(repo)
         all_repos = self.sa.query(Repository)\
             .filter(Repository.user_id == c.user.user_id)\
             .order_by(func.lower(Repository.repo_name))\
             .all()
         c.user_repos = HgModel().get_repos(all_repos)
         if c.user.username == 'default':
             h.flash(_("You can't edit this user since it's"

rhodecode/controllers/settings.py

➞

Show inline comments

@@ @@ -78,7 +78,7 @@ class SettingsController(BaseController) @@
         try:
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']
@@ @@ -96,7 +96,7 @@ class SettingsController(BaseController) @@
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of repository %s') \
+            h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
         return redirect(url('repo_settings_home', repo_name=changed_name))
@@ @@ -126,7 +126,7 @@ class SettingsController(BaseController) @@
             action_logger(self.rhodecode_user, 'user_deleted_repo',
                               repo_name, '', self.sa)
             repo_model.delete(repo)
-            invalidate_cache('cached_repo_list')
+            invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
         except Exception:
             h.flash(_('An error occurred during deletion of %s') % repo_name,

rhodecode/lib/base.py

➞

Show inline comments

@@ @@ -9,20 +9,20 @@ from rhodecode import __version__ @@
 from rhodecode.lib import auth
 from rhodecode.lib.utils import get_repo_slug
 from rhodecode.model import meta
 from rhodecode.model.hg import _get_repos_cached, \
     _get_repos_switcher_cached
 from rhodecode.model.hg import HgModel
 from vcs import BACKENDS
 class BaseController(WSGIController):
     def __before__(self):
         c.rhodecode_version = __version__
         c.rhodecode_name = config['rhodecode_title']
         c.repo_name = get_repo_slug(request)
         c.cached_repo_list = _get_repos_cached()
         c.repo_switcher_list = _get_repos_switcher_cached(c.cached_repo_list)
         c.cached_repo_list = HgModel().get_repos()
         c.backends = BACKENDS.keys()
         if c.repo_name:
-            cached_repo = c.cached_repo_list.get(c.repo_name)
+            cached_repo = HgModel().get(c.repo_name)
             if cached_repo:
                 c.repository_tags = cached_repo.tags

rhodecode/lib/middleware/simplegit.py

➞

Show inline comments

@@ @@ -206,5 +206,4 @@ class SimpleGit(object): @@
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('cached_repo_list')
         invalidate_cache('full_changelog', repo_name)
         invalidate_cache('get_repo_cached_%s' % repo_name)

rhodecode/lib/middleware/simplehg.py

➞

Show inline comments

@@ @@ -200,8 +200,7 @@ class SimpleHg(object): @@
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('cached_repo_list')
         invalidate_cache('full_changelog', repo_name)
         invalidate_cache('get_repo_cached_%s' % repo_name)
     def __load_web_settings(self, hgserve, extras={}):

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -275,25 +275,11 @@ def set_rhodecode_config(config): @@
         config[k] = v
 def invalidate_cache(name, *args):
     """Invalidates given name cache"""
     from beaker.cache import region_invalidate
     log.info('INVALIDATING CACHE FOR %s', name)
     """propagate our arguments to make sure invalidation works. First
     argument has to be the name of cached func name give to cache decorator
     without that the invalidation would not work"""
     tmp = [name]
     tmp.extend(args)
     args = tuple(tmp)
     if name == 'cached_repo_list':
         from rhodecode.model.hg import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
     if name == 'full_changelog':
         from rhodecode.model.hg import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
     """
     Puts cache invalidation task into db for
     further global cache invalidation
     """
     pass
 class EmptyChangeset(BaseChangeset):
     """
@@ @@ -352,7 +338,6 @@ def repo2db_mapper(initial_repo_list, re @@
+                         }
             rm.create(form_data, user, just_db=True)
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
@@ @@ -360,10 +345,6 @@ def repo2db_mapper(initial_repo_list, re @@
                 sa.delete(repo)
                 sa.commit()
     meta.Session.remove()
 class OrderedDict(dict, DictMixin):
     def __init__(self, *args, **kwds):

rhodecode/model/hg.py

➞

Show inline comments

@@ @@ -22,57 +22,25 @@ Created on April 9, 2010 @@
 Model for RhodeCode
 @author: marcink
 """
 from beaker.cache import cache_region
+from beaker.cache import cache_region, region_invalidate
 from mercurial import ui
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import invalidate_cache
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import Repository, User, RhodeCodeUi
 from sqlalchemy.orm import joinedload
 from vcs import get_repo as vcs_get_repo, get_backend
 from vcs.backends.hg import MercurialRepository
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
 import logging
-import sys
+import os
 import time
 log = logging.getLogger(__name__)
 try:
     from vcs.backends.hg import MercurialRepository
     from vcs.backends.git import GitRepository
 except ImportError:
     sys.stderr.write('You have to import vcs module')
     raise Exception('Unable to import vcs')
 def _get_repos_cached_initial(app_globals, initial):
     """return cached dict with repos
     """
     g = app_globals
     return HgModel().repo_scan(g.paths[0][1], g.baseui, initial)
 @cache_region('long_term', 'cached_repo_list')
 def _get_repos_cached():
     """return cached dict with repos
     """
     log.info('getting all repositories list')
     from pylons import app_globals as g
     return HgModel().repo_scan(g.paths[0][1], g.baseui)
 @cache_region('super_short_term', 'cached_repos_switcher_list')
 def _get_repos_switcher_cached(cached_repo_list):
     repos_lst = []
     for repo in [x for x in cached_repo_list.values()]:
         if HasRepoPermissionAny('repository.write', 'repository.read',
                     'repository.admin')(repo.name, 'main page check'):
             repos_lst.append((repo.name, repo.dbrepo.private,))
     return sorted(repos_lst, key=lambda k:k[0].lower())
 @cache_region('long_term', 'full_changelog')
 def _full_changelog_cached(repo_name):
     log.info('getting full changelog for %s', repo_name)
     return list(reversed(list(HgModel().get_repo(repo_name))))
 class HgModel(object):
     """
     Mercurial Model
@@ @@ -84,6 +52,16 @@ class HgModel(object): @@
         else:
             self.sa = sa
     @LazyProperty
     def repos_path(self):
         """
         Get's the repositories root path from database
         """
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
         return q.ui_value
     def repo_scan(self, repos_path, baseui, initial=False):
         """
         Listing of repositories in given path. This path should not be a
@@ @@ -91,77 +69,59 @@ class HgModel(object): @@
         :param repos_path: path to directory containing repositories
         :param baseui
-        :param initial: initial scann
+        :param initial: initial scan
         """
         log.info('scanning for repositories in %s', repos_path)
         if not isinstance(baseui, ui.ui):
             baseui = ui.ui()
         from rhodecode.lib.utils import get_repos
         repos = get_repos(repos_path)
         repos_list = {}
-        for name, path in repos:
+        for name, path in get_repos(repos_path):
             try:
                 #name = name.split('/')[-1]
                 if repos_list.has_key(name):
                     raise RepositoryError('Duplicate repository name %s found in'
                                     ' %s' % (name, path))
                     raise RepositoryError('Duplicate repository name %s '
                                     'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg':
                         repos_list[name] = MercurialRepository(path[1], baseui=baseui)
                         repos_list[name].name = name
                         repos_list[name] = klass(path[1], baseui=baseui)
                     if path[0] == 'git':
                         repos_list[name] = GitRepository(path[1])
                         repos_list[name].name = name
                     dbrepo = None
                     if not initial:
                         #for initial scann on application first run we don't
                         #have db repos yet.
                         dbrepo = self.sa.query(Repository)\
                             .options(joinedload(Repository.fork))\
                             .filter(Repository.repo_name == name)\
                             .scalar()
                     if dbrepo:
                         log.info('Adding db instance to cached list')
                         repos_list[name].dbrepo = dbrepo
                         repos_list[name].description = dbrepo.description
                         if dbrepo.user:
                             repos_list[name].contact = dbrepo.user.full_contact
                         else:
                             repos_list[name].contact = self.sa.query(User)\
                             .filter(User.admin == True).first().full_contact
                         repos_list[name] = klass(path[1])
             except OSError:
                 continue
         return repos_list
     def get_repos(self):
         for name, repo in _get_repos_cached().items():
     def get_repos(self, all_repos=None):
         """
         Get all repos from db and for each such repo make backend and
         fetch dependent data from db
         """
         if not all_repos:
             all_repos = self.sa.query(Repository).all()
             if isinstance(repo, MercurialRepository) and repo._get_hidden():
                 #skip hidden web repository
                 continue
         for r in all_repos:
             repo = self.get(r.repo_name)
             if repo is not None:
             last_change = repo.last_change
             tip = h.get_changeset_safe(repo, 'tip')
             tmp_d = {}
             tmp_d['name'] = repo.name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['description'] = repo.description
+                tmp_d['description'] = repo.dbrepo.description
             tmp_d['description_sort'] = tmp_d['description']
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision
             tmp_d['rev'] = tip.revision
-            tmp_d['contact'] = repo.contact
+                tmp_d['contact'] = repo.dbrepo.user.full_contact
             tmp_d['contact_sort'] = tmp_d['contact']
             tmp_d['repo_archives'] = list(repo._get_archives())
             tmp_d['last_msg'] = tip.message
@@ @@ -169,15 +129,40 @@ class HgModel(object): @@
             yield tmp_d
     def get_repo(self, repo_name):
         try:
             repo = _get_repos_cached()[repo_name]
             return repo
         except KeyError:
             #i we're here and we got key errors let's try to invalidate the
             #cahce and try again
             invalidate_cache('cached_repo_list')
             repo = _get_repos_cached()[repo_name]
         return self.get(repo_name)
     def get(self, repo_name):
         """
         Get's repository from given name, creates BackendInstance and
         propagates it's data from database with all additional information
         :param repo_name:
         """
         if not HasRepoPermissionAny('repository.read', 'repository.write',
                             'repository.admin')(repo_name, 'get repo check'):
             return
         @cache_region('long_term', 'get_repo_cached_%s' % repo_name)
         def _get_repo(repo_name):
             repo = vcs_get_repo(os.path.join(self.repos_path, repo_name),
                                 alias=None, create=False)
             #skip hidden web repository
             if isinstance(repo, MercurialRepository) and repo._get_hidden():
                 return
             dbrepo = self.sa.query(Repository)\
                 .options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .filter(Repository.repo_name == repo_name)\
                 .scalar()
             repo.dbrepo = dbrepo
             return repo
         invalidate = False
         if invalidate:
             log.info('INVALIDATING CACHE FOR %s', repo_name)
             region_invalidate(_get_repo, None, repo_name)
         return _get_repo(repo_name)

rhodecode/templates/admin/users/user_edit_my_account.html

➞

Show inline comments

@@ @@ -100,32 +100,32 @@ @@
 		     %for repo in c.user_repos:
 		        <tr>
 		            <td>
                      %if repo.dbrepo.repo_type =='hg':
+                     %if repo['repo'].dbrepo.repo_type =='hg':
                        <img class="icon" title="${_('Mercurial repository')}" alt="${_('Mercurial repository')}" src="/images/icons/hgicon.png"/>
                      %elif repo.dbrepo.repo_type =='git':
+                     %elif repo['repo'].dbrepo.repo_type =='git':
                        <img class="icon" title="${_('Git repository')}" alt="${_('Git repository')}" src="/images/icons/giticon.png"/>
                      %else:
                      %endif
 		             %if repo.dbrepo.private:
+		             %if repo['repo'].dbrepo.private:
 		                <img class="icon" alt="${_('private')}" src="/images/icons/lock.png"/>
 		             %else:
 		                <img class="icon" alt="${_('public')}" src="/images/icons/lock_open.png"/>
 		             %endif
 		            ${h.link_to(repo.name, h.url('summary_home',repo_name=repo.name),class_="repo_name")}
 		            %if repo.dbrepo.fork:
 		            	<a href="${h.url('summary_home',repo_name=repo.dbrepo.fork.repo_name)}">
 		            ${h.link_to(repo['repo'].name, h.url('summary_home',repo_name=repo['repo'].name),class_="repo_name")}
 		            %if repo['repo'].dbrepo.fork:
 		            	<a href="${h.url('summary_home',repo_name=repo['repo'].dbrepo.fork.repo_name)}">
 		            	<img class="icon" alt="${_('public')}"
 		            	title="${_('Fork of')} ${repo.dbrepo.fork.repo_name}"
+		            	title="${_('Fork of')} ${repo['repo'].dbrepo.fork.repo_name}"
 		            	src="/images/icons/arrow_divide.png"/></a>
 		            %endif
 		            </td>
 		            <td><span class="tooltip" tooltip_title="${repo.last_change}">${("r%s:%s") % (h.get_changeset_safe(repo,'tip').revision,h.short_id(h.get_changeset_safe(repo,'tip').raw_id))}</span></td>
 		            <td><a href="${h.url('repo_settings_home',repo_name=repo.name)}" title="${_('edit')}"><img class="icon" alt="${_('private')}" src="/images/icons/application_form_edit.png"/></a></td>
 		            <td><span class="tooltip" tooltip_title="${repo['repo'].last_change}">${("r%s:%s") % (h.get_changeset_safe(repo['repo'],'tip').revision,h.short_id(h.get_changeset_safe(repo['repo'],'tip').raw_id))}</span></td>
 		            <td><a href="${h.url('repo_settings_home',repo_name=repo['repo'].name)}" title="${_('edit')}"><img class="icon" alt="${_('private')}" src="/images/icons/application_form_edit.png"/></a></td>
 		            <td>
 	                  ${h.form(url('repo_settings_delete', repo_name=repo.name),method='delete')}
 	                    ${h.submit('remove_%s' % repo.name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")}
 	                  ${h.form(url('repo_settings_delete', repo_name=repo['repo'].name),method='delete')}
 	                    ${h.submit('remove_%s' % repo['repo'].name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")}
 	                  ${h.end_form()}
 		            </td>
 		        </tr>

rhodecode/templates/base/base.html

➞

Show inline comments

@@ @@ -98,11 +98,12 @@ @@
                     <span>&darr;</span>
 					</a>
 					<ul class="repo_switcher">
                         %for repo,private in c.repo_switcher_list:
                           %if private:
                              <li>${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="private_repo")}</li>
                         %for repo in c.cached_repo_list:
                           %if repo['repo'].dbrepo.private:
                              <li>${h.link_to(repo['repo'].name,h.url('summary_home',repo_name=repo['repo'].name),class_="private_repo %s" % repo['repo'].dbrepo.repo_type)}</li>
                           %else:
                              <li>${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="public_repo")}</li>
+                             <li>${h.link_to(repo['repo'].name,h.url('summary_home',repo_name=repo['repo'].name),class_="public_repo %s" % repo['repo'].dbrepo.repo_type)}</li>
                           %endif
                         %endfor
 					</ul>

0 comments (0 inline, 0 general)