# HG changeset patch # User Marcin Kuzminski # Date 2010-11-07 15:02:56 # Node ID 070f327436325ae37a182f1c883756b32510b10e # Parent 50d9b1afaca7a9620d00dc51992f336adaf1aec2 Moved out reposcan into hg Model. Rewrote repo scann and caching of repositories, all is in hgModel. Changed invalidate cache calls. mergeds main repo list and repo switcher list into one new based on hgModel. diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -6,7 +6,7 @@ from rhodecode.config.routing import mak from rhodecode.lib.auth import set_available_permissions, set_base_path from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config from rhodecode.model import init_model -from rhodecode.model.hg import _get_repos_cached_initial +from rhodecode.model.hg import HgModel from sqlalchemy import engine_from_config import logging import os @@ -69,7 +69,8 @@ def load_environment(global_conf, app_co #init baseui config['pylons.app_globals'].baseui = make_ui('db') - repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial)) + g = config['pylons.app_globals'] + repo2db_mapper(HgModel().repo_scan(g.paths[0][1], g.baseui, initial)) set_available_permissions(config) set_base_path(config) set_rhodecode_config(config) diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -74,7 +74,6 @@ class ReposController(BaseController): try: form_result = _form.to_python(dict(request.POST)) repo_model.create(form_result, c.rhodecode_user) - invalidate_cache('cached_repo_list') h.flash(_('created repository %s') % form_result['repo_name'], category='success') @@ -133,7 +132,7 @@ class ReposController(BaseController): try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated succesfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -182,7 +181,7 @@ class ReposController(BaseController): action_logger(self.rhodecode_user, 'admin_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception, e: diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -33,12 +33,13 @@ from rhodecode.lib.auth import LoginRequ from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \ set_rhodecode_config, get_hg_settings, get_hg_ui_settings -from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi +from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi, Repository from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \ ApplicationUiSettingsForm from rhodecode.model.hg import HgModel from rhodecode.model.user import UserModel from rhodecode.lib.celerylib import tasks, run_task +from sqlalchemy import func import formencode import logging import traceback @@ -98,9 +99,12 @@ class SettingsController(BaseController) rm_obsolete = request.POST.get('destroy', False) log.debug('Rescanning directories with destroy=%s', rm_obsolete) - initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui) + initial = HgModel().repo_scan(g.paths[0][1], g.baseui) + for repo_name in initial.keys(): + invalidate_cache('get_repo_cached_%s' % repo_name) + repo2db_mapper(initial, rm_obsolete) - invalidate_cache('cached_repo_list') + h.flash(_('Repositories successfully rescanned'), category='success') if setting_id == 'whoosh': @@ -238,12 +242,14 @@ class SettingsController(BaseController) """ GET /_admin/my_account Displays info about my account """ + # url('admin_settings_my_account') c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) - c.user_repos = [] - for repo in c.cached_repo_list.values(): - if repo.dbrepo.user.username == c.user.username: - c.user_repos.append(repo) + all_repos = self.sa.query(Repository)\ + .filter(Repository.user_id == c.user.user_id)\ + .order_by(func.lower(Repository.repo_name))\ + .all() + c.user_repos = HgModel().get_repos(all_repos) if c.user.username == 'default': h.flash(_("You can't edit this user since it's" diff --git a/rhodecode/controllers/settings.py b/rhodecode/controllers/settings.py --- a/rhodecode/controllers/settings.py +++ b/rhodecode/controllers/settings.py @@ -78,7 +78,7 @@ class SettingsController(BaseController) try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated successfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -96,7 +96,7 @@ class SettingsController(BaseController) encoding="UTF-8") except Exception: log.error(traceback.format_exc()) - h.flash(_('error occured during update of repository %s') \ + h.flash(_('error occurred during update of repository %s') \ % repo_name, category='error') return redirect(url('repo_settings_home', repo_name=changed_name)) @@ -126,7 +126,7 @@ class SettingsController(BaseController) action_logger(self.rhodecode_user, 'user_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception: h.flash(_('An error occurred during deletion of %s') % repo_name, diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -9,20 +9,20 @@ from rhodecode import __version__ from rhodecode.lib import auth from rhodecode.lib.utils import get_repo_slug from rhodecode.model import meta -from rhodecode.model.hg import _get_repos_cached, \ - _get_repos_switcher_cached +from rhodecode.model.hg import HgModel from vcs import BACKENDS + class BaseController(WSGIController): def __before__(self): c.rhodecode_version = __version__ c.rhodecode_name = config['rhodecode_title'] c.repo_name = get_repo_slug(request) - c.cached_repo_list = _get_repos_cached() - c.repo_switcher_list = _get_repos_switcher_cached(c.cached_repo_list) + c.cached_repo_list = HgModel().get_repos() c.backends = BACKENDS.keys() + if c.repo_name: - cached_repo = c.cached_repo_list.get(c.repo_name) + cached_repo = HgModel().get(c.repo_name) if cached_repo: c.repository_tags = cached_repo.tags diff --git a/rhodecode/lib/middleware/simplegit.py b/rhodecode/lib/middleware/simplegit.py --- a/rhodecode/lib/middleware/simplegit.py +++ b/rhodecode/lib/middleware/simplegit.py @@ -83,15 +83,15 @@ class SimpleGit(object): self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_git(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS GIT REQUEST #=================================================================== @@ -104,7 +104,7 @@ class SimpleGit(object): REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -206,5 +206,4 @@ class SimpleGit(object): """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name) diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -52,20 +52,20 @@ class SimpleHg(object): self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_mercurial(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS MERCURIAL REQUEST #=================================================================== username = REMOTE_USER(environ) - + if not username: self.authenticate.realm = self.config['rhodecode_realm'] result = self.authenticate(environ) @@ -74,7 +74,7 @@ class SimpleHg(object): REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -114,7 +114,7 @@ class SimpleHg(object): 'repository.admin')\ (user, repo_name): return HTTPForbidden()(environ, start_response) - + self.extras = {'ip':self.ipaddr, 'username':self.username, 'action':self.action, @@ -200,8 +200,7 @@ class SimpleHg(object): """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name) def __load_web_settings(self, hgserve, extras={}): @@ -209,12 +208,12 @@ class SimpleHg(object): hgserve.repo.ui = self.baseui hgrc = os.path.join(self.repo_path, '.hg', 'hgrc') - + #inject some additional parameters that will be available in ui #for hooks for k, v in extras.items(): hgserve.repo.ui.setconfig('rhodecode_extras', k, v) - + repoui = make_ui('file', hgrc, False) if repoui: @@ -222,7 +221,7 @@ class SimpleHg(object): for section in ui_sections: for k, v in repoui.configitems(section): hgserve.repo.ui.setconfig(section, k, v) - + return hgserve diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -275,25 +275,11 @@ def set_rhodecode_config(config): config[k] = v def invalidate_cache(name, *args): - """Invalidates given name cache""" - - from beaker.cache import region_invalidate - log.info('INVALIDATING CACHE FOR %s', name) - - """propagate our arguments to make sure invalidation works. First - argument has to be the name of cached func name give to cache decorator - without that the invalidation would not work""" - tmp = [name] - tmp.extend(args) - args = tuple(tmp) - - if name == 'cached_repo_list': - from rhodecode.model.hg import _get_repos_cached - region_invalidate(_get_repos_cached, None, *args) - - if name == 'full_changelog': - from rhodecode.model.hg import _full_changelog_cached - region_invalidate(_full_changelog_cached, None, *args) + """ + Puts cache invalidation task into db for + further global cache invalidation + """ + pass class EmptyChangeset(BaseChangeset): """ @@ -352,7 +338,6 @@ def repo2db_mapper(initial_repo_list, re } rm.create(form_data, user, just_db=True) - if remove_obsolete: #remove from database those repositories that are not in the filesystem for repo in sa.query(Repository).all(): @@ -360,10 +345,6 @@ def repo2db_mapper(initial_repo_list, re sa.delete(repo) sa.commit() - - meta.Session.remove() - - class OrderedDict(dict, DictMixin): def __init__(self, *args, **kwds): diff --git a/rhodecode/model/hg.py b/rhodecode/model/hg.py --- a/rhodecode/model/hg.py +++ b/rhodecode/model/hg.py @@ -22,57 +22,25 @@ Created on April 9, 2010 Model for RhodeCode @author: marcink """ -from beaker.cache import cache_region +from beaker.cache import cache_region, region_invalidate from mercurial import ui from rhodecode.lib import helpers as h -from rhodecode.lib.utils import invalidate_cache from rhodecode.lib.auth import HasRepoPermissionAny +from rhodecode.lib.utils import get_repos from rhodecode.model import meta -from rhodecode.model.db import Repository, User +from rhodecode.model.caching_query import FromCache +from rhodecode.model.db import Repository, User, RhodeCodeUi from sqlalchemy.orm import joinedload +from vcs import get_repo as vcs_get_repo, get_backend +from vcs.backends.hg import MercurialRepository from vcs.exceptions import RepositoryError, VCSError +from vcs.utils.lazy import LazyProperty import logging -import sys +import os import time log = logging.getLogger(__name__) -try: - from vcs.backends.hg import MercurialRepository - from vcs.backends.git import GitRepository -except ImportError: - sys.stderr.write('You have to import vcs module') - raise Exception('Unable to import vcs') - -def _get_repos_cached_initial(app_globals, initial): - """return cached dict with repos - """ - g = app_globals - return HgModel().repo_scan(g.paths[0][1], g.baseui, initial) - -@cache_region('long_term', 'cached_repo_list') -def _get_repos_cached(): - """return cached dict with repos - """ - log.info('getting all repositories list') - from pylons import app_globals as g - return HgModel().repo_scan(g.paths[0][1], g.baseui) - -@cache_region('super_short_term', 'cached_repos_switcher_list') -def _get_repos_switcher_cached(cached_repo_list): - repos_lst = [] - for repo in [x for x in cached_repo_list.values()]: - if HasRepoPermissionAny('repository.write', 'repository.read', - 'repository.admin')(repo.name, 'main page check'): - repos_lst.append((repo.name, repo.dbrepo.private,)) - - return sorted(repos_lst, key=lambda k:k[0].lower()) - -@cache_region('long_term', 'full_changelog') -def _full_changelog_cached(repo_name): - log.info('getting full changelog for %s', repo_name) - return list(reversed(list(HgModel().get_repo(repo_name)))) - class HgModel(object): """ Mercurial Model @@ -84,6 +52,16 @@ class HgModel(object): else: self.sa = sa + + @LazyProperty + def repos_path(self): + """ + Get's the repositories root path from database + """ + q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + + return q.ui_value + def repo_scan(self, repos_path, baseui, initial=False): """ Listing of repositories in given path. This path should not be a @@ -91,93 +69,100 @@ class HgModel(object): :param repos_path: path to directory containing repositories :param baseui - :param initial: initial scann + :param initial: initial scan """ log.info('scanning for repositories in %s', repos_path) if not isinstance(baseui, ui.ui): baseui = ui.ui() - - from rhodecode.lib.utils import get_repos - repos = get_repos(repos_path) - - repos_list = {} - for name, path in repos: + for name, path in get_repos(repos_path): try: - #name = name.split('/')[-1] if repos_list.has_key(name): - raise RepositoryError('Duplicate repository name %s found in' - ' %s' % (name, path)) + raise RepositoryError('Duplicate repository name %s ' + 'found in %s' % (name, path)) else: + + klass = get_backend(path[0]) + if path[0] == 'hg': - repos_list[name] = MercurialRepository(path[1], baseui=baseui) - repos_list[name].name = name + repos_list[name] = klass(path[1], baseui=baseui) if path[0] == 'git': - repos_list[name] = GitRepository(path[1]) - repos_list[name].name = name - - dbrepo = None - if not initial: - #for initial scann on application first run we don't - #have db repos yet. - dbrepo = self.sa.query(Repository)\ - .options(joinedload(Repository.fork))\ - .filter(Repository.repo_name == name)\ - .scalar() - - if dbrepo: - log.info('Adding db instance to cached list') - repos_list[name].dbrepo = dbrepo - repos_list[name].description = dbrepo.description - if dbrepo.user: - repos_list[name].contact = dbrepo.user.full_contact - else: - repos_list[name].contact = self.sa.query(User)\ - .filter(User.admin == True).first().full_contact + repos_list[name] = klass(path[1]) except OSError: continue return repos_list - def get_repos(self): - for name, repo in _get_repos_cached().items(): + def get_repos(self, all_repos=None): + """ + Get all repos from db and for each such repo make backend and + fetch dependent data from db + """ + if not all_repos: + all_repos = self.sa.query(Repository).all() - if isinstance(repo, MercurialRepository) and repo._get_hidden(): - #skip hidden web repository - continue + for r in all_repos: + + repo = self.get(r.repo_name) - last_change = repo.last_change - tip = h.get_changeset_safe(repo, 'tip') + if repo is not None: + last_change = repo.last_change + tip = h.get_changeset_safe(repo, 'tip') - tmp_d = {} - tmp_d['name'] = repo.name - tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = repo.description - tmp_d['description_sort'] = tmp_d['description'] - tmp_d['last_change'] = last_change - tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) - tmp_d['tip'] = tip.raw_id - tmp_d['tip_sort'] = tip.revision - tmp_d['rev'] = tip.revision - tmp_d['contact'] = repo.contact - tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(repo._get_archives()) - tmp_d['last_msg'] = tip.message - tmp_d['repo'] = repo - yield tmp_d + tmp_d = {} + tmp_d['name'] = repo.name + tmp_d['name_sort'] = tmp_d['name'].lower() + tmp_d['description'] = repo.dbrepo.description + tmp_d['description_sort'] = tmp_d['description'] + tmp_d['last_change'] = last_change + tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) + tmp_d['tip'] = tip.raw_id + tmp_d['tip_sort'] = tip.revision + tmp_d['rev'] = tip.revision + tmp_d['contact'] = repo.dbrepo.user.full_contact + tmp_d['contact_sort'] = tmp_d['contact'] + tmp_d['repo_archives'] = list(repo._get_archives()) + tmp_d['last_msg'] = tip.message + tmp_d['repo'] = repo + yield tmp_d def get_repo(self, repo_name): - try: - repo = _get_repos_cached()[repo_name] - return repo - except KeyError: - #i we're here and we got key errors let's try to invalidate the - #cahce and try again - invalidate_cache('cached_repo_list') - repo = _get_repos_cached()[repo_name] + return self.get(repo_name) + + def get(self, repo_name): + """ + Get's repository from given name, creates BackendInstance and + propagates it's data from database with all additional information + :param repo_name: + """ + if not HasRepoPermissionAny('repository.read', 'repository.write', + 'repository.admin')(repo_name, 'get repo check'): + return + + @cache_region('long_term', 'get_repo_cached_%s' % repo_name) + def _get_repo(repo_name): + + repo = vcs_get_repo(os.path.join(self.repos_path, repo_name), + alias=None, create=False) + + #skip hidden web repository + if isinstance(repo, MercurialRepository) and repo._get_hidden(): + return + + dbrepo = self.sa.query(Repository)\ + .options(joinedload(Repository.fork))\ + .options(joinedload(Repository.user))\ + .filter(Repository.repo_name == repo_name)\ + .scalar() + repo.dbrepo = dbrepo return repo + invalidate = False + if invalidate: + log.info('INVALIDATING CACHE FOR %s', repo_name) + region_invalidate(_get_repo, None, repo_name) + return _get_repo(repo_name) diff --git a/rhodecode/templates/admin/users/user_edit_my_account.html b/rhodecode/templates/admin/users/user_edit_my_account.html --- a/rhodecode/templates/admin/users/user_edit_my_account.html +++ b/rhodecode/templates/admin/users/user_edit_my_account.html @@ -100,32 +100,32 @@ %for repo in c.user_repos: - %if repo.dbrepo.repo_type =='hg': + %if repo['repo'].dbrepo.repo_type =='hg': ${_('Mercurial repository')} - %elif repo.dbrepo.repo_type =='git': + %elif repo['repo'].dbrepo.repo_type =='git': ${_('Git repository')} %else: %endif - %if repo.dbrepo.private: + %if repo['repo'].dbrepo.private: ${_('private')} %else: ${_('public')} %endif - ${h.link_to(repo.name, h.url('summary_home',repo_name=repo.name),class_="repo_name")} - %if repo.dbrepo.fork: - + ${h.link_to(repo['repo'].name, h.url('summary_home',repo_name=repo['repo'].name),class_="repo_name")} + %if repo['repo'].dbrepo.fork: + ${_('public')} %endif - ${("r%s:%s") % (h.get_changeset_safe(repo,'tip').revision,h.short_id(h.get_changeset_safe(repo,'tip').raw_id))} - ${_('private')} + ${("r%s:%s") % (h.get_changeset_safe(repo['repo'],'tip').revision,h.short_id(h.get_changeset_safe(repo['repo'],'tip').raw_id))} + ${_('private')} - ${h.form(url('repo_settings_delete', repo_name=repo.name),method='delete')} - ${h.submit('remove_%s' % repo.name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} + ${h.form(url('repo_settings_delete', repo_name=repo['repo'].name),method='delete')} + ${h.submit('remove_%s' % repo['repo'].name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} ${h.end_form()} diff --git a/rhodecode/templates/base/base.html b/rhodecode/templates/base/base.html --- a/rhodecode/templates/base/base.html +++ b/rhodecode/templates/base/base.html @@ -98,11 +98,12 @@ ↓