# HG changeset patch # User Marcin Kuzminski # Date 2010-11-15 02:26:19 # Node ID cb0d9ce6ac5cd11aafb9387f558730d39acf9f62 # Parent 7486da5f06280fbf21f725bc0963f681d933e71d #50 on point cache invalidation changes. Created cacheInvalidation table cleaned up sa sessions from models, since it wasn't really needed. diff --git a/rhodecode/controllers/admin/permissions.py b/rhodecode/controllers/admin/permissions.py --- a/rhodecode/controllers/admin/permissions.py +++ b/rhodecode/controllers/admin/permissions.py @@ -31,7 +31,7 @@ from rhodecode.lib import helpers as h from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator from rhodecode.lib.base import BaseController, render from rhodecode.model.forms import UserForm, DefaultPermissionsForm -from rhodecode.model.permission_model import PermissionModel +from rhodecode.model.permission import PermissionModel from rhodecode.model.user import UserModel import formencode import logging diff --git a/rhodecode/controllers/admin/repos.py b/rhodecode/controllers/admin/repos.py --- a/rhodecode/controllers/admin/repos.py +++ b/rhodecode/controllers/admin/repos.py @@ -133,7 +133,7 @@ class ReposController(BaseController): form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) invalidate_cache('get_repo_cached_%s' % repo_name) - h.flash(_('Repository %s updated succesfully' % repo_name), + h.flash(_('Repository %s updated successfully' % repo_name), category='success') changed_name = form_result['repo_name'] action_logger(self.rhodecode_user, 'admin_updated_repo', @@ -152,7 +152,7 @@ class ReposController(BaseController): except Exception: log.error(traceback.format_exc()) - h.flash(_('error occured during update of repository %s') \ + h.flash(_('error occurred during update of repository %s') \ % repo_name, category='error') return redirect(url('edit_repo', repo_name=changed_name)) diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -248,7 +248,7 @@ class SettingsController(BaseController) """ # url('admin_settings_my_account') - c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) + c.user = UserModel().get(c.rhodecode_user.user_id, cache=False) all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ .order_by(func.lower(Repository.repo_name))\ @@ -289,7 +289,7 @@ class SettingsController(BaseController) except formencode.Invalid, errors: c.user = user_model.get(c.rhodecode_user.user_id, cache=False) - c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) + c.user = UserModel().get(c.rhodecode_user.user_id, cache=False) all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ .order_by(func.lower(Repository.repo_name))\ diff --git a/rhodecode/controllers/admin/users.py b/rhodecode/controllers/admin/users.py --- a/rhodecode/controllers/admin/users.py +++ b/rhodecode/controllers/admin/users.py @@ -17,7 +17,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. -from rhodecode.lib.utils import action_logger """ Created on April 4, 2010 users controller for pylons diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py --- a/rhodecode/lib/auth.py +++ b/rhodecode/lib/auth.py @@ -143,7 +143,7 @@ def fill_perms(user): #=========================================================================== # fetch default permissions #=========================================================================== - default_user = UserModel(sa).get_by_username('default', cache=True) + default_user = UserModel().get_by_username('default', cache=True) default_perms = sa.query(RepoToPerm, Repository, Permission)\ .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\ diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -1,16 +1,18 @@ from celery.decorators import task +import os +import traceback +from time import mktime + from operator import itemgetter from pylons.i18n.translation import _ from rhodecode.lib.celerylib import run_task, locked_task from rhodecode.lib.helpers import person from rhodecode.lib.smtp_mailer import SmtpMailer from rhodecode.lib.utils import OrderedDict -from time import mktime -import os -import traceback from vcs.backends import get_repo -from rhodecode.model.scm import ScmModel +from rhodecode.model.db import RhodeCodeUi + try: import json except ImportError: @@ -44,6 +46,11 @@ def get_session(): return sa +def get_repos_path(): + sa = get_session() + q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + return q.ui_value + @task @locked_task def whoosh_index(repo_location, full_index): @@ -62,7 +69,7 @@ def get_commits_stats(repo_name, ts_min_ commits_by_day_author_aggregate = {} commits_by_day_aggregate = {} - repos_path = ScmModel().repos_path + repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) repo = get_repo(p) @@ -271,10 +278,10 @@ def create_repo_fork(form_data, cur_user from rhodecode.model.repo import RepoModel from vcs import get_backend log = create_repo_fork.get_logger() - repo_model = RepoModel(get_session()) + repo_model = RepoModel() repo_model.create(form_data, cur_user, just_db=True, fork=True) repo_name = form_data['repo_name'] - repos_path = ScmModel().repos_path + repos_path = get_repos_path() repo_path = os.path.join(repos_path, repo_name) repo_fork_path = os.path.join(repos_path, form_data['fork_name']) alias = form_data['repo_type'] @@ -291,7 +298,7 @@ def __get_codes_stats(repo_name): 's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws'] - repos_path = ScmModel().repos_path + repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) repo = get_repo(p) tip = repo.get_changeset() diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -89,7 +89,7 @@ def action_logger(user, action, repo, ip if hasattr(user, 'user_id'): user_obj = user elif isinstance(user, basestring): - user_obj = UserModel(sa).get_by_username(user, cache=False) + user_obj = UserModel().get_by_username(user, cache=False) else: raise Exception('You have to provide user object or username') @@ -97,7 +97,7 @@ def action_logger(user, action, repo, ip if repo: repo_name = repo.lstrip('/') - repository = RepoModel(sa).get(repo_name, cache=False) + repository = RepoModel().get(repo_name, cache=False) if not repository: raise Exception('You have to provide valid repository') else: @@ -293,12 +293,16 @@ def set_rhodecode_config(config): for k, v in hgsettings.items(): config[k] = v -def invalidate_cache(name, *args): +def invalidate_cache(cache_key, *args): """ Puts cache invalidation task into db for further global cache invalidation """ - pass + from rhodecode.model.scm import ScmModel + + if cache_key.startswith('get_repo_cached_'): + name = cache_key.split('get_repo_cached_')[-1] + ScmModel().mark_for_invalidation(name) class EmptyChangeset(BaseChangeset): """ @@ -340,7 +344,7 @@ def repo2db_mapper(initial_repo_list, re """ sa = meta.Session() - rm = RepoModel(sa) + rm = RepoModel() user = sa.query(User).filter(User.admin == True).first() for name, repo in initial_repo_list.items(): diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -90,9 +90,8 @@ class Repository(Base): repo_to_perm = relation('RepoToPerm', cascade='all') stats = relation('Statistics', cascade='all') - def __repr__(self): - return "" % (self.repo_id, self.repo_name) + return "" % (self.repo_id, self.repo_name) class Permission(Base): __tablename__ = 'permissions' @@ -140,10 +139,17 @@ class Statistics(Base): class CacheInvalidation(Base): __tablename__ = 'cache_invalidation' - __table_args__ = {'useexisting':True} + __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True}) cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True) cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) - cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None) + cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False) + def __init__(self, cache_key, cache_args=''): + self.cache_key = cache_key + self.cache_args = cache_args + self.cache_active = False + + def __repr__(self): + return "" % (self.cache_id, self.cache_key) diff --git a/rhodecode/model/permission_model.py b/rhodecode/model/permission.py rename from rhodecode/model/permission_model.py rename to rhodecode/model/permission.py --- a/rhodecode/model/permission_model.py +++ b/rhodecode/model/permission.py @@ -33,11 +33,8 @@ log = logging.getLogger(__name__) class PermissionModel(object): - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa + def __init__(self): + self.sa = Session() def get_permission(self, permission_id, cache=False): perm = self.sa.query(Permission) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -36,11 +36,8 @@ log = logging.getLogger(__name__) class RepoModel(object): - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa + def __init__(self): + self.sa = Session() def get(self, repo_id, cache=False): repo = self.sa.query(Repository)\ @@ -67,7 +64,7 @@ class RepoModel(object): #update permissions for username, perm in form_data['perms_updates']: r2p = self.sa.query(RepoToPerm)\ - .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\ + .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\ .filter(RepoToPerm.repository == self.get(repo_name))\ .one() @@ -80,7 +77,7 @@ class RepoModel(object): for username, perm in form_data['perms_new']: r2p = RepoToPerm() r2p.repository = self.get(repo_name) - r2p.user = UserModel(self.sa).get_by_username(username, cache=False) + r2p.user = UserModel().get_by_username(username, cache=False) r2p.permission_id = self.sa.query(Permission).filter( Permission.permission_name == perm)\ @@ -134,7 +131,7 @@ class RepoModel(object): #create default permission repo_to_perm = RepoToPerm() default = 'repository.read' - for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms: + for p in UserModel().get_by_username('default', cache=False).user_perms: if p.permission.permission_name.startswith('repository.'): default = p.permission.permission_name break @@ -146,7 +143,7 @@ class RepoModel(object): .one().permission_id repo_to_perm.repository_id = new_repo.repo_id - repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id + repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id self.sa.add(repo_to_perm) self.sa.commit() diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -28,12 +28,15 @@ from rhodecode.lib import helpers as h from rhodecode.lib.auth import HasRepoPermissionAny from rhodecode.lib.utils import get_repos from rhodecode.model import meta -from rhodecode.model.db import Repository, User, RhodeCodeUi +from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation +from rhodecode.model.caching_query import FromCache from sqlalchemy.orm import joinedload +from sqlalchemy.orm.session import make_transient from vcs import get_backend from vcs.utils.helpers import get_scm from vcs.exceptions import RepositoryError, VCSError from vcs.utils.lazy import LazyProperty +import traceback import logging import os import time @@ -45,12 +48,8 @@ class ScmModel(object): Mercurial Model """ - def __init__(self, sa=None): - if not sa: - self.sa = meta.Session() - else: - self.sa = sa - + def __init__(self): + self.sa = meta.Session() @LazyProperty def repos_path(self): @@ -143,7 +142,7 @@ class ScmModel(object): 'repository.admin')(repo_name, 'get repo check'): return - @cache_region('long_term', 'get_repo_cached_%s' % repo_name) + @cache_region('long_term') def _get_repo(repo_name): repo_path = os.path.join(self.repos_path, repo_name) @@ -165,13 +164,76 @@ class ScmModel(object): .options(joinedload(Repository.user))\ .filter(Repository.repo_name == repo_name)\ .scalar() + make_transient(dbrepo) repo.dbrepo = dbrepo return repo - invalidate = False + invalidate = self._should_invalidate(repo_name) if invalidate: - log.info('INVALIDATING CACHE FOR %s', repo_name) + log.info('invalidating cache for repository %s', repo_name) region_invalidate(_get_repo, None, repo_name) + self._mark_invalidated(invalidate) return _get_repo(repo_name) + + + def mark_for_invalidation(self, repo_name): + """ + Puts cache invalidation task into db for + further global cache invalidation + + :param repo_name: this repo that should invalidation take place + """ + log.debug('marking %s for invalidation', repo_name) + cache = self.sa.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == repo_name).scalar() + + if cache: + #mark this cache as inactive + cache.cache_active = False + else: + log.debug('cache key not found in invalidation db -> creating one') + cache = CacheInvalidation(repo_name) + + try: + self.sa.add(cache) + self.sa.commit() + except: + log.error(traceback.format_exc()) + self.sa.rollback() + + + + + + def _should_invalidate(self, repo_name): + """ + Looks up database for invalidation signals for this repo_name + :param repo_name: + """ + + ret = self.sa.query(CacheInvalidation)\ + .options(FromCache('sql_cache_short', + 'get_invalidation_%s' % repo_name))\ + .filter(CacheInvalidation.cache_key == repo_name)\ + .filter(CacheInvalidation.cache_active == False)\ + .scalar() + + return ret + + def _mark_invalidated(self, cache_key): + """ + Marks all occurences of cache to invaldation as already invalidated + @param repo_name: + """ + if cache_key: + log.debug('marking %s as already invalidated', cache_key) + try: + cache_key.cache_active = True + self.sa.add(cache_key) + self.sa.commit() + except: + log.error(traceback.format_exc()) + self.sa.rollback() + diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -36,11 +36,8 @@ class DefaultUserException(Exception):pa class UserModel(object): - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa + def __init__(self): + self.sa = Session() def get(self, user_id, cache=False): user = self.sa.query(User)