kallithea Changeset - cb0d9ce6ac5c

Changeset - cb0d9ce6ac5c

Parent rev.

Child rev.

[Not reviewed]

beta

1 11 1

Marcin Kuzminski - 15 years ago 2010-11-15 02:26:19
marcin@python-works.com

#50 on point cache invalidation changes.
Created cacheInvalidation table
cleaned up sa sessions from models, since it wasn't really needed.

12 files changed with 122 insertions and 53 deletions:

rhodecode/controllers/admin/permissions.py

rhodecode/controllers/admin/repos.py

rhodecode/controllers/admin/settings.py

rhodecode/controllers/admin/users.py

rhodecode/lib/auth.py

rhodecode/lib/celerylib/tasks.py

rhodecode/lib/utils.py

rhodecode/model/db.py

rhodecode/model/permission.py

rhodecode/model/repo.py

rhodecode/model/scm.py

rhodecode/model/user.py

0 comments (0 inline, 0 general)

rhodecode/controllers/admin/permissions.py

➞

Show inline comments

@@ @@ -31,7 +31,7 @@ from rhodecode.lib import helpers as h @@
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.model.forms import UserForm, DefaultPermissionsForm
-from rhodecode.model.permission_model import PermissionModel
+from rhodecode.model.permission import PermissionModel
 from rhodecode.model.user import UserModel
 import formencode
 import logging

rhodecode/controllers/admin/repos.py

➞

Show inline comments

@@ @@ -133,7 +133,7 @@ class ReposController(BaseController): @@
             form_result = _form.to_python(dict(request.POST))
             repo_model.update(repo_name, form_result)
             invalidate_cache('get_repo_cached_%s' % repo_name)
-            h.flash(_('Repository %s updated succesfully' % repo_name),
+            h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = form_result['repo_name']
             action_logger(self.rhodecode_user, 'admin_updated_repo',
@@ @@ -152,7 +152,7 @@ class ReposController(BaseController): @@
         except Exception:
             log.error(traceback.format_exc())
-            h.flash(_('error occured during update of repository %s') \
+            h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
         return redirect(url('edit_repo', repo_name=changed_name))

rhodecode/controllers/admin/settings.py

➞

Show inline comments

@@ @@ -248,7 +248,7 @@ class SettingsController(BaseController) @@
         """
         # url('admin_settings_my_account')
-        c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
         c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
         all_repos = self.sa.query(Repository)\
             .filter(Repository.user_id == c.user.user_id)\
             .order_by(func.lower(Repository.repo_name))\
@@ @@ -289,7 +289,7 @@ class SettingsController(BaseController) @@
         except formencode.Invalid, errors:
             c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
-            c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
             c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.user_id == c.user.user_id)\
                 .order_by(func.lower(Repository.repo_name))\

rhodecode/controllers/admin/users.py

➞

Show inline comments

@@ @@ -17,7 +17,6 @@ @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 from rhodecode.lib.utils import action_logger
 """
 Created on April 4, 2010
 users controller for pylons

rhodecode/lib/auth.py

➞

Show inline comments

@@ @@ -143,7 +143,7 @@ def fill_perms(user): @@
     #===========================================================================
     # fetch default permissions
     #===========================================================================
-    default_user = UserModel(sa).get_by_username('default', cache=True)
     default_user = UserModel().get_by_username('default', cache=True)
     default_perms = sa.query(RepoToPerm, Repository, Permission)\
         .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\

rhodecode/lib/celerylib/tasks.py

➞

Show inline comments

 from celery.decorators import task
 import os
 import traceback
 from time import mktime
 from operator import itemgetter
 from pylons.i18n.translation import _
 from rhodecode.lib.celerylib import run_task, locked_task
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict
 from time import mktime
 import os
 import traceback
 from vcs.backends import get_repo
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import RhodeCodeUi
 try:
     import json
 except ImportError:
@@ @@ -44,6 +46,11 @@ def get_session(): @@
     return sa
 def get_repos_path():
     sa = get_session()
     q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
     return q.ui_value
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
@@ @@ -62,7 +69,7 @@ def get_commits_stats(repo_name, ts_min_ @@
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
@@ @@ -271,10 +278,10 @@ def create_repo_fork(form_data, cur_user @@
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
     log = create_repo_fork.get_logger()
-    repo_model = RepoModel(get_session())
     repo_model = RepoModel()
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
@@ @@ -291,7 +298,7 @@ def __get_codes_stats(repo_name): @@
     's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
-    repos_path = ScmModel().repos_path
+    repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     tip = repo.get_changeset()

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -89,7 +89,7 @@ def action_logger(user, action, repo, ip @@
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
-            user_obj = UserModel(sa).get_by_username(user, cache=False)
             user_obj = UserModel().get_by_username(user, cache=False)
         else:
             raise Exception('You have to provide user object or username')
@@ @@ -97,7 +97,7 @@ def action_logger(user, action, repo, ip @@
         if repo:
             repo_name = repo.lstrip('/')
-            repository = RepoModel(sa).get(repo_name, cache=False)
             repository = RepoModel().get(repo_name, cache=False)
             if not repository:
                 raise Exception('You have to provide valid repository')
         else:
@@ @@ -293,12 +293,16 @@ def set_rhodecode_config(config): @@
     for k, v in hgsettings.items():
         config[k] = v
-def invalidate_cache(name, *args):
+def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for
     further global cache invalidation
     """
     pass
     from rhodecode.model.scm import ScmModel
     if cache_key.startswith('get_repo_cached_'):
         name = cache_key.split('get_repo_cached_')[-1]
         ScmModel().mark_for_invalidation(name)
 class EmptyChangeset(BaseChangeset):
     """
@@ @@ -340,7 +344,7 @@ def repo2db_mapper(initial_repo_list, re @@
     """
     sa = meta.Session()
-    rm = RepoModel(sa)
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
     for name, repo in initial_repo_list.items():

rhodecode/model/db.py

➞

Show inline comments

@@ @@ -90,9 +90,8 @@ class Repository(Base): @@
     repo_to_perm = relation('RepoToPerm', cascade='all')
     stats = relation('Statistics', cascade='all')
     def __repr__(self):
-        return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
         return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 class Permission(Base):
     __tablename__ = 'permissions'
@@ @@ -140,10 +139,17 @@ class Statistics(Base): @@
 class CacheInvalidation(Base):
     __tablename__ = 'cache_invalidation'
     __table_args__ = {'useexisting':True}
+    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
     cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
-    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None)
+    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False)
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
     def __repr__(self):
         return "<CacheInvaidation('%s:%s')>" % (self.cache_id, self.cache_key)

rhodecode/model/permission.py

➞

Show inline comments

@@ @@ -33,11 +33,8 @@ log = logging.getLogger(__name__) @@
 class PermissionModel(object):
     def __init__(self, sa=None):
         if not sa:
             self.sa = Session()
         else:
             self.sa = sa
     def __init__(self):
         self.sa = Session()
     def get_permission(self, permission_id, cache=False):
         perm = self.sa.query(Permission)

rhodecode/model/repo.py

➞

Show inline comments

@@ @@ -36,11 +36,8 @@ log = logging.getLogger(__name__) @@
 class RepoModel(object):
     def __init__(self, sa=None):
         if not sa:
             self.sa = Session()
         else:
             self.sa = sa
     def __init__(self):
         self.sa = Session()
     def get(self, repo_id, cache=False):
         repo = self.sa.query(Repository)\
@@ @@ -67,7 +64,7 @@ class RepoModel(object): @@
             #update permissions
             for username, perm in form_data['perms_updates']:
                 r2p = self.sa.query(RepoToPerm)\
-                        .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\
                         .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\
                         .filter(RepoToPerm.repository == self.get(repo_name))\
                         .one()
@@ @@ -80,7 +77,7 @@ class RepoModel(object): @@
             for username, perm in form_data['perms_new']:
                 r2p = RepoToPerm()
                 r2p.repository = self.get(repo_name)
-                r2p.user = UserModel(self.sa).get_by_username(username, cache=False)
                 r2p.user = UserModel().get_by_username(username, cache=False)
                 r2p.permission_id = self.sa.query(Permission).filter(
                                         Permission.permission_name == perm)\
@@ @@ -134,7 +131,7 @@ class RepoModel(object): @@
             #create default permission
             repo_to_perm = RepoToPerm()
             default = 'repository.read'
-            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
             for p in UserModel().get_by_username('default', cache=False).user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     default = p.permission.permission_name
                     break
@@ @@ -146,7 +143,7 @@ class RepoModel(object): @@
                     .one().permission_id
             repo_to_perm.repository_id = new_repo.repo_id
-            repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id
             repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id
             self.sa.add(repo_to_perm)
             self.sa.commit()

rhodecode/model/scm.py

➞

Show inline comments

@@ @@ -28,12 +28,15 @@ from rhodecode.lib import helpers as h @@
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User, RhodeCodeUi
 from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation
 from rhodecode.model.caching_query import FromCache
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.session import make_transient
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
 import traceback
 import logging
 import os
 import time
@@ @@ -45,12 +48,8 @@ class ScmModel(object): @@
     Mercurial Model
     """
     def __init__(self, sa=None):
         if not sa:
             self.sa = meta.Session()
         else:
             self.sa = sa
     def __init__(self):
         self.sa = meta.Session()
     @LazyProperty
     def repos_path(self):
@@ @@ -143,7 +142,7 @@ class ScmModel(object): @@
                             'repository.admin')(repo_name, 'get repo check'):
             return
-        @cache_region('long_term', 'get_repo_cached_%s' % repo_name)
         @cache_region('long_term')
         def _get_repo(repo_name):
             repo_path = os.path.join(self.repos_path, repo_name)
@@ @@ -165,13 +164,76 @@ class ScmModel(object): @@
                 .options(joinedload(Repository.user))\
                 .filter(Repository.repo_name == repo_name)\
                 .scalar()
             make_transient(dbrepo)
             repo.dbrepo = dbrepo
             return repo
-        invalidate = False
+        invalidate = self._should_invalidate(repo_name)
         if invalidate:
-            log.info('INVALIDATING CACHE FOR %s', repo_name)
+            log.info('invalidating cache for repository %s', repo_name)
             region_invalidate(_get_repo, None, repo_name)
             self._mark_invalidated(invalidate)
         return _get_repo(repo_name)
     def mark_for_invalidation(self, repo_name):
         """
         Puts cache invalidation task into db for
         further global cache invalidation
         :param repo_name: this repo that should invalidation take place
         """
         log.debug('marking %s for invalidation', repo_name)
         cache = self.sa.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == repo_name).scalar()
         if cache:
             #mark this cache as inactive
             cache.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             cache = CacheInvalidation(repo_name)
         try:
             self.sa.add(cache)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
     def _should_invalidate(self, repo_name):
         """
         Looks up database for invalidation signals for this repo_name
         :param repo_name:
         """
         ret = self.sa.query(CacheInvalidation)\
             .options(FromCache('sql_cache_short',
                            'get_invalidation_%s' % repo_name))\
             .filter(CacheInvalidation.cache_key == repo_name)\
             .filter(CacheInvalidation.cache_active == False)\
             .scalar()
         return ret
     def _mark_invalidated(self, cache_key):
         """
         Marks all occurences of cache to invaldation as already invalidated
         @param repo_name:
         """
         if cache_key:
             log.debug('marking %s as already invalidated', cache_key)
         try:
             cache_key.cache_active = True
             self.sa.add(cache_key)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()

rhodecode/model/user.py

➞

Show inline comments

@@ @@ -36,11 +36,8 @@ class DefaultUserException(Exception):pa @@
 class UserModel(object):
     def __init__(self, sa=None):
         if not sa:
             self.sa = Session()
         else:
             self.sa = sa
     def __init__(self):
         self.sa = Session()
     def get(self, user_id, cache=False):
         user = self.sa.query(User)

0 comments (0 inline, 0 general)