Changeset - cb0d9ce6ac5c
[Not reviewed]
beta
1 11 1
Marcin Kuzminski - 15 years ago 2010-11-15 02:26:19
marcin@python-works.com
#50 on point cache invalidation changes.
Created cacheInvalidation table
cleaned up sa sessions from models, since it wasn't really needed.
12 files changed with 122 insertions and 53 deletions:
0 comments (0 inline, 0 general)
rhodecode/controllers/admin/permissions.py
Show inline comments
 
@@ -28,13 +28,13 @@ from pylons import request, session, tmp
 
from pylons.controllers.util import abort, redirect
 
from pylons.i18n.translation import _
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.model.forms import UserForm, DefaultPermissionsForm
 
from rhodecode.model.permission_model import PermissionModel
 
from rhodecode.model.permission import PermissionModel
 
from rhodecode.model.user import UserModel
 
import formencode
 
import logging
 
import traceback
 

	
 
log = logging.getLogger(__name__)
rhodecode/controllers/admin/repos.py
Show inline comments
 
@@ -130,13 +130,13 @@ class ReposController(BaseController):
 
        _form = RepoForm(edit=True, old_data={'repo_name':repo_name})()
 

	
 
        try:
 
            form_result = _form.to_python(dict(request.POST))
 
            repo_model.update(repo_name, form_result)
 
            invalidate_cache('get_repo_cached_%s' % repo_name)
 
            h.flash(_('Repository %s updated succesfully' % repo_name),
 
            h.flash(_('Repository %s updated successfully' % repo_name),
 
                    category='success')
 
            changed_name = form_result['repo_name']
 
            action_logger(self.rhodecode_user, 'admin_updated_repo',
 
                              changed_name, '', self.sa)
 

	
 
        except formencode.Invalid, errors:
 
@@ -149,13 +149,13 @@ class ReposController(BaseController):
 
                errors=errors.error_dict or {},
 
                prefix_error=False,
 
                encoding="UTF-8")
 

	
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            h.flash(_('error occured during update of repository %s') \
 
            h.flash(_('error occurred during update of repository %s') \
 
                    % repo_name, category='error')
 

	
 
        return redirect(url('edit_repo', repo_name=changed_name))
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def delete(self, repo_name):
rhodecode/controllers/admin/settings.py
Show inline comments
 
@@ -245,13 +245,13 @@ class SettingsController(BaseController)
 
    def my_account(self):
 
        """
 
        GET /_admin/my_account Displays info about my account 
 
        """
 

	
 
        # url('admin_settings_my_account')
 
        c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
 
        c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
 
        all_repos = self.sa.query(Repository)\
 
            .filter(Repository.user_id == c.user.user_id)\
 
            .order_by(func.lower(Repository.repo_name))\
 
            .all()
 
        c.user_repos = ScmModel().get_repos(all_repos)
 

	
 
@@ -286,13 +286,13 @@ class SettingsController(BaseController)
 
            user_model.update_my_account(uid, form_result)
 
            h.flash(_('Your account was updated succesfully'),
 
                    category='success')
 

	
 
        except formencode.Invalid, errors:
 
            c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
 
            c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
 
            c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
 
            all_repos = self.sa.query(Repository)\
 
                .filter(Repository.user_id == c.user.user_id)\
 
                .order_by(func.lower(Repository.repo_name))\
 
                .all()
 
            c.user_repos = ScmModel().get_repos(all_repos)
 

	
rhodecode/controllers/admin/users.py
Show inline comments
 
@@ -14,13 +14,12 @@
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
from rhodecode.lib.utils import action_logger
 
"""
 
Created on April 4, 2010
 
users controller for pylons
 
@author: marcink
 
"""
 

	
rhodecode/lib/auth.py
Show inline comments
 
@@ -140,13 +140,13 @@ def fill_perms(user):
 
    user.permissions['repositories'] = {}
 
    user.permissions['global'] = set()
 

	
 
    #===========================================================================
 
    # fetch default permissions
 
    #===========================================================================
 
    default_user = UserModel(sa).get_by_username('default', cache=True)
 
    default_user = UserModel().get_by_username('default', cache=True)
 

	
 
    default_perms = sa.query(RepoToPerm, Repository, Permission)\
 
        .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
 
        .join((Permission, RepoToPerm.permission_id == Permission.permission_id))\
 
        .filter(RepoToPerm.user == default_user).all()
 

	
rhodecode/lib/celerylib/tasks.py
Show inline comments
 
from celery.decorators import task
 

	
 
import os
 
import traceback
 
from time import mktime
 

	
 
from operator import itemgetter
 
from pylons.i18n.translation import _
 
from rhodecode.lib.celerylib import run_task, locked_task
 
from rhodecode.lib.helpers import person
 
from rhodecode.lib.smtp_mailer import SmtpMailer
 
from rhodecode.lib.utils import OrderedDict
 
from time import mktime
 
import os
 
import traceback
 
from vcs.backends import get_repo
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.model.db import RhodeCodeUi
 

	
 
try:
 
    import json
 
except ImportError:
 
    #python 2.5 compatibility
 
    import simplejson as json
 

	
 
@@ -41,12 +43,17 @@ def get_session():
 
        #If we don't use celery reuse our current application Session
 
        from rhodecode.model.meta import Session
 
        sa = Session()
 

	
 
    return sa
 

	
 
def get_repos_path():
 
    sa = get_session()
 
    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
    return q.ui_value
 

	
 
@task
 
@locked_task
 
def whoosh_index(repo_location, full_index):
 
    log = whoosh_index.get_logger()
 
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
 
    index_location = dict(config.items('app:main'))['index_dir']
 
@@ -59,13 +66,13 @@ def get_commits_stats(repo_name, ts_min_
 
    from rhodecode.model.db import Statistics, Repository
 
    log = get_commits_stats.get_logger()
 
    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
 

	
 
    commits_by_day_author_aggregate = {}
 
    commits_by_day_aggregate = {}
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    p = os.path.join(repos_path, repo_name)
 
    repo = get_repo(p)
 

	
 
    skip_date_limit = True
 
    parse_limit = 250 #limit for single task changeset parsing optimal for
 
    last_rev = 0
 
@@ -268,16 +275,16 @@ def send_email(recipients, subject, body
 

	
 
@task
 
def create_repo_fork(form_data, cur_user):
 
    from rhodecode.model.repo import RepoModel
 
    from vcs import get_backend
 
    log = create_repo_fork.get_logger()
 
    repo_model = RepoModel(get_session())
 
    repo_model = RepoModel()
 
    repo_model.create(form_data, cur_user, just_db=True, fork=True)
 
    repo_name = form_data['repo_name']
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    repo_path = os.path.join(repos_path, repo_name)
 
    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
 
    alias = form_data['repo_type']
 

	
 
    log.info('creating repo fork %s as %s', repo_name, repo_path)
 
    backend = get_backend(alias)
 
@@ -288,13 +295,13 @@ def __get_codes_stats(repo_name):
 
    'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el',
 
    'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml',
 
    'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst',
 
    's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 

	
 

	
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    p = os.path.join(repos_path, repo_name)
 
    repo = get_repo(p)
 
    tip = repo.get_changeset()
 
    code_stats = {}
 

	
 
    def aggregate(cs):
rhodecode/lib/utils.py
Show inline comments
 
@@ -86,21 +86,21 @@ def action_logger(user, action, repo, ip
 
        sa = meta.Session()
 

	
 
    try:
 
        if hasattr(user, 'user_id'):
 
            user_obj = user
 
        elif isinstance(user, basestring):
 
            user_obj = UserModel(sa).get_by_username(user, cache=False)
 
            user_obj = UserModel().get_by_username(user, cache=False)
 
        else:
 
            raise Exception('You have to provide user object or username')
 

	
 

	
 
        if repo:
 
            repo_name = repo.lstrip('/')
 

	
 
            repository = RepoModel(sa).get(repo_name, cache=False)
 
            repository = RepoModel().get(repo_name, cache=False)
 
            if not repository:
 
                raise Exception('You have to provide valid repository')
 
        else:
 
            raise Exception('You have to provide repository to action logger')
 

	
 

	
 
@@ -290,18 +290,22 @@ def make_ui(read_from='file', path=None,
 
def set_rhodecode_config(config):
 
    hgsettings = get_hg_settings()
 

	
 
    for k, v in hgsettings.items():
 
        config[k] = v
 

	
 
def invalidate_cache(name, *args):
 
def invalidate_cache(cache_key, *args):
 
    """
 
    Puts cache invalidation task into db for 
 
    further global cache invalidation
 
    """
 
    pass
 
    from rhodecode.model.scm import ScmModel
 

	
 
    if cache_key.startswith('get_repo_cached_'):
 
        name = cache_key.split('get_repo_cached_')[-1]
 
        ScmModel().mark_for_invalidation(name)
 

	
 
class EmptyChangeset(BaseChangeset):
 
    """
 
    An dummy empty changeset. It's possible to pass hash when creating
 
    an EmptyChangeset
 
    """
 
@@ -337,13 +341,13 @@ class EmptyChangeset(BaseChangeset):
 
def repo2db_mapper(initial_repo_list, remove_obsolete=False):
 
    """
 
    maps all found repositories into db
 
    """
 

	
 
    sa = meta.Session()
 
    rm = RepoModel(sa)
 
    rm = RepoModel()
 
    user = sa.query(User).filter(User.admin == True).first()
 

	
 
    for name, repo in initial_repo_list.items():
 
        if not rm.get(name, cache=False):
 
            log.info('repository %s not found creating default', name)
 

	
rhodecode/model/db.py
Show inline comments
 
@@ -87,15 +87,14 @@ class Repository(Base):
 

	
 
    user = relation('User')
 
    fork = relation('Repository', remote_side=repo_id)
 
    repo_to_perm = relation('RepoToPerm', cascade='all')
 
    stats = relation('Statistics', cascade='all')
 

	
 

	
 
    def __repr__(self):
 
        return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
 
        return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 

	
 
class Permission(Base):
 
    __tablename__ = 'permissions'
 
    __table_args__ = {'useexisting':True}
 
    permission_id = Column("permission_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    permission_name = Column("permission_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
@@ -137,13 +136,20 @@ class Statistics(Base):
 
    languages = Column("languages", BLOB(), nullable=False)#JSON data
 

	
 
    repository = relation('Repository', single_parent=True)
 

	
 
class CacheInvalidation(Base):
 
    __tablename__ = 'cache_invalidation'
 
    __table_args__ = {'useexisting':True}
 
    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
 
    cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None)
 
    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False)
 

	
 

	
 
    def __init__(self, cache_key, cache_args=''):
 
        self.cache_key = cache_key
 
        self.cache_args = cache_args
 
        self.cache_active = False
 

	
 
    def __repr__(self):
 
        return "<CacheInvaidation('%s:%s')>" % (self.cache_id, self.cache_key)
rhodecode/model/permission.py
Show inline comments
 
file renamed from rhodecode/model/permission_model.py to rhodecode/model/permission.py
 
@@ -30,17 +30,14 @@ import logging
 
import traceback
 
log = logging.getLogger(__name__)
 

	
 

	
 
class PermissionModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get_permission(self, permission_id, cache=False):
 
        perm = self.sa.query(Permission)
 
        if cache:
 
            perm = perm.options(FromCache("sql_cache_short",
 
                                          "get_permission_%s" % permission_id))
rhodecode/model/repo.py
Show inline comments
 
@@ -33,17 +33,14 @@ import os
 
import shutil
 
import traceback
 
log = logging.getLogger(__name__)
 

	
 
class RepoModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get(self, repo_id, cache=False):
 
        repo = self.sa.query(Repository)\
 
            .filter(Repository.repo_name == repo_id)
 

	
 
        if cache:
 
@@ -64,26 +61,26 @@ class RepoModel(object):
 
    def update(self, repo_name, form_data):
 
        try:
 

	
 
            #update permissions
 
            for username, perm in form_data['perms_updates']:
 
                r2p = self.sa.query(RepoToPerm)\
 
                        .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\
 
                        .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\
 
                        .filter(RepoToPerm.repository == self.get(repo_name))\
 
                        .one()
 

	
 
                r2p.permission_id = self.sa.query(Permission).filter(
 
                                                Permission.permission_name ==
 
                                                perm).one().permission_id
 
                self.sa.add(r2p)
 

	
 
            #set new permissions
 
            for username, perm in form_data['perms_new']:
 
                r2p = RepoToPerm()
 
                r2p.repository = self.get(repo_name)
 
                r2p.user = UserModel(self.sa).get_by_username(username, cache=False)
 
                r2p.user = UserModel().get_by_username(username, cache=False)
 

	
 
                r2p.permission_id = self.sa.query(Permission).filter(
 
                                        Permission.permission_name == perm)\
 
                                        .one().permission_id
 
                self.sa.add(r2p)
 

	
 
@@ -131,25 +128,25 @@ class RepoModel(object):
 
            new_repo.user_id = cur_user.user_id
 
            self.sa.add(new_repo)
 

	
 
            #create default permission
 
            repo_to_perm = RepoToPerm()
 
            default = 'repository.read'
 
            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
 
            for p in UserModel().get_by_username('default', cache=False).user_perms:
 
                if p.permission.permission_name.startswith('repository.'):
 
                    default = p.permission.permission_name
 
                    break
 

	
 
            default_perm = 'repository.none' if form_data['private'] else default
 

	
 
            repo_to_perm.permission_id = self.sa.query(Permission)\
 
                    .filter(Permission.permission_name == default_perm)\
 
                    .one().permission_id
 

	
 
            repo_to_perm.repository_id = new_repo.repo_id
 
            repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id
 
            repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id
 

	
 
            self.sa.add(repo_to_perm)
 
            self.sa.commit()
 
            if not just_db:
 
                self.__create_repo(repo_name, form_data['repo_type'])
 
        except:
rhodecode/model/scm.py
Show inline comments
 
@@ -25,35 +25,34 @@ Model for RhodeCode
 
from beaker.cache import cache_region, region_invalidate
 
from mercurial import ui
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import HasRepoPermissionAny
 
from rhodecode.lib.utils import get_repos
 
from rhodecode.model import meta
 
from rhodecode.model.db import Repository, User, RhodeCodeUi
 
from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation
 
from rhodecode.model.caching_query import FromCache
 
from sqlalchemy.orm import joinedload
 
from sqlalchemy.orm.session import make_transient
 
from vcs import get_backend
 
from vcs.utils.helpers import get_scm
 
from vcs.exceptions import RepositoryError, VCSError
 
from vcs.utils.lazy import LazyProperty
 
import traceback
 
import logging
 
import os
 
import time
 

	
 
log = logging.getLogger(__name__)
 

	
 
class ScmModel(object):
 
    """
 
    Mercurial Model
 
    """
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = meta.Session()
 
        else:
 
            self.sa = sa
 

	
 
    def __init__(self):
 
        self.sa = meta.Session()
 

	
 
    @LazyProperty
 
    def repos_path(self):
 
        """
 
        Get's the repositories root path from database
 
        """
 
@@ -140,13 +139,13 @@ class ScmModel(object):
 
        :param repo_name:
 
        """
 
        if not HasRepoPermissionAny('repository.read', 'repository.write',
 
                            'repository.admin')(repo_name, 'get repo check'):
 
            return
 

	
 
        @cache_region('long_term', 'get_repo_cached_%s' % repo_name)
 
        @cache_region('long_term')
 
        def _get_repo(repo_name):
 

	
 
            repo_path = os.path.join(self.repos_path, repo_name)
 
            alias = get_scm(repo_path)[0]
 

	
 
            log.debug('Creating instance of %s repository', alias)
 
@@ -162,16 +161,79 @@ class ScmModel(object):
 

	
 
            dbrepo = self.sa.query(Repository)\
 
                .options(joinedload(Repository.fork))\
 
                .options(joinedload(Repository.user))\
 
                .filter(Repository.repo_name == repo_name)\
 
                .scalar()
 
            make_transient(dbrepo)
 
            repo.dbrepo = dbrepo
 
            return repo
 

	
 
        invalidate = False
 
        invalidate = self._should_invalidate(repo_name)
 
        if invalidate:
 
            log.info('INVALIDATING CACHE FOR %s', repo_name)
 
            log.info('invalidating cache for repository %s', repo_name)
 
            region_invalidate(_get_repo, None, repo_name)
 
            self._mark_invalidated(invalidate)
 

	
 
        return _get_repo(repo_name)
 

	
 

	
 

	
 
    def mark_for_invalidation(self, repo_name):
 
        """
 
        Puts cache invalidation task into db for 
 
        further global cache invalidation
 
        
 
        :param repo_name: this repo that should invalidation take place
 
        """
 
        log.debug('marking %s for invalidation', repo_name)
 
        cache = self.sa.query(CacheInvalidation)\
 
            .filter(CacheInvalidation.cache_key == repo_name).scalar()
 

	
 
        if cache:
 
            #mark this cache as inactive
 
            cache.cache_active = False
 
        else:
 
            log.debug('cache key not found in invalidation db -> creating one')
 
            cache = CacheInvalidation(repo_name)
 

	
 
        try:
 
            self.sa.add(cache)
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 

	
 

	
 

	
 

	
 

	
 
    def _should_invalidate(self, repo_name):
 
        """
 
        Looks up database for invalidation signals for this repo_name
 
        :param repo_name:
 
        """
 

	
 
        ret = self.sa.query(CacheInvalidation)\
 
            .options(FromCache('sql_cache_short',
 
                           'get_invalidation_%s' % repo_name))\
 
            .filter(CacheInvalidation.cache_key == repo_name)\
 
            .filter(CacheInvalidation.cache_active == False)\
 
            .scalar()
 

	
 
        return ret
 

	
 
    def _mark_invalidated(self, cache_key):
 
        """
 
        Marks all occurences of cache to invaldation as already invalidated
 
        @param repo_name:
 
        """
 
        if cache_key:
 
            log.debug('marking %s as already invalidated', cache_key)
 
        try:
 
            cache_key.cache_active = True
 
            self.sa.add(cache_key)
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 

	
rhodecode/model/user.py
Show inline comments
 
@@ -33,17 +33,14 @@ import traceback
 
log = logging.getLogger(__name__)
 

	
 
class DefaultUserException(Exception):pass
 

	
 
class UserModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get(self, user_id, cache=False):
 
        user = self.sa.query(User)
 
        if cache:
 
            user = user.options(FromCache("sql_cache_short",
 
                                          "get_user_%s" % user_id))
0 comments (0 inline, 0 general)