Changeset - 00498b3e4c88
[Not reviewed]
beta
0 5 0
Marcin Kuzminski - 13 years ago 2012-11-23 16:59:30
marcin@python-works.com
Cleaned the way cache keys are invalidated
should fix issue #504 RhodeCode is showing different
versions of README on different summary page loads
5 files changed with 45 insertions and 23 deletions:
0 comments (0 inline, 0 general)
rhodecode/lib/utils.py
Show inline comments
 
@@ -439,26 +439,28 @@ def repo2db_mapper(initial_repo_list, re
 
                just_db=True
 
            )
 
            # we added that repo just now, and make sure it has githook
 
            # installed
 
            if new_repo.repo_type == 'git':
 
                ScmModel().install_git_hook(new_repo.scm_instance)
 
        elif install_git_hook:
 
            if db_repo.repo_type == 'git':
 
                ScmModel().install_git_hook(db_repo.scm_instance)
 
        # during starting install all cache keys for all repositories in the
 
        # system, this will register all repos and multiple instances
 
        key, _prefix, _org_key = CacheInvalidation._get_key(name)
 
        log.debug("Creating a cache key for %s instance_id:`%s`" % (name, _prefix))
 
        CacheInvalidation._get_or_create_key(key, _prefix, _org_key, commit=False)
 
        CacheInvalidation.invalidate(name)
 
        log.debug("Creating a cache key for %s instance_id=>`%s`"
 
                  % (name, _prefix or '-'))
 

	
 
    sa.commit()
 
    removed = []
 
    if remove_obsolete:
 
        # remove from database those repositories that are not in the filesystem
 
        for repo in sa.query(Repository).all():
 
            if repo.repo_name not in initial_repo_list.keys():
 
                log.debug("Removing non-existing repository found in db `%s`" %
 
                          repo.repo_name)
 
                try:
 
                    sa.delete(repo)
 
                    sa.commit()
 
                    removed.append(repo.repo_name)
rhodecode/model/db.py
Show inline comments
 
@@ -928,25 +928,25 @@ class Repository(Base, BaseModel):
 
    #==========================================================================
 
    # SCM CACHE INSTANCE
 
    #==========================================================================
 

	
 
    @property
 
    def invalidate(self):
 
        return CacheInvalidation.invalidate(self.repo_name)
 

	
 
    def set_invalidate(self):
 
        """
 
        set a cache for invalidation for this instance
 
        """
 
        CacheInvalidation.set_invalidate(self.repo_name)
 
        CacheInvalidation.set_invalidate(repo_name=self.repo_name)
 

	
 
    @LazyProperty
 
    def scm_instance(self):
 
        return self.scm_instance_cached()
 
        return self.__get_instance()
 

	
 
    def scm_instance_cached(self, cache_map=None):
 
        @cache_region('long_term')
 
        def _c(repo_name):
 
            return self.__get_instance()
 
        rn = self.repo_name
 
        log.debug('Getting cached instance of repo')
 
@@ -1429,84 +1429,94 @@ class CacheInvalidation(Base, BaseModel)
 
    def clear_cache(cls):
 
        cls.query().delete()
 

	
 
    @classmethod
 
    def _get_key(cls, key):
 
        """
 
        Wrapper for generating a key, together with a prefix
 

	
 
        :param key:
 
        """
 
        import rhodecode
 
        prefix = ''
 
        org_key = key
 
        iid = rhodecode.CONFIG.get('instance_id')
 
        if iid:
 
            prefix = iid
 
        #remove specific suffixes like _README or _RSS
 
        key = remove_suffix(key, '_README')
 
        key = remove_suffix(key, '_RSS')
 
        key = remove_suffix(key, '_ATOM')
 
        return "%s%s" % (prefix, key), prefix, key
 

	
 
        return "%s%s" % (prefix, key), prefix, org_key
 

	
 
    @classmethod
 
    def get_by_key(cls, key):
 
        return cls.query().filter(cls.cache_key == key).scalar()
 

	
 
    @classmethod
 
    def _get_or_create_key(cls, key, prefix, org_key, commit=True):
 
    def get_by_repo_name(cls, repo_name):
 
        return cls.query().filter(cls.cache_args == repo_name).all()
 

	
 
    @classmethod
 
    def _get_or_create_key(cls, key, repo_name, commit=True):
 
        inv_obj = Session().query(cls).filter(cls.cache_key == key).scalar()
 
        if not inv_obj:
 
            try:
 
                inv_obj = CacheInvalidation(key, org_key)
 
                inv_obj = CacheInvalidation(key, repo_name)
 
                Session().add(inv_obj)
 
                if commit:
 
                    Session().commit()
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                Session().rollback()
 
        return inv_obj
 

	
 
    @classmethod
 
    def invalidate(cls, key):
 
        """
 
        Returns Invalidation object if this given key should be invalidated
 
        None otherwise. `cache_active = False` means that this cache
 
        state is not valid and needs to be invalidated
 

	
 
        :param key:
 
        """
 
        repo_name = key
 
        repo_name = remove_suffix(repo_name, '_README')
 
        repo_name = remove_suffix(repo_name, '_RSS')
 
        repo_name = remove_suffix(repo_name, '_ATOM')
 

	
 
        # adds instance prefix
 
        key, _prefix, _org_key = cls._get_key(key)
 
        inv = cls._get_or_create_key(key, _prefix, _org_key)
 
        inv = cls._get_or_create_key(key, repo_name)
 

	
 
        if inv and inv.cache_active is False:
 
            return inv
 

	
 
    @classmethod
 
    def set_invalidate(cls, key):
 
    def set_invalidate(cls, key=None, repo_name=None):
 
        """
 
        Mark this Cache key for invalidation
 
        Mark this Cache key for invalidation, either by key or whole
 
        cache sets based on repo_name
 

	
 
        :param key:
 
        """
 

	
 
        if key:
 
        key, _prefix, _org_key = cls._get_key(key)
 
        inv_objs = Session().query(cls).filter(cls.cache_args == _org_key).all()
 
        log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),
 
                                                             _org_key))
 
            inv_objs = Session().query(cls).filter(cls.cache_key == key).all()
 
        elif repo_name:
 
            inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
 

	
 
        log.debug('marking %s key[s] for invalidation based on key=%s,repo_name=%s'
 
                  % (len(inv_objs), key, repo_name))
 
        try:
 
            for inv_obj in inv_objs:
 
                if inv_obj:
 
                print inv_obj
 
                    inv_obj.cache_active = False
 

	
 
                Session().add(inv_obj)
 
            Session().commit()
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            Session().rollback()
 

	
 
    @classmethod
 
    def set_valid(cls, key):
 
        """
 
        Mark this cache key as active and currently cached
 

	
 
        :param key:
rhodecode/model/scm.py
Show inline comments
 
@@ -282,25 +282,25 @@ class ScmModel(BaseModel):
 
                .filter(RepoGroup.group_parent_id == None).all()
 
        group_iter = GroupList(all_groups)
 

	
 
        return group_iter
 

	
 
    def mark_for_invalidation(self, repo_name):
 
        """
 
        Puts cache invalidation task into db for
 
        further global cache invalidation
 

	
 
        :param repo_name: this repo that should invalidation take place
 
        """
 
        CacheInvalidation.set_invalidate(repo_name)
 
        CacheInvalidation.set_invalidate(repo_name=repo_name)
 

	
 
    def toggle_following_repo(self, follow_repo_id, user_id):
 

	
 
        f = self.sa.query(UserFollowing)\
 
            .filter(UserFollowing.follows_repo_id == follow_repo_id)\
 
            .filter(UserFollowing.user_id == user_id).scalar()
 

	
 
        if f is not None:
 
            try:
 
                self.sa.delete(f)
 
                action_logger(UserTemp(user_id),
 
                              'stopped_following_repo',
rhodecode/public/css/style.css
Show inline comments
 
@@ -1618,24 +1618,25 @@ div.form div.fields div.field div.button
 
	margin: 0;
 
	padding: 0;
 
	border: 1px solid #eee;
 
    -webkit-border-radius: 4px;
 
    -moz-border-radius: 4px;
 
    border-radius: 4px;	
 
}
 
 
#content div.box table th {
 
	background: #eee;
 
	border-bottom: 1px solid #ddd;
 
	padding: 5px 0px 5px 5px;
 
	text-align: left;
 
}
 
 
#content div.box table th.left {
 
	text-align: left;
 
}
 
 
#content div.box table th.right {
 
	text-align: right;
 
}
 
 
#content div.box table th.center {
 
	text-align: center;
rhodecode/templates/admin/repos/repo_edit.html
Show inline comments
 
@@ -187,29 +187,38 @@
 
        ${h.form(url('repo_cache', repo_name=c.repo_info.repo_name),method='delete')}
 
        <div class="form">
 
           <div class="fields">
 
               ${h.submit('reset_cache_%s' % c.repo_info.repo_name,_('Invalidate repository cache'),class_="ui-btn",onclick="return confirm('"+_('Confirm to invalidate repository cache')+"');")}
 
              <div class="field" style="border:none;color:#888">
 
              <ul>
 
                  <li>${_('Manually invalidate cache for this repository. On first access repository will be cached again')}
 
                  </li>
 
              </ul>
 
              </div>
 
              <div class="field" style="border:none;">
 
                ${_('List of cached values')}
 
                  <ul>
 
                   <table>
 
                   <tr>
 
                    <th>${_('Prefix')}</th>
 
                    <th>${_('Key')}</th>
 
                    <th>${_('Active')}</th>
 
                    </tr>
 
                  %for cache in c.repo_info.cache_keys:
 
                      <li>INSTANCE ID:${cache.prefix or '-'} ${cache.cache_args} CACHED: ${h.bool2icon(cache.cache_active)}</li>
 
                      <tr>
 
                        <td>${cache.prefix or '-'}</td> 
 
                        <td>${cache.cache_key}</td>
 
                        <td>${h.bool2icon(cache.cache_active)}</td>
 
                      </tr>
 
                  %endfor
 
                  </ul>
 
                  </table>
 
              </div>
 
           </div>
 
        </div>
 
        ${h.end_form()}
 

	
 
        <h3>${_('Public journal')}</h3>
 
        ${h.form(url('repo_public_journal', repo_name=c.repo_info.repo_name),method='put')}
 
        <div class="form">
 
          ${h.hidden('auth_token',str(h.get_token()))}
 
          <div class="field">
 
          %if c.in_public_journal:
 
            ${h.submit('set_public_%s' % c.repo_info.repo_name,_('Remove from public journal'),class_="ui-btn")}
0 comments (0 inline, 0 general)