kallithea Changeset - 00498b3e4c88

Changeset - 00498b3e4c88

Parent rev.

Child rev.

[Not reviewed]

beta

0 5 0

Marcin Kuzminski - 13 years ago 2012-11-23 16:59:30
marcin@python-works.com

Cleaned the way cache keys are invalidated
should fix issue #504 RhodeCode is showing different
versions of README on different summary page loads

5 files changed with 45 insertions and 23 deletions:

rhodecode/lib/utils.py

rhodecode/model/db.py

rhodecode/model/scm.py

rhodecode/public/css/style.css

rhodecode/templates/admin/repos/repo_edit.html

0 comments (0 inline, 0 general)

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -403,98 +403,100 @@ def repo2db_mapper(initial_repo_list, re @@
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hook: if this is True, also check and install githook
         for a repo if missing
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.scm import ScmModel
     sa = meta.Session()
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
     if user is None:
         raise Exception('Missing administrative account!')
     added = []
 #    # clear cache keys
 #    log.debug("Clearing cache keys now...")
 #    CacheInvalidation.clear_cache()
 #    sa.commit()
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         db_repo = rm.get_by_repo_name(name)
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository %s not found, creating now' % name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
             new_repo = rm.create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repos_group=getattr(group, 'group_id', None),
                 owner=user,
                 just_db=True
+            )
             # we added that repo just now, and make sure it has githook
             # installed
             if new_repo.repo_type == 'git':
                 ScmModel().install_git_hook(new_repo.scm_instance)
         elif install_git_hook:
             if db_repo.repo_type == 'git':
                 ScmModel().install_git_hook(db_repo.scm_instance)
         # during starting install all cache keys for all repositories in the
         # system, this will register all repos and multiple instances
         key, _prefix, _org_key = CacheInvalidation._get_key(name)
         log.debug("Creating a cache key for %s instance_id:`%s`" % (name, _prefix))
         CacheInvalidation._get_or_create_key(key, _prefix, _org_key, commit=False)
         CacheInvalidation.invalidate(name)
         log.debug("Creating a cache key for %s instance_id=>`%s`"
                   % (name, _prefix or '-'))
     sa.commit()
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non-existing repository found in db `%s`" %
                           repo.repo_name)
                 try:
                     sa.delete(repo)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
     return added, removed
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
 ))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings

rhodecode/model/db.py

➞

Show inline comments

@@ @@ -892,97 +892,97 @@ class Repository(Base, BaseModel): @@
             grouped[cmt.revision].append(cmt)
         return grouped
     def statuses(self, revisions=None):
         """
         Returns statuses for this repository
         :param revisions: list of revisions to get statuses for
         :type revisions: list
         """
         statuses = ChangesetStatus.query()\
             .filter(ChangesetStatus.repo == self)\
             .filter(ChangesetStatus.version == 0)
         if revisions:
             statuses = statuses.filter(ChangesetStatus.revision.in_(revisions))
         grouped = {}
         #maybe we have open new pullrequest without a status ?
         stat = ChangesetStatus.STATUS_UNDER_REVIEW
         status_lbl = ChangesetStatus.get_status_lbl(stat)
         for pr in PullRequest.query().filter(PullRequest.org_repo == self).all():
             for rev in pr.revisions:
                 pr_id = pr.pull_request_id
                 pr_repo = pr.other_repo.repo_name
                 grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
         for stat in statuses.all():
             pr_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo]
         return grouped
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
     @property
     def invalidate(self):
         return CacheInvalidation.invalidate(self.repo_name)
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
         CacheInvalidation.set_invalidate(self.repo_name)
+        CacheInvalidation.set_invalidate(repo_name=self.repo_name)
     @LazyProperty
     def scm_instance(self):
         return self.scm_instance_cached()
         return self.__get_instance()
     def scm_instance_cached(self, cache_map=None):
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
         rn = self.repo_name
         log.debug('Getting cached instance of repo')
         if cache_map:
             # get using prefilled cache_map
             invalidate_repo = cache_map[self.repo_name]
             if invalidate_repo:
                 invalidate_repo = (None if invalidate_repo.cache_active
                                    else invalidate_repo)
         else:
             # get from invalidate
             invalidate_repo = self.invalidate
         if invalidate_repo is not None:
             region_invalidate(_c, None, rn)
             # update our cache
             CacheInvalidation.set_valid(invalidate_repo.cache_key)
         return _c(rn)
     def __get_instance(self):
         repo_full_path = self.repo_full_path
         try:
             alias = get_scm(repo_full_path)[0]
             log.debug('Creating instance of %s repository' % alias)
             backend = get_backend(alias)
         except VCSError:
             log.error(traceback.format_exc())
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
         if alias == 'hg':
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             # skip hidden web repository
             if repo._get_hidden():
@@ @@ -1393,156 +1393,166 @@ class UserFollowing(Base, BaseModel): @@
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (
         UniqueConstraint('cache_key'),
         Index('key_idx', 'cache_key'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
+    )
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
     def __init__(self, cache_key, cache_args=''):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
     @property
     def prefix(self):
         _split = self.cache_key.split(self.cache_args, 1)
         if _split and len(_split) == 2:
             return _split[0]
         return ''
     @classmethod
     def clear_cache(cls):
         cls.query().delete()
     @classmethod
     def _get_key(cls, key):
         """
         Wrapper for generating a key, together with a prefix
         :param key:
         """
         import rhodecode
         prefix = ''
         org_key = key
         iid = rhodecode.CONFIG.get('instance_id')
         if iid:
             prefix = iid
         #remove specific suffixes like _README or _RSS
         key = remove_suffix(key, '_README')
         key = remove_suffix(key, '_RSS')
         key = remove_suffix(key, '_ATOM')
         return "%s%s" % (prefix, key), prefix, key
         return "%s%s" % (prefix, key), prefix, org_key
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.cache_key == key).scalar()
     @classmethod
     def _get_or_create_key(cls, key, prefix, org_key, commit=True):
     def get_by_repo_name(cls, repo_name):
         return cls.query().filter(cls.cache_args == repo_name).all()
     @classmethod
     def _get_or_create_key(cls, key, repo_name, commit=True):
         inv_obj = Session().query(cls).filter(cls.cache_key == key).scalar()
         if not inv_obj:
             try:
-                inv_obj = CacheInvalidation(key, org_key)
+                inv_obj = CacheInvalidation(key, repo_name)
                 Session().add(inv_obj)
                 if commit:
                     Session().commit()
             except Exception:
                 log.error(traceback.format_exc())
                 Session().rollback()
         return inv_obj
     @classmethod
     def invalidate(cls, key):
         """
         Returns Invalidation object if this given key should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
         :param key:
         """
         repo_name = key
         repo_name = remove_suffix(repo_name, '_README')
         repo_name = remove_suffix(repo_name, '_RSS')
         repo_name = remove_suffix(repo_name, '_ATOM')
         # adds instance prefix
         key, _prefix, _org_key = cls._get_key(key)
-        inv = cls._get_or_create_key(key, _prefix, _org_key)
+        inv = cls._get_or_create_key(key, repo_name)
         if inv and inv.cache_active is False:
             return inv
     @classmethod
     def set_invalidate(cls, key):
+    def set_invalidate(cls, key=None, repo_name=None):
         """
         Mark this Cache key for invalidation
         Mark this Cache key for invalidation, either by key or whole
         cache sets based on repo_name
         :param key:
         """
+        if key:
         key, _prefix, _org_key = cls._get_key(key)
         inv_objs = Session().query(cls).filter(cls.cache_args == _org_key).all()
         log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),
                                                              _org_key))
             inv_objs = Session().query(cls).filter(cls.cache_key == key).all()
         elif repo_name:
             inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
         log.debug('marking %s key[s] for invalidation based on key=%s,repo_name=%s'
                   % (len(inv_objs), key, repo_name))
         try:
             for inv_obj in inv_objs:
-                if inv_obj:
+                print inv_obj
                     inv_obj.cache_active = False
                 Session().add(inv_obj)
             Session().commit()
         except Exception:
             log.error(traceback.format_exc())
             Session().rollback()
     @classmethod
     def set_valid(cls, key):
         """
         Mark this cache key as active and currently cached
         :param key:
         """
         inv_obj = cls.get_by_key(key)
         inv_obj.cache_active = True
         Session().add(inv_obj)
         Session().commit()
     @classmethod
     def get_cache_map(cls):
         class cachemapdict(dict):
             def __init__(self, *args, **kwargs):
                 fixkey = kwargs.get('fixkey')
                 if fixkey:
                     del kwargs['fixkey']
                 self.fixkey = fixkey
                 super(cachemapdict, self).__init__(*args, **kwargs)
             def __getattr__(self, name):
                 key = name
                 if self.fixkey:
                     key, _prefix, _org_key = cls._get_key(key)
                 if key in self.__dict__:
                     return self.__dict__[key]
                 else:
                     return self[key]
             def __getitem__(self, key):
                 if self.fixkey:
                     key, _prefix, _org_key = cls._get_key(key)
                 try:
                     return super(cachemapdict, self).__getitem__(key)
                 except KeyError:
                     return
         cache_map = cachemapdict(fixkey=True)

rhodecode/model/scm.py

➞

Show inline comments

@@ @@ -246,97 +246,97 @@ class ScmModel(BaseModel): @@
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos[name] = klass(path[1])
             except OSError:
                 continue
         return repos
     def get_repos(self, all_repos=None, sort_key=None, simple=False):
         """
         Get all repos from db and for each repo create it's
         backend instance and fill that backed with information from database
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
         :param sort_key: initial sorting of repos
         :param simple: use SimpleCachedList - one without the SCM info
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                         .filter(Repository.group_id == None)\
                         .order_by(func.lower(Repository.repo_name)).all()
         if simple:
             repo_iter = SimpleCachedRepoList(all_repos,
                                              repos_path=self.repos_path,
                                              order_by=sort_key)
         else:
             repo_iter = CachedRepoList(all_repos,
                                        repos_path=self.repos_path,
                                        order_by=sort_key)
         return repo_iter
     def get_repos_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         group_iter = GroupList(all_groups)
         return group_iter
     def mark_for_invalidation(self, repo_name):
         """
         Puts cache invalidation task into db for
         further global cache invalidation
         :param repo_name: this repo that should invalidation take place
         """
         CacheInvalidation.set_invalidate(repo_name)
+        CacheInvalidation.set_invalidate(repo_name=repo_name)
     def toggle_following_repo(self, follow_repo_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except:
             log.error(traceback.format_exc())
             raise
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id

rhodecode/public/css/style.css

➞

Show inline comments

@@ @@ -1582,96 +1582,97 @@ input.disabled { @@
 	margin: 0;
 	padding: 0;
+}
 #content div.box div.form div.fields div.field div.textarea table td table td
+	{
 	font-size: 11px;
 	padding: 5px 5px 5px 0;
+}
 #content div.box div.form div.fields div.field input[type=text]:focus,
 #content div.box div.form div.fields div.field input[type=password]:focus,
 #content div.box div.form div.fields div.field input[type=file]:focus,
 #content div.box div.form div.fields div.field textarea:focus,
 #content div.box div.form div.fields div.field select:focus,
 .reviewer_ac input:focus
+	{
 	background: #f6f6f6;
 	border-color: #666;
+}
 .reviewer_ac {
 	padding:10px
+}
 div.form div.fields div.field div.button {
 	margin: 0;
 	padding: 0 0 0 8px;
+}
 #content div.box table.noborder {
 	border: 1px solid transparent;
+}
 #content div.box table {
 	width: 100%;
 	border-collapse: separate;
 	margin: 0;
 	padding: 0;
 	border: 1px solid #eee;
     -webkit-border-radius: 4px;
     -moz-border-radius: 4px;
     border-radius: 4px;
+}
 #content div.box table th {
 	background: #eee;
 	border-bottom: 1px solid #ddd;
 	padding: 5px 0px 5px 5px;
 	text-align: left;
+}
 #content div.box table th.left {
 	text-align: left;
+}
 #content div.box table th.right {
 	text-align: right;
+}
 #content div.box table th.center {
 	text-align: center;
+}
 #content div.box table th.selected {
 	vertical-align: middle;
 	padding: 0;
+}
 #content div.box table td {
 	background: #fff;
 	border-bottom: 1px solid #cdcdcd;
 	vertical-align: middle;
 	padding: 5px;
+}
 #content div.box table tr.selected td {
 	background: #FFC;
+}
 #content div.box table td.selected {
 	width: 3%;
 	text-align: center;
 	vertical-align: middle;
 	padding: 0;
+}
 #content div.box table td.action {
 	width: 45%;
 	text-align: left;
+}
 #content div.box table td.date {
 	width: 33%;
 	text-align: center;
+}
 #content div.box div.action {

rhodecode/templates/admin/repos/repo_edit.html

➞

Show inline comments

@@ @@ -151,101 +151,110 @@ @@
     <div class="title">
         <h5>${_('Administration')}</h5>
     </div>
         <h3>${_('Statistics')}</h3>
         ${h.form(url('repo_stats', repo_name=c.repo_info.repo_name),method='delete')}
         <div class="form">
            <div class="fields">
                ${h.submit('reset_stats_%s' % c.repo_info.repo_name,_('Reset current statistics'),class_="ui-btn",onclick="return confirm('"+_('Confirm to remove current statistics')+"');")}
                <div class="field" style="border:none;color:#888">
                <ul>
                     <li>${_('Fetched to rev')}: ${c.stats_revision}/${c.repo_last_rev}</li>
                     <li>${_('Stats gathered')}: ${c.stats_percentage}%</li>
                </ul>
                </div>
            </div>
         </div>
         ${h.end_form()}
         %if c.repo_info.clone_uri:
         <h3>${_('Remote')}</h3>
         ${h.form(url('repo_pull', repo_name=c.repo_info.repo_name),method='put')}
         <div class="form">
            <div class="fields">
                ${h.submit('remote_pull_%s' % c.repo_info.repo_name,_('Pull changes from remote location'),class_="ui-btn",onclick="return confirm('"+_('Confirm to pull changes from remote side')+"');")}
                <div class="field" style="border:none">
                <ul>
                     <li><a href="${c.repo_info.clone_uri}">${c.repo_info.clone_uri}</a></li>
                </ul>
                </div>
            </div>
         </div>
         ${h.end_form()}
         %endif
         <h3>${_('Cache')}</h3>
         ${h.form(url('repo_cache', repo_name=c.repo_info.repo_name),method='delete')}
         <div class="form">
            <div class="fields">
                ${h.submit('reset_cache_%s' % c.repo_info.repo_name,_('Invalidate repository cache'),class_="ui-btn",onclick="return confirm('"+_('Confirm to invalidate repository cache')+"');")}
               <div class="field" style="border:none;color:#888">
               <ul>
                   <li>${_('Manually invalidate cache for this repository. On first access repository will be cached again')}
                   </li>
               </ul>
               </div>
               <div class="field" style="border:none;">
                 ${_('List of cached values')}
                   <ul>
                    <table>
                    <tr>
                     <th>${_('Prefix')}</th>
                     <th>${_('Key')}</th>
                     <th>${_('Active')}</th>
                     </tr>
                   %for cache in c.repo_info.cache_keys:
                       <li>INSTANCE ID:${cache.prefix or '-'} ${cache.cache_args} CACHED: ${h.bool2icon(cache.cache_active)}</li>
                       <tr>
                         <td>${cache.prefix or '-'}</td>
                         <td>${cache.cache_key}</td>
                         <td>${h.bool2icon(cache.cache_active)}</td>
                       </tr>
                   %endfor
-                  </ul>
+                  </table>
               </div>
            </div>
         </div>
         ${h.end_form()}
         <h3>${_('Public journal')}</h3>
         ${h.form(url('repo_public_journal', repo_name=c.repo_info.repo_name),method='put')}
         <div class="form">
           ${h.hidden('auth_token',str(h.get_token()))}
           <div class="field">
           %if c.in_public_journal:
             ${h.submit('set_public_%s' % c.repo_info.repo_name,_('Remove from public journal'),class_="ui-btn")}
           %else:
             ${h.submit('set_public_%s' % c.repo_info.repo_name,_('Add to public journal'),class_="ui-btn")}
           %endif
           </div>
          <div class="field" style="border:none;color:#888">
          <ul>
               <li>${_('All actions made on this repository will be accessible to everyone in public journal')}
               </li>
          </ul>
          </div>
         </div>
         ${h.end_form()}
         <h3>${_('Locking')}</h3>
         ${h.form(url('repo_locking', repo_name=c.repo_info.repo_name),method='put')}
         <div class="form">
            <div class="fields">
               %if c.repo_info.locked[0]:
                ${h.submit('set_unlock' ,_('Unlock locked repo'),class_="ui-btn",onclick="return confirm('"+_('Confirm to unlock repository')+"');")}
                ${'Locked by %s on %s' % (h.person_by_id(c.repo_info.locked[0]),h.fmt_date(h.time_to_datetime(c.repo_info.locked[1])))}
               %else:
                 ${h.submit('set_lock',_('lock repo'),class_="ui-btn",onclick="return confirm('"+_('Confirm to lock repository')+"');")}
                 ${_('Repository is not locked')}
               %endif
            </div>
            <div class="field" style="border:none;color:#888">
            <ul>
                 <li>${_('Force locking on repository. Works only when anonymous access is disabled')}
                 </li>
            </ul>
            </div>
         </div>
         ${h.end_form()}
         <h3>${_('Set as fork of')}</h3>
         ${h.form(url('repo_as_fork', repo_name=c.repo_info.repo_name),method='put')}

0 comments (0 inline, 0 general)