Changeset - aa1891074dd6
[Not reviewed]
default
0 5 0
Mads Kiilerich - 10 years ago 2015-10-08 23:21:58
madski@unity3d.com
cache: when invalidating a cache, always just delete all 'live cache' records instead of marking them inactive

Keep it simple. Adding the record again might be slightly more expensive than
just updating the active flag but instead we get get a simpler model and
automatic cleanup without using the cache-keys paster command.
5 files changed with 10 insertions and 15 deletions:
0 comments (0 inline, 0 general)
kallithea/controllers/admin/repos.py
Show inline comments
 
@@ -556,49 +556,49 @@ class ReposController(BaseRepoController
 
            repo = Repository.get_by_repo_name(repo_name)
 

	
 
            if repo.enable_locking:
 
                if repo.locked[0]:
 
                    Repository.unlock(repo)
 
                    h.flash(_('Repository has been unlocked'), category='success')
 
                else:
 
                    Repository.lock(repo, c.authuser.user_id)
 
                    h.flash(_('Repository has been locked'), category='success')
 

	
 
        except Exception as e:
 
            log.error(traceback.format_exc())
 
            h.flash(_('An error occurred during unlocking'),
 
                    category='error')
 
        return redirect(url('summary_home', repo_name=repo_name))
 

	
 
    @HasRepoPermissionAllDecorator('repository.admin')
 
    def edit_caches(self, repo_name):
 
        """GET /repo_name/settings: Form to edit an existing item"""
 
        # url('edit_repo', repo_name=ID)
 
        c.repo_info = self._load_repo(repo_name)
 
        c.active = 'caches'
 
        if request.POST:
 
            try:
 
                ScmModel().mark_for_invalidation(repo_name, delete=True)
 
                ScmModel().mark_for_invalidation(repo_name)
 
                Session().commit()
 
                h.flash(_('Cache invalidation successful'),
 
                        category='success')
 
            except Exception as e:
 
                log.error(traceback.format_exc())
 
                h.flash(_('An error occurred during cache invalidation'),
 
                        category='error')
 

	
 
            return redirect(url('edit_repo_caches', repo_name=c.repo_name))
 
        return render('admin/repos/repo_edit.html')
 

	
 
    @HasRepoPermissionAllDecorator('repository.admin')
 
    def edit_remote(self, repo_name):
 
        """GET /repo_name/settings: Form to edit an existing item"""
 
        # url('edit_repo', repo_name=ID)
 
        c.repo_info = self._load_repo(repo_name)
 
        c.active = 'remote'
 
        if request.POST:
 
            try:
 
                ScmModel().pull_changes(repo_name, self.authuser.username)
 
                h.flash(_('Pulled from remote location'), category='success')
 
            except Exception as e:
 
                log.error(traceback.format_exc())
 
                h.flash(_('An error occurred during pull from remote location'),
kallithea/controllers/admin/settings.py
Show inline comments
 
@@ -185,49 +185,49 @@ class SettingsController(BaseController)
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def settings_mapping(self):
 
        """GET /admin/settings/mapping: All items in the collection"""
 
        # url('admin_settings_mapping')
 
        c.active = 'mapping'
 
        if request.POST:
 
            rm_obsolete = request.POST.get('destroy', False)
 
            install_git_hooks = request.POST.get('hooks', False)
 
            overwrite_git_hooks = request.POST.get('hooks_overwrite', False);
 
            invalidate_cache = request.POST.get('invalidate', False)
 
            log.debug('rescanning repo location with destroy obsolete=%s, '
 
                      'install git hooks=%s and '
 
                      'overwrite git hooks=%s' % (rm_obsolete, install_git_hooks, overwrite_git_hooks))
 

	
 
            if invalidate_cache:
 
                log.debug('invalidating all repositories cache')
 
                for repo in Repository.get_all():
 
                    ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
                    ScmModel().mark_for_invalidation(repo.repo_name)
 

	
 
            filesystem_repos = ScmModel().repo_scan()
 
            added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,
 
                                            install_git_hooks=install_git_hooks,
 
                                            user=c.authuser.username,
 
                                            overwrite_git_hooks=overwrite_git_hooks)
 
            h.flash(h.literal(_('Repositories successfully rescanned. Added: %s. Removed: %s.') %
 
                (', '.join(h.link_to(safe_unicode(repo_name), h.url('summary_home', repo_name=repo_name))
 
                 for repo_name in added) or '-',
 
                 ', '.join(h.escape(safe_unicode(repo_name)) for repo_name in removed) or '-')),
 
                category='success')
 
            return redirect(url('admin_settings_mapping'))
 

	
 
        defaults = Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def settings_global(self):
kallithea/model/db.py
Show inline comments
 
@@ -2082,64 +2082,60 @@ class CacheInvalidation(Base, BaseModel)
 
        """
 
        # prefix, repo_name, suffix
 
        return self._cache_key_partition()[2]
 

	
 
    @classmethod
 
    def clear_cache(cls):
 
        """
 
        Delete all cache keys from database.
 
        Should only be run when all instances are down and all entries thus stale.
 
        """
 
        cls.query().delete()
 
        Session().commit()
 

	
 
    @classmethod
 
    def _get_cache_key(cls, key):
 
        """
 
        Wrapper for generating a unique cache key for this instance and "key".
 
        key must / will start with a repo_name which will be stored in .cache_args .
 
        """
 
        import kallithea
 
        prefix = kallithea.CONFIG.get('instance_id', '')
 
        return "%s%s" % (prefix, key)
 

	
 
    @classmethod
 
    def set_invalidate(cls, repo_name, delete=False):
 
    def set_invalidate(cls, repo_name):
 
        """
 
        Mark all caches of a repo as invalid in the database.
 
        """
 
        inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
 
        log.debug('for repo %s got %s invalidation objects',
 
                  safe_str(repo_name), inv_objs)
 

	
 
        for inv_obj in inv_objs:
 
            log.debug('marking %s key for invalidation based on repo_name=%s',
 
                      inv_obj, safe_str(repo_name))
 
            if delete:
 
                Session().delete(inv_obj)
 
            else:
 
                inv_obj.cache_active = False
 
                Session().add(inv_obj)
 
            Session().delete(inv_obj)
 
        Session().commit()
 

	
 
    @classmethod
 
    def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
 
        """
 
        Mark this cache key as active and currently cached.
 
        Return True if the existing cache registration still was valid.
 
        Return False to indicate that it had been invalidated and caches should be refreshed.
 
        """
 

	
 
        key = (repo_name + '_' + kind) if kind else repo_name
 
        cache_key = cls._get_cache_key(key)
 

	
 
        if valid_cache_keys and cache_key in valid_cache_keys:
 
            return True
 

	
 
        inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
 
        if not inv_obj:
 
            inv_obj = CacheInvalidation(cache_key, repo_name)
 
        if inv_obj.cache_active:
 
            return True
 
        inv_obj.cache_active = True
 
        Session().add(inv_obj)
 
        Session().commit()
kallithea/model/scm.py
Show inline comments
 
@@ -310,55 +310,55 @@ class ScmModel(BaseModel):
 
        :param sort_key: initial sorting of repos
 
        :param simple: use SimpleCachedList - one without the SCM info
 
        """
 
        if all_repos is None:
 
            all_repos = self.sa.query(Repository)\
 
                        .filter(Repository.group_id == None)\
 
                        .order_by(func.lower(Repository.repo_name)).all()
 
        if simple:
 
            repo_iter = SimpleCachedRepoList(all_repos,
 
                                             repos_path=self.repos_path,
 
                                             order_by=sort_key)
 
        else:
 
            repo_iter = CachedRepoList(all_repos,
 
                                       repos_path=self.repos_path,
 
                                       order_by=sort_key)
 

	
 
        return repo_iter
 

	
 
    def get_repo_groups(self, all_groups=None):
 
        if all_groups is None:
 
            all_groups = RepoGroup.query()\
 
                .filter(RepoGroup.group_parent_id == None).all()
 
        return [x for x in RepoGroupList(all_groups)]
 

	
 
    def mark_for_invalidation(self, repo_name, delete=False):
 
    def mark_for_invalidation(self, repo_name):
 
        """
 
        Mark caches of this repo invalid in the database.
 

	
 
        :param repo_name: the repo for which caches should be marked invalid
 
        """
 
        CacheInvalidation.set_invalidate(repo_name, delete=delete)
 
        CacheInvalidation.set_invalidate(repo_name)
 
        repo = Repository.get_by_repo_name(repo_name)
 
        if repo is not None:
 
            repo.update_changeset_cache()
 

	
 
    def toggle_following_repo(self, follow_repo_id, user_id):
 

	
 
        f = self.sa.query(UserFollowing)\
 
            .filter(UserFollowing.follows_repo_id == follow_repo_id)\
 
            .filter(UserFollowing.user_id == user_id).scalar()
 

	
 
        if f is not None:
 
            try:
 
                self.sa.delete(f)
 
                action_logger(UserTemp(user_id),
 
                              'stopped_following_repo',
 
                              RepoTemp(follow_repo_id))
 
                return
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                raise
 

	
 
        try:
 
            f = UserFollowing()
 
            f.user_id = user_id
kallithea/tests/other/manual_test_vcs_operations.py
Show inline comments
 
@@ -252,73 +252,72 @@ class TestVCSOperations(BaseTestCase):
 

	
 
        # commit some stuff into this repo
 
        stdout, stderr = _add_files_and_push('git', DEST)
 

	
 
        print [(x.repo_full_path,x.repo_path) for x in Repository.get_all()]
 
        _check_proper_git_push(stdout, stderr)
 

	
 
    def test_push_invalidates_cache_hg(self):
 
        key = CacheInvalidation.query().filter(CacheInvalidation.cache_key
 
                                               ==HG_REPO).scalar()
 
        if not key:
 
            key = CacheInvalidation(HG_REPO, HG_REPO)
 

	
 
        key.cache_active = True
 
        Session().add(key)
 
        Session().commit()
 

	
 
        DEST = _get_tmp_dir()
 
        clone_url = _construct_url(HG_REPO, dest=DEST)
 
        stdout, stderr = Command('/tmp').execute('hg clone', clone_url)
 

	
 
        stdout, stderr = _add_files_and_push('hg', DEST, files_no=1)
 

	
 
        key = CacheInvalidation.query().filter(CacheInvalidation.cache_key
 
                                               ==HG_REPO).one()
 
        self.assertEqual(key.cache_active, False)
 
                                               ==HG_REPO).all()
 
        self.assertEqual(key, [])
 

	
 
    def test_push_invalidates_cache_git(self):
 
        key = CacheInvalidation.query().filter(CacheInvalidation.cache_key
 
                                               ==GIT_REPO).scalar()
 
        if not key:
 
            key = CacheInvalidation(GIT_REPO, GIT_REPO)
 

	
 
        key.cache_active = True
 
        Session().add(key)
 
        Session().commit()
 

	
 
        DEST = _get_tmp_dir()
 
        clone_url = _construct_url(GIT_REPO, dest=DEST)
 
        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
 

	
 
        # commit some stuff into this repo
 
        stdout, stderr = _add_files_and_push('git', DEST, files_no=1)
 
        _check_proper_git_push(stdout, stderr)
 

	
 
        key = CacheInvalidation.query().filter(CacheInvalidation.cache_key
 
                                               ==GIT_REPO).one()
 
        print CacheInvalidation.get_all()
 
        self.assertEqual(key.cache_active, False)
 
                                               ==GIT_REPO).all()
 
        self.assertEqual(key, [])
 

	
 
    def test_push_wrong_credentials_hg(self):
 
        DEST = _get_tmp_dir()
 
        clone_url = _construct_url(HG_REPO, dest=DEST)
 
        stdout, stderr = Command('/tmp').execute('hg clone', clone_url)
 

	
 
        stdout, stderr = _add_files_and_push('hg', DEST, user='bad',
 
                                             passwd='name')
 

	
 
        assert 'abort: authorization failed' in stderr
 

	
 
    def test_push_wrong_credentials_git(self):
 
        DEST = _get_tmp_dir()
 
        clone_url = _construct_url(GIT_REPO, dest=DEST)
 
        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
 

	
 
        stdout, stderr = _add_files_and_push('git', DEST, user='bad',
 
                                             passwd='name')
 

	
 
        assert 'fatal: Authentication failed' in stderr
 

	
 
    def test_push_back_to_wrong_url_hg(self):
 
        DEST = _get_tmp_dir()
 
        clone_url = _construct_url(HG_REPO, dest=DEST)
0 comments (0 inline, 0 general)