Changeset - 6843cabe9925
[Not reviewed]
beta
0 5 0
Marcin Kuzminski - 13 years ago 2013-04-03 02:35:01
marcin@python-works.com
removed duplicated logic of how we invalidate caches for repos
5 files changed with 10 insertions and 24 deletions:
0 comments (0 inline, 0 general)
rhodecode/controllers/admin/repos.py
Show inline comments
 
@@ -19,49 +19,49 @@
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import logging
 
import traceback
 
import formencode
 
from formencode import htmlfill
 

	
 
from webob.exc import HTTPInternalServerError, HTTPForbidden
 
from pylons import request, session, tmpl_context as c, url
 
from pylons.controllers.util import redirect
 
from pylons.i18n.translation import _
 
from sqlalchemy.exc import IntegrityError
 

	
 
import rhodecode
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
 
    HasPermissionAnyDecorator, HasRepoPermissionAllDecorator, NotAnonymous,\
 
    HasPermissionAny, HasReposGroupPermissionAny, HasRepoPermissionAnyDecorator
 
from rhodecode.lib.base import BaseRepoController, render
 
from rhodecode.lib.utils import invalidate_cache, action_logger, repo_name_slug
 
from rhodecode.lib.utils import action_logger, repo_name_slug
 
from rhodecode.lib.helpers import get_token
 
from rhodecode.model.meta import Session
 
from rhodecode.model.db import User, Repository, UserFollowing, RepoGroup,\
 
    RhodeCodeSetting, RepositoryField
 
from rhodecode.model.forms import RepoForm, RepoFieldForm, RepoPermsForm
 
from rhodecode.model.scm import ScmModel, GroupList
 
from rhodecode.model.repo import RepoModel
 
from rhodecode.lib.compat import json
 
from sqlalchemy.sql.expression import func
 
from rhodecode.lib.exceptions import AttachedForksError
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class ReposController(BaseRepoController):
 
    """
 
    REST Controller styled on the Atom Publishing Protocol"""
 
    # To properly map this controller, ensure your config/routing.py
 
    # file has a resource setup:
 
    #     map.resource('repo', 'repos')
 

	
 
    @LoginRequired()
 
    def __before__(self):
 
        c.admin_user = session.get('admin_user')
 
@@ -241,49 +241,49 @@ class ReposController(BaseRepoController
 
    def update(self, repo_name):
 
        """
 
        PUT /repos/repo_name: Update an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="PUT" />
 
        # Or using helpers:
 
        #    h.form(url('repo', repo_name=ID),
 
        #           method='put')
 
        # url('repo', repo_name=ID)
 
        self.__load_defaults()
 
        repo_model = RepoModel()
 
        changed_name = repo_name
 
        #override the choices with extracted revisions !
 
        choices, c.landing_revs = ScmModel().get_repo_landing_revs(repo_name)
 
        c.landing_revs_choices = choices
 
        repo = Repository.get_by_repo_name(repo_name)
 
        _form = RepoForm(edit=True, old_data={'repo_name': repo_name,
 
                                              'repo_group': repo.group.get_dict() \
 
                                              if repo.group else {}},
 
                         repo_groups=c.repo_groups_choices,
 
                         landing_revs=c.landing_revs_choices)()
 
        try:
 
            form_result = _form.to_python(dict(request.POST))
 
            repo = repo_model.update(repo_name, **form_result)
 
            invalidate_cache('get_repo_cached_%s' % repo_name)
 
            ScmModel().mark_for_invalidation(repo_name)
 
            h.flash(_('Repository %s updated successfully') % repo_name,
 
                    category='success')
 
            changed_name = repo.repo_name
 
            action_logger(self.rhodecode_user, 'admin_updated_repo',
 
                              changed_name, self.ip_addr, self.sa)
 
            Session().commit()
 
        except formencode.Invalid, errors:
 
            defaults = self.__load_data(repo_name)
 
            defaults.update(errors.value)
 
            return htmlfill.render(
 
                render('admin/repos/repo_edit.html'),
 
                defaults=defaults,
 
                errors=errors.error_dict or {},
 
                prefix_error=False,
 
                encoding="UTF-8")
 

	
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            h.flash(_('Error occurred during update of repository %s') \
 
                    % repo_name, category='error')
 
        return redirect(url('edit_repo', repo_name=changed_name))
 

	
 
    @HasRepoPermissionAllDecorator('repository.admin')
 
    def delete(self, repo_name):
 
@@ -294,49 +294,49 @@ class ReposController(BaseRepoController
 
        # Or using helpers:
 
        #    h.form(url('repo', repo_name=ID),
 
        #           method='delete')
 
        # url('repo', repo_name=ID)
 

	
 
        repo_model = RepoModel()
 
        repo = repo_model.get_by_repo_name(repo_name)
 
        if not repo:
 
            h.not_mapped_error(repo_name)
 
            return redirect(url('repos'))
 
        try:
 
            _forks = repo.forks.count()
 
            handle_forks = None
 
            if _forks and request.POST.get('forks'):
 
                do = request.POST['forks']
 
                if do == 'detach_forks':
 
                    handle_forks = 'detach'
 
                    h.flash(_('Detached %s forks') % _forks, category='success')
 
                elif do == 'delete_forks':
 
                    handle_forks = 'delete'
 
                    h.flash(_('Deleted %s forks') % _forks, category='success')
 
            repo_model.delete(repo, forks=handle_forks)
 
            action_logger(self.rhodecode_user, 'admin_deleted_repo',
 
                  repo_name, self.ip_addr, self.sa)
 
            invalidate_cache('get_repo_cached_%s' % repo_name)
 
            ScmModel().mark_for_invalidation(repo_name)
 
            h.flash(_('Deleted repository %s') % repo_name, category='success')
 
            Session().commit()
 
        except AttachedForksError:
 
            h.flash(_('Cannot delete %s it still contains attached forks')
 
                        % repo_name, category='warning')
 

	
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            h.flash(_('An error occurred during deletion of %s') % repo_name,
 
                    category='error')
 

	
 
        return redirect(url('repos'))
 

	
 
    @HasRepoPermissionAllDecorator('repository.admin')
 
    def set_repo_perm_member(self, repo_name):
 
        form = RepoPermsForm()().to_python(request.POST)
 

	
 
        perms_new = form['perms_new']
 
        perms_updates = form['perms_updates']
 
        cur_repo = repo_name
 

	
 
        # update permissions
 
        for member, perm, member_type in perms_updates:
 
            if member_type == 'user':
rhodecode/controllers/admin/settings.py
Show inline comments
 
@@ -20,63 +20,62 @@
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import logging
 
import traceback
 
import formencode
 
import pkg_resources
 
import platform
 

	
 
from sqlalchemy import func
 
from formencode import htmlfill
 
from pylons import request, session, tmpl_context as c, url, config
 
from pylons.controllers.util import abort, redirect
 
from pylons.i18n.translation import _
 

	
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
 
    HasPermissionAnyDecorator, NotAnonymous, HasPermissionAny,\
 
    HasReposGroupPermissionAll, HasReposGroupPermissionAny, AuthUser
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.lib.celerylib import tasks, run_task
 
from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \
 
    set_rhodecode_config, repo_name_slug, check_git_version
 
from rhodecode.lib.utils import repo2db_mapper, set_rhodecode_config, \
 
    check_git_version
 
from rhodecode.model.db import RhodeCodeUi, Repository, RepoGroup, \
 
    RhodeCodeSetting, PullRequest, PullRequestReviewers
 
from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \
 
    ApplicationUiSettingsForm, ApplicationVisualisationForm
 
from rhodecode.model.scm import ScmModel, GroupList
 
from rhodecode.model.user import UserModel
 
from rhodecode.model.repo import RepoModel
 
from rhodecode.model.db import User
 
from rhodecode.model.notification import EmailNotificationModel
 
from rhodecode.model.meta import Session
 
from rhodecode.lib.utils2 import str2bool, safe_unicode
 
from rhodecode.lib.compat import json
 
from webob.exc import HTTPForbidden
 
log = logging.getLogger(__name__)
 

	
 

	
 
class SettingsController(BaseController):
 
    """REST Controller styled on the Atom Publishing Protocol"""
 
    # To properly map this controller, ensure your config/routing.py
 
    # file has a resource setup:
 
    #     map.resource('setting', 'settings', controller='admin/settings',
 
    #         path_prefix='/admin', name_prefix='admin_')
 

	
 
    @LoginRequired()
 
    def __before__(self):
 
        c.admin_user = session.get('admin_user')
 
        c.admin_username = session.get('admin_username')
 
        c.modules = sorted([(p.project_name, p.version)
 
                            for p in pkg_resources.working_set]
 
                           + [('git', check_git_version())],
 
                           key=lambda k: k[0].lower())
 
        c.py_version = platform.python_version()
 
        c.platform = platform.platform()
 
        super(SettingsController, self).__before__()
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def index(self, format='html'):
 
@@ -98,49 +97,49 @@ class SettingsController(BaseController)
 
        """POST /admin/settings: Create a new item"""
 
        # url('admin_settings')
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def new(self, format='html'):
 
        """GET /admin/settings/new: Form to create a new item"""
 
        # url('admin_new_setting')
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def update(self, setting_id):
 
        """PUT /admin/settings/setting_id: Update an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="PUT" />
 
        # Or using helpers:
 
        #    h.form(url('admin_setting', setting_id=ID),
 
        #           method='put')
 
        # url('admin_setting', setting_id=ID)
 

	
 
        if setting_id == 'mapping':
 
            rm_obsolete = request.POST.get('destroy', False)
 
            log.debug('Rescanning directories with destroy=%s' % rm_obsolete)
 
            initial = ScmModel().repo_scan()
 
            log.debug('invalidating all repositories')
 
            for repo_name in initial.keys():
 
                invalidate_cache('get_repo_cached_%s' % repo_name)
 
                ScmModel().mark_for_invalidation(repo_name)
 

	
 
            added, removed = repo2db_mapper(initial, rm_obsolete)
 
            _repr = lambda l: ', '.join(map(safe_unicode, l)) or '-'
 
            h.flash(_('Repositories successfully '
 
                      'rescanned added: %s ; removed: %s') %
 
                    (_repr(added), _repr(removed)),
 
                    category='success')
 

	
 
        if setting_id == 'whoosh':
 
            repo_location = self._get_hg_ui_settings()['paths_root_path']
 
            full_index = request.POST.get('full_index', False)
 
            run_task(tasks.whoosh_index, repo_location, full_index)
 
            h.flash(_('Whoosh reindex task scheduled'), category='success')
 

	
 
        if setting_id == 'global':
 

	
 
            application_form = ApplicationSettingsForm()()
 
            try:
 
                form_result = application_form.to_python(dict(request.POST))
 
            except formencode.Invalid, errors:
 
                return htmlfill.render(
 
                     render('admin/settings/settings.html'),
 
                     defaults=errors.value,
 
                     errors=errors.error_dict or {},
rhodecode/lib/base.py
Show inline comments
 
"""The base Controller API
 

	
 
Provides the BaseController class for subclassing.
 
"""
 
import logging
 
import time
 
import traceback
 

	
 
from paste.auth.basic import AuthBasicAuthenticator
 
from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden
 
from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 

	
 
from pylons import config, tmpl_context as c, request, session, url
 
from pylons.controllers import WSGIController
 
from pylons.controllers.util import redirect
 
from pylons.templating import render_mako as render
 

	
 
from rhodecode import __version__, BACKENDS
 

	
 
from rhodecode.lib.utils2 import str2bool, safe_unicode, AttributeDict,\
 
    safe_str, safe_int
 
from rhodecode.lib.auth import AuthUser, get_container_username, authfunc,\
 
    HasPermissionAnyMiddleware, CookieStoreWrapper
 
from rhodecode.lib.utils import get_repo_slug, invalidate_cache
 
from rhodecode.lib.utils import get_repo_slug
 
from rhodecode.model import meta
 

	
 
from rhodecode.model.db import Repository, RhodeCodeUi, User, RhodeCodeSetting
 
from rhodecode.model.notification import NotificationModel
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.model.meta import Session
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def _filter_proxy(ip):
 
    """
 
    HEADERS can have mutliple ips inside the left-most being the original
 
    client, and each successive proxy that passed the request adding the IP
 
    address where it received the request from.
 

	
 
    :param ip:
 
    """
 
    if ',' in ip:
 
        _ips = ip.split(',')
 
        _first_ip = _ips[0].strip()
 
        log.debug('Got multiple IPs %s, using %s' % (','.join(_ips), _first_ip))
 
        return _first_ip
 
    return ip
 
@@ -128,49 +128,49 @@ class BaseVCSController(object):
 

	
 
        :param repo_name:
 
        """
 
        try:
 
            data = repo_name.split('/')
 
            if len(data) >= 2:
 
                by_id = data[1].split('_')
 
                if len(by_id) == 2 and by_id[1].isdigit():
 
                    _repo_name = Repository.get(by_id[1]).repo_name
 
                    data[1] = _repo_name
 
        except Exception:
 
            log.debug('Failed to extract repo_name from id %s' % (
 
                      traceback.format_exc()
 
                      )
 
            )
 

	
 
        return '/'.join(data)
 

	
 
    def _invalidate_cache(self, repo_name):
 
        """
 
        Set's cache for this repository for invalidation on next access
 

	
 
        :param repo_name: full repo name, also a cache key
 
        """
 
        invalidate_cache('get_repo_cached_%s' % repo_name)
 
        ScmModel().mark_for_invalidation(repo_name)
 

	
 
    def _check_permission(self, action, user, repo_name, ip_addr=None):
 
        """
 
        Checks permissions using action (push/pull) user and repository
 
        name
 

	
 
        :param action: push or pull action
 
        :param user: user instance
 
        :param repo_name: repository name
 
        """
 
        #check IP
 
        authuser = AuthUser(user_id=user.user_id, ip_addr=ip_addr)
 
        if not authuser.ip_allowed:
 
            return False
 
        else:
 
            log.info('Access for IP:%s allowed' % (ip_addr))
 
        if action == 'push':
 
            if not HasPermissionAnyMiddleware('repository.write',
 
                                              'repository.admin')(user,
 
                                                                  repo_name):
 
                return False
 

	
 
        else:
 
            #any other action need at least read permission
rhodecode/lib/utils.py
Show inline comments
 
@@ -335,61 +335,48 @@ def make_ui(read_from='file', path=None,
 
                baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
 
                                 safe_str(ui_.ui_value))
 
            if ui_.ui_key == 'push_ssl':
 
                # force set push_ssl requirement to False, rhodecode
 
                # handles that
 
                baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
 
                                 False)
 
        if clear_session:
 
            meta.Session.remove()
 
    return baseui
 

	
 

	
 
def set_rhodecode_config(config):
 
    """
 
    Updates pylons config with new settings from database
 

	
 
    :param config:
 
    """
 
    hgsettings = RhodeCodeSetting.get_app_settings()
 

	
 
    for k, v in hgsettings.items():
 
        config[k] = v
 

	
 

	
 
def invalidate_cache(cache_key, *args):
 
    """
 
    Puts cache invalidation task into db for
 
    further global cache invalidation
 
    """
 

	
 
    from rhodecode.model.scm import ScmModel
 

	
 
    if cache_key.startswith('get_repo_cached_'):
 
        name = cache_key.split('get_repo_cached_')[-1]
 
        ScmModel().mark_for_invalidation(name)
 

	
 

	
 
def map_groups(path):
 
    """
 
    Given a full path to a repository, create all nested groups that this
 
    repo is inside. This function creates parent-child relationships between
 
    groups and creates default perms for all new groups.
 

	
 
    :param paths: full path to repository
 
    """
 
    sa = meta.Session()
 
    groups = path.split(Repository.url_sep())
 
    parent = None
 
    group = None
 

	
 
    # last element is repo in nested groups structure
 
    groups = groups[:-1]
 
    rgm = ReposGroupModel(sa)
 
    for lvl, group_name in enumerate(groups):
 
        group_name = '/'.join(groups[:lvl] + [group_name])
 
        group = RepoGroup.get_by_group_name(group_name)
 
        desc = '%s group' % group_name
 

	
 
        # skip folders that are now removed repos
 
        if REMOVED_REPO_PAT.match(group_name):
 
            break
rhodecode/tests/functional/test_summary.py
Show inline comments
 
from rhodecode.tests import *
 
from rhodecode.tests.fixture import Fixture
 
from rhodecode.model.db import Repository
 
from rhodecode.lib.utils import invalidate_cache
 
from rhodecode.model.repo import RepoModel
 
from rhodecode.model.meta import Session
 
from rhodecode.model.scm import ScmModel
 

	
 
fixture = Fixture()
 

	
 

	
 
class TestSummaryController(TestController):
 

	
 
    def test_index(self):
 
        self.log_user()
 
        ID = Repository.get_by_repo_name(HG_REPO).repo_id
 
        response = self.app.get(url(controller='summary',
 
                                    action='index',
 
                                    repo_name=HG_REPO))
 

	
 
        #repo type
 
        response.mustcontain(
 
            """<img style="margin-bottom:2px" class="icon" """
 
            """title="Mercurial repository" alt="Mercurial repository" """
 
            """src="/images/icons/hgicon.png"/>"""
 
        )
 
        response.mustcontain(
 
            """<img style="margin-bottom:2px" class="icon" """
 
            """title="Public repository" alt="Public """
 
            """repository" src="/images/icons/lock_open.png"/>"""
 
        )
 

	
 
        #codes stats
 
        self._enable_stats()
 

	
 
        invalidate_cache('get_repo_cached_%s' % HG_REPO)
 
        ScmModel().mark_for_invalidation(HG_REPO)
 
        response = self.app.get(url(controller='summary', action='index',
 
                                    repo_name=HG_REPO))
 
        response.mustcontain(
 
            """var data = [["py", {"count": 42, "desc": ["Python"]}], """
 
            """["rst", {"count": 11, "desc": ["Rst"]}], """
 
            """["sh", {"count": 2, "desc": ["Bash"]}], """
 
            """["makefile", {"count": 1, "desc": ["Makefile", "Makefile"]}],"""
 
            """ ["cfg", {"count": 1, "desc": ["Ini"]}], """
 
            """["css", {"count": 1, "desc": ["Css"]}], """
 
            """["bat", {"count": 1, "desc": ["Batch"]}]];"""
 
        )
 

	
 
        # clone url...
 
        response.mustcontain('''id="clone_url" readonly="readonly" value="http://test_admin@localhost:80/%s"''' % HG_REPO)
 
        response.mustcontain('''id="clone_url_id" readonly="readonly" value="http://test_admin@localhost:80/_%s"''' % ID)
 

	
 
    def test_index_git(self):
 
        self.log_user()
 
        ID = Repository.get_by_repo_name(GIT_REPO).repo_id
 
        response = self.app.get(url(controller='summary',
 
                                    action='index',
 
                                    repo_name=GIT_REPO))
 

	
 
        #repo type
0 comments (0 inline, 0 general)