Changeset - cb0d9ce6ac5c
[Not reviewed]
beta
1 11 1
Marcin Kuzminski - 15 years ago 2010-11-15 02:26:19
marcin@python-works.com
#50 on point cache invalidation changes.
Created cacheInvalidation table
cleaned up sa sessions from models, since it wasn't really needed.
12 files changed with 122 insertions and 53 deletions:
0 comments (0 inline, 0 general)
rhodecode/controllers/admin/permissions.py
Show inline comments
 
@@ -10,49 +10,49 @@
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 27, 2010
 
permissions controller for pylons
 
@author: marcink
 
"""
 

	
 
from formencode import htmlfill
 
from pylons import request, session, tmpl_context as c, url
 
from pylons.controllers.util import abort, redirect
 
from pylons.i18n.translation import _
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.model.forms import UserForm, DefaultPermissionsForm
 
from rhodecode.model.permission_model import PermissionModel
 
from rhodecode.model.permission import PermissionModel
 
from rhodecode.model.user import UserModel
 
import formencode
 
import logging
 
import traceback
 

	
 
log = logging.getLogger(__name__)
 

	
 
class PermissionsController(BaseController):
 
    """REST Controller styled on the Atom Publishing Protocol"""
 
    # To properly map this controller, ensure your config/routing.py
 
    # file has a resource setup:
 
    #     map.resource('permission', 'permissions')
 

	
 
    @LoginRequired()
 
    @HasPermissionAllDecorator('hg.admin')
 
    def __before__(self):
 
        c.admin_user = session.get('admin_user')
 
        c.admin_username = session.get('admin_username')
 
        super(PermissionsController, self).__before__()
 

	
 
        self.perms_choices = [('repository.none', _('None'),),
 
                              ('repository.read', _('Read'),),
 
                              ('repository.write', _('Write'),),
 
                              ('repository.admin', _('Admin'),)]
rhodecode/controllers/admin/repos.py
Show inline comments
 
@@ -112,68 +112,68 @@ class ReposController(BaseController):
 
    def new(self, format='html'):
 
        """GET /repos/new: Form to create a new item"""
 
        new_repo = request.GET.get('repo', '')
 
        c.new_repo = h.repo_name_slug(new_repo)
 

	
 
        return render('admin/repos/repo_add.html')
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def update(self, repo_name):
 
        """PUT /repos/repo_name: Update an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="PUT" />
 
        # Or using helpers:
 
        #    h.form(url('repo', repo_name=ID),
 
        #           method='put')
 
        # url('repo', repo_name=ID)
 
        repo_model = RepoModel()
 
        changed_name = repo_name
 
        _form = RepoForm(edit=True, old_data={'repo_name':repo_name})()
 

	
 
        try:
 
            form_result = _form.to_python(dict(request.POST))
 
            repo_model.update(repo_name, form_result)
 
            invalidate_cache('get_repo_cached_%s' % repo_name)
 
            h.flash(_('Repository %s updated succesfully' % repo_name),
 
            h.flash(_('Repository %s updated successfully' % repo_name),
 
                    category='success')
 
            changed_name = form_result['repo_name']
 
            action_logger(self.rhodecode_user, 'admin_updated_repo',
 
                              changed_name, '', self.sa)
 

	
 
        except formencode.Invalid, errors:
 
            c.repo_info = repo_model.get(repo_name)
 
            c.users_array = repo_model.get_users_js()
 
            errors.value.update({'user':c.repo_info.user.username})
 
            return htmlfill.render(
 
                render('admin/repos/repo_edit.html'),
 
                defaults=errors.value,
 
                errors=errors.error_dict or {},
 
                prefix_error=False,
 
                encoding="UTF-8")
 

	
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            h.flash(_('error occured during update of repository %s') \
 
            h.flash(_('error occurred during update of repository %s') \
 
                    % repo_name, category='error')
 

	
 
        return redirect(url('edit_repo', repo_name=changed_name))
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def delete(self, repo_name):
 
        """DELETE /repos/repo_name: Delete an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="DELETE" />
 
        # Or using helpers:
 
        #    h.form(url('repo', repo_name=ID),
 
        #           method='delete')
 
        # url('repo', repo_name=ID)
 

	
 
        repo_model = RepoModel()
 
        repo = repo_model.get(repo_name)
 
        if not repo:
 
            h.flash(_('%s repository is not mapped to db perhaps'
 
                      ' it was moved or renamed  from the filesystem'
 
                      ' please run the application again'
 
                      ' in order to rescan repositories') % repo_name,
 
                      category='error')
 

	
 
            return redirect(url('repos'))
rhodecode/controllers/admin/settings.py
Show inline comments
 
@@ -227,90 +227,90 @@ class SettingsController(BaseController)
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="DELETE" />
 
        # Or using helpers:
 
        #    h.form(url('admin_setting', setting_id=ID),
 
        #           method='delete')
 
        # url('admin_setting', setting_id=ID)
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def show(self, setting_id, format='html'):
 
        """GET /admin/settings/setting_id: Show a specific item"""
 
        # url('admin_setting', setting_id=ID)
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def edit(self, setting_id, format='html'):
 
        """GET /admin/settings/setting_id/edit: Form to edit an existing item"""
 
        # url('admin_edit_setting', setting_id=ID)
 

	
 

	
 
    def my_account(self):
 
        """
 
        GET /_admin/my_account Displays info about my account 
 
        """
 

	
 
        # url('admin_settings_my_account')
 
        c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
 
        c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
 
        all_repos = self.sa.query(Repository)\
 
            .filter(Repository.user_id == c.user.user_id)\
 
            .order_by(func.lower(Repository.repo_name))\
 
            .all()
 
        c.user_repos = ScmModel().get_repos(all_repos)
 

	
 
        if c.user.username == 'default':
 
            h.flash(_("You can't edit this user since it's"
 
              " crucial for entire application"), category='warning')
 
            return redirect(url('users'))
 

	
 
        defaults = c.user.__dict__
 
        return htmlfill.render(
 
            render('admin/users/user_edit_my_account.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False
 
        )
 

	
 
    def my_account_update(self):
 
        """PUT /_admin/my_account_update: Update an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="PUT" />
 
        # Or using helpers:
 
        #    h.form(url('admin_settings_my_account_update'),
 
        #           method='put')
 
        # url('admin_settings_my_account_update', id=ID)
 
        user_model = UserModel()
 
        uid = c.rhodecode_user.user_id
 
        _form = UserForm(edit=True, old_data={'user_id':uid,
 
                                              'email':c.rhodecode_user.email})()
 
        form_result = {}
 
        try:
 
            form_result = _form.to_python(dict(request.POST))
 
            user_model.update_my_account(uid, form_result)
 
            h.flash(_('Your account was updated succesfully'),
 
                    category='success')
 

	
 
        except formencode.Invalid, errors:
 
            c.user = user_model.get(c.rhodecode_user.user_id, cache=False)
 
            c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False)
 
            c.user = UserModel().get(c.rhodecode_user.user_id, cache=False)
 
            all_repos = self.sa.query(Repository)\
 
                .filter(Repository.user_id == c.user.user_id)\
 
                .order_by(func.lower(Repository.repo_name))\
 
                .all()
 
            c.user_repos = ScmModel().get_repos(all_repos)
 

	
 
            return htmlfill.render(
 
                render('admin/users/user_edit_my_account.html'),
 
                defaults=errors.value,
 
                errors=errors.error_dict or {},
 
                prefix_error=False,
 
                encoding="UTF-8")
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            h.flash(_('error occured during update of user %s') \
 
                    % form_result.get('username'), category='error')
 

	
 
        return redirect(url('my_account'))
 

	
 
    @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
 
    def create_repository(self):
 
        """GET /_admin/create_repository: Form to create a new item"""
 
        new_repo = request.GET.get('repo', '')
 
        c.new_repo = h.repo_name_slug(new_repo)
rhodecode/controllers/admin/users.py
Show inline comments
 
#!/usr/bin/env python
 
# encoding: utf-8
 
# users controller for pylons
 
# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
 
# 
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
from rhodecode.lib.utils import action_logger
 
"""
 
Created on April 4, 2010
 
users controller for pylons
 
@author: marcink
 
"""
 

	
 
from formencode import htmlfill
 
from pylons import request, session, tmpl_context as c, url
 
from pylons.controllers.util import abort, redirect
 
from pylons.i18n.translation import _
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.model.db import User, UserLog
 
from rhodecode.model.forms import UserForm
 
from rhodecode.model.user import UserModel, DefaultUserException
 
import formencode
 
import logging
 
import traceback
 

	
 
log = logging.getLogger(__name__)
 

	
 
class UsersController(BaseController):
 
    """REST Controller styled on the Atom Publishing Protocol"""
rhodecode/lib/auth.py
Show inline comments
 
@@ -122,49 +122,49 @@ def set_available_permissions(config):
 
    except:
 
        pass
 
    finally:
 
        meta.Session.remove()
 

	
 
    config['available_permissions'] = [x.permission_name for x in all_perms]
 

	
 
def set_base_path(config):
 
    config['base_path'] = config['pylons.app_globals'].base_path
 

	
 

	
 
def fill_perms(user):
 
    """
 
    Fills user permission attribute with permissions taken from database
 
    :param user:
 
    """
 

	
 
    sa = meta.Session()
 
    user.permissions['repositories'] = {}
 
    user.permissions['global'] = set()
 

	
 
    #===========================================================================
 
    # fetch default permissions
 
    #===========================================================================
 
    default_user = UserModel(sa).get_by_username('default', cache=True)
 
    default_user = UserModel().get_by_username('default', cache=True)
 

	
 
    default_perms = sa.query(RepoToPerm, Repository, Permission)\
 
        .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
 
        .join((Permission, RepoToPerm.permission_id == Permission.permission_id))\
 
        .filter(RepoToPerm.user == default_user).all()
 

	
 
    if user.is_admin:
 
        #=======================================================================
 
        # #admin have all default rights set to admin        
 
        #=======================================================================
 
        user.permissions['global'].add('hg.admin')
 

	
 
        for perm in default_perms:
 
            p = 'repository.admin'
 
            user.permissions['repositories'][perm.RepoToPerm.repository.repo_name] = p
 

	
 
    else:
 
        #=======================================================================
 
        # set default permissions
 
        #=======================================================================
 

	
 
        #default global
 
        default_global_perms = sa.query(UserToPerm)\
 
            .filter(UserToPerm.user == sa.query(User).filter(User.username ==
rhodecode/lib/celerylib/tasks.py
Show inline comments
 
from celery.decorators import task
 

	
 
import os
 
import traceback
 
from time import mktime
 

	
 
from operator import itemgetter
 
from pylons.i18n.translation import _
 
from rhodecode.lib.celerylib import run_task, locked_task
 
from rhodecode.lib.helpers import person
 
from rhodecode.lib.smtp_mailer import SmtpMailer
 
from rhodecode.lib.utils import OrderedDict
 
from time import mktime
 
import os
 
import traceback
 
from vcs.backends import get_repo
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.model.db import RhodeCodeUi
 

	
 
try:
 
    import json
 
except ImportError:
 
    #python 2.5 compatibility
 
    import simplejson as json
 

	
 
try:
 
    from celeryconfig import PYLONS_CONFIG as config
 
    celery_on = True
 
except ImportError:
 
    #if celeryconfig is not present let's just load our pylons
 
    #config instead
 
    from pylons import config
 
    celery_on = False
 

	
 

	
 
__all__ = ['whoosh_index', 'get_commits_stats',
 
           'reset_user_password', 'send_email']
 

	
 
def get_session():
 
    if celery_on:
 
        from sqlalchemy import engine_from_config
 
        from sqlalchemy.orm import sessionmaker, scoped_session
 
        engine = engine_from_config(dict(config.items('app:main')),
 
                                    'sqlalchemy.db1.')
 
        sa = scoped_session(sessionmaker(bind=engine))
 
    else:
 
        #If we don't use celery reuse our current application Session
 
        from rhodecode.model.meta import Session
 
        sa = Session()
 

	
 
    return sa
 

	
 
def get_repos_path():
 
    sa = get_session()
 
    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
    return q.ui_value
 

	
 
@task
 
@locked_task
 
def whoosh_index(repo_location, full_index):
 
    log = whoosh_index.get_logger()
 
    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
 
    index_location = dict(config.items('app:main'))['index_dir']
 
    WhooshIndexingDaemon(index_location=index_location,
 
                         repo_location=repo_location).run(full_index=full_index)
 

	
 
@task
 
@locked_task
 
def get_commits_stats(repo_name, ts_min_y, ts_max_y):
 
    from rhodecode.model.db import Statistics, Repository
 
    log = get_commits_stats.get_logger()
 
    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
 

	
 
    commits_by_day_author_aggregate = {}
 
    commits_by_day_aggregate = {}
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    p = os.path.join(repos_path, repo_name)
 
    repo = get_repo(p)
 

	
 
    skip_date_limit = True
 
    parse_limit = 250 #limit for single task changeset parsing optimal for
 
    last_rev = 0
 
    last_cs = None
 
    timegetter = itemgetter('time')
 

	
 
    sa = get_session()
 

	
 
    dbrepo = sa.query(Repository)\
 
        .filter(Repository.repo_name == repo_name).scalar()
 
    cur_stats = sa.query(Statistics)\
 
        .filter(Statistics.repository == dbrepo).scalar()
 
    if cur_stats:
 
        last_rev = cur_stats.stat_on_revision
 
    if not repo.revisions:
 
        return True
 

	
 
    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
 
        #pass silently without any work if we're not on first revision or current
 
        #state of parsing revision(from db marker) is the last revision
 
        return True
 
@@ -250,66 +257,66 @@ def send_email(recipients, subject, body
 

	
 
    mail_from = email_config.get('app_email_from')
 
    user = email_config.get('smtp_username')
 
    passwd = email_config.get('smtp_password')
 
    mail_server = email_config.get('smtp_server')
 
    mail_port = email_config.get('smtp_port')
 
    tls = str2bool(email_config.get('smtp_use_tls'))
 
    ssl = str2bool(email_config.get('smtp_use_ssl'))
 

	
 
    try:
 
        m = SmtpMailer(mail_from, user, passwd, mail_server,
 
                       mail_port, ssl, tls)
 
        m.send(recipients, subject, body)
 
    except:
 
        log.error('Mail sending failed')
 
        log.error(traceback.format_exc())
 
        return False
 
    return True
 

	
 
@task
 
def create_repo_fork(form_data, cur_user):
 
    from rhodecode.model.repo import RepoModel
 
    from vcs import get_backend
 
    log = create_repo_fork.get_logger()
 
    repo_model = RepoModel(get_session())
 
    repo_model = RepoModel()
 
    repo_model.create(form_data, cur_user, just_db=True, fork=True)
 
    repo_name = form_data['repo_name']
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    repo_path = os.path.join(repos_path, repo_name)
 
    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
 
    alias = form_data['repo_type']
 

	
 
    log.info('creating repo fork %s as %s', repo_name, repo_path)
 
    backend = get_backend(alias)
 
    backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 

	
 
def __get_codes_stats(repo_name):
 
    LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx',
 
    'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el',
 
    'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml',
 
    'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst',
 
    's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 

	
 

	
 
    repos_path = ScmModel().repos_path
 
    repos_path = get_repos_path()
 
    p = os.path.join(repos_path, repo_name)
 
    repo = get_repo(p)
 
    tip = repo.get_changeset()
 
    code_stats = {}
 

	
 
    def aggregate(cs):
 
        for f in cs[2]:
 
            k = f.mimetype
 
            if f.extension in LANGUAGES_EXTENSIONS:
 
                if code_stats.has_key(k):
 
                    code_stats[k] += 1
 
                else:
 
                    code_stats[k] = 1
 

	
 
    map(aggregate, tip.walk('/'))
 

	
 
    return code_stats or {}
 

	
 

	
 

	
 

	
rhodecode/lib/utils.py
Show inline comments
 
@@ -68,57 +68,57 @@ def is_git(environ):
 
        return True
 
    return False
 

	
 
def action_logger(user, action, repo, ipaddr='', sa=None):
 
    """
 
    Action logger for various action made by users
 
    
 
    :param user: user that made this action, can be a string unique username or
 
        object containing user_id attribute
 
    :param action: action to log, should be on of predefined unique actions for
 
        easy translations
 
    :param repo: repository that action was made on
 
    :param ipaddr: optional ip address from what the action was made
 
    :param sa: optional sqlalchemy session
 
    
 
    """
 

	
 
    if not sa:
 
        sa = meta.Session()
 

	
 
    try:
 
        if hasattr(user, 'user_id'):
 
            user_obj = user
 
        elif isinstance(user, basestring):
 
            user_obj = UserModel(sa).get_by_username(user, cache=False)
 
            user_obj = UserModel().get_by_username(user, cache=False)
 
        else:
 
            raise Exception('You have to provide user object or username')
 

	
 

	
 
        if repo:
 
            repo_name = repo.lstrip('/')
 

	
 
            repository = RepoModel(sa).get(repo_name, cache=False)
 
            repository = RepoModel().get(repo_name, cache=False)
 
            if not repository:
 
                raise Exception('You have to provide valid repository')
 
        else:
 
            raise Exception('You have to provide repository to action logger')
 

	
 

	
 
        user_log = UserLog()
 
        user_log.user_id = user_obj.user_id
 
        user_log.action = action
 
        user_log.repository_name = repo_name
 
        user_log.repository = repository
 
        user_log.action_date = datetime.datetime.now()
 
        user_log.user_ip = ipaddr
 
        sa.add(user_log)
 
        sa.commit()
 

	
 
        log.info('Adding user %s, action %s on %s',
 
                                        user_obj.username, action, repo)
 
    except:
 
        log.error(traceback.format_exc())
 
        sa.rollback()
 

	
 
def get_repos(path, recursive=False, initial=False):
 
    """
 
@@ -272,96 +272,100 @@ def make_ui(read_from='file', path=None,
 
        cfg = config.config()
 
        cfg.read(path)
 
        for section in ui_sections:
 
            for k, v in cfg.items(section):
 
                baseui.setconfig(section, k, v)
 
                log.debug('settings ui from file[%s]%s:%s', section, k, v)
 

	
 
    elif read_from == 'db':
 
        hg_ui = get_hg_ui_cached()
 
        for ui_ in hg_ui:
 
            if ui_.ui_active:
 
                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
                baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 

	
 

	
 
    return baseui
 

	
 

	
 
def set_rhodecode_config(config):
 
    hgsettings = get_hg_settings()
 

	
 
    for k, v in hgsettings.items():
 
        config[k] = v
 

	
 
def invalidate_cache(name, *args):
 
def invalidate_cache(cache_key, *args):
 
    """
 
    Puts cache invalidation task into db for 
 
    further global cache invalidation
 
    """
 
    pass
 
    from rhodecode.model.scm import ScmModel
 

	
 
    if cache_key.startswith('get_repo_cached_'):
 
        name = cache_key.split('get_repo_cached_')[-1]
 
        ScmModel().mark_for_invalidation(name)
 

	
 
class EmptyChangeset(BaseChangeset):
 
    """
 
    An dummy empty changeset. It's possible to pass hash when creating
 
    an EmptyChangeset
 
    """
 

	
 
    def __init__(self, cs='0' * 40):
 
        self._empty_cs = cs
 
        self.revision = -1
 
        self.message = ''
 
        self.author = ''
 
        self.date = ''
 

	
 
    @LazyProperty
 
    def raw_id(self):
 
        """
 
        Returns raw string identifying this changeset, useful for web
 
        representation.
 
        """
 
        return self._empty_cs
 

	
 
    @LazyProperty
 
    def short_id(self):
 
        return self.raw_id[:12]
 

	
 
    def get_file_changeset(self, path):
 
        return self
 

	
 
    def get_file_content(self, path):
 
        return u''
 

	
 
    def get_file_size(self, path):
 
        return 0
 

	
 
def repo2db_mapper(initial_repo_list, remove_obsolete=False):
 
    """
 
    maps all found repositories into db
 
    """
 

	
 
    sa = meta.Session()
 
    rm = RepoModel(sa)
 
    rm = RepoModel()
 
    user = sa.query(User).filter(User.admin == True).first()
 

	
 
    for name, repo in initial_repo_list.items():
 
        if not rm.get(name, cache=False):
 
            log.info('repository %s not found creating default', name)
 

	
 
            form_data = {
 
                         'repo_name':name,
 
                         'repo_type':repo.alias,
 
                         'description':repo.description \
 
                            if repo.description != 'unknown' else \
 
                                        '%s repository' % name,
 
                         'private':False
 
                         }
 
            rm.create(form_data, user, just_db=True)
 

	
 
    if remove_obsolete:
 
        #remove from database those repositories that are not in the filesystem
 
        for repo in sa.query(Repository).all():
 
            if repo.repo_name not in initial_repo_list.keys():
 
                sa.delete(repo)
 
                sa.commit()
 

	
 
class OrderedDict(dict, DictMixin):
rhodecode/model/db.py
Show inline comments
 
@@ -69,81 +69,87 @@ class UserLog(Base):
 
    repository_name = Column("repository_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    user_ip = Column("user_ip", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    action = Column("action", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    action_date = Column("action_date", DATETIME(timezone=False), nullable=True, unique=None, default=None)
 

	
 
    user = relation('User')
 
    repository = relation('Repository')
 

	
 
class Repository(Base):
 
    __tablename__ = 'repositories'
 
    __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
 
    repo_id = Column("repo_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    repo_name = Column("repo_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
 
    repo_type = Column("repo_type", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
 
    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
 
    private = Column("private", BOOLEAN(), nullable=True, unique=None, default=None)
 
    description = Column("description", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    fork_id = Column("fork_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
 

	
 
    user = relation('User')
 
    fork = relation('Repository', remote_side=repo_id)
 
    repo_to_perm = relation('RepoToPerm', cascade='all')
 
    stats = relation('Statistics', cascade='all')
 

	
 

	
 
    def __repr__(self):
 
        return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
 
        return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name)
 

	
 
class Permission(Base):
 
    __tablename__ = 'permissions'
 
    __table_args__ = {'useexisting':True}
 
    permission_id = Column("permission_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    permission_name = Column("permission_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    permission_longname = Column("permission_longname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 

	
 
    def __repr__(self):
 
        return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 

	
 
class RepoToPerm(Base):
 
    __tablename__ = 'repo_to_perm'
 
    __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
 
    repo_to_perm_id = Column("repo_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
 
    permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
 
    repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
 

	
 
    user = relation('User')
 
    permission = relation('Permission')
 
    repository = relation('Repository')
 

	
 
class UserToPerm(Base):
 
    __tablename__ = 'user_to_perm'
 
    __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
 
    user_to_perm_id = Column("user_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
 
    permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
 

	
 
    user = relation('User')
 
    permission = relation('Permission')
 

	
 
class Statistics(Base):
 
    __tablename__ = 'statistics'
 
    __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
 
    stat_id = Column("stat_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
 
    stat_on_revision = Column("stat_on_revision", INTEGER(), nullable=False)
 
    commit_activity = Column("commit_activity", BLOB(), nullable=False)#JSON data
 
    commit_activity_combined = Column("commit_activity_combined", BLOB(), nullable=False)#JSON data
 
    languages = Column("languages", BLOB(), nullable=False)#JSON data
 

	
 
    repository = relation('Repository', single_parent=True)
 

	
 
class CacheInvalidation(Base):
 
    __tablename__ = 'cache_invalidation'
 
    __table_args__ = {'useexisting':True}
 
    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
 
    cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
 
    cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None)
 
    cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False)
 

	
 

	
 
    def __init__(self, cache_key, cache_args=''):
 
        self.cache_key = cache_key
 
        self.cache_args = cache_args
 
        self.cache_active = False
 

	
 
    def __repr__(self):
 
        return "<CacheInvaidation('%s:%s')>" % (self.cache_id, self.cache_key)
rhodecode/model/permission.py
Show inline comments
 
file renamed from rhodecode/model/permission_model.py to rhodecode/model/permission.py
 
@@ -12,53 +12,50 @@
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on Aug 20, 2010
 
Model for permissions
 
@author: marcink
 
"""
 

	
 
from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm
 
from rhodecode.model.caching_query import FromCache
 
from rhodecode.model.meta import Session
 
import logging
 
import traceback
 
log = logging.getLogger(__name__)
 

	
 

	
 
class PermissionModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get_permission(self, permission_id, cache=False):
 
        perm = self.sa.query(Permission)
 
        if cache:
 
            perm = perm.options(FromCache("sql_cache_short",
 
                                          "get_permission_%s" % permission_id))
 
        return perm.get(permission_id)
 

	
 
    def get_permission_by_name(self, name, cache=False):
 
        perm = self.sa.query(Permission)\
 
            .filter(Permission.permission_name == name)
 
        if cache:
 
            perm = perm.options(FromCache("sql_cache_short",
 
                                          "get_permission_%s" % name))
 
        return perm.scalar()
 

	
 
    def update(self, form_result):
 
        perm_user = self.sa.query(User)\
 
                .filter(User.username == form_result['perm_user_name']).scalar()
 
        u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all()
 
        if len(u2p) != 3:
 
            raise Exception('Defined: %s should be 3  permissions for default'
 
                            ' user. This should not happen please verify'
 
                            ' your database' % len(u2p))
rhodecode/model/repo.py
Show inline comments
 
@@ -15,93 +15,90 @@
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on Jun 5, 2010
 
model for handling repositories actions
 
:author: marcink
 
"""
 
from vcs.backends import get_repo, get_backend
 
from datetime import datetime
 
from pylons import app_globals as g
 
from rhodecode.model.db import Repository, RepoToPerm, User, Permission
 
from rhodecode.model.meta import Session
 
from rhodecode.model.user import UserModel
 
from rhodecode.model.caching_query import FromCache
 
import logging
 
import os
 
import shutil
 
import traceback
 
log = logging.getLogger(__name__)
 

	
 
class RepoModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get(self, repo_id, cache=False):
 
        repo = self.sa.query(Repository)\
 
            .filter(Repository.repo_name == repo_id)
 

	
 
        if cache:
 
            repo = repo.options(FromCache("sql_cache_short",
 
                                          "get_repo_%s" % repo))
 
        return repo.scalar()
 

	
 
    def get_users_js(self):
 

	
 
        users = self.sa.query(User).filter(User.active == True).all()
 
        u_tmpl = '''{id:%s, fname:"%s", lname:"%s", nname:"%s"},'''
 
        users_array = '[%s];' % '\n'.join([u_tmpl % (u.user_id, u.name,
 
                                                    u.lastname, u.username)
 
                                        for u in users])
 
        return users_array
 

	
 

	
 
    def update(self, repo_name, form_data):
 
        try:
 

	
 
            #update permissions
 
            for username, perm in form_data['perms_updates']:
 
                r2p = self.sa.query(RepoToPerm)\
 
                        .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\
 
                        .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\
 
                        .filter(RepoToPerm.repository == self.get(repo_name))\
 
                        .one()
 

	
 
                r2p.permission_id = self.sa.query(Permission).filter(
 
                                                Permission.permission_name ==
 
                                                perm).one().permission_id
 
                self.sa.add(r2p)
 

	
 
            #set new permissions
 
            for username, perm in form_data['perms_new']:
 
                r2p = RepoToPerm()
 
                r2p.repository = self.get(repo_name)
 
                r2p.user = UserModel(self.sa).get_by_username(username, cache=False)
 
                r2p.user = UserModel().get_by_username(username, cache=False)
 

	
 
                r2p.permission_id = self.sa.query(Permission).filter(
 
                                        Permission.permission_name == perm)\
 
                                        .one().permission_id
 
                self.sa.add(r2p)
 

	
 
            #update current repo
 
            cur_repo = self.get(repo_name, cache=False)
 

	
 
            for k, v in form_data.items():
 
                if k == 'user':
 
                    cur_repo.user_id = v
 
                else:
 
                    setattr(cur_repo, k, v)
 

	
 
            self.sa.add(cur_repo)
 

	
 
            if repo_name != form_data['repo_name']:
 
                #rename our data
 
                self.__rename_repo(repo_name, form_data['repo_name'])
 

	
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
@@ -113,61 +110,61 @@ class RepoModel(object):
 
            if fork:
 
                #force str since hg doesn't go with unicode
 
                repo_name = str(form_data['fork_name'])
 
                org_name = str(form_data['repo_name'])
 

	
 
            else:
 
                org_name = repo_name = str(form_data['repo_name'])
 
            new_repo = Repository()
 
            for k, v in form_data.items():
 
                if k == 'repo_name':
 
                    v = repo_name
 
                setattr(new_repo, k, v)
 

	
 
            if fork:
 
                parent_repo = self.sa.query(Repository)\
 
                        .filter(Repository.repo_name == org_name).scalar()
 
                new_repo.fork = parent_repo
 

	
 
            new_repo.user_id = cur_user.user_id
 
            self.sa.add(new_repo)
 

	
 
            #create default permission
 
            repo_to_perm = RepoToPerm()
 
            default = 'repository.read'
 
            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
 
            for p in UserModel().get_by_username('default', cache=False).user_perms:
 
                if p.permission.permission_name.startswith('repository.'):
 
                    default = p.permission.permission_name
 
                    break
 

	
 
            default_perm = 'repository.none' if form_data['private'] else default
 

	
 
            repo_to_perm.permission_id = self.sa.query(Permission)\
 
                    .filter(Permission.permission_name == default_perm)\
 
                    .one().permission_id
 

	
 
            repo_to_perm.repository_id = new_repo.repo_id
 
            repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id
 
            repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id
 

	
 
            self.sa.add(repo_to_perm)
 
            self.sa.commit()
 
            if not just_db:
 
                self.__create_repo(repo_name, form_data['repo_type'])
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 
            raise
 

	
 
    def create_fork(self, form_data, cur_user):
 
        from rhodecode.lib.celerylib import tasks, run_task
 
        run_task(tasks.create_repo_fork, form_data, cur_user)
 

	
 
    def delete(self, repo):
 
        try:
 
            self.sa.delete(repo)
 
            self.__delete_repo(repo)
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 
            raise
 

	
rhodecode/model/scm.py
Show inline comments
 
@@ -7,71 +7,70 @@
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 9, 2010
 
Model for RhodeCode
 
@author: marcink
 
"""
 
from beaker.cache import cache_region, region_invalidate
 
from mercurial import ui
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import HasRepoPermissionAny
 
from rhodecode.lib.utils import get_repos
 
from rhodecode.model import meta
 
from rhodecode.model.db import Repository, User, RhodeCodeUi
 
from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation
 
from rhodecode.model.caching_query import FromCache
 
from sqlalchemy.orm import joinedload
 
from sqlalchemy.orm.session import make_transient
 
from vcs import get_backend
 
from vcs.utils.helpers import get_scm
 
from vcs.exceptions import RepositoryError, VCSError
 
from vcs.utils.lazy import LazyProperty
 
import traceback
 
import logging
 
import os
 
import time
 

	
 
log = logging.getLogger(__name__)
 

	
 
class ScmModel(object):
 
    """
 
    Mercurial Model
 
    """
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = meta.Session()
 
        else:
 
            self.sa = sa
 

	
 
    def __init__(self):
 
        self.sa = meta.Session()
 

	
 
    @LazyProperty
 
    def repos_path(self):
 
        """
 
        Get's the repositories root path from database
 
        """
 
        q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 

	
 
        return q.ui_value
 

	
 
    def repo_scan(self, repos_path, baseui, initial=False):
 
        """
 
        Listing of repositories in given path. This path should not be a 
 
        repository itself. Return a dictionary of repository objects
 
        
 
        :param repos_path: path to directory containing repositories
 
        :param baseui
 
        :param initial: initial scan
 
        """
 
        log.info('scanning for repositories in %s', repos_path)
 

	
 
        if not isinstance(baseui, ui.ui):
 
            baseui = ui.ui()
 
        repos_list = {}
 
@@ -122,56 +121,119 @@ class ScmModel(object):
 
                tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
 
                tmp_d['tip'] = tip.raw_id
 
                tmp_d['tip_sort'] = tip.revision
 
                tmp_d['rev'] = tip.revision
 
                tmp_d['contact'] = repo.dbrepo.user.full_contact
 
                tmp_d['contact_sort'] = tmp_d['contact']
 
                tmp_d['repo_archives'] = list(repo._get_archives())
 
                tmp_d['last_msg'] = tip.message
 
                tmp_d['repo'] = repo
 
                yield tmp_d
 

	
 
    def get_repo(self, repo_name):
 
        return self.get(repo_name)
 

	
 
    def get(self, repo_name):
 
        """
 
        Get's repository from given name, creates BackendInstance and
 
        propagates it's data from database with all additional information
 
        :param repo_name:
 
        """
 
        if not HasRepoPermissionAny('repository.read', 'repository.write',
 
                            'repository.admin')(repo_name, 'get repo check'):
 
            return
 

	
 
        @cache_region('long_term', 'get_repo_cached_%s' % repo_name)
 
        @cache_region('long_term')
 
        def _get_repo(repo_name):
 

	
 
            repo_path = os.path.join(self.repos_path, repo_name)
 
            alias = get_scm(repo_path)[0]
 

	
 
            log.debug('Creating instance of %s repository', alias)
 
            backend = get_backend(alias)
 

	
 
            if alias == 'hg':
 
                repo = backend(repo_path, create=False, baseui=None)
 
                #skip hidden web repository
 
                if repo._get_hidden():
 
                    return
 
            else:
 
                repo = backend(repo_path, create=False)
 

	
 
            dbrepo = self.sa.query(Repository)\
 
                .options(joinedload(Repository.fork))\
 
                .options(joinedload(Repository.user))\
 
                .filter(Repository.repo_name == repo_name)\
 
                .scalar()
 
            make_transient(dbrepo)
 
            repo.dbrepo = dbrepo
 
            return repo
 

	
 
        invalidate = False
 
        invalidate = self._should_invalidate(repo_name)
 
        if invalidate:
 
            log.info('INVALIDATING CACHE FOR %s', repo_name)
 
            log.info('invalidating cache for repository %s', repo_name)
 
            region_invalidate(_get_repo, None, repo_name)
 
            self._mark_invalidated(invalidate)
 

	
 
        return _get_repo(repo_name)
 

	
 

	
 

	
 
    def mark_for_invalidation(self, repo_name):
 
        """
 
        Puts cache invalidation task into db for 
 
        further global cache invalidation
 
        
 
        :param repo_name: this repo that should invalidation take place
 
        """
 
        log.debug('marking %s for invalidation', repo_name)
 
        cache = self.sa.query(CacheInvalidation)\
 
            .filter(CacheInvalidation.cache_key == repo_name).scalar()
 

	
 
        if cache:
 
            #mark this cache as inactive
 
            cache.cache_active = False
 
        else:
 
            log.debug('cache key not found in invalidation db -> creating one')
 
            cache = CacheInvalidation(repo_name)
 

	
 
        try:
 
            self.sa.add(cache)
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 

	
 

	
 

	
 

	
 

	
 
    def _should_invalidate(self, repo_name):
 
        """
 
        Looks up database for invalidation signals for this repo_name
 
        :param repo_name:
 
        """
 

	
 
        ret = self.sa.query(CacheInvalidation)\
 
            .options(FromCache('sql_cache_short',
 
                           'get_invalidation_%s' % repo_name))\
 
            .filter(CacheInvalidation.cache_key == repo_name)\
 
            .filter(CacheInvalidation.cache_active == False)\
 
            .scalar()
 

	
 
        return ret
 

	
 
    def _mark_invalidated(self, cache_key):
 
        """
 
        Marks all occurences of cache to invaldation as already invalidated
 
        @param repo_name:
 
        """
 
        if cache_key:
 
            log.debug('marking %s as already invalidated', cache_key)
 
        try:
 
            cache_key.cache_active = True
 
            self.sa.add(cache_key)
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 

	
rhodecode/model/user.py
Show inline comments
 
@@ -15,53 +15,50 @@
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 9, 2010
 
Model for users
 
:author: marcink
 
"""
 

	
 
from pylons.i18n.translation import _
 
from rhodecode.model.caching_query import FromCache
 
from rhodecode.model.db import User
 
from rhodecode.model.meta import Session
 
import logging
 
import traceback
 

	
 
log = logging.getLogger(__name__)
 

	
 
class DefaultUserException(Exception):pass
 

	
 
class UserModel(object):
 

	
 
    def __init__(self, sa=None):
 
        if not sa:
 
            self.sa = Session()
 
        else:
 
            self.sa = sa
 
    def __init__(self):
 
        self.sa = Session()
 

	
 
    def get(self, user_id, cache=False):
 
        user = self.sa.query(User)
 
        if cache:
 
            user = user.options(FromCache("sql_cache_short",
 
                                          "get_user_%s" % user_id))
 
        return user.get(user_id)
 

	
 

	
 
    def get_by_username(self, username, cache=False):
 
        user = self.sa.query(User)\
 
            .filter(User.username == username)
 
        if cache:
 
            user = user.options(FromCache("sql_cache_short",
 
                                          "get_user_%s" % username))
 
        return user.scalar()
 

	
 
    def create(self, form_data):
 
        try:
 
            new_user = User()
 
            for k, v in form_data.items():
 
                setattr(new_user, k, v)
 

	
 
            self.sa.add(new_user)
0 comments (0 inline, 0 general)