Changeset - 405b80e4ccd5
[Not reviewed]
beta
0 7 0
Marcin Kuzminski - 15 years ago 2011-02-12 21:21:23
marcin@python-works.com
Major refactoring, removed when possible calls to app globals.
Refactored models to fetch the paths needed for scans from database directly
small fixes in base
7 files changed with 47 insertions and 55 deletions:
0 comments (0 inline, 0 general)
rhodecode/config/environment.py
Show inline comments
 
"""Pylons environment configuration"""
 

	
 
import os
 
import logging
 

	
 
from mako.lookup import TemplateLookup
 
from pylons.configuration import PylonsConfig
 
from pylons.error import handle_mako_error
 
from sqlalchemy import engine_from_config
 

	
 
import rhodecode.lib.app_globals as app_globals
 
import rhodecode.lib.helpers
 

	
 
from rhodecode.config.routing import make_map
 
from rhodecode.lib import celerypylons
 
from rhodecode.lib.auth import set_available_permissions, set_base_path
 
from rhodecode.lib.auth import set_available_permissions
 
from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config
 
from rhodecode.model import init_model
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.lib.timerproxy import TimerProxy
 

	
 
log = logging.getLogger(__name__)
 

	
 
def load_environment(global_conf, app_conf, initial=False):
 
    """Configure the Pylons environment via the ``pylons.config``
 
    object
 
    """
 
    config = PylonsConfig()
 

	
 
    # Pylons paths
 
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
    paths = dict(root=root,
 
                 controllers=os.path.join(root, 'controllers'),
 
                 static_files=os.path.join(root, 'public'),
 
                 templates=[os.path.join(root, 'templates')])
 

	
 
    # Initialize config with the basic options
 
    config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
 

	
 
    config['routes.map'] = make_map(config)
 
@@ -51,36 +51,34 @@ def load_environment(global_conf, app_co
 
        error_handler=handle_mako_error,
 
        module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
 
        input_encoding='utf-8', default_filters=['escape'],
 
        imports=['from webhelpers.html import escape'])
 

	
 
    #sets the c attribute access when don't existing attribute are accessed
 
    config['pylons.strict_tmpl_context'] = True
 
    test = os.path.split(config['__file__'])[-1] == 'test.ini'
 
    if test:
 
        from rhodecode.lib.utils import create_test_env, create_test_index
 
        from rhodecode.tests import  TESTS_TMP_PATH
 
        create_test_env(TESTS_TMP_PATH, config)
 
        create_test_index(TESTS_TMP_PATH, True)
 

	
 
    #MULTIPLE DB configs
 
    # Setup the SQLAlchemy database engine
 
    if config['debug'] and not test:
 
        #use query time debugging.
 
        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.',
 
                                                            proxy=TimerProxy())
 
    else:
 
        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
 

	
 
    init_model(sa_engine_db1)
 
    #init baseui
 
    config['pylons.app_globals'].baseui = make_ui('db')
 

	
 
    g = config['pylons.app_globals']
 
    repo2db_mapper(ScmModel().repo_scan(g.paths[0][1], g.baseui))
 
    repos_path = make_ui('db').configitems('paths')[0][1]
 
    repo2db_mapper(ScmModel().repo_scan(repos_path))
 
    set_available_permissions(config)
 
    set_base_path(config)
 
    config['base_path'] = repos_path
 
    set_rhodecode_config(config)
 
    # CONFIGURATION OPTIONS HERE (note: all config options will override
 
    # any Pylons config options)
 

	
 
    return config
rhodecode/controllers/admin/settings.py
Show inline comments
 
@@ -10,67 +10,66 @@
 
    :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com>    
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 

	
 
import logging
 
import traceback
 
import formencode
 

	
 
from sqlalchemy import func
 
from formencode import htmlfill
 
from pylons import request, session, tmpl_context as c, url, app_globals as g, \
 
    config
 
from pylons import request, session, tmpl_context as c, url, config
 
from pylons.controllers.util import abort, redirect
 
from pylons.i18n.translation import _
 

	
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
 
    HasPermissionAnyDecorator, NotAnonymous
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.lib.celerylib import tasks, run_task
 
from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \
 
    set_rhodecode_config, repo_name_slug
 
from rhodecode.model.db import RhodeCodeUi, Repository
 
from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \
 
    ApplicationUiSettingsForm
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.model.settings import SettingsModel
 
from rhodecode.model.user import UserModel
 

	
 
from rhodecode.model.repo import RepoModel
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class SettingsController(BaseController):
 
    """REST Controller styled on the Atom Publishing Protocol"""
 
    # To properly map this controller, ensure your config/routing.py
 
    # file has a resource setup:
 
    #     map.resource('setting', 'settings', controller='admin/settings', 
 
    #         path_prefix='/admin', name_prefix='admin_')
 

	
 

	
 
    @LoginRequired()
 
    def __before__(self):
 
        c.admin_user = session.get('admin_user')
 
        c.admin_username = session.get('admin_username')
 
        super(SettingsController, self).__before__()
 

	
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def index(self, format='html'):
 
        """GET /admin/settings: All items in the collection"""
 
        # url('admin_settings')
 

	
 
@@ -84,50 +83,49 @@ class SettingsController(BaseController)
 
        )
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def create(self):
 
        """POST /admin/settings: Create a new item"""
 
        # url('admin_settings')
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def new(self, format='html'):
 
        """GET /admin/settings/new: Form to create a new item"""
 
        # url('admin_new_setting')
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def update(self, setting_id):
 
        """PUT /admin/settings/setting_id: Update an existing item"""
 
        # Forms posted to this method should contain a hidden field:
 
        #    <input type="hidden" name="_method" value="PUT" />
 
        # Or using helpers:
 
        #    h.form(url('admin_setting', setting_id=ID),
 
        #           method='put')
 
        # url('admin_setting', setting_id=ID)
 
        if setting_id == 'mapping':
 
            rm_obsolete = request.POST.get('destroy', False)
 
            log.debug('Rescanning directories with destroy=%s', rm_obsolete)
 

	
 
            initial = ScmModel().repo_scan(g.paths[0][1], g.baseui)
 
            initial = ScmModel().repo_scan()
 
            for repo_name in initial.keys():
 
                invalidate_cache('get_repo_cached_%s' % repo_name)
 

	
 
            repo2db_mapper(initial, rm_obsolete)
 

	
 
            h.flash(_('Repositories successfully rescanned'), category='success')
 

	
 
        if setting_id == 'whoosh':
 
            repo_location = self.get_hg_ui_settings()['paths_root_path']
 
            full_index = request.POST.get('full_index', False)
 
            task = run_task(tasks.whoosh_index, repo_location, full_index)
 

	
 
            h.flash(_('Whoosh reindex task scheduled'), category='success')
 
        if setting_id == 'global':
 

	
 
            application_form = ApplicationSettingsForm()()
 
            try:
 
                form_result = application_form.to_python(dict(request.POST))
 
                settings_model = SettingsModel()
 

	
 
                try:
 
                    hgsettings1 = settings_model.get('title')
 
                    hgsettings1.app_settings_value = form_result['rhodecode_title']
 

	
rhodecode/lib/app_globals.py
Show inline comments
 
"""The application's Globals object"""
 

	
 
from beaker.cache import CacheManager
 
from beaker.util import parse_cache_config_options
 
from vcs.utils.lazy import LazyProperty
 

	
 
class Globals(object):
 
    """Globals acts as a container for objects available throughout the
 
    life of the application
 

	
 
    """
 

	
 
    def __init__(self, config):
 
        """One instance of Globals is created during application
 
        initialization and is available during requests via the
 
        'app_globals' variable
 

	
 
        """
 
        self.cache = CacheManager(**parse_cache_config_options(config))
 
        self.available_permissions = None   # propagated after init_model
 
        self.baseui = None                  # propagated after init_model        
 

	
 
    @LazyProperty
 
    def paths(self):
 
        if self.baseui:
 
            return self.baseui.configitems('paths')
 

	
 
    @LazyProperty
 
    def base_path(self):
 
        if self.baseui:
 
            return self.paths[0][1]
rhodecode/lib/auth.py
Show inline comments
 
@@ -192,52 +192,48 @@ class  AuthUser(object):
 

	
 
    def __repr__(self):
 
        return "<AuthUser('id:%s:%s')>" % (self.user_id, self.username)
 

	
 
def set_available_permissions(config):
 
    """This function will propagate pylons globals with all available defined
 
    permission given in db. We don't want to check each time from db for new 
 
    permissions since adding a new permission also requires application restart
 
    ie. to decorate new views with the newly created permission
 
    
 
    :param config: current pylons config instance
 
    
 
    """
 
    log.info('getting information about all available permissions')
 
    try:
 
        sa = meta.Session()
 
        all_perms = sa.query(Permission).all()
 
    except:
 
        pass
 
    finally:
 
        meta.Session.remove()
 

	
 
    config['available_permissions'] = [x.permission_name for x in all_perms]
 

	
 
def set_base_path(config):
 
    config['base_path'] = config['pylons.app_globals'].base_path
 

	
 

	
 
def fill_perms(user):
 
    """Fills user permission attribute with permissions taken from database
 
    works for permissions given for repositories, and for permissions that
 
    as part of beeing group member
 
    
 
    :param user: user instance to fill his perms
 
    """
 

	
 
    sa = meta.Session()
 
    user.permissions['repositories'] = {}
 
    user.permissions['global'] = set()
 

	
 
    #===========================================================================
 
    # fetch default permissions
 
    #===========================================================================
 
    default_user = UserModel().get_by_username('default', cache=True)
 

	
 
    default_perms = sa.query(RepoToPerm, Repository, Permission)\
 
        .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
 
        .join((Permission, RepoToPerm.permission_id == Permission.permission_id))\
 
        .filter(RepoToPerm.user == default_user).all()
 

	
 
    if user.is_admin:
 
        #=======================================================================
rhodecode/lib/base.py
Show inline comments
 
"""The base Controller API
 

	
 
Provides the BaseController class for subclassing.
 
"""
 
from pylons import config, tmpl_context as c, request, session
 
from pylons.controllers import WSGIController
 
from pylons.templating import render_mako as render
 
from rhodecode import __version__
 
from rhodecode.lib import auth
 
from rhodecode.lib.utils import get_repo_slug
 
from rhodecode.model import meta
 
from rhodecode.model.scm import ScmModel
 
from rhodecode import BACKENDS
 

	
 
class BaseController(WSGIController):
 

	
 
    def __before__(self):
 
        c.rhodecode_version = __version__
 
        c.rhodecode_name = config.get('rhodecode_title')
 
        c.ga_code = config.get('rhodecode_ga_code')
 
        c.repo_name = get_repo_slug(request)
 
        c.cached_repo_list = ScmModel().get_repos()
 
        c.backends = BACKENDS.keys()
 
        self.cut_off_limit = int(config.get('cut_off_limit'))
 

	
 
        self.sa = meta.Session()
 
        scm_model = ScmModel(self.sa)
 
        c.cached_repo_list = scm_model.get_repos()
 
        #c.unread_journal = scm_model.get_unread_journal()
 

	
 
        if c.repo_name:
 
            cached_repo = scm_model.get(c.repo_name)
 
            if cached_repo:
 
                c.repository_tags = cached_repo.tags
 
                c.repository_branches = cached_repo.branches
 
                c.repository_followers = scm_model.get_followers(cached_repo.dbrepo.repo_id)
 
                c.repository_forks = scm_model.get_forks(cached_repo.dbrepo.repo_id)
 
            else:
 
                c.repository_tags = {}
 
                c.repository_branches = {}
 
                c.repository_followers = 0
 
                c.repository_forks = 0
 

	
 

	
 
    def __call__(self, environ, start_response):
 
        """Invoke the Controller"""
 
        # WSGIController.__call__ dispatches to the Controller method
 
        # the request is routed to. This routing information is
 
        # available in environ['pylons.routes_dict']
 
        try:
 
            #putting this here makes sure that we update permissions every time
 
            self.rhodecode_user = c.rhodecode_user = auth.get_user(session)
rhodecode/lib/indexers/daemon.py
Show inline comments
 
@@ -60,49 +60,49 @@ formatter = logging.Formatter("%(asctime
 

	
 
# add formatter to ch
 
ch.setFormatter(formatter)
 

	
 
# add ch to logger
 
log.addHandler(ch)
 

	
 
class WhooshIndexingDaemon(object):
 
    """
 
    Deamon for atomic jobs
 
    """
 

	
 
    def __init__(self, indexname='HG_INDEX', index_location=None,
 
                 repo_location=None, sa=None, repo_list=None):
 
        self.indexname = indexname
 

	
 
        self.index_location = index_location
 
        if not index_location:
 
            raise Exception('You have to provide index location')
 

	
 
        self.repo_location = repo_location
 
        if not repo_location:
 
            raise Exception('You have to provide repositories location')
 

	
 
        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location, None)
 
        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
 

	
 
        if repo_list:
 
            filtered_repo_paths = {}
 
            for repo_name, repo in self.repo_paths.items():
 
                if repo_name in repo_list:
 
                    filtered_repo_paths[repo.name] = repo
 

	
 
            self.repo_paths = filtered_repo_paths
 

	
 

	
 
        self.initial = False
 
        if not os.path.isdir(self.index_location):
 
            os.makedirs(self.index_location)
 
            log.info('Cannot run incremental index since it does not'
 
                     ' yet exist running full build')
 
            self.initial = True
 

	
 
    def get_paths(self, repo):
 
        """recursive walk in root dir and return a set of all path in that dir
 
        based on repository walk function
 
        """
 
        index_paths_ = set()
 
        try:
 
            tip = repo.get_changeset('tip')
rhodecode/model/repo.py
Show inline comments
 
@@ -9,100 +9,111 @@
 
    :author: marcink
 
    :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com>    
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
import os
 
import shutil
 
import logging
 
import traceback
 
from datetime import datetime
 

	
 
from sqlalchemy.orm import joinedload
 

	
 
from vcs.utils.lazy import LazyProperty
 
from vcs.backends import get_backend
 

	
 
from rhodecode.model import BaseModel
 
from rhodecode.model.caching_query import FromCache
 
from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
 
    Statistics, UsersGroup, UsersGroupToPerm
 
    Statistics, UsersGroup, UsersGroupToPerm, RhodeCodeUi
 
from rhodecode.model.user import UserModel
 
from rhodecode.model.users_group import UsersGroupMember, UsersGroupModel
 

	
 
from vcs.backends import get_backend
 

	
 
log = logging.getLogger(__name__)
 

	
 
class RepoModel(BaseModel):
 

	
 
    def __init__(self, sa=None):
 
        try:
 
            from pylons import app_globals
 
            self._base_path = app_globals.base_path
 
        except:
 
            self._base_path = None
 

	
 
        super(RepoModel, self).__init__(sa)
 
    @LazyProperty
 
    def repos_path(self):
 
        """Get's the repositories root path from database
 
        """
 

	
 
    @property
 
    def base_path(self):
 
        if self._base_path is None:
 
            raise Exception('Base Path is empty, try set this after'
 
                            'class initialization when not having '
 
                            'app_globals available')
 
        return self._base_path
 

	
 
        super(RepoModel, self).__init__()
 

	
 
        q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
        return q.ui_value
 

	
 
    def get(self, repo_id, cache=False):
 
        repo = self.sa.query(Repository)\
 
            .filter(Repository.repo_id == repo_id)
 

	
 
        if cache:
 
            repo = repo.options(FromCache("sql_cache_short",
 
                                          "get_repo_%s" % repo_id))
 
        return repo.scalar()
 

	
 

	
 
    def get_by_repo_name(self, repo_name, cache=False):
 
        repo = self.sa.query(Repository)\
 
            .filter(Repository.repo_name == repo_name)
 

	
 
        if cache:
 
            repo = repo.options(FromCache("sql_cache_short",
 
                                          "get_repo_%s" % repo_name))
 
        return repo.scalar()
 

	
 

	
 
    def get_full(self, repo_name, cache=False, invalidate=False):
 
        repo = self.sa.query(Repository)\
 
            .options(joinedload(Repository.fork))\
 
            .options(joinedload(Repository.user))\
 
            .options(joinedload(Repository.followers))\
 
            .options(joinedload(Repository.repo_to_perm))\
 
            .options(joinedload(Repository.users_group_to_perm))\
 
            .filter(Repository.repo_name == repo_name)\
 

	
 
        if cache:
 
            repo = repo.options(FromCache("sql_cache_long",
 
                                          "get_repo_full_%s" % repo_name))
 
        if invalidate:
 
            repo.invalidate()
 

	
 
        return repo.scalar()
 

	
 

	
 
    def get_users_js(self):
 

	
 
        users = self.sa.query(User).filter(User.active == True).all()
 
        u_tmpl = '''{id:%s, fname:"%s", lname:"%s", nname:"%s"},'''
 
        users_array = '[%s]' % '\n'.join([u_tmpl % (u.user_id, u.name,
 
                                                    u.lastname, u.username)
 
                                        for u in users])
 
        return users_array
 

	
 

	
 
    def get_users_groups_js(self):
 
        users_groups = self.sa.query(UsersGroup)\
 
            .filter(UsersGroup.users_group_active == True).all()
 

	
 
        g_tmpl = '''{id:%s, grname:"%s",grmembers:"%s"},'''
 

	
 
        users_groups_array = '[%s]' % '\n'.join([g_tmpl % \
 
                                    (gr.users_group_id, gr.users_group_name,
 
                                     len(gr.members))
 
                                        for gr in users_groups])
 
        return users_groups_array
 

	
 
    def update(self, repo_name, form_data):
 
        try:
 
@@ -269,63 +280,63 @@ class RepoModel(BaseModel):
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 
            raise
 

	
 
    def delete_stats(self, repo_name):
 
        try:
 
            self.sa.query(Statistics)\
 
                .filter(Statistics.repository == \
 
                        self.get_by_repo_name(repo_name)).delete()
 
            self.sa.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 
            raise
 

	
 

	
 
    def __create_repo(self, repo_name, alias):
 
        """
 
        makes repository on filesystem
 
        :param repo_name:
 
        :param alias:
 
        """
 
        from rhodecode.lib.utils import check_repo
 
        repo_path = os.path.join(self.base_path, repo_name)
 
        if check_repo(repo_name, self.base_path):
 
        repo_path = os.path.join(self.repos_path, repo_name)
 
        if check_repo(repo_name, self.repos_path):
 
            log.info('creating repo %s in %s', repo_name, repo_path)
 
            backend = get_backend(alias)
 
            backend(repo_path, create=True)
 

	
 
    def __rename_repo(self, old, new):
 
        """
 
        renames repository on filesystem
 
        :param old: old name
 
        :param new: new name
 
        """
 
        log.info('renaming repo from %s to %s', old, new)
 

	
 
        old_path = os.path.join(self.base_path, old)
 
        new_path = os.path.join(self.base_path, new)
 
        old_path = os.path.join(self.repos_path, old)
 
        new_path = os.path.join(self.repos_path, new)
 
        if os.path.isdir(new_path):
 
            raise Exception('Was trying to rename to already existing dir %s',
 
                            new_path)
 
        shutil.move(old_path, new_path)
 

	
 
    def __delete_repo(self, repo):
 
        """
 
        removes repo from filesystem, the removal is acctually made by
 
        added rm__ prefix into dir, and rename internat .hg/.git dirs so this
 
        repository is no longer valid for rhodecode, can be undeleted later on
 
        by reverting the renames on this repository
 
        :param repo: repo object
 
        """
 
        rm_path = os.path.join(self.base_path, repo.repo_name)
 
        rm_path = os.path.join(self.repos_path, repo.repo_name)
 
        log.info("Removing %s", rm_path)
 
        #disable hg/git
 
        alias = repo.repo_type
 
        shutil.move(os.path.join(rm_path, '.%s' % alias),
 
                    os.path.join(rm_path, 'rm__.%s' % alias))
 
        #disable repo
 
        shutil.move(rm_path, os.path.join(self.base_path, 'rm__%s__%s' \
 
        shutil.move(rm_path, os.path.join(self.repos_path, 'rm__%s__%s' \
 
                                          % (datetime.today().isoformat(),
 
                                             repo.repo_name)))
0 comments (0 inline, 0 general)