Changeset - c1c1cf772337
[Not reviewed]
default
0 6 0
Marcin Kuzminski - 15 years ago 2010-10-12 16:39:53
marcin@python-works.com
moved out sqlalchemy cache from meta to the config files.
added caching query for permissions.
6 files changed with 56 insertions and 39 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -28,56 +28,67 @@ debug = true
 
threadpool_workers = 5
 

	
 
##max request before
 
threadpool_max_requests = 6
 

	
 
##option to use threads of process
 
use_threadpool = false
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 5000
 

	
 
[app:main]
 
use = egg:rhodecode
 
full_stack = true
 
static_files = true
 
lang=en
 
cache_dir = %(here)s/data
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/%(here)s/data/cache/data
 
beaker.cache.lock_dir=/%(here)s/data/cache/lock
 
beaker.cache.regions=super_short_term,short_term,long_term
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=5
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## "dbm", "file", "memcached", "database", and "memory". 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = g654dcno0-9873jhgfreyu
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
production.ini
Show inline comments
 
@@ -28,56 +28,67 @@ debug = true
 
threadpool_workers = 5
 

	
 
##max request before thread respawn
 
threadpool_max_requests = 2
 

	
 
##option to use threads of process
 
use_threadpool = true
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 8001
 

	
 
[app:main]
 
use = egg:rhodecode
 
full_stack = true
 
static_files = false
 
lang=en
 
cache_dir = %(here)s/data
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/%(here)s/data/cache/data
 
beaker.cache.lock_dir=/%(here)s/data/cache/lock
 
beaker.cache.regions=super_short_term,short_term,long_term
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=5
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = g654dcno0-9873jhgfreyu
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
rhodecode/config/deployment.ini_tmpl
Show inline comments
 
@@ -29,56 +29,67 @@ threadpool_workers = 5
 

	
 
##max request before thread respawn
 
threadpool_max_requests = 2
 

	
 
##option to use threads of process
 
use_threadpool = true
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 5000
 

	
 
[app:main]
 
use = egg:rhodecode
 
full_stack = true
 
static_files = true
 
lang=en
 
cache_dir = %(here)s/data
 
app_instance_uuid = ${app_instance_uuid}
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/%(here)s/data/cache/data
 
beaker.cache.lock_dir=/%(here)s/data/cache/lock
 
beaker.cache.regions=super_short_term,short_term,long_term
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=5
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = ${app_instance_secret}
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
rhodecode/lib/auth.py
Show inline comments
 
@@ -6,48 +6,49 @@
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 4, 2010
 

	
 
@author: marcink
 
"""
 
from beaker.cache import cache_region
 
from pylons import config, session, url, request
 
from pylons.controllers.util import abort, redirect
 
from rhodecode.lib.utils import get_repo_slug
 
from rhodecode.model import meta
 
from rhodecode.model.caching_query import FromCache
 
from rhodecode.model.db import User, RepoToPerm, Repository, Permission, \
 
    UserToPerm
 
from sqlalchemy.exc import OperationalError
 
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 
import bcrypt
 
from decorator import decorator
 
import logging
 
import random
 

	
 
log = logging.getLogger(__name__) 
 

	
 
class PasswordGenerator(object):
 
    """This is a simple class for generating password from
 
        different sets of characters
 
        usage:
 
        passwd_gen = PasswordGenerator()
 
        #print 8-letter password containing only big and small letters of alphabet
 
        print passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)        
 
    """
 
    ALPHABETS_NUM = r'''1234567890'''#[0]
 
    ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''#[1]
 
    ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''#[2]
 
    ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''    #[3]
 
    ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM + ALPHABETS_SPECIAL#[4]
 
@@ -120,78 +121,83 @@ def set_available_permissions(config):
 
    permission given in db. We don't wannt to check each time from db for new 
 
    permissions since adding a new permission also requires application restart
 
    ie. to decorate new views with the newly created permission
 
    @param config:
 
    """
 
    log.info('getting information about all available permissions')
 
    try:
 
        sa = meta.Session
 
        all_perms = sa.query(Permission).all()
 
    finally:
 
        meta.Session.remove()
 
    
 
    config['available_permissions'] = [x.permission_name for x in all_perms]
 

	
 
def set_base_path(config):
 
    config['base_path'] = config['pylons.app_globals'].base_path
 

	
 
def fill_data(user):
 
    """
 
    Fills user data with those from database and log out user if not present
 
    in database
 
    @param user:
 
    """
 
    sa = meta.Session
 
    dbuser = sa.query(User).get(user.user_id)
 
    dbuser = sa.query(User).options(FromCache('sql_cache_short',
 
                                              'getuser_%s' % user.user_id))\
 
        .get(user.user_id)
 
    if dbuser:
 
        user.username = dbuser.username
 
        user.is_admin = dbuser.admin
 
        user.name = dbuser.name
 
        user.lastname = dbuser.lastname
 
        user.email = dbuser.email
 
    else:
 
        user.is_authenticated = False
 
    meta.Session.remove()
 
    return user
 
            
 
def fill_perms(user):
 
    """
 
    Fills user permission attribute with permissions taken from database
 
    @param user:
 
    """
 
    
 
    sa = meta.Session
 
    user.permissions['repositories'] = {}
 
    user.permissions['global'] = set()
 
    
 
    #===========================================================================
 
    # fetch default permissions
 
    #===========================================================================
 
    default_user = sa.query(User)\
 
        .options(FromCache('sql_cache_short','getuser_%s' % 'default'))\
 
        .filter(User.username == 'default').scalar()
 
                                            
 
    default_perms = sa.query(RepoToPerm, Repository, Permission)\
 
        .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
 
        .join((Permission, RepoToPerm.permission_id == Permission.permission_id))\
 
        .filter(RepoToPerm.user == sa.query(User).filter(User.username == 
 
                                            'default').scalar()).all()
 
        .filter(RepoToPerm.user == default_user).all()
 
                                            
 
    if user.is_admin:
 
        #=======================================================================
 
        # #admin have all default rights set to admin        
 
        #=======================================================================
 
        user.permissions['global'].add('hg.admin')
 
        
 
        for perm in default_perms:
 
            p = 'repository.admin'
 
            user.permissions['repositories'][perm.RepoToPerm.repository.repo_name] = p
 
    
 
    else:
 
        #=======================================================================
 
        # set default permissions
 
        #=======================================================================
 
        
 
        #default global
 
        default_global_perms = sa.query(UserToPerm)\
 
            .filter(UserToPerm.user == sa.query(User).filter(User.username == 
 
            'default').one())
 
        
 
        for perm in default_global_perms:
 
            user.permissions['global'].add(perm.permission.permission_name)
 
                    
rhodecode/model/caching_query.py
Show inline comments
 
"""caching_query.py
 

	
 
Represent persistence structures which allow the usage of
 
Beaker caching with SQLAlchemy.
 

	
 
The three new concepts introduced here are:
 

	
 
 * CachingQuery - a Query subclass that caches and
 
   retrieves results in/from Beaker.
 
 * FromCache - a query option that establishes caching
 
   parameters on a Query
 
 * RelationshipCache - a variant of FromCache which is specific
 
   to a query invoked during a lazy load.
 
 * _params_from_query - extracts value parameters from 
 
   a Query.
 

	
 
The rest of what's here are standard SQLAlchemy and
 
Beaker constructs.
 
   
 
"""
 
from beaker.exceptions import BeakerException
 
from sqlalchemy.orm.interfaces import MapperOption
 
from sqlalchemy.orm.query import Query
 
from sqlalchemy.sql import visitors
 
import beaker
 

	
 
class CachingQuery(Query):
 
    """A Query subclass which optionally loads full results from a Beaker 
 
    cache region.
 
    
 
    The CachingQuery stores additional state that allows it to consult
 
    a Beaker cache before accessing the database:
 
    
 
    * A "region", which is a cache region argument passed to a 
 
      Beaker CacheManager, specifies a particular cache configuration
 
      (including backend implementation, expiration times, etc.)
 
    * A "namespace", which is a qualifying name that identifies a
 
      group of keys within the cache.  A query that filters on a name 
 
      might use the name "by_name", a query that filters on a date range 
 
      to a joined table might use the name "related_date_range".
 
      
 
    When the above state is present, a Beaker cache is retrieved.
 
    
 
    The "namespace" name is first concatenated with 
 
    a string composed of the individual entities and columns the Query 
 
    requests, i.e. such as ``Query(User.id, User.name)``.
 
    
 
    The Beaker cache is then loaded from the cache manager based
 
    on the region and composed namespace.  The key within the cache
 
@@ -85,69 +87,75 @@ class CachingQuery(Query):
 
    def get_value(self, merge=True, createfunc=None):
 
        """Return the value from the cache for this query.
 

	
 
        Raise KeyError if no value present and no
 
        createfunc specified.
 

	
 
        """
 
        cache, cache_key = _get_cache_parameters(self)
 
        ret = cache.get_value(cache_key, createfunc=createfunc)
 
        if merge:
 
            ret = self.merge_result(ret, load=False)
 
        return ret
 

	
 
    def set_value(self, value):
 
        """Set the value in the cache for this query."""
 

	
 
        cache, cache_key = _get_cache_parameters(self)
 
        cache.put(cache_key, value)        
 

	
 
def query_callable(manager):
 
    def query(*arg, **kw):
 
        return CachingQuery(manager, *arg, **kw)
 
    return query
 
    
 
def get_cache_region(name, region):
 
        if region not in beaker.cache.cache_regions:
 
            raise BeakerException('Cache region not configured: %s' % region)
 
        kw = beaker.cache.cache_regions[region]
 
        return beaker.cache.Cache._get_cache(name, kw)
 
        
 
def _get_cache_parameters(query):
 
    """For a query with cache_region and cache_namespace configured,
 
    return the correspoinding Cache instance and cache key, based
 
    on this query's current criterion and parameter values.
 

	
 
    """
 
    if not hasattr(query, '_cache_parameters'):
 
        raise ValueError("This Query does not have caching parameters configured.")
 

	
 
    region, namespace, cache_key = query._cache_parameters
 
    
 
    namespace = _namespace_from_query(namespace, query)
 

	
 
    if cache_key is None:
 
        # cache key - the value arguments from this query's parameters.
 
        args = _params_from_query(query)
 
        cache_key = " ".join([str(x) for x in args])
 

	
 
    # get cache
 
    cache = query.cache_manager.get_cache_region(namespace, region)
 

	
 
    #cache = query.cache_manager.get_cache_region(namespace, region)
 
    cache = get_cache_region(namespace, region)
 
    # optional - hash the cache_key too for consistent length
 
    # import uuid
 
    # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
 

	
 
    return cache, cache_key
 

	
 
def _namespace_from_query(namespace, query):
 
    # cache namespace - the token handed in by the 
 
    # option + class we're querying against
 
    namespace = " ".join([namespace] + [str(x) for x in query._entities])
 

	
 
    # memcached wants this
 
    namespace = namespace.replace(' ', '_')
 

	
 
    return namespace
 

	
 
def _set_cache_parameters(query, region, namespace, cache_key):
 
    
 
    if hasattr(query, '_cache_parameters'):
 
        region, namespace, cache_key = query._cache_parameters
 
        raise ValueError("This query is already configured "
 
                        "for region %r namespace %r" % 
 
                        (region, namespace)
 
                    )
rhodecode/model/meta.py
Show inline comments
 
@@ -7,57 +7,27 @@ import os
 
from os.path import join as jn, dirname as dn, abspath
 
import time
 

	
 
# Beaker CacheManager.  A home base for cache configurations.
 
cache_manager = cache.CacheManager()
 

	
 
__all__ = ['Base', 'Session']
 
#
 
# SQLAlchemy session manager. Updated by model.init_model()
 
#
 
Session = scoped_session(
 
                sessionmaker(
 
                    query_cls=caching_query.query_callable(cache_manager)
 
                )
 
          )
 

	
 
# The declarative Base
 
Base = declarative_base()
 
#For another db...
 
#Base2 = declarative_base()
 

	
 
#===============================================================================
 
# CACHE OPTIONS
 
#===============================================================================
 
cache_base = jn(dn(dn(dn(abspath(__file__)))), 'data')
 
cache_dir = jn(cache_base, 'cache')
 

	
 
if not os.path.isdir(cache_base):
 
    os.mkdir(cache_base)
 

	
 
if not os.path.isdir(cache_dir):
 
    os.mkdir(cache_dir)
 
# set start_time to current time
 
# to re-cache everything
 
# upon application startup
 
start_time = time.time()
 
# configure the "sqlalchemy" cache region.
 
cache_manager.regions['sql_cache_short'] = {
 
        'type':'memory',
 
        'data_dir':cache_dir,
 
        'expire':10,
 
        'start_time':start_time
 
    }
 
cache_manager.regions['sql_cache_med'] = {
 
        'type':'memory',
 
        'data_dir':cache_dir,
 
        'expire':360,
 
        'start_time':start_time
 
    }
 
cache_manager.regions['sql_cache_long'] = {
 
        'type':'file',
 
        'data_dir':cache_dir,
 
        'expire':3600,
 
        'start_time':start_time
 
    }
 
#Configured globally in .ini files
 
#to use cache use this in query
 
#.options(FromCache("sqlalchemy_cache_type", "cachekey"))
0 comments (0 inline, 0 general)