Changeset - 6484963056cd
[Not reviewed]
default
0 5 0
Marcin Kuzminski - 15 years ago 2010-07-14 12:31:11
marcin@python-works.com
implemented cache for repeated queries in simplehg mercurial requests
5 files changed with 109 insertions and 90 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -35,29 +35,31 @@ port = 5000
 
[app:main]
 
use = egg:pylons_app
 
full_stack = true
 
static_files = true
 
lang=en
 
cache_dir = %(here)s/data
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/%(here)s/data/cache/data
 
beaker.cache.lock_dir=/%(here)s/data/cache/lock
 
beaker.cache.regions=short_term,long_term
 
beaker.cache.regions=super_short_term,short_term,long_term
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 

	
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
#set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
production.ini
Show inline comments
 
@@ -30,35 +30,36 @@ use_threadpool = true
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 8001
 

	
 
[app:main]
 
use = egg:pylons_app
 
full_stack = true
 
static_files = false
 
lang=en
 
cache_dir = %(here)s/data
 

	
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/%(here)s/data/cache/data
 
beaker.cache.lock_dir=/%(here)s/data/cache/lock
 
beaker.cache.regions=short_term,long_term
 
beaker.cache.regions=super_short_term,short_term,long_term
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
pylons_app/lib/auth.py
Show inline comments
 
@@ -13,50 +13,56 @@
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
"""
 
Created on April 4, 2010
 

	
 
@author: marcink
 
"""
 

	
 
from beaker.cache import cache_region
 
from functools import wraps
 
from pylons import session, url, request
 
from pylons import config, session, url, request
 
from pylons.controllers.util import abort, redirect
 
from pylons_app.lib.utils import get_repo_slug
 
from pylons_app.model import meta
 
from pylons_app.model.db import User, Repo2Perm, Repository, Permission
 
from pylons_app.lib.utils import get_repo_slug
 
from sqlalchemy.exc import OperationalError
 
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 
import crypt
 
import logging
 
from pylons import config
 

	
 
log = logging.getLogger(__name__) 
 

	
 
def get_crypt_password(password):
 
    """
 
    Cryptographic function used for password hashing
 
    @param password: password to hash
 
    """
 
    return crypt.crypt(password, '6a')
 

	
 

	
 
@cache_region('super_short_term', 'cached_user')
 
def get_user_cached(username):
 
    sa = meta.Session
 
    user = sa.query(User).filter(User.username == username).one()
 
    return user
 

	
 
def authfunc(environ, username, password):
 
    sa = meta.Session
 
    password_crypt = get_crypt_password(password)
 
    try:
 
        user = sa.query(User).filter(User.username == username).one()
 
        user = get_user_cached(username)
 
    except (NoResultFound, MultipleResultsFound, OperationalError) as e:
 
        log.error(e)
 
        user = None
 
        
 
    if user:
 
        if user.active:
 
            if user.username == username and user.password == password_crypt:
 
                log.info('user %s authenticated correctly', username)
 
                return True
 
        else:
 
            log.error('user %s is disabled', username)
 
            
pylons_app/lib/middleware/simplehg.py
Show inline comments
 
#!/usr/bin/env python
 
# encoding: utf-8
 
# middleware to handle mercurial api calls
 
# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
 
 
 

	
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 

	
 
"""
 
Created on 2010-04-28
 

	
 
@author: marcink
 
SimpleHG middleware for handling mercurial protocol request (push/clone etc.)
 
It's implemented with basic auth function
 
"""
 
from datetime import datetime
 
from itertools import chain
 
from mercurial.error import RepoError
 
from mercurial.hgweb import hgweb
 
from mercurial.hgweb.request import wsgiapplication
 
from mercurial.error import RepoError
 
from paste.auth.basic import AuthBasicAuthenticator
 
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware
 
from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware, \
 
    get_user_cached
 
from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache, \
 
    check_repo_fast
 
from pylons_app.model import meta
 
from pylons_app.model.db import UserLog, User
 
import pylons_app.lib.helpers as h
 
from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 
import logging
 
import os
 
import pylons_app.lib.helpers as h
 
import traceback
 
 
 
log = logging.getLogger(__name__)
 

	
 
class SimpleHg(object):
 

	
 
    def __init__(self, application, config):
 
        self.application = application
 
        self.config = config
 
        #authenticate this mercurial request using 
 
        realm = self.config['hg_app_auth_realm']
 
        self.authenticate = AuthBasicAuthenticator(realm, authfunc)
 
        
 
    def __call__(self, environ, start_response):
 
        if not is_mercurial(environ):
 
            return self.application(environ, start_response)
 
        else:
 
            #===================================================================
 
            # AUTHENTICATE THIS MERCURIAL REQUEST
 
            #===================================================================
 
            username = REMOTE_USER(environ)
 
            if not username:
 
                result = self.authenticate(environ)
 
                if isinstance(result, str):
 
                    AUTH_TYPE.update(environ, 'basic')
 
                    REMOTE_USER.update(environ, result)
 
                else:
 
                    return result.wsgi_application(environ, start_response)
 
            
 

	
 
        #===================================================================
 
        # AUTHENTICATE THIS MERCURIAL REQUEST
 
        #===================================================================
 
        username = REMOTE_USER(environ)
 
        if not username:
 
            result = self.authenticate(environ)
 
            if isinstance(result, str):
 
                AUTH_TYPE.update(environ, 'basic')
 
                REMOTE_USER.update(environ, result)
 
            else:
 
                return result.wsgi_application(environ, start_response)
 
        
 
        try:
 
            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
 
        except:
 
            log.error(traceback.format_exc())
 
            return HTTPInternalServerError()(environ, start_response)
 
        
 
        #===================================================================
 
        # CHECK PERMISSIONS FOR THIS REQUEST
 
        #===================================================================
 
        action = self.__get_action(environ)
 
        if action:
 
            username = self.__get_environ_user(environ)
 
            try:
 
                repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
 
                user = self.__get_user(username)
 
            except:
 
                log.error(traceback.format_exc())
 
                return HTTPInternalServerError()(environ, start_response)
 
            #check permissions for this repository
 
            if action == 'pull':
 
                if not HasPermissionAnyMiddleware('repository.read',
 
                                                  'repository.write',
 
                                                  'repository.admin')\
 
                                                    (user, repo_name):
 
                    return HTTPForbidden()(environ, start_response)
 
            if action == 'push':
 
                if not HasPermissionAnyMiddleware('repository.write',
 
                                                  'repository.admin')\
 
                                                    (user, repo_name):
 
                    return HTTPForbidden()(environ, start_response)
 
            
 
            #===================================================================
 
            # CHECK PERMISSIONS FOR THIS REQUEST
 
            #===================================================================
 
            action = self.__get_action(environ)
 
            if action:
 
                username = self.__get_environ_user(environ)
 
                try:
 
                    sa = meta.Session
 
                    user = sa.query(User)\
 
                        .filter(User.username == username).one()
 
                except:
 
                    log.error(traceback.format_exc())
 
                    return HTTPInternalServerError()(environ, start_response)
 
                #check permissions for this repository
 
                if action == 'pull':
 
                    if not HasPermissionAnyMiddleware('repository.read',
 
                                                      'repository.write',
 
                                                      'repository.admin')\
 
                                                        (user, repo_name):
 
                        return HTTPForbidden()(environ, start_response)
 
                if action == 'push':
 
                    if not HasPermissionAnyMiddleware('repository.write',
 
                                                      'repository.admin')\
 
                                                        (user, repo_name):
 
                        return HTTPForbidden()(environ, start_response)
 
                
 
                #log action    
 
                proxy_key = 'HTTP_X_REAL_IP'
 
                def_key = 'REMOTE_ADDR'
 
                ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
 
                self.__log_user_action(user, action, repo_name, ipaddr)            
 
            
 
            #===================================================================
 
            # MERCURIAL REQUEST HANDLING
 
            #===================================================================
 
            environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
 
            self.baseui = make_ui('db')
 
            self.basepath = self.config['base_path']
 
            self.repo_path = os.path.join(self.basepath, repo_name)
 
            #log action    
 
            proxy_key = 'HTTP_X_REAL_IP'
 
            def_key = 'REMOTE_ADDR'
 
            ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
 
            self.__log_user_action(user, action, repo_name, ipaddr)            
 
        
 
        #===================================================================
 
        # MERCURIAL REQUEST HANDLING
 
        #===================================================================
 
        environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
 
        self.baseui = make_ui('db')
 
        self.basepath = self.config['base_path']
 
        self.repo_path = os.path.join(self.basepath, repo_name)
 

	
 
            #quick check if that dir exists...
 
            if check_repo_fast(repo_name, self.basepath):
 
        #quick check if that dir exists...
 
        if check_repo_fast(repo_name, self.basepath):
 
            return HTTPNotFound()(environ, start_response)
 
        try:
 
            app = wsgiapplication(self.__make_app)
 
        except RepoError as e:
 
            if str(e).find('not found') != -1:
 
                return HTTPNotFound()(environ, start_response)
 
            try:
 
                app = wsgiapplication(self.__make_app)
 
            except RepoError as e:
 
                if str(e).find('not found') != -1:
 
                    return HTTPNotFound()(environ, start_response)
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                return HTTPInternalServerError()(environ, start_response)
 
            
 
            #invalidate cache on push
 
            if action == 'push':
 
                self.__invalidate_cache(repo_name)
 
                messages = []
 
                messages.append('thank you for using hg-app')
 
            
 
                return self.msg_wrapper(app, environ, start_response, messages)
 
            else:
 
                return app(environ, start_response)           
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            return HTTPInternalServerError()(environ, start_response)
 
        
 
        #invalidate cache on push
 
        if action == 'push':
 
            self.__invalidate_cache(repo_name)
 
            messages = []
 
            messages.append('thank you for using hg-app')
 
        
 
            return self.msg_wrapper(app, environ, start_response, messages)
 
        else:
 
            return app(environ, start_response)           
 

	
 

	
 
    def msg_wrapper(self, app, environ, start_response, messages=[]):
 
        """
 
        Wrapper for custom messages that come out of mercurial respond messages
 
        is a list of messages that the user will see at the end of response 
 
        from merurial protocol actions that involves remote answers
 
        @param app:
 
        @param environ:
 
        @param start_response:
 
        """
 
        def custom_messages(msg_list):
 
            for msg in msg_list:
 
                yield msg + '\n'
 
        org_response = app(environ, start_response)
 
        return chain(org_response, custom_messages(messages))
 

	
 
    def __make_app(self):
 
        hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
 
        return  self.__load_web_settings(hgserve)
 
    
 
    def __get_environ_user(self, environ):
 
        return environ.get('REMOTE_USER')
 
    
 
    def __get_user(self, username):
 
        return get_user_cached(username)
 
        
 
        
 
                        
 
    def __get_size(self, repo_path, content_size):
 
        size = int(content_size)
 
        for path, dirs, files in os.walk(repo_path):
 
            if path.find('.hg') == -1:
 
                for f in files:
 
                    size += os.path.getsize(os.path.join(path, f))
 
        return size
 
        return h.format_byte_size(size)
 
        
 
    def __get_action(self, environ):
 
        """
 
        Maps mercurial request commands into a pull or push command.
pylons_app/lib/utils.py
Show inline comments
 
@@ -7,24 +7,25 @@
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
from beaker.cache import cache_region
 

	
 
"""
 
Created on April 18, 2010
 
Utilities for hg app
 
@author: marcink
 
"""
 

	
 
import os
 
import logging
 
from mercurial import ui, config, hg
 
from mercurial.error import RepoError
 
from pylons_app.model.db import Repository, User, HgAppUi
 
@@ -66,24 +67,31 @@ def check_repo(repo_name, base_path, ver
 
            return False
 
        r = hg.repository(ui.ui(), repo_path)
 
        if verify:
 
            hg.verify(r)
 
        #here we hnow that repo exists it was verified
 
        log.info('%s repo is already created', repo_name)
 
        return False
 
    except RepoError:
 
        #it means that there is no valid repo there...
 
        log.info('%s repo is free for creation', repo_name)
 
        return True
 

	
 

	
 
@cache_region('super_short_term', 'cached_hg_ui')
 
def get_hg_ui_cached():
 
    from pylons_app.model.meta import Session
 
    sa = Session()
 
    return sa.query(HgAppUi).all()    
 

	
 
def make_ui(read_from='file', path=None, checkpaths=True):        
 
    """
 
    A function that will read python rc files or database
 
    and make an mercurial ui object from read options
 
    
 
    @param path: path to mercurial config file
 
    @param checkpaths: check the path
 
    @param read_from: read from 'file' or 'db'
 
    """
 
    #propagated from mercurial documentation
 
    sections = ['alias', 'auth',
 
                'decode/encode', 'defaults',
 
@@ -103,28 +111,25 @@ def make_ui(read_from='file', path=None,
 
            log.warning('Unable to read config file %s' % path)
 
            return False
 
        
 
        cfg = config.config()
 
        cfg.read(path)
 
        for section in sections:
 
            for k, v in cfg.items(section):
 
                baseui.setconfig(section, k, v)
 
        if checkpaths:check_repo_dir(cfg.items('paths'))                
 
              
 
        
 
    elif read_from == 'db':
 
        from pylons_app.model.meta import Session
 
        sa = Session()
 
            
 
        hg_ui = sa.query(HgAppUi).all()
 
        hg_ui = get_hg_ui_cached()
 
        for ui_ in hg_ui:
 
            baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
        
 
    
 
    return baseui
 

	
 

	
 
def set_hg_app_config(config):
 
    config['hg_app_auth_realm'] = 'realm'
 
    config['hg_app_name'] = 'app name'
 

	
 
def invalidate_cache(name, *args):
0 comments (0 inline, 0 general)