kallithea Changeset - 6484963056cd

Changeset - 6484963056cd

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Marcin Kuzminski - 15 years ago 2010-07-14 12:31:11
marcin@python-works.com

implemented cache for repeated queries in simplehg mercurial requests

5 files changed with 109 insertions and 90 deletions:

development.ini

production.ini

pylons_app/lib/auth.py

pylons_app/lib/middleware/simplehg.py

pylons_app/lib/utils.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -44,11 +44,13 @@ cache_dir = %(here)s/data @@
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
 beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##

production.ini

➞

Show inline comments

@@ @@ -39,17 +39,18 @@ static_files = false @@
 lang=en
 cache_dir = %(here)s/data
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
 beaker.cache.regions=short_term,long_term
+beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##

pylons_app/lib/auth.py

➞

Show inline comments

@@ @@ -22,18 +22,18 @@ Created on April 4, 2010 @@
 @author: marcink
 """
+from beaker.cache import cache_region
 from functools import wraps
 from pylons import session, url, request
+from pylons import config, session, url, request
 from pylons.controllers.util import abort, redirect
 from pylons_app.lib.utils import get_repo_slug
 from pylons_app.model import meta
 from pylons_app.model.db import User, Repo2Perm, Repository, Permission
 from pylons_app.lib.utils import get_repo_slug
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 import crypt
 import logging
-from pylons import config
 log = logging.getLogger(__name__)
 def get_crypt_password(password):
@@ @@ -43,11 +43,17 @@ def get_crypt_password(password): @@
     """
     return crypt.crypt(password, '6a')
 @cache_region('super_short_term', 'cached_user')
 def get_user_cached(username):
     sa = meta.Session
     user = sa.query(User).filter(User.username == username).one()
     return user
 def authfunc(environ, username, password):
     sa = meta.Session
     password_crypt = get_crypt_password(password)
     try:
-        user = sa.query(User).filter(User.username == username).one()
+        user = get_user_cached(username)
     except (NoResultFound, MultipleResultsFound, OperationalError) as e:
         log.error(e)
         user = None

pylons_app/lib/middleware/simplehg.py

➞

Show inline comments

@@ @@ -2,7 +2,7 @@ @@
 # encoding: utf-8
 # middleware to handle mercurial api calls
 # Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
@@ @@ -27,21 +27,23 @@ It's implemented with basic auth functio @@
 """
 from datetime import datetime
 from itertools import chain
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from mercurial.error import RepoError
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware
 from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware, \
     get_user_cached
 from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache, \
     check_repo_fast
 from pylons_app.model import meta
 from pylons_app.model.db import UserLog, User
 import pylons_app.lib.helpers as h
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
 import logging
 import os
 import pylons_app.lib.helpers as h
 import traceback
 log = logging.getLogger(__name__)
 class SimpleHg(object):
@@ @@ -56,86 +58,84 @@ class SimpleHg(object): @@
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
         else:
             #===================================================================
             # AUTHENTICATE THIS MERCURIAL REQUEST
             #===================================================================
             username = REMOTE_USER(environ)
             if not username:
                 result = self.authenticate(environ)
                 if isinstance(result, str):
                     AUTH_TYPE.update(environ, 'basic')
                     REMOTE_USER.update(environ, result)
                 else:
                     return result.wsgi_application(environ, start_response)
         #===================================================================
         # AUTHENTICATE THIS MERCURIAL REQUEST
         #===================================================================
         username = REMOTE_USER(environ)
         if not username:
             result = self.authenticate(environ)
             if isinstance(result, str):
                 AUTH_TYPE.update(environ, 'basic')
                 REMOTE_USER.update(environ, result)
             else:
                 return result.wsgi_application(environ, start_response)
         try:
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
         except:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
         #===================================================================
         # CHECK PERMISSIONS FOR THIS REQUEST
         #===================================================================
         action = self.__get_action(environ)
         if action:
             username = self.__get_environ_user(environ)
             try:
-                repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+                user = self.__get_user(username)
             except:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
             #check permissions for this repository
             if action == 'pull':
                 if not HasPermissionAnyMiddleware('repository.read',
                                                   'repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
             if action == 'push':
                 if not HasPermissionAnyMiddleware('repository.write',
                                                   'repository.admin')\
                                                     (user, repo_name):
                     return HTTPForbidden()(environ, start_response)
             #===================================================================
             # CHECK PERMISSIONS FOR THIS REQUEST
             #===================================================================
             action = self.__get_action(environ)
             if action:
                 username = self.__get_environ_user(environ)
                 try:
                     sa = meta.Session
                     user = sa.query(User)\
                         .filter(User.username == username).one()
                 except:
                     log.error(traceback.format_exc())
                     return HTTPInternalServerError()(environ, start_response)
                 #check permissions for this repository
                 if action == 'pull':
                     if not HasPermissionAnyMiddleware('repository.read',
                                                       'repository.write',
                                                       'repository.admin')\
                                                         (user, repo_name):
                         return HTTPForbidden()(environ, start_response)
                 if action == 'push':
                     if not HasPermissionAnyMiddleware('repository.write',
                                                       'repository.admin')\
                                                         (user, repo_name):
                         return HTTPForbidden()(environ, start_response)
                 #log action
                 proxy_key = 'HTTP_X_REAL_IP'
                 def_key = 'REMOTE_ADDR'
                 ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
                 self.__log_user_action(user, action, repo_name, ipaddr)
             #===================================================================
             # MERCURIAL REQUEST HANDLING
             #===================================================================
             environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
             self.baseui = make_ui('db')
             self.basepath = self.config['base_path']
             self.repo_path = os.path.join(self.basepath, repo_name)
             #log action
             proxy_key = 'HTTP_X_REAL_IP'
             def_key = 'REMOTE_ADDR'
             ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
             self.__log_user_action(user, action, repo_name, ipaddr)
         #===================================================================
         # MERCURIAL REQUEST HANDLING
         #===================================================================
         environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
         self.baseui = make_ui('db')
         self.basepath = self.config['base_path']
         self.repo_path = os.path.join(self.basepath, repo_name)
             #quick check if that dir exists...
             if check_repo_fast(repo_name, self.basepath):
         #quick check if that dir exists...
         if check_repo_fast(repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = wsgiapplication(self.__make_app)
         except RepoError as e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
             try:
                 app = wsgiapplication(self.__make_app)
             except RepoError as e:
                 if str(e).find('not found') != -1:
                     return HTTPNotFound()(environ, start_response)
             except Exception:
                 log.error(traceback.format_exc())
                 return HTTPInternalServerError()(environ, start_response)
             #invalidate cache on push
             if action == 'push':
                 self.__invalidate_cache(repo_name)
                 messages = []
                 messages.append('thank you for using hg-app')
                 return self.msg_wrapper(app, environ, start_response, messages)
             else:
                 return app(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
         #invalidate cache on push
         if action == 'push':
             self.__invalidate_cache(repo_name)
             messages = []
             messages.append('thank you for using hg-app')
             return self.msg_wrapper(app, environ, start_response, messages)
         else:
             return app(environ, start_response)
     def msg_wrapper(self, app, environ, start_response, messages=[]):
@@ @@ -160,6 +160,11 @@ class SimpleHg(object): @@
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
     def __get_user(self, username):
         return get_user_cached(username)
     def __get_size(self, repo_path, content_size):
         size = int(content_size)
         for path, dirs, files in os.walk(repo_path):

pylons_app/lib/utils.py

➞

Show inline comments

@@ @@ -16,6 +16,7 @@ @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 from beaker.cache import cache_region
 """
 Created on April 18, 2010
@@ @@ -75,6 +76,13 @@ def check_repo(repo_name, base_path, ver @@
         log.info('%s repo is free for creation', repo_name)
         return True
 @cache_region('super_short_term', 'cached_hg_ui')
 def get_hg_ui_cached():
     from pylons_app.model.meta import Session
     sa = Session()
     return sa.query(HgAppUi).all()
 def make_ui(read_from='file', path=None, checkpaths=True):
     """
     A function that will read python rc files or database
@@ @@ -112,10 +120,7 @@ def make_ui(read_from='file', path=None, @@
     elif read_from == 'db':
         from pylons_app.model.meta import Session
         sa = Session()
         hg_ui = sa.query(HgAppUi).all()
         hg_ui = get_hg_ui_cached()
         for ui_ in hg_ui:
             baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)

0 comments (0 inline, 0 general)