Changeset - 48727add84c9
[Not reviewed]
default
1 7 1
Marcin Kuzminski - 15 years ago 2010-05-30 22:08:21
marcin@python-works.com
Made repos path config configurable from pylons app configs. update Readme
8 files changed with 19 insertions and 8 deletions:
0 comments (0 inline, 0 general)
README.txt
Show inline comments
 
Pylons based replacement for hgwebdir. Fully customizable, 
 
with authentication, permissions. Based on vcs library.
 
- has it's own middleware to handle mercurial protocol request each request can
 
  be logged and authenticated +threaded performance unlikely to hgweb
 
- mako templates let's you cusmotize look and feel of appplication.
 
- diffs annotations and source code all colored by pygments.
 
- admin interface for performing user/permission managments as well as repository
 
  managment
 
- added cache with invalidation on push/repo managment for high performance and
 
  always upto date data.
 
- rss /atom feed customizable
 
- future support for git
 
- based on pylons 1.0 / sqlalchemy 0.6
 

	
 
===
 
This software is still in beta mode. I don't guarantee that it'll work.
 
I started this project since i was tired of sad looks, and zero controll over
 
our company regular hgwebdir.
 

	
 

	
 
== INSTALATION
 
run dbmanage.py from pylons_app/lib it should create all needed table and
 
an admin account, Edit file repositories.config and change the path for you 
 
mercurial repositories, remember about permissions.
 
\ No newline at end of file
development.ini
Show inline comments
 
@@ -19,48 +19,49 @@ debug = true
 
#error_message = 'mercurial crash !'
 

	
 
[server:main]
 
##nr of threads to spawn
 
threadpool_workers = 5
 

	
 
##max request before
 
threadpool_max_requests = 2
 

	
 
##option to use threads of process
 
use_threadpool = true
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 5000
 

	
 
[app:main]
 
use = egg:pylons_app
 
full_stack = true
 
static_files = false
 
lang=en
 
cache_dir = %(here)s/data
 
##a name for our application
 
hg_app_name = Python-works
 
hg_app_repo_conf = repositories.config
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/tmp/cache/data
 
beaker.cache.lock_dir=/tmp/cache/lock
 
beaker.cache.regions=short_term,long_term
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 

	
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
#set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
production.ini
Show inline comments
 
@@ -19,48 +19,49 @@ debug = true
 
#error_message = 'mercurial crash !'
 

	
 
[server:main]
 
##nr of threads to spawn
 
threadpool_workers = 5
 

	
 
##max request before
 
threadpool_max_requests = 2
 

	
 
##option to use threads of process
 
use_threadpool = true
 

	
 
use = egg:Paste#http
 
host = 127.0.0.1
 
port = 8001
 

	
 
[app:main]
 
use = egg:pylons_app
 
full_stack = true
 
static_files = false
 
lang=en
 
cache_dir = %(here)s/data
 
##a name for our application
 
hg_app_name = Python-works
 
hg_app_repo_conf = repositories.config
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/tmp/cache/data
 
beaker.cache.lock_dir=/tmp/cache/lock
 
beaker.cache.regions=short_term,long_term
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
pylons_app/lib/app_globals.py
Show inline comments
 
"""The application's Globals object"""
 

	
 
from beaker.cache import CacheManager
 
from beaker.util import parse_cache_config_options
 
from pylons_app.lib.utils import make_ui
 

	
 
class Globals(object):
 

	
 
    """Globals acts as a container for objects available throughout the
 
    life of the application
 

	
 
    """
 

	
 
    def __init__(self, config):
 
        """One instance of Globals is created during application
 
        initialization and is available during requests via the
 
        'app_globals' variable
 

	
 
        """
 
        self.cache = CacheManager(**parse_cache_config_options(config))
 
        self.baseui = make_ui('hgwebdir.config')
 
        self.baseui = make_ui(config['hg_app_repo_conf'])
 
        self.paths = self.baseui.configitems('paths')
 
        self.base_path = self.paths[0][1].replace('*', '')
 
        self.changeset_annotation_colors = {}
 
        self.available_permissions = None # propagated after init_model
pylons_app/lib/backup_manager.py
Show inline comments
 
'''BACKUP MANAGER'''
 
import logging
 
from mercurial import config
 
import tarfile
 
import os
 
import datetime
 
import sys
 
import subprocess
 
logging.basicConfig(level=logging.DEBUG,
 
                    format="%(asctime)s %(levelname)-5.5s %(message)s")
 

	
 
class BackupManager(object):
 
    def __init__(self):
 
    def __init__(self, id_rsa_path, repo_conf):
 
        self.repos_path = None
 
        self.backup_file_name = None
 
        self.id_rsa_path = '/home/pylons/id_rsa'
 
        self.id_rsa_path = id_rsa_path
 
        self.check_id_rsa()
 
        cur_dir = os.path.realpath(__file__)
 
        dn = os.path.dirname
 
        self.backup_file_path = os.path.join(dn(dn(dn(cur_dir))), 'data')
 
        cfg = config.config()
 
        try:
 
            cfg.read(os.path.join(dn(dn(dn(cur_dir))), 'hgwebdir.config'))
 
            cfg.read(os.path.join(dn(dn(dn(cur_dir))), repo_conf))
 
        except IOError:
 
            logging.error('Could not read hgwebdir.config')
 
            logging.error('Could not read %s', repo_conf)
 
            sys.exit()
 
        self.set_repos_path(cfg.items('paths'))
 
        logging.info('starting backup for %s', self.repos_path)
 
        logging.info('backup target %s', self.backup_file_path)
 

	
 
        if not os.path.isdir(self.repos_path):
 
            raise Exception('Not a valid directory in %s' % self.repos_path)
 

	
 
    def check_id_rsa(self):
 
        if not os.path.isfile(self.id_rsa_path):
 
            logging.error('Could not load id_rsa key file in %s',
 
                          self.id_rsa_path)
 
            sys.exit()
 

	
 
    def set_repos_path(self, paths):
 
        repos_path = paths[0][1].split('/')
 
        if repos_path[-1] in ['*', '**']:
 
            repos_path = repos_path[:-1]
 
        if repos_path[0] != '/':
 
            repos_path[0] = '/'
 
        self.repos_path = os.path.join(*repos_path)
 

	
 
    def backup_repos(self):
 
        today = datetime.datetime.now().weekday() + 1
 
@@ -59,30 +59,30 @@ class BackupManager(object):
 

	
 

	
 

	
 
    def transfer_files(self):
 
        params = {
 
                  'id_rsa_key': self.id_rsa_path,
 
                  'backup_file_path':self.backup_file_path,
 
                  'backup_file_name':self.backup_file_name,
 
                  }
 
        cmd = ['scp', '-l', '40000', '-i', '%(id_rsa_key)s' % params,
 
               '%(backup_file_path)s/%(backup_file_name)s' % params,
 
               'root@192.168.2.102:/backups/mercurial' % params]
 

	
 
        subprocess.call(cmd)
 
        logging.info('Transfered file %s to %s', self.backup_file_name, cmd[4])
 
        
 
    
 
    def rm_file(self):
 
        logging.info('Removing file %s', self.backup_file_name)
 
        os.remove(os.path.join(self.backup_file_path, self.backup_file_name))
 
    
 

	
 

	
 
if __name__ == "__main__":
 
    B_MANAGER = BackupManager()
 
    B_MANAGER = BackupManager('/home/pylons/id_rsa', 'repositories.config')
 
    B_MANAGER.backup_repos()
 
    B_MANAGER.transfer_files()
 
    B_MANAGER.rm_file()
 

	
 

	
pylons_app/lib/middleware/simplehg.py
Show inline comments
 
@@ -35,49 +35,49 @@ class SimpleHg(object):
 
        
 
    def __call__(self, environ, start_response):
 
        if not is_mercurial(environ):
 
            return self.application(environ, start_response)
 
        else:
 
            #===================================================================
 
            # AUTHENTICATE THIS MERCURIAL REQUEST
 
            #===================================================================
 
            username = REMOTE_USER(environ)
 
            if not username:
 
                result = self.authenticate(environ)
 
                if isinstance(result, str):
 
                    AUTH_TYPE.update(environ, 'basic')
 
                    REMOTE_USER.update(environ, result)
 
                else:
 
                    return result.wsgi_application(environ, start_response)
 
            
 
            try:
 
                repo_name = environ['PATH_INFO'].split('/')[1]
 
            except:
 
                return HTTPNotFound()(environ, start_response)
 
            
 
            #since we wrap into hgweb, just reset the path
 
            environ['PATH_INFO'] = '/'
 
            self.baseui = make_ui()
 
            self.baseui = make_ui(self.config['hg_app_repo_conf'])
 
            self.basepath = self.baseui.configitems('paths')[0][1]\
 
                                                            .replace('*', '')
 
            self.repo_path = os.path.join(self.basepath, repo_name)
 
            try:
 
                app = wsgiapplication(self.__make_app)
 
            except Exception as e:
 
                return HTTPNotFound()(environ, start_response)
 
            action = self.__get_action(environ)            
 
            #invalidate cache on push
 
            if action == 'push':
 
                self.__invalidate_cache(repo_name)
 
            
 
            if action:
 
                username = self.__get_environ_user(environ)
 
                self.__log_user_action(username, action, repo_name)
 
                         
 
            return app(environ, start_response)            
 

	
 
    def __make_app(self):
 
        hgserve = hgweb(self.repo_path)
 
        return  self.__load_web_settings(hgserve)
 
    
 
    def __get_environ_user(self, environ):
 
        return environ.get('REMOTE_USER')
pylons_app/lib/utils.py
Show inline comments
 
@@ -25,54 +25,57 @@ def check_repo_dir(paths):
 
    repos_path = paths[0][1].split('/')
 
    if repos_path[-1] in ['*', '**']:
 
        repos_path = repos_path[:-1]
 
    if repos_path[0] != '/':
 
        repos_path[0] = '/'
 
    if not os.path.isdir(os.path.join(*repos_path)):
 
        raise Exception('Not a valid repository in %s' % paths[0][1])
 

	
 
def check_repo(repo_name, base_path):
 

	
 
    repo_path = os.path.join(base_path, repo_name)
 

	
 
    try:
 
        r = hg.repository(ui.ui(), repo_path)
 
        hg.verify(r)
 
        #here we hnow that repo exists it was verified
 
        log.info('%s repo is already created', repo_name)
 
        return False
 
        #raise Exception('Repo exists')
 
    except RepoError:
 
        log.info('%s repo is free for creation', repo_name)
 
        #it means that there is no valid repo there...
 
        return True
 
                
 
def make_ui(path='hgwebdir.config', checkpaths=True):        
 
def make_ui(path=None, checkpaths=True):        
 
    """
 
    A funcion that will read python rc files and make an ui from read options
 
    
 
    @param path: path to mercurial config file
 
    """
 
    if not path:
 
        log.error('repos config path is empty !')
 
    
 
    if not os.path.isfile(path):
 
        log.warning('Unable to read config file %s' % path)
 
        return False
 
    #propagated from mercurial documentation
 
    sections = [
 
                'alias',
 
                'auth',
 
                'decode/encode',
 
                'defaults',
 
                'diff',
 
                'email',
 
                'extensions',
 
                'format',
 
                'merge-patterns',
 
                'merge-tools',
 
                'hooks',
 
                'http_proxy',
 
                'smtp',
 
                'patch',
 
                'paths',
 
                'profiling',
 
                'server',
 
                'trusted',
 
                'ui',
repositories.config
Show inline comments
 
file renamed from hgwebdir.config to repositories.config
0 comments (0 inline, 0 general)