Changeset - bc9a73adc216
[Not reviewed]
beta
0 2 0
Marcin Kuzminski - 15 years ago 2010-12-22 03:39:32
marcin@python-works.com
Added recursive scanning for repositories in directory
2 files changed with 25 insertions and 18 deletions:
0 comments (0 inline, 0 general)
rhodecode/lib/utils.py
Show inline comments
 
@@ -65,121 +65,128 @@ def action_logger(user, action, repo, ip
 
    :param repo: string name of repository or object containing repo_id,
 
        that action was made on
 
    :param ipaddr: optional ip address from what the action was made
 
    :param sa: optional sqlalchemy session
 
    
 
    """
 

	
 
    if not sa:
 
        sa = meta.Session()
 

	
 
    try:
 
        um = UserModel()
 
        if hasattr(user, 'user_id'):
 
            user_obj = user
 
        elif isinstance(user, basestring):
 
            user_obj = um.get_by_username(user, cache=False)
 
        else:
 
            raise Exception('You have to provide user object or username')
 

	
 

	
 
        rm = RepoModel()
 
        if hasattr(repo, 'repo_id'):
 
            repo_obj = rm.get(repo.repo_id, cache=False)
 
            repo_name = repo_obj.repo_name
 
        elif  isinstance(repo, basestring):
 
            repo_name = repo.lstrip('/')
 
            repo_obj = rm.get_by_repo_name(repo_name, cache=False)
 
        else:
 
            raise Exception('You have to provide repository to action logger')
 

	
 

	
 
        user_log = UserLog()
 
        user_log.user_id = user_obj.user_id
 
        user_log.action = action
 

	
 
        user_log.repository_id = repo_obj.repo_id
 
        user_log.repository_name = repo_name
 

	
 
        user_log.action_date = datetime.datetime.now()
 
        user_log.user_ip = ipaddr
 
        sa.add(user_log)
 
        sa.commit()
 

	
 
        log.info('Adding user %s, action %s on %s', user_obj, action, repo)
 
    except:
 
        log.error(traceback.format_exc())
 
        sa.rollback()
 

	
 
def get_repos(path, recursive=False, initial=False):
 
def get_repos(path, recursive=False):
 
    """
 
    Scans given path for repos and return (name,(type,path)) tuple 
 
    
 
    :param prefix:
 
    :param path:
 
    :param recursive:
 
    :param initial:
 
    :param path: path to scann for repositories
 
    :param recursive: recursive search and return names with subdirs in front 
 
    """
 
    from vcs.utils.helpers import get_scm
 
    from vcs.exceptions import VCSError
 

	
 
    if path.endswith('/'):
 
        #add ending slash for better results
 
        path = path[:-1]
 

	
 
    def _get_repos(p):
 
        for dirpath in os.listdir(p):
 
            if os.path.isfile(os.path.join(p, dirpath)):
 
                continue
 
            cur_path = os.path.join(p, dirpath)
 
    try:
 
        scm = get_scm(path)
 
    except:
 
        pass
 
    else:
 
        raise Exception('The given path %s should not be a repository got %s',
 
                        path, scm)
 
                scm_info = get_scm(cur_path)
 
                yield scm_info[1].split(path)[-1].lstrip('/'), scm_info
 
            except VCSError:
 
                if not recursive:
 
                    continue
 
                #check if this dir containts other repos for recursive scan
 
                rec_path = os.path.join(p, dirpath)
 
                if os.path.isdir(rec_path):
 
                    for inner_scm in _get_repos(rec_path):
 
                        yield inner_scm
 

	
 
    for dirpath in os.listdir(path):
 
        try:
 
            yield dirpath, get_scm(os.path.join(path, dirpath))
 
        except VCSError:
 
            pass
 
    return _get_repos(path)
 

	
 
def check_repo_fast(repo_name, base_path):
 
    """
 
    Check given path for existence of directory
 
    :param repo_name:
 
    :param base_path:
 
    
 
    :return False: if this directory is present
 
    """
 
    if os.path.isdir(os.path.join(base_path, repo_name)):return False
 
    return True
 

	
 
def check_repo(repo_name, base_path, verify=True):
 

	
 
    repo_path = os.path.join(base_path, repo_name)
 

	
 
    try:
 
        if not check_repo_fast(repo_name, base_path):
 
            return False
 
        r = hg.repository(ui.ui(), repo_path)
 
        if verify:
 
            hg.verify(r)
 
        #here we hnow that repo exists it was verified
 
        log.info('%s repo is already created', repo_name)
 
        return False
 
    except RepoError:
 
        #it means that there is no valid repo there...
 
        log.info('%s repo is free for creation', repo_name)
 
        return True
 

	
 
def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
 
    while True:
 
        ok = raw_input(prompt)
 
        if ok in ('y', 'ye', 'yes'): return True
 
        if ok in ('n', 'no', 'nop', 'nope'): return False
 
        retries = retries - 1
 
        if retries < 0: raise IOError
 
        print complaint
 

	
 
#propagated from mercurial documentation
 
ui_sections = ['alias', 'auth',
 
                'decode/encode', 'defaults',
 
                'diff', 'email',
 
                'extensions', 'format',
 
                'merge-patterns', 'merge-tools',
 
                'hooks', 'http_proxy',
 
                'smtp', 'patch',
 
                'paths', 'profiling',
rhodecode/model/scm.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
"""
 
    rhodecode.model.scm
 
    ~~~~~~~~~~~~~~~~~~~
 

	
 
    Scm model for RhodeCode
 

	
 
    :created_on: Apr 9, 2010
 
    :author: marcink
 
    :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>    
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software; you can redistribute it and/or
 
# modify it under the terms of the GNU General Public License
 
# as published by the Free Software Foundation; version 2
 
# of the License or (at your opinion) any later version of the license.
 
# 
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 
import os
 
import time
 
import traceback
 
import logging
 

	
 
from vcs import get_backend
 
from vcs.utils.helpers import get_scm
 
from vcs.exceptions import RepositoryError, VCSError
 
from vcs.utils.lazy import LazyProperty
 

	
 
from mercurial import ui
 

	
 
from beaker.cache import cache_region, region_invalidate
 

	
 
from rhodecode import BACKENDS
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.auth import HasRepoPermissionAny
 
from rhodecode.lib.utils import get_repos, make_ui, action_logger
 
from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, action_logger
 
from rhodecode.model import BaseModel
 
from rhodecode.model.user import UserModel
 

	
 
from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
 
    UserFollowing, UserLog
 
from rhodecode.model.caching_query import FromCache
 

	
 
from sqlalchemy.orm import joinedload
 
from sqlalchemy.orm.session import make_transient
 
from sqlalchemy.exc import DatabaseError
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class UserTemp(object):
 
    def __init__(self, user_id):
 
        self.user_id = user_id
 
class RepoTemp(object):
 
    def __init__(self, repo_id):
 
        self.repo_id = repo_id
 

	
 
class ScmModel(BaseModel):
 
    """Generic Scm Model
 
    """
 

	
 
    @LazyProperty
 
    def repos_path(self):
 
        """Get's the repositories root path from database
 
        """
 

	
 
        q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 

	
 
        return q.ui_value
 

	
 
    def repo_scan(self, repos_path, baseui):
 
        """Listing of repositories in given path. This path should not be a 
 
        repository itself. Return a dictionary of repository objects
 
        
 
        :param repos_path: path to directory containing repositories
 
        :param baseui: baseui instance to instantiate MercurialRepostitory with
 
        """
 

	
 
        log.info('scanning for repositories in %s', repos_path)
 

	
 
        if not isinstance(baseui, ui.ui):
 
            baseui = make_ui('db')
 
        repos_list = {}
 

	
 
        for name, path in get_repos(repos_path):
 
        for name, path in get_filesystem_repos(repos_path, recursive=True):
 
            try:
 
                if repos_list.has_key(name):
 
                    raise RepositoryError('Duplicate repository name %s '
 
                                    'found in %s' % (name, path))
 
                else:
 

	
 
                    klass = get_backend(path[0])
 

	
 
                    if path[0] == 'hg' and path[0] in BACKENDS.keys():
 
                        repos_list[name] = klass(path[1], baseui=baseui)
 

	
 
                    if path[0] == 'git' and path[0] in BACKENDS.keys():
 
                        repos_list[name] = klass(path[1])
 
            except OSError:
 
                continue
 

	
 
        return repos_list
 

	
 
    def get_repos(self, all_repos=None):
 
        """Get all repos from db and for each repo create it's backend instance.
 
        and fill that backed with information from database
 
        
 
        :param all_repos: give specific repositories list, good for filtering
 
        """
 

	
 
        if all_repos is None:
 
            all_repos = self.sa.query(Repository)\
 
                .order_by(Repository.repo_name).all()
 

	
 
        #get the repositories that should be invalidated
 
        invalidation_list = [str(x.cache_key) for x in \
 
                             self.sa.query(CacheInvalidation.cache_key)\
 
                             .filter(CacheInvalidation.cache_active == False)\
 
                             .all()]
 

	
 
        for r in all_repos:
 

	
 
            repo = self.get(r.repo_name, invalidation_list)
 

	
 
            if repo is not None:
 
                last_change = repo.last_change
 
                tip = h.get_changeset_safe(repo, 'tip')
 

	
 
                tmp_d = {}
 
                tmp_d['name'] = repo.name
 
                tmp_d['name_sort'] = tmp_d['name'].lower()
 
                tmp_d['description'] = repo.dbrepo.description
 
                tmp_d['description_sort'] = tmp_d['description']
0 comments (0 inline, 0 general)