for dirpath in os.listdir(p):

            if os.path.isfile(os.path.join(p, dirpath)):

                continue

            cur_path = os.path.join(p, dirpath)

            try:

                scm_info = get_scm(cur_path)

                yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info

            except VCSError:

                if not recursive:

                    continue

                #check if this dir containts other repos for recursive scan

                rec_path = os.path.join(p, dirpath)

                if os.path.isdir(rec_path):

                    for inner_scm in _get_repos(rec_path):

                        yield inner_scm

    return _get_repos(path)

def is_valid_repo(repo_name, base_path, scm=None):

    """

    Returns True if given path is a valid repository False otherwise.

    If scm param is given also compare if given scm is the same as expected 

    from scm parameter

    :param repo_name:

    :param base_path:

    :param scm:

    :return True: if given path is a valid repository

    """

    full_path = os.path.join(safe_str(base_path), safe_str(repo_name))

    try:

        scm_ = get_scm(full_path)

        if scm:

            return scm_[0] == scm

        return True

    except VCSError:

        return False

def is_valid_repos_group(repos_group_name, base_path):

    """

    Returns True if given path is a repos group False otherwise

    :param repo_name:

    :param base_path:

    """

    full_path = os.path.join(safe_str(base_path), safe_str(repos_group_name))

    # check if it's not a repo

    if is_valid_repo(repos_group_name, base_path):

        return False

    try:

        # we need to check bare git repos at higher level

        # since we might match branches/hooks/info/objects or possible

        # other things inside bare git repo

        get_scm(os.path.dirname(full_path))

        return False

    except VCSError:

        pass

    # check if it's a valid path

    if os.path.isdir(full_path):

        return True

    return False

def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):

    while True:

        ok = raw_input(prompt)

        if ok in ('y', 'ye', 'yes'):

            return True

        if ok in ('n', 'no', 'nop', 'nope'):

            return False

        retries = retries - 1

        if retries < 0:

            raise IOError

        print complaint

#propagated from mercurial documentation

ui_sections = ['alias', 'auth',

                'decode/encode', 'defaults',

                'diff', 'email',

                'extensions', 'format',

                'merge-patterns', 'merge-tools',

                'hooks', 'http_proxy',

                'smtp', 'patch',

                'paths', 'profiling',

                'server', 'trusted',

                'ui', 'web', ]

    """

    A function that will read python rc files or database

    and make an mercurial ui object from read options

    :param path: path to mercurial config file

    :param checkpaths: check the path

    :param read_from: read from 'file' or 'db'

    """

    baseui = ui.ui()

    # clean the baseui object

    baseui._ocfg = config.config()

    baseui._ucfg = config.config()

    baseui._tcfg = config.config()

    if read_from == 'file':

        if not os.path.isfile(path):

            log.debug('hgrc file is not present at %s skipping...' % path)

            return False

        log.debug('reading hgrc from %s' % path)

        cfg = config.config()

        cfg.read(path)

        for section in ui_sections:

            for k, v in cfg.items(section):

                log.debug('settings ui from file[%s]%s:%s' % (section, k, v))

                baseui.setconfig(section, k, v)

    elif read_from == 'db':

        sa = meta.Session()

        ret = sa.query(RhodeCodeUi)\

            .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\

            .all()

        hg_ui = ret

        for ui_ in hg_ui:

            if ui_.ui_active:

                log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,

                          ui_.ui_key, ui_.ui_value)

                baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)

            if ui_.ui_key == 'push_ssl':

                # force set push_ssl requirement to False, rhodecode

                # handles that

                baseui.setconfig(ui_.ui_section, ui_.ui_key, False)

        meta.Session.remove()

    return baseui

def set_rhodecode_config(config):

    """

    Updates pylons config with new settings from database

    :param config:

    """

    hgsettings = RhodeCodeSetting.get_app_settings()

    for k, v in hgsettings.items():

        config[k] = v

def invalidate_cache(cache_key, *args):

    """

    Puts cache invalidation task into db for

    further global cache invalidation

    """

    from rhodecode.model.scm import ScmModel

    if cache_key.startswith('get_repo_cached_'):

        name = cache_key.split('get_repo_cached_')[-1]

        ScmModel().mark_for_invalidation(name)

def map_groups(path):

    """

    Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s' % (lvl,

                                                                   group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hook=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hook: if this is True, also check and install githook

        for a repo if missing

    """

    from rhodecode.model.repo import RepoModel

    from rhodecode.model.scm import ScmModel

    sa = meta.Session()

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        db_repo = rm.get_by_repo_name(name)

        # found repo that is on filesystem not in RhodeCode database

        if not db_repo:

            log.info('repository %s not found creating now' % name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                _(u'Repository named %(repo)s already exists'),

            'repository_in_group_exists': _(u'Repository "%(repo)s" already '

                                            'exists in group "%(group)s"'),

            'same_group_exists': _(u'Repositories group with name "%(repo)s" '

                                   'already exists')

        }

        def _to_python(self, value, state):

            repo_name = repo_name_slug(value.get('repo_name', ''))

            repo_group = value.get('repo_group')

            if repo_group:

                gr = RepoGroup.get(repo_group)

                group_path = gr.full_path

                group_name = gr.group_name

                # value needs to be aware of group name in order to check

                # db key This is an actual just the name to store in the

                # database

                repo_name_full = group_path + RepoGroup.url_sep() + repo_name

            else:

                group_name = group_path = ''

                repo_name_full = repo_name

            value['repo_name'] = repo_name

            value['repo_name_full'] = repo_name_full

            value['group_path'] = group_path

            value['group_name'] = group_name

            return value

        def validate_python(self, value, state):

            repo_name = value.get('repo_name')

            repo_name_full = value.get('repo_name_full')

            group_path = value.get('group_path')

            group_name = value.get('group_name')

            if repo_name in [ADMIN_PREFIX, '']:

                msg = M(self, 'invalid_repo_name', state, repo=repo_name)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(repo_name=msg)

                )

            rename = old_data.get('repo_name') != repo_name_full

            create = not edit

            if rename or create:

                if group_path != '':

                    if Repository.get_by_repo_name(repo_name_full):

                        msg = M(self, 'repository_in_group_exists', state,

                                repo=repo_name, group=group_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

                elif RepoGroup.get_by_group_name(repo_name_full):

                        msg = M(self, 'same_group_exists', state,

                                repo=repo_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

                elif Repository.get_by_repo_name(repo_name_full):

                        msg = M(self, 'repository_exists', state,

                                repo=repo_name)

                        raise formencode.Invalid(msg, value, state,

                            error_dict=dict(repo_name=msg)

                        )

            return value

    return _validator

def ValidForkName(*args, **kwargs):

    return ValidRepoName(*args, **kwargs)

def SlugifyName():

    class _validator(formencode.validators.FancyValidator):

        def _to_python(self, value, state):

            return repo_name_slug(value)

        def validate_python(self, value, state):

            pass

    return _validator

def ValidCloneUri():

    from rhodecode.lib.utils import make_ui

    def url_handler(repo_type, url, ui=None):

        if repo_type == 'hg':

            from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository

            from mercurial.httppeer import httppeer

            if url.startswith('http'):

                ## initially check if it's at least the proper URL

                ## or does it pass basic auth

                MercurialRepository._check_url(url)

            elif url.startswith('svn+http'):

                from hgsubversion.svnrepo import svnremoterepo

            elif url.startswith('git+http'):

                raise NotImplementedError()

        elif repo_type == 'git':

            from rhodecode.lib.vcs.backends.git.repository import GitRepository

            if url.startswith('http'):

                ## initially check if it's at least the proper URL

                ## or does it pass basic auth

                GitRepository._check_url(url)

            elif url.startswith('svn+http'):

                raise NotImplementedError()

            elif url.startswith('hg+http'):

                raise NotImplementedError()

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'clone_uri': _(u'invalid clone url'),

            'invalid_clone_uri': _(u'Invalid clone url, provide a '

                                    'valid clone http(s)/svn+http(s) url')

        }

        def validate_python(self, value, state):

            repo_type = value.get('repo_type')

            url = value.get('clone_uri')

            if not url:

                pass

            else:

                try:

                except Exception:

                    log.exception('Url validation failed')

                    msg = M(self, 'clone_uri')

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(clone_uri=msg)

                    )

    return _validator

def ValidForkType(old_data={}):

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'invalid_fork_type': _(u'Fork have to be the same type as parent')

        }

        def validate_python(self, value, state):

            if old_data['repo_type'] != value:

                msg = M(self, 'invalid_fork_type', state)

                raise formencode.Invalid(msg, value, state,

                    error_dict=dict(repo_type=msg)

                )

    return _validator

def ValidPerms(type_='repo'):

    if type_ == 'group':

        EMPTY_PERM = 'group.none'

    elif type_ == 'repo':

        EMPTY_PERM = 'repository.none'

    class _validator(formencode.validators.FancyValidator):

        messages = {

            'perm_new_member_name':

                _(u'This username or users group name is not valid')

        }

        def to_python(self, value, state):

            perms_update = []

            perms_new = []

            # build a list of permission to update and new permission to create

            for k, v in value.items():

                # means new added member to permissions

                if k.startswith('perm_new_member'):

                    new_perm = value.get('perm_new_member', False)

                    new_member = value.get('perm_new_member_name', False)

                    new_type = value.get('perm_new_member_type')

                    if new_member and new_perm:

                        if (new_member, new_perm, new_type) not in perms_new:

                            perms_new.append((new_member, new_perm, new_type))

                elif k.startswith('u_perm_') or k.startswith('g_perm_'):

                    member = k[7:]

                    t = {'u': 'user',

                         'g': 'users_group'

                    }[k[0]]

                    if member == 'default':

                        if value.get('private'):

                            # set none for default when updating to

                            # private repo

                            v = EMPTY_PERM

                    perms_update.append((member, v, t))

            value['perms_updates'] = perms_update

            value['perms_new'] = perms_new

            # update permissions

            for k, v, t in perms_new:

                try:

                    if t is 'user':

                        self.user_db = User.query()\

                            .filter(User.active == True)\

                            .filter(User.username == k).one()

                    if t is 'users_group':

                        self.user_db = UsersGroup.query()\

                            .filter(UsersGroup.users_group_active == True)\

                            .filter(UsersGroup.users_group_name == k).one()

                except Exception:

                    log.exception('Updated permission failed')

                    msg = M(self, 'perm_new_member_type', state)

                    raise formencode.Invalid(msg, value, state,

                        error_dict=dict(perm_new_member_name=msg)

                    )

            return value

    return _validator

def ValidSettings():

    class _validator(formencode.validators.FancyValidator):

        def _to_python(self, value, state):

            # settings  form can't edit user

            if 'user' in value:

                del value['user']

            return value

        def validate_python(self, value, state):