Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    owner = User.get_first_admin()

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s'

                      % (lvl, group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            group.user = owner

            sa.add(group)

            perm_obj = rgm._create_default_perms(group)

            sa.add(perm_obj)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hook=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hook: if this is True, also check and install githook

        for a repo if missing

    """

    from rhodecode.model.repo import RepoModel

    from rhodecode.model.scm import ScmModel

    sa = meta.Session()

    rm = RepoModel()

    user = User.get_first_admin()

    added = []

    ##creation defaults

    defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_locking = defs.get('repo_enable_locking')

    enable_downloads = defs.get('repo_enable_downloads')

    private = defs.get('repo_private')

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        db_repo = rm.get_by_repo_name(name)

        # found repo that is on filesystem not in RhodeCode database

        if not db_repo:

            log.info('repository %s not found, creating now' % name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = rm.create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repos_group=getattr(group, 'group_id', None),

                owner=user,

                just_db=True,

                enable_locking=enable_locking,

                enable_downloads=enable_downloads,

                enable_statistics=enable_statistics,

                private=private

            )

            # we added that repo just now, and make sure it has githook

            # installed

            if new_repo.repo_type == 'git':

                ScmModel().install_git_hook(new_repo.scm_instance)

            new_repo.update_changeset_cache()

        elif install_git_hook:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hook(db_repo.scm_instance)

    sa.commit()

    removed = []

    if remove_obsolete:

        # remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            if repo.repo_name not in initial_repo_list.keys():

                log.debug("Removing non-existing repository found in db `%s`" %

                          repo.repo_name)

                try:

                    removed.append(repo.repo_name)

                    RepoModel(sa).delete(repo, forks='detach', fs_remove=False)

                    sa.commit()

                except Exception:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

    import rhodecode

    from rhodecode.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = rhodecode.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...' % rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:

            log.debug('settings custom INDEX_EXTENSIONS')

            conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])

        #ADDITIONAL MAPPINGS

        log.debug('adding extra into INDEX_EXTENSIONS')

        conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))

        # auto check if the module is not missing any data, set to default if is

        # this will help autoupdate new feature of rcext module

        from rhodecode.config import rcextensions

        for k in dir(rcextensions):

            if not k.startswith('_') and not hasattr(EXT, k):

                setattr(EXT, k, getattr(rcextensions, k))

def get_custom_lexer(extension):

    """

    returns a custom lexer if it's defined in rcextensions module, or None

    if there's no custom lexer defined

    """

    import rhodecode

    from pygments import lexers

    #check if we didn't define this extension as other lexer

    if rhodecode.EXTENSIONS and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:

        _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]

        return lexers.get_lexer_by_name(_lexer_name)

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

        self._compare_ok(id_, expected, given=response.body)

    def test_api_get_user(self):

        id_, params = _build_data(self.apikey, 'get_user',

                                  userid=TEST_USER_ADMIN_LOGIN)

        response = api_call(self, params)

        usr = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)

        ret = usr.get_api_data()

        ret['permissions'] = AuthUser(usr.user_id).permissions

        expected = ret

        self._compare_ok(id_, expected, given=response.body)

    def test_api_get_user_that_does_not_exist(self):

        id_, params = _build_data(self.apikey, 'get_user',

                                  userid='trololo')

        response = api_call(self, params)

        expected = "user `%s` does not exist" % 'trololo'

        self._compare_error(id_, expected, given=response.body)

    def test_api_get_user_without_giving_userid(self):

        id_, params = _build_data(self.apikey, 'get_user')

        response = api_call(self, params)

        usr = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)

        ret = usr.get_api_data()

        ret['permissions'] = AuthUser(usr.user_id).permissions

        expected = ret

        self._compare_ok(id_, expected, given=response.body)

    def test_api_get_user_without_giving_userid_non_admin(self):

        id_, params = _build_data(self.apikey_regular, 'get_user')

        response = api_call(self, params)

        usr = UserModel().get_by_username(self.TEST_USER_LOGIN)

        ret = usr.get_api_data()

        ret['permissions'] = AuthUser(usr.user_id).permissions

        expected = ret

        self._compare_ok(id_, expected, given=response.body)

    def test_api_get_user_with_giving_userid_non_admin(self):

        id_, params = _build_data(self.apikey_regular, 'get_user',

                                  userid=self.TEST_USER_LOGIN)

        response = api_call(self, params)

        expected = 'userid is not the same as your user'

        self._compare_error(id_, expected, given=response.body)

    def test_api_pull(self):

        #TODO: issues with rhodecode_extras here.. not sure why !

        pass

#        repo_name = 'test_pull'

#        r = fixture.create_repo(repo_name, repo_type=self.REPO_TYPE)

#        r.clone_uri = TEST_self.REPO

#        Session.add(r)

#        Session.commit()

#

#        id_, params = _build_data(self.apikey, 'pull',

#                                  repoid=repo_name,)

#        response = self.app.post(API_URL, content_type='application/json',

#                                 params=params)

#

#        expected = 'Pulled from `%s`' % repo_name

#        self._compare_ok(id_, expected, given=response.body)

#

#        fixture.destroy_repo(repo_name)

    def test_api_pull_error(self):

        id_, params = _build_data(self.apikey, 'pull',

                                  repoid=self.REPO,)

        response = api_call(self, params)

        expected = 'Unable to pull changes from `%s`' % self.REPO

        self._compare_error(id_, expected, given=response.body)

    def test_api_rescan_repos(self):

        id_, params = _build_data(self.apikey, 'rescan_repos')

        response = api_call(self, params)

        expected = {'added': [], 'removed': []}

        self._compare_ok(id_, expected, given=response.body)

    @mock.patch.object(ScmModel, 'repo_scan', crash)

    def test_api_rescann_error(self):

        id_, params = _build_data(self.apikey, 'rescan_repos',)

        response = api_call(self, params)

        expected = 'Error occurred during rescan repositories action'

        self._compare_error(id_, expected, given=response.body)

    def test_api_invalidate_cache(self):

        id_, params = _build_data(self.apikey, 'invalidate_cache',

                                  repoid=self.REPO)

        response = api_call(self, params)

        expected = ("Caches of repository `%s` was invalidated" % (self.REPO))

        self._compare_ok(id_, expected, given=response.body)

    @mock.patch.object(ScmModel, 'mark_for_invalidation', crash)

    def test_api_invalidate_cache_error(self):

        id_, params = _build_data(self.apikey, 'invalidate_cache',

                                  repoid=self.REPO)

        response = api_call(self, params)

        expected = 'Error occurred during cache invalidation action'

        self._compare_error(id_, expected, given=response.body)

    def test_api_lock_repo_lock_aquire(self):

        id_, params = _build_data(self.apikey, 'lock',

                                  userid=TEST_USER_ADMIN_LOGIN,

                                  repoid=self.REPO,

                                  locked=True)

        response = api_call(self, params)

        expected = ('User `%s` set lock state for repo `%s` to `%s`'

                   % (TEST_USER_ADMIN_LOGIN, self.REPO, True))

        self._compare_ok(id_, expected, given=response.body)

    def test_api_lock_repo_lock_aquire_by_non_admin(self):

        repo_name = 'api_delete_me'

        fixture.create_repo(repo_name, repo_type=self.REPO_TYPE,

                            cur_user=self.TEST_USER_LOGIN)

        try:

            id_, params = _build_data(self.apikey_regular, 'lock',

                                      repoid=repo_name,

                                      locked=True)

            response = api_call(self, params)

            expected = ('User `%s` set lock state for repo `%s` to `%s`'

                       % (self.TEST_USER_LOGIN, repo_name, True))

            self._compare_ok(id_, expected, given=response.body)

        finally:

            fixture.destroy_repo(repo_name)

    def test_api_lock_repo_lock_aquire_non_admin_with_userid(self):

        repo_name = 'api_delete_me'

        fixture.create_repo(repo_name, repo_type=self.REPO_TYPE,

                            cur_user=self.TEST_USER_LOGIN)

        try:

            id_, params = _build_data(self.apikey_regular, 'lock',

                                      userid=TEST_USER_ADMIN_LOGIN,

                                      repoid=repo_name,

                                      locked=True)

            response = api_call(self, params)

            expected = 'userid is not the same as your user'

            self._compare_error(id_, expected, given=response.body)

        finally:

            fixture.destroy_repo(repo_name)

    def test_api_lock_repo_lock_aquire_non_admin_not_his_repo(self):

        id_, params = _build_data(self.apikey_regular, 'lock',

                                  repoid=self.REPO,

                                  locked=True)

        response = api_call(self, params)

        expected = 'repository `%s` does not exist' % (self.REPO)

        self._compare_error(id_, expected, given=response.body)

    def test_api_lock_repo_lock_release(self):

        id_, params = _build_data(self.apikey, 'lock',

                                  userid=TEST_USER_ADMIN_LOGIN,

                                  repoid=self.REPO,

                                  locked=False)

        response = api_call(self, params)

        expected = ('User `%s` set lock state for repo `%s` to `%s`'

                   % (TEST_USER_ADMIN_LOGIN, self.REPO, False))

        self._compare_ok(id_, expected, given=response.body)

    def test_api_lock_repo_lock_aquire_optional_userid(self):

        id_, params = _build_data(self.apikey, 'lock',

                                  repoid=self.REPO,

                                  locked=True)

        response = api_call(self, params)

        expected = ('User `%s` set lock state for repo `%s` to `%s`'

                   % (TEST_USER_ADMIN_LOGIN, self.REPO, True))

        self._compare_ok(id_, expected, given=response.body)

    def test_api_lock_repo_lock_optional_locked(self):

        from rhodecode.lib.utils2 import  time_to_datetime

        _locked_since = json.dumps(time_to_datetime(Repository\

                                    .get_by_repo_name(self.REPO).locked[1]))

        id_, params = _build_data(self.apikey, 'lock',

                                  repoid=self.REPO)

        response = api_call(self, params)

        expected = ('Repo `%s` locked by `%s`. Locked=`True`. Locked since: `%s`'

                   % (self.REPO, TEST_USER_ADMIN_LOGIN, _locked_since))

        self._compare_ok(id_, expected, given=response.body)

    @mock.patch.object(Repository, 'lock', crash)

    def test_api_lock_error(self):