Converting from RhodeCode

-------------------------

Currently, you have two options for working with an existing RhodeCode

database:

- keep the database unconverted (intended for testing and evaluation)

- convert the database in a one-time step

Maintaining interoperability

~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Interoperability with RhodeCode 2.2.X installations is provided so you don't

have to immediately commit to switching to Kallithea. This option will most

likely go away once the two projects have diverged significantly.

To run Kallithea on a RhodeCode database, run::

   echo "BRAND = 'rhodecode'" > kallithea/brand.py

This location will depend on where you installed Kallithea. If you installed

via::

   python setup.py install

then you will find this location at

``$VIRTUAL_ENV/lib/python2.7/site-packages/Kallithea-0.1-py2.7.egg/kallithea``.

One-time conversion

~~~~~~~~~~~~~~~~~~~

Alternatively, if you would like to convert the database for good, you can use

a helper script provided by Kallithea. This script will operate directly on the

database, using the database string you can find in your ``production.ini`` (or

``development.ini``) file. For example, if using SQLite::

   cd /path/to/kallithea

   cp /path/to/rhodecode/rhodecode.db kallithea.db

   pip install sqlalchemy-migrate

   python kallithea/bin/rebranddb.py sqlite:///kallithea.db

.. Note::

   If you started out using the branding interoperability approach mentioned

   above, watch out for stray brand.pyc after removing brand.py.

.. _virtualenv: http://pypi.python.org/pypi/virtualenv

.. _Python: http://www.python.org/

.. _Sphinx: http://sphinx.pocoo.org/

.. _Mercurial: http://mercurial.selenic.com/

.. _Bitbucket: http://bitbucket.org/

.. _GitHub: http://github.com/

.. _Subversion: http://subversion.tigris.org/

.. _Git: http://git-scm.com/

.. _Celery: http://celeryproject.org/

.. _vcs: http://pypi.python.org/pypi/vcs

.. _Software Freedom Conservancy: http://sfconservancy.org/

    if test:

        if test_env is None:

            test_env = not int(os.environ.get('KALLITHEA_NO_TMP_PATH', 0))

        if test_index is None:

            test_index = not int(os.environ.get('KALLITHEA_WHOOSH_TEST_DISABLE', 0))

        if os.environ.get('TEST_DB'):

            # swap config if we pass enviroment variable

            config['sqlalchemy.db1.url'] = os.environ.get('TEST_DB')

        from kallithea.lib.utils import create_test_env, create_test_index

        from kallithea.tests import TESTS_TMP_PATH

        #set KALLITHEA_NO_TMP_PATH=1 to disable re-creating the database and

        #test repos

        if test_env:

            create_test_env(TESTS_TMP_PATH, config)

        #set KALLITHEA_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests

        if test_index:

            create_test_index(TESTS_TMP_PATH, config, True)

    DbManage.check_waitress()

    # MULTIPLE DB configs

    # Setup the SQLAlchemy database engine

    sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')

    init_model(sa_engine_db1)

    set_available_permissions(config)

    repos_path = make_ui('db').configitems('paths')[0][1]

    config['base_path'] = repos_path

    set_app_settings(config)

    instance_id = kallithea.CONFIG.get('instance_id')

    if instance_id == '*':

        instance_id = '%s-%s' % (platform.uname()[1], os.getpid())

        kallithea.CONFIG['instance_id'] = instance_id

    # CONFIGURATION OPTIONS HERE (note: all config options will override

    # any Pylons config options)

    # store config reference into our module to skip import magic of

    # pylons

    kallithea.CONFIG.update(config)

    set_vcs_config(kallithea.CONFIG)

    #check git version

    check_git_version()

    if str2bool(config.get('initial_repo_scan', True)):

        repo2db_mapper(ScmModel().repo_scan(repos_path),

    return config

                    try:

                        import hgsubversion  # pragma: no cover

                    except ImportError:

                        raise HgsubversionImportError

                Session().add(sett)

#                sett = Ui.get_by_key('hggit')

#                if not sett:

#                    #make one if it's not there !

#                    sett = Ui()

#                    sett.ui_key = 'hggit'

#                    sett.ui_section = 'extensions'

#

#                sett.ui_active = form_result['extensions_hggit']

#                Session().add(sett)

                Session().commit()

                h.flash(_('Updated VCS settings'), category='success')

            except HgsubversionImportError:

                log.error(traceback.format_exc())

                h.flash(_('Unable to activate hgsubversion support. '

                          'The "hgsubversion" library is missing'),

                        category='error')

            except Exception:

                log.error(traceback.format_exc())

                h.flash(_('Error occurred while updating '

                          'application settings'), category='error')

        defaults = Setting.get_app_settings()

        defaults.update(self._get_hg_ui_settings())

        return htmlfill.render(

            render('admin/settings/settings.html'),

            defaults=defaults,

            encoding="UTF-8",

            force_defaults=False)

    @HasPermissionAllDecorator('hg.admin')

    def settings_mapping(self):

        """GET /admin/settings/mapping: All items in the collection"""

        # url('admin_settings_mapping')

        c.active = 'mapping'

        if request.POST:

            rm_obsolete = request.POST.get('destroy', False)

            install_git_hooks = request.POST.get('hooks', False)

            invalidate_cache = request.POST.get('invalidate', False)

            if invalidate_cache:

                log.debug('invalidating all repositories cache')

                for repo in Repository.get_all():

                    ScmModel().mark_for_invalidation(repo.repo_name, delete=True)

            filesystem_repos = ScmModel().repo_scan()

            added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,

            h.flash(h.literal(_('Repositories successfully rescanned. Added: %s. Removed: %s.') %

                (', '.join(h.link_to(safe_unicode(repo_name), h.url('summary_home', repo_name=repo_name))

                 for repo_name in added) or '-',

                 ', '.join(h.escape(safe_unicode(repo_name)) for repo_name in removed) or '-')),

                category='success')

            return redirect(url('admin_settings_mapping'))

        defaults = Setting.get_app_settings()

        defaults.update(self._get_hg_ui_settings())

        return htmlfill.render(

            render('admin/settings/settings.html'),

            defaults=defaults,

            encoding="UTF-8",

            force_defaults=False)

    @HasPermissionAllDecorator('hg.admin')

    def settings_global(self):

        """GET /admin/settings/global: All items in the collection"""

        # url('admin_settings_global')

        c.active = 'global'

        if request.POST:

            application_form = ApplicationSettingsForm()()

            try:

                form_result = application_form.to_python(dict(request.POST))

            except formencode.Invalid as errors:

                return htmlfill.render(

                    render('admin/settings/settings.html'),

                    defaults=errors.value,

                    errors=errors.error_dict or {},

                    prefix_error=False,

                    encoding="UTF-8",

                    force_defaults=False)

            try:

                sett1 = Setting.create_or_update('title',

                                            form_result['title'])

                Session().add(sett1)

                sett2 = Setting.create_or_update('realm',

                                            form_result['realm'])

                Session().add(sett2)

                sett3 = Setting.create_or_update('ga_code',

                                            form_result['ga_code'])

                Session().add(sett3)

                sett4 = Setting.create_or_update('captcha_public_key',

    conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')

    conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()

    conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',

                                                        'utf8'), sep=',')

def map_groups(path):

    """

    Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = RepoGroupModel(sa)

    owner = User.get_first_admin()

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s',

                      lvl, group_name)

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            group.user = owner

            sa.add(group)

            perm_obj = rgm._create_default_perms(group)

            sa.add(perm_obj)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

        for a repo if missing

    """

    from kallithea.model.repo import RepoModel

    from kallithea.model.scm import ScmModel

    sa = meta.Session()

    repo_model = RepoModel()

    if user is None:

        user = User.get_first_admin()

    added = []

    ##creation defaults

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_locking = defs.get('repo_enable_locking')

    enable_downloads = defs.get('repo_enable_downloads')

    private = defs.get('repo_private')

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        unicode_name = safe_unicode(name)

        db_repo = repo_model.get_by_repo_name(unicode_name)

        # found repo that is on filesystem not in Kallithea database

        if not db_repo:

            log.info('repository %s not found, creating now', name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = repo_model._create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repo_group=getattr(group, 'group_id', None),

                owner=user,

                enable_locking=enable_locking,

                enable_downloads=enable_downloads,

                enable_statistics=enable_statistics,

                private=private,

                state=Repository.STATE_CREATED

            )

            sa.commit()

            # we added that repo just now, and make sure it has githook

            # installed, and updated server info

            if new_repo.repo_type == 'git':

                git_repo = new_repo.scm_instance

                # update repository server-info

                log.debug('Running update server info')

                git_repo._update_server_info()

            new_repo.update_changeset_cache()

            if db_repo.repo_type == 'git':

    removed = []

    # remove from database those repositories that are not in the filesystem

    for repo in sa.query(Repository).all():

        if repo.repo_name not in initial_repo_list.keys():

            if remove_obsolete:

                log.debug("Removing non-existing repository found in db `%s`",

                          repo.repo_name)

                try:

                    RepoModel(sa).delete(repo, forks='detach', fs_remove=False)

                    sa.commit()

                except Exception:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

            removed.append(repo.repo_name)

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

        :param alias:

        :param parent:

        :param clone_uri:

        :param repo_store_location:

        """

        from kallithea.lib.utils import is_valid_repo, is_valid_repo_group

        from kallithea.model.scm import ScmModel

        if '/' in repo_name:

            raise ValueError('repo_name must not contain groups got `%s`' % repo_name)

        if isinstance(repo_group, RepoGroup):

            new_parent_path = os.sep.join(repo_group.full_path_splitted)

        else:

            new_parent_path = repo_group or ''

        if repo_store_location:

            _paths = [repo_store_location]

        else:

            _paths = [self.repos_path, new_parent_path, repo_name]

            # we need to make it str for mercurial

        repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))

        # check if this path is not a repository

        if is_valid_repo(repo_path, self.repos_path):

            raise Exception('This path %s is a valid repository' % repo_path)

        # check if this path is a group

        if is_valid_repo_group(repo_path, self.repos_path):

            raise Exception('This path %s is a valid group' % repo_path)

        log.info('creating repo %s in %s from url: `%s`',

            repo_name, safe_unicode(repo_path),

            obfuscate_url_pw(clone_uri))

        backend = get_backend(repo_type)

        if repo_type == 'hg':

            baseui = make_ui('db', clear_session=False)

            # patch and reset hooks section of UI config to not run any

            # hooks on creating remote repo

            for k, v in baseui.configitems('hooks'):

                baseui.setconfig('hooks', k, None)

            repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui)

        elif repo_type == 'git':

            repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)

            # add kallithea hook into this repo

        else:

            raise Exception('Not supported repo_type %s expected hg/git' % repo_type)

        log.debug('Created repo %s with %s backend',

                  safe_unicode(repo_name), safe_unicode(repo_type))

        return repo

    def _rename_filesystem_repo(self, old, new):

        """

        renames repository on filesystem

        :param old: old name

        :param new: new name

        """

        log.info('renaming repo from %s to %s', old, new)

        old_path = os.path.join(self.repos_path, old)

        new_path = os.path.join(self.repos_path, new)

        if os.path.isdir(new_path):

            raise Exception(

                'Was trying to rename to already existing dir %s' % new_path

            )

        shutil.move(old_path, new_path)

    def _delete_filesystem_repo(self, repo):

        """

        removes repo from filesystem, the removal is actually done by

        renaming dir to a 'rm__*' prefix which Kallithea will skip.

        It can be undeleted later by reverting the rename.

        :param repo: repo object

        """

        rm_path = os.path.join(self.repos_path, repo.repo_name)

        log.info("Removing %s", rm_path)

        _now = datetime.now()

        _ms = str(_now.microsecond).rjust(6, '0')

        _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),

                             repo.just_name)

        if repo.group:

            args = repo.group.full_path_splitted + [_d]

            _d = os.path.join(*args)

        shutil.move(rm_path, os.path.join(self.repos_path, _d))

        self.mark_for_invalidation(repo.repo_name)

        if trigger_push_hook:

            self._handle_push(scm_instance,

                              username=user.username,

                              action='push_local',

                              repo_name=repo.repo_name,

                              revisions=[tip.raw_id])

        return tip

    def get_unread_journal(self):

        return self.sa.query(UserLog).count()

    def get_repo_landing_revs(self, repo=None):

        """

        Generates select option with tags branches and bookmarks (for hg only)

        grouped by type

        :param repo:

        """

        hist_l = []

        choices = []

        repo = self.__get_repo(repo)

        hist_l.append(['rev:tip', _('latest tip')])

        choices.append('rev:tip')

        if repo is None:

            return choices, hist_l

        repo = repo.scm_instance

        branches_group = ([(u'branch:%s' % k, k) for k, v in

                           repo.branches.iteritems()], _("Branches"))

        hist_l.append(branches_group)

        choices.extend([x[0] for x in branches_group[0]])

        if repo.alias == 'hg':

            bookmarks_group = ([(u'book:%s' % k, k) for k, v in

                                repo.bookmarks.iteritems()], _("Bookmarks"))

            hist_l.append(bookmarks_group)

            choices.extend([x[0] for x in bookmarks_group[0]])

        tags_group = ([(u'tag:%s' % k, k) for k, v in

                       repo.tags.iteritems()], _("Tags"))

        hist_l.append(tags_group)

        choices.extend([x[0] for x in tags_group[0]])

        return choices, hist_l

        """

        Creates a kallithea hook inside a git repository

        :param repo: Instance of VCS repo

        :param force_create: Create even if same name hook exists

        """

        loc = jn(repo.path, 'hooks')

        if not repo.bare:

            loc = jn(repo.path, '.git', 'hooks')

        if not os.path.isdir(loc):

            os.makedirs(loc)

        tmpl_post = pkg_resources.resource_string(

            'kallithea', jn('config', 'post_receive_tmpl.py')

        )

        tmpl_pre = pkg_resources.resource_string(

            'kallithea', jn('config', 'pre_receive_tmpl.py')

        )

        for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:

            _hook_file = jn(loc, '%s-receive' % h_type)

            has_hook = False

            log.debug('Installing git hook in repo %s', repo)

            if os.path.exists(_hook_file):

                # let's take a look at this hook, maybe it's kallithea ?

                log.debug('hook exists, checking if it is from kallithea')

                with open(_hook_file, 'rb') as f:

                    data = f.read()

                    matches = re.compile(r'(?:%s)\s*=\s*(.*)'

                                         % 'KALLITHEA_HOOK_VER').search(data)

                    if matches:

                        try:

                            ver = matches.groups()[0]

                            log.debug('got %s it is kallithea', ver)

                            has_hook = True

                        except Exception:

                            log.error(traceback.format_exc())

            else:

                # there is no hook in this dir, so we want to create one

                has_hook = True

            if has_hook or force_create:

                log.debug('writing %s hook file !', h_type)

                try:

                    with open(_hook_file, 'wb') as f:

                        tmpl = tmpl.replace('_TMPL_', kallithea.__version__)

                        f.write(tmpl)

${h.form(url('admin_settings_mapping'), method='post')}

    <div class="form">

        <div class="fields">

            <div class="field">

                <div class="label label-checkbox">

                    <label for="destroy">${_('Rescan option')}:</label>

                </div>

                <div class="checkboxes">

                    <div class="checkbox">

                        ${h.checkbox('destroy',True)}

                        <label for="destroy">${_('Delete records of missing repositories')}</label>

                    </div>

                    <span class="help-block">${_('Check this option to remove all comments, pull requests and other records related to repositories that no longer exist in the filesystem.')}</span>

                    <div class="checkbox">

                        ${h.checkbox('invalidate',True)}

                        <label for="invalidate"> ${_('Invalidate cache for all repositories')}</label>

                    </div>

                    <span class="help-block">${_('Check this to reload data and clear cache keys for all repositories.')}</span>

                    <div class="checkbox">

                        ${h.checkbox('hooks',True)}

                        <label for="hooks"> ${_('Install Git hooks')} </label>

                    </div>

                    <span class="help-block">${_("Verify if Kallithea's Git hooks are installed for each repository. Current hooks will be updated to the latest version.")}</span>

                </div>

            </div>

            <div class="buttons">

            ${h.submit('rescan',_('Rescan Repositories'),class_="btn")}

            </div>

        </div>

    </div>

${h.end_form()}