.. _upgrade:

Upgrade

=======

Upgrading from Cheese Shop

--------------------------

.. note::

   Firstly, it is recommended that you **always** perform a database backup 

   before doing an upgrade.

The easiest way to upgrade ``rhodecode`` is to run::

 easy_install -U rhodecode

Or::

 pip install --upgrade rhodecode

Then make sure you run the following command from the installation directory::

 paster make-config RhodeCode production.ini

This will display any changes made by the new version of RhodeCode to your

current configuration. It will try to perform an automerge. It's always better

to make a backup of your configuration file before hand and recheck the 

content after the automerge.

.. note::

It is also recommended that you rebuild the whoosh index after upgrading since 

The final step is to upgrade the database. To do this simply run::

    paster upgrade-db production.ini

This will upgrade the schema and update some of the defaults in the database,

and will always recheck the settings of the application, if there are no new 

options that need to be set.

.. _virtualenv: http://pypi.python.org/pypi/virtualenv  

.. _python: http://www.python.org/

.. _mercurial: http://mercurial.selenic.com/

.. _celery: http://celeryproject.org/

.. _rabbitmq: http://www.rabbitmq.com/

    :created_on: Apr 25, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

import traceback

from collections import defaultdict

from webob.exc import HTTPForbidden

from pylons import tmpl_context as c, url, request, response

from pylons.i18n.translation import _

from pylons.controllers.util import redirect

from pylons.decorators import jsonify

from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetError, \

    ChangesetDoesNotExistError

from rhodecode.lib.vcs.nodes import FileNode

import rhodecode.lib.helpers as h

from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

from rhodecode.lib.base import BaseRepoController, render

from rhodecode.lib.utils import EmptyChangeset

from rhodecode.lib.compat import OrderedDict

from rhodecode.lib import diffs

from rhodecode.model.db import ChangesetComment

from rhodecode.model.comment import ChangesetCommentsModel

from rhodecode.model.meta import Session

from rhodecode.lib.diffs import wrapped_diff

log = logging.getLogger(__name__)

def anchor_url(revision, path):

    fid = h.FID(revision, path)

def get_ignore_ws(fid, GET):

    ig_ws_global = request.GET.get('ignorews')

    ig_ws = filter(lambda k: k.startswith('WS'), GET.getall(fid))

    if ig_ws:

        try:

            return int(ig_ws[0].split(':')[-1])

        except:

            pass

    return ig_ws_global

def _ignorews_url(fileid=None):

    params = defaultdict(list)

    lbl = _('show white space')

    ig_ws = get_ignore_ws(fileid, request.GET)

    ln_ctx = get_line_ctx(fileid, request.GET)

    # global option

    if fileid is None:

        if ig_ws is None:

            params['ignorews'] += [1]

            lbl = _('ignore white space')

        ctx_key = 'context'

        ctx_val = ln_ctx

    # per file options

    else:

        if ig_ws is None:

            params[fileid] += ['WS:1']

            lbl = _('ignore white space')

        ctx_key = fileid

        ctx_val = 'C:%s' % ln_ctx

    # if we have passed in ln_ctx pass it along to our params

    if ln_ctx:

        params[ctx_key] += [ctx_val]

    params['anchor'] = fileid

    img = h.image(h.url('/images/icons/text_strikethrough.png'), lbl, class_='icon')

    return h.link_to(img, h.url.current(**params), title=lbl, class_='tooltip')

def get_line_ctx(fid, GET):

    ln_ctx_global = request.GET.get('context')

    ln_ctx = filter(lambda k: k.startswith('C'), GET.getall(fid))

    if ln_ctx:

# -*- coding: utf-8 -*-

"""

    rhodecode.controllers.summary

    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Summary controller for Rhodecode

    :created_on: Apr 18, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import traceback

import calendar

import logging

from time import mktime

from datetime import timedelta, date

from urlparse import urlparse

from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \

    NodeDoesNotExistError

from pylons import tmpl_context as c, request, url, config

from pylons.i18n.translation import _

from beaker.cache import cache_region, region_invalidate

from rhodecode.model.db import Statistics, CacheInvalidation

from rhodecode.lib import ALL_READMES, ALL_EXTS

from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

from rhodecode.lib.base import BaseRepoController, render

from rhodecode.lib.utils import EmptyChangeset

from rhodecode.lib.markup_renderer import MarkupRenderer

from rhodecode.lib.celerylib import run_task

from rhodecode.lib.celerylib.tasks import get_commits_stats, \

    LANGUAGES_EXTENSIONS_MAP

from rhodecode.lib.helpers import RepoPage

from rhodecode.lib.compat import json, OrderedDict

log = logging.getLogger(__name__)

README_FILES = [''.join([x[0][0], x[1][0]]) for x in

                    sorted(list(product(ALL_READMES, ALL_EXTS)),

                           key=lambda y:y[0][1] + y[1][1])]

class SummaryController(BaseRepoController):

    @LoginRequired()

    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',

                                   'repository.admin')

    def __before__(self):

        super(SummaryController, self).__before__()

    def index(self, repo_name):

        c.dbrepo = dbrepo = c.rhodecode_db_repo

        c.following = self.scm_model.is_following_repo(repo_name,

                                                self.rhodecode_user.user_id)

        def url_generator(**kw):

            return url('shortlog_home', repo_name=repo_name, size=10, **kw)

        c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,

                                     items_per_page=10, url=url_generator)

        if self.rhodecode_user.username == 'default':

            raise KeyError("'firstkey(): ordered dictionary is empty'")

    def lastkey(self):

        if self:

            return self._dict_impl().__getattribute__(self, 'lt')

        else:

            raise KeyError("'lastkey(): ordered dictionary is empty'")

    def as_dict(self):

        return self._dict_impl()(self.items())

    def _repr(self):

        """_repr(): low level repr of the whole data contained in the odict.

        Useful for debugging.

        """

        dict_impl = self._dict_impl()

        form = "odict low level repr lh,lt,data: %r, %r, %s"

        return form % (dict_impl.__getattribute__(self, 'lh'),

                       dict_impl.__getattribute__(self, 'lt'),

                       dict_impl.__repr__(self))

class OrderedDict(_odict, dict):

    def _dict_impl(self):

        return dict

#==============================================================================

# OrderedSet

#==============================================================================

from sqlalchemy.util import OrderedSet

#==============================================================================

# kill FUNCTIONS

#==============================================================================

if __platform__ in PLATFORM_WIN:

    import ctypes

    def kill(pid, sig):

        """kill function for Win32"""

        kernel32 = ctypes.windll.kernel32

        handle = kernel32.OpenProcess(1, 0, pid)

        return (0 != kernel32.TerminateProcess(handle, 0))

else:

    kill = os.kill

# -*- coding: utf-8 -*-

"""

    rhodecode.lib.utils

    ~~~~~~~~~~~~~~~~~~~

    Utilities library for RhodeCode

    :created_on: Apr 18, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import logging

import datetime

import traceback

import paste

import beaker

import tarfile

import shutil

from os.path import abspath

from os.path import dirname as dn, join as jn

from paste.script.command import Command, BadCommand

from mercurial import ui, config

from webhelpers.text import collapse, remove_formatting, strip_tags

from rhodecode.lib.vcs import get_backend

from rhodecode.lib.vcs.backends.base import BaseChangeset

from rhodecode.lib.vcs.utils.lazy import LazyProperty

from rhodecode.lib.vcs.utils.helpers import get_scm

from rhodecode.lib.vcs.exceptions import VCSError

from rhodecode.lib.caching_query import FromCache

from rhodecode.model import meta

from rhodecode.model.db import Repository, User, RhodeCodeUi, \

    UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm

from rhodecode.model.meta import Session

from rhodecode.model.repos_group import ReposGroupModel

log = logging.getLogger(__name__)

def recursive_replace(str_, replace=' '):

    """Recursive replace of given sign to just one instance

    :param str_: given string

    :param replace: char to find and replace multiple instances

    Examples::

    >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')

    'Mighty-Mighty-Bo-sstones'

    """

    if str_.find(replace * 2) == -1:

        return str_

    else:

        str_ = str_.replace(replace * 2, replace)

        return recursive_replace(str_, replace)

def repo_name_slug(value):

    """Return slug of name of repository

    This function is called on each creation/modification

    of repository to prevent bad names in repo

    """

    slug = remove_formatting(value)

    slug = strip_tags(slug)

    for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:

        slug = slug.replace(c, '-')

    slug = recursive_replace(slug, '-')

    slug = collapse(slug, '-')

    return slug

def get_repo_slug(request):

    _repo = request.environ['pylons.routes_dict'].get('repo_name')

    if _repo:

        _repo = _repo.rstrip('/')

    return _repo

def get_repos_group_slug(request):

    _group = request.environ['pylons.routes_dict'].get('group_name')

    if _group:

        _group = _group.rstrip('/')

    return _group

        representation.

        """

        return self._empty_cs

    @LazyProperty

    def branch(self):

        return get_backend(self.alias).DEFAULT_BRANCH_NAME

    @LazyProperty

    def short_id(self):

        return self.raw_id[:12]

    def get_file_changeset(self, path):

        return self

    def get_file_content(self, path):

        return u''

    def get_file_size(self, path):

        return 0

def map_groups(groups):

    """

    Checks for groups existence, and creates groups structures.

    It returns last group in structure

    :param groups: list of groups structure

    """

    sa = meta.Session

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

#        # WTF that doesn't work !?

#        if group is None:

#            group = rgm.create(group_name, desc, parent, just_db=True)

#            sa.commit()

        if group is None:

            log.debug('creating group level: %s group_name: %s' % (lvl, group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.commit()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    """

    from rhodecode.model.repo import RepoModel

    sa = meta.Session

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

    for name, repo in initial_repo_list.items():

        group = map_groups(name.split(Repository.url_sep()))

        if not rm.get_by_repo_name(name, cache=False):

            log.info('repository %s not found creating default' % name)

            added.append(name)

            form_data = {

             'repo_name': name,

             'repo_name_full': name,

             'repo_type': repo.alias,

             'description': repo.description \

                if repo.description != 'unknown' else '%s repository' % name,

             'private': False,

             'group_id': getattr(group, 'group_id', None)

            }

            rm.create(form_data, user, just_db=True)

    sa.commit()

    removed = []

    if remove_obsolete:

        #remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            value['perms_updates'] = perms_update

            value['perms_new'] = perms_new

            # update permissions

            for k, v, t in perms_new:

                try:

                    if t is 'user':

                        self.user_db = User.query()\

                            .filter(User.active == True)\

                            .filter(User.username == k).one()

                    if t is 'users_group':

                        self.user_db = UsersGroup.query()\

                            .filter(UsersGroup.users_group_active == True)\

                            .filter(UsersGroup.users_group_name == k).one()

                except Exception:

                    msg = self.message('perm_new_member_name',

                                         state=State_obj)

                    raise formencode.Invalid(

                        msg, value, state, error_dict={'perm_new_member_name': msg}

                    )

            return value

    return _ValidPerms

class ValidSettings(formencode.validators.FancyValidator):

    def to_python(self, value, state):

        # settings  form can't edit user

        if 'user' in value:

            del['value']['user']

        return value

class ValidPath(formencode.validators.FancyValidator):

    def to_python(self, value, state):

        if not os.path.isdir(value):

            msg = _('This is not a valid path')

            raise formencode.Invalid(msg, value, state,

                                     error_dict={'paths_root_path': msg})

        return value

def UniqSystemEmail(old_data):

    class _UniqSystemEmail(formencode.validators.FancyValidator):

        def to_python(self, value, state):

            value = value.lower()

                user = User.get_by_email(value, case_insensitive=True)

                if user:

                    raise formencode.Invalid(

                        _("This e-mail address is already taken"), value, state

                    )

            return value

    return _UniqSystemEmail

class ValidSystemEmail(formencode.validators.FancyValidator):

    def to_python(self, value, state):

        value = value.lower()

        user = User.get_by_email(value, case_insensitive=True)

        if  user is None:

            raise formencode.Invalid(

                _("This e-mail address doesn't exist."), value, state

            )

        return value

class LdapLibValidator(formencode.validators.FancyValidator):

    def to_python(self, value, state):

        try:

            import ldap

        except ImportError:

            raise LdapImportError

        return value

class AttrLoginValidator(formencode.validators.FancyValidator):

    def to_python(self, value, state):

        if not value or not isinstance(value, (str, unicode)):

            raise formencode.Invalid(

                _("The LDAP Login attribute of the CN must be specified - "

                  "this is the name of the attribute that is equivalent "

                  "to 'username'"), value, state

            )

        return value

#==============================================================================

# -*- coding: utf-8 -*-

"""

    rhodecode.model.scm

    ~~~~~~~~~~~~~~~~~~~

    Scm model for RhodeCode

    :created_on: Apr 9, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import time

import traceback

import logging

import cStringIO

from rhodecode.lib.vcs import get_backend

from rhodecode.lib.vcs.exceptions import RepositoryError

from rhodecode.lib.vcs.utils.lazy import LazyProperty

from rhodecode.lib.vcs.nodes import FileNode

from rhodecode import BACKENDS

from rhodecode.lib import helpers as h

from rhodecode.lib import safe_str

from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny

from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \

from rhodecode.model import BaseModel

from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \

    UserFollowing, UserLog, User, RepoGroup

log = logging.getLogger(__name__)

class UserTemp(object):

    def __init__(self, user_id):

        self.user_id = user_id

    def __repr__(self):

        return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)

class RepoTemp(object):

    def __init__(self, repo_id):

        self.repo_id = repo_id

    def __repr__(self):

        return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)

class CachedRepoList(object):

    def __init__(self, db_repo_list, repos_path, order_by=None):

        self.db_repo_list = db_repo_list

        self.repos_path = repos_path

        self.order_by = order_by

        self.reversed = (order_by or '').startswith('-')

    def __len__(self):

        return len(self.db_repo_list)

    def __repr__(self):

        return '<%s (%s)>' % (self.__class__.__name__, self.__len__())

    def __iter__(self):

        for dbr in self.db_repo_list:

            scmr = dbr.scm_instance_cached

            # check permission at this level

            if not HasRepoPermissionAny(

                'repository.read', 'repository.write', 'repository.admin'

            )(dbr.repo_name, 'get repo check'):

                continue

            if scmr is None:

                log.error(

            yield dbgr

class ScmModel(BaseModel):

    """

    Generic Scm Model

    """

    def __get_repo(self, instance):

        cls = Repository

        if isinstance(instance, cls):

            return instance

        elif isinstance(instance, int) or str(instance).isdigit():

            return cls.get(instance)

        elif isinstance(instance, basestring):

            return cls.get_by_repo_name(instance)

        elif instance:

            raise Exception('given object must be int, basestr or Instance'

                            ' of %s got %s' % (type(cls), type(instance)))

    @LazyProperty

    def repos_path(self):

        """

        Get's the repositories root path from database

        """

        q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()

        return q.ui_value

    def repo_scan(self, repos_path=None):

        """

        Listing of repositories in given path. This path should not be a

        repository itself. Return a dictionary of repository objects

        :param repos_path: path to directory containing repositories

        """

        if repos_path is None:

            repos_path = self.repos_path

        log.info('scanning for repositories in %s' % repos_path)

        baseui = make_ui('db')

        repos = {}

        for name, path in get_filesystem_repos(repos_path, recursive=True):

            # name need to be decomposed and put back together using the /

            # since this is internal storage separator for rhodecode

            name = Repository.url_sep().join(name.split(os.sep))

            try:

                if name in repos:

                    raise RepositoryError('Duplicate repository name %s '

                                          'found in %s' % (name, path))

                else:

                    klass = get_backend(path[0])

                    if path[0] == 'hg' and path[0] in BACKENDS.keys():

                        repos[name] = klass(safe_str(path[1]), baseui=baseui)

                    if path[0] == 'git' and path[0] in BACKENDS.keys():

                        repos[name] = klass(path[1])

            except OSError:

                continue

        return repos

    def get_repos(self, all_repos=None, sort_key=None):

        """

        Get all repos from db and for each repo create it's

        backend instance and fill that backed with information from database

        :param all_repos: list of repository names as strings

            give specific repositories list, good for filtering

        """

        if all_repos is None:

            all_repos = self.sa.query(Repository)\

                        .filter(Repository.group_id == None)\

                        .order_by(Repository.repo_name).all()

        repo_iter = CachedRepoList(all_repos, repos_path=self.repos_path,

                                   order_by=sort_key)

        return repo_iter

    def get_repos_groups(self, all_groups=None):

        if all_groups is None:

            all_groups = RepoGroup.query()\

                .filter(RepoGroup.group_parent_id == None).all()

        group_iter = GroupList(all_groups)

        return group_iter

## -*- coding: utf-8 -*-

<%inherit file="/base/base.html"/>

<%def name="title()">

    ${_('Repositories administration')} - ${c.rhodecode_name}

</%def>

<%def name="breadcrumbs_links()">

    ${h.link_to(_('Admin'),h.url('admin_home'))} » ${_('Repositories')}

</%def>

<%def name="page_nav()">

	${self.menu('admin')}

</%def>

<%def name="main()">

<div class="box">

    <div class="title">

        ${self.breadcrumbs()}

        <ul class="links">

          <li>

            <span>${h.link_to(_(u'ADD REPOSITORY'),h.url('new_repo'))}</span>

          </li>

        </ul>

    </div>

    <div class="table">

        <div id='repos_list_wrap' class="yui-skin-sam">

        <%cnt=0%>

        <%namespace name="dt" file="/_data_table/_dt_elements.html"/>

        <table id="repos_list">

         <thead>

          <tr>

            <th class="left"></th>

  	        <th class="left">${_('Name')}</th>

  	        <th class="left">${_('Description')}</th>

  	        <th class="left">${_('Last change')}</th>

  	        <th class="left">${_('Tip')}</th>

  	        <th class="left">${_('Contact')}</th>

            <th class="left">${_('Action')}</th>

          </tr>

         </thead>