.. _upgrade:

Upgrade

=======

Upgrading from Cheese Shop

--------------------------

.. note::

   Firstly, it is recommended that you **always** perform a database backup 

   before doing an upgrade.

The easiest way to upgrade ``rhodecode`` is to run::

 easy_install -U rhodecode

Or::

 pip install --upgrade rhodecode

Then make sure you run the following command from the installation directory::

 paster make-config RhodeCode production.ini

This will display any changes made by the new version of RhodeCode to your

current configuration. It will try to perform an automerge. It's always better

to make a backup of your configuration file before hand and recheck the 

content after the automerge.

.. note::

It is also recommended that you rebuild the whoosh index after upgrading since 

The final step is to upgrade the database. To do this simply run::

    paster upgrade-db production.ini

This will upgrade the schema and update some of the defaults in the database,

and will always recheck the settings of the application, if there are no new 

options that need to be set.

.. _virtualenv: http://pypi.python.org/pypi/virtualenv  

.. _python: http://www.python.org/

.. _mercurial: http://mercurial.selenic.com/

.. _celery: http://celeryproject.org/

.. _rabbitmq: http://www.rabbitmq.com/

# -*- coding: utf-8 -*-

"""

    rhodecode.controllers.changeset

    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    changeset controller for pylons showoing changes beetween

    revisions

    :created_on: Apr 25, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

import traceback

from collections import defaultdict

from webob.exc import HTTPForbidden

from pylons import tmpl_context as c, url, request, response

from pylons.i18n.translation import _

from pylons.controllers.util import redirect

from pylons.decorators import jsonify

from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetError, \

    ChangesetDoesNotExistError

from rhodecode.lib.vcs.nodes import FileNode

import rhodecode.lib.helpers as h

from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

from rhodecode.lib.base import BaseRepoController, render

from rhodecode.lib.utils import EmptyChangeset

from rhodecode.lib.compat import OrderedDict

from rhodecode.lib import diffs

from rhodecode.model.db import ChangesetComment

from rhodecode.model.comment import ChangesetCommentsModel

from rhodecode.model.meta import Session

from rhodecode.lib.diffs import wrapped_diff

log = logging.getLogger(__name__)

def anchor_url(revision, path):

    fid = h.FID(revision, path)

def get_ignore_ws(fid, GET):

    ig_ws_global = request.GET.get('ignorews')

    ig_ws = filter(lambda k: k.startswith('WS'), GET.getall(fid))

    if ig_ws:

        try:

            return int(ig_ws[0].split(':')[-1])

        except:

            pass

    return ig_ws_global

def _ignorews_url(fileid=None):

    params = defaultdict(list)

    lbl = _('show white space')

    ig_ws = get_ignore_ws(fileid, request.GET)

    ln_ctx = get_line_ctx(fileid, request.GET)

    # global option

    if fileid is None:

        if ig_ws is None:

            params['ignorews'] += [1]

            lbl = _('ignore white space')

        ctx_key = 'context'

        ctx_val = ln_ctx

    # per file options

    else:

        if ig_ws is None:

            params[fileid] += ['WS:1']

            lbl = _('ignore white space')

        ctx_key = fileid

        ctx_val = 'C:%s' % ln_ctx

    # if we have passed in ln_ctx pass it along to our params

    if ln_ctx:

        params[ctx_key] += [ctx_val]

    params['anchor'] = fileid

    img = h.image(h.url('/images/icons/text_strikethrough.png'), lbl, class_='icon')

    return h.link_to(img, h.url.current(**params), title=lbl, class_='tooltip')

def get_line_ctx(fid, GET):

    ln_ctx_global = request.GET.get('context')

    ln_ctx = filter(lambda k: k.startswith('C'), GET.getall(fid))

    if ln_ctx:

        retval = ln_ctx[0].split(':')[-1]

    else:

        retval = ln_ctx_global

    try:

        return int(retval)

    except:

        return

def _context_url(fileid=None):

    """

    Generates url for context lines

    :param fileid:

    """

    ig_ws = get_ignore_ws(fileid, request.GET)

    ln_ctx = (get_line_ctx(fileid, request.GET) or 3) * 2

    params = defaultdict(list)

    # global option

    if fileid is None:

        if ln_ctx > 0:

            params['context'] += [ln_ctx]

        if ig_ws:

            ig_ws_key = 'ignorews'

            ig_ws_val = 1

    # per file option

    else:

        params[fileid] += ['C:%s' % ln_ctx]

        ig_ws_key = fileid

        ig_ws_val = 'WS:%s' % 1

    if ig_ws:

        params[ig_ws_key] += [ig_ws_val]

    lbl = _('%s line context') % ln_ctx

    params['anchor'] = fileid

    img = h.image(h.url('/images/icons/table_add.png'), lbl, class_='icon')

    return h.link_to(img, h.url.current(**params), title=lbl, class_='tooltip')

class ChangesetController(BaseRepoController):

# -*- coding: utf-8 -*-

"""

    rhodecode.controllers.summary

    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Summary controller for Rhodecode

    :created_on: Apr 18, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import traceback

import calendar

import logging

from time import mktime

from datetime import timedelta, date

from urlparse import urlparse

from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \

    NodeDoesNotExistError

from pylons import tmpl_context as c, request, url, config

from pylons.i18n.translation import _

from beaker.cache import cache_region, region_invalidate

from rhodecode.model.db import Statistics, CacheInvalidation

from rhodecode.lib import ALL_READMES, ALL_EXTS

from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

from rhodecode.lib.base import BaseRepoController, render

from rhodecode.lib.utils import EmptyChangeset

from rhodecode.lib.markup_renderer import MarkupRenderer

from rhodecode.lib.celerylib import run_task

from rhodecode.lib.celerylib.tasks import get_commits_stats, \

    LANGUAGES_EXTENSIONS_MAP

from rhodecode.lib.helpers import RepoPage

from rhodecode.lib.compat import json, OrderedDict

log = logging.getLogger(__name__)

README_FILES = [''.join([x[0][0], x[1][0]]) for x in

                    sorted(list(product(ALL_READMES, ALL_EXTS)),

                           key=lambda y:y[0][1] + y[1][1])]

class SummaryController(BaseRepoController):

    @LoginRequired()

    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',

                                   'repository.admin')

    def __before__(self):

        super(SummaryController, self).__before__()

    def index(self, repo_name):

        c.dbrepo = dbrepo = c.rhodecode_db_repo

        c.following = self.scm_model.is_following_repo(repo_name,

                                                self.rhodecode_user.user_id)

        def url_generator(**kw):

            return url('shortlog_home', repo_name=repo_name, size=10, **kw)

        c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,

                                     items_per_page=10, url=url_generator)

        if self.rhodecode_user.username == 'default':

            # for default(anonymous) user we don't need to pass credentials

            username = ''

            password = ''

        else:

            username = str(self.rhodecode_user.username)

            password = '@'

        parsed_url = urlparse(url.current(qualified=True))

        default_clone_uri = '{scheme}://{user}{pass}{netloc}{path}'

        uri_tmpl = config.get('clone_uri', default_clone_uri)

        uri_tmpl = uri_tmpl.replace('{', '%(').replace('}', ')s')

        uri_dict = {

           'user': username,

           'pass': password,

           'scheme': parsed_url.scheme,

           'netloc': parsed_url.netloc,

           'path': parsed_url.path

        }

        uri = uri_tmpl % uri_dict

        # generate another clone url by id

        uri_dict.update({'path': '/_%s' % c.dbrepo.repo_id})

        uri_id = uri_tmpl % uri_dict

        c.clone_repo_url = uri

        c.clone_repo_url_id = uri_id

        c.repo_tags = OrderedDict()

        for name, hash in c.rhodecode_repo.tags.items()[:10]:

            try:

                c.repo_tags[name] = c.rhodecode_repo.get_changeset(hash)

            except ChangesetError:

                c.repo_tags[name] = EmptyChangeset(hash)

        c.repo_branches = OrderedDict()

        for name, hash in c.rhodecode_repo.branches.items()[:10]:

            try:

                c.repo_branches[name] = c.rhodecode_repo.get_changeset(hash)

            except ChangesetError:

                c.repo_branches[name] = EmptyChangeset(hash)

        td = date.today() + timedelta(days=1)

        td_1m = td - timedelta(days=calendar.mdays[td.month])

        td_1y = td - timedelta(days=365)

        ts_min_m = mktime(td_1m.timetuple())

        ts_min_y = mktime(td_1y.timetuple())

        """To iterate on keys in reversed order.

        """

        dict_impl = self._dict_impl()

        curr_key = dict_impl.__getattribute__(self, 'lt')

        while curr_key != _nil:

            yield curr_key

            curr_key = dict_impl.__getitem__(self, curr_key)[0]

    __reversed__ = riterkeys

    def rkeys(self):

        """List of the keys in reversed order.

        """

        return list(self.riterkeys())

    def ritervalues(self):

        """To iterate on values in reversed order.

        """

        dict_impl = self._dict_impl()

        curr_key = dict_impl.__getattribute__(self, 'lt')

        while curr_key != _nil:

            curr_key, val, _ = dict_impl.__getitem__(self, curr_key)

            yield val

    def rvalues(self):

        """List of the values in reversed order.

        """

        return list(self.ritervalues())

    def riteritems(self):

        """To iterate on (key, value) in reversed order.

        """

        dict_impl = self._dict_impl()

        curr_key = dict_impl.__getattribute__(self, 'lt')

        while curr_key != _nil:

            pred_key, val, _ = dict_impl.__getitem__(self, curr_key)

            yield curr_key, val

            curr_key = pred_key

    def ritems(self):

        """List of the (key, value) in reversed order.

        """

        return list(self.riteritems())

    def firstkey(self):

        if self:

            return self._dict_impl().__getattribute__(self, 'lh')

        else:

            raise KeyError("'firstkey(): ordered dictionary is empty'")

    def lastkey(self):

        if self:

            return self._dict_impl().__getattribute__(self, 'lt')

        else:

            raise KeyError("'lastkey(): ordered dictionary is empty'")

    def as_dict(self):

        return self._dict_impl()(self.items())

    def _repr(self):

        """_repr(): low level repr of the whole data contained in the odict.

        Useful for debugging.

        """

        dict_impl = self._dict_impl()

        form = "odict low level repr lh,lt,data: %r, %r, %s"

        return form % (dict_impl.__getattribute__(self, 'lh'),

                       dict_impl.__getattribute__(self, 'lt'),

                       dict_impl.__repr__(self))

class OrderedDict(_odict, dict):

    def _dict_impl(self):

        return dict

#==============================================================================

# OrderedSet

#==============================================================================

from sqlalchemy.util import OrderedSet

#==============================================================================

# kill FUNCTIONS

#==============================================================================

if __platform__ in PLATFORM_WIN:

    import ctypes

    def kill(pid, sig):

        """kill function for Win32"""

        kernel32 = ctypes.windll.kernel32

        handle = kernel32.OpenProcess(1, 0, pid)

        return (0 != kernel32.TerminateProcess(handle, 0))

else:

    kill = os.kill

# -*- coding: utf-8 -*-

"""

    rhodecode.lib.utils

    ~~~~~~~~~~~~~~~~~~~

    Utilities library for RhodeCode

    :created_on: Apr 18, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import logging

import datetime

import traceback

import paste

import beaker

import tarfile

import shutil

from os.path import abspath

from os.path import dirname as dn, join as jn

from paste.script.command import Command, BadCommand

from mercurial import ui, config

from webhelpers.text import collapse, remove_formatting, strip_tags

from rhodecode.lib.vcs import get_backend

from rhodecode.lib.vcs.backends.base import BaseChangeset

from rhodecode.lib.vcs.utils.lazy import LazyProperty

from rhodecode.lib.vcs.utils.helpers import get_scm

from rhodecode.lib.vcs.exceptions import VCSError

from rhodecode.lib.caching_query import FromCache

from rhodecode.model import meta

from rhodecode.model.db import Repository, User, RhodeCodeUi, \

    UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm

from rhodecode.model.meta import Session

from rhodecode.model.repos_group import ReposGroupModel

log = logging.getLogger(__name__)

def recursive_replace(str_, replace=' '):

    """Recursive replace of given sign to just one instance

    :param str_: given string

    :param replace: char to find and replace multiple instances

    Examples::

    >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')

    'Mighty-Mighty-Bo-sstones'

    """

    if str_.find(replace * 2) == -1:

        return str_

    else:

        str_ = str_.replace(replace * 2, replace)

        return recursive_replace(str_, replace)

def repo_name_slug(value):

    """Return slug of name of repository

    This function is called on each creation/modification

    of repository to prevent bad names in repo

    """

    slug = remove_formatting(value)

    slug = strip_tags(slug)

    for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:

        slug = slug.replace(c, '-')

    slug = recursive_replace(slug, '-')

    slug = collapse(slug, '-')

    return slug

def get_repo_slug(request):

    _repo = request.environ['pylons.routes_dict'].get('repo_name')

    if _repo:

        _repo = _repo.rstrip('/')

    return _repo

def get_repos_group_slug(request):

    _group = request.environ['pylons.routes_dict'].get('group_name')

    if _group:

        _group = _group.rstrip('/')

    return _group

def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):

    """

    Action logger for various actions made by users

    :param user: user that made this action, can be a unique username string or

        object containing user_id attribute

    :param action: action to log, should be on of predefined unique actions for

        easy translations

    :param repo: string name of repository or object containing repo_id,

        that action was made on

    :param ipaddr: optional ip address from what the action was made

    :param sa: optional sqlalchemy session

    """

    if not sa:

        sa = meta.Session

    try:

        if hasattr(user, 'user_id'):

            user_obj = user

        elif isinstance(user, basestring):

            user_obj = User.get_by_username(user)

        else:

            raise Exception('You have to provide user object or username')

        if hasattr(repo, 'repo_id'):

            repo_obj = Repository.get(repo.repo_id)

            repo_name = repo_obj.repo_name

        elif  isinstance(repo, basestring):

            repo_name = repo.lstrip('/')

            repo_obj = Repository.get_by_repo_name(repo_name)

        else:

            raise Exception('You have to provide repository to action logger')

        user_log = UserLog()

        user_log.user_id = user_obj.user_id

        user_log.action = action

        user_log.repository_id = repo_obj.repo_id

        user_log.repository_name = repo_name

        user_log.action_date = datetime.datetime.now()

        user_log.user_ip = ipaddr

        sa.add(user_log)

        log.info('Adding user %s, action %s on %s' % (user_obj, action, repo))

def set_rhodecode_config(config):

    """

    Updates pylons config with new settings from database

    :param config:

    """

    hgsettings = RhodeCodeSetting.get_app_settings()

    for k, v in hgsettings.items():

        config[k] = v

def invalidate_cache(cache_key, *args):

    """

    Puts cache invalidation task into db for

    further global cache invalidation

    """

    from rhodecode.model.scm import ScmModel

    if cache_key.startswith('get_repo_cached_'):

        name = cache_key.split('get_repo_cached_')[-1]

        ScmModel().mark_for_invalidation(name)

class EmptyChangeset(BaseChangeset):

    """

    An dummy empty changeset. It's possible to pass hash when creating

    an EmptyChangeset

    """

    def __init__(self, cs='0' * 40, repo=None, requested_revision=None,

                 alias=None):

        self._empty_cs = cs

        self.revision = -1

        self.message = ''

        self.author = ''

        self.date = ''

        self.repository = repo

        self.requested_revision = requested_revision

        self.alias = alias

    @LazyProperty

    def raw_id(self):

        """

        Returns raw string identifying this changeset, useful for web

        representation.

        """

        return self._empty_cs

    @LazyProperty

    def branch(self):

        return get_backend(self.alias).DEFAULT_BRANCH_NAME

    @LazyProperty

    def short_id(self):

        return self.raw_id[:12]

    def get_file_changeset(self, path):

        return self

    def get_file_content(self, path):

        return u''

    def get_file_size(self, path):

        return 0

def map_groups(groups):

    """

    Checks for groups existence, and creates groups structures.

    It returns last group in structure

    :param groups: list of groups structure

    """

    sa = meta.Session

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = ReposGroupModel(sa)

    for lvl, group_name in enumerate(groups):

        group_name = '/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

#        # WTF that doesn't work !?

#        if group is None:

#            group = rgm.create(group_name, desc, parent, just_db=True)

#            sa.commit()

        if group is None:

            log.debug('creating group level: %s group_name: %s' % (lvl, group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.commit()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    """

    from rhodecode.model.repo import RepoModel

    sa = meta.Session

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

    for name, repo in initial_repo_list.items():

        group = map_groups(name.split(Repository.url_sep()))

        if not rm.get_by_repo_name(name, cache=False):

            log.info('repository %s not found creating default' % name)

            added.append(name)

            form_data = {

             'repo_name': name,

             'repo_name_full': name,

             'repo_type': repo.alias,

             'description': repo.description \

                if repo.description != 'unknown' else '%s repository' % name,

             'private': False,

             'group_id': getattr(group, 'group_id', None)

            }

            rm.create(form_data, user, just_db=True)

    sa.commit()

    removed = []

    if remove_obsolete:

        #remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            if repo.repo_name not in initial_repo_list.keys():

                removed.append(repo.repo_name)

                sa.delete(repo)

                sa.commit()

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    :param config: test config

    :param full_index:

    """

    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

    class _ValidForkType(formencode.validators.FancyValidator):

        def to_python(self, value, state):

            if old_data['repo_type'] != value:

                raise formencode.Invalid(_('Fork have to be the same '

                                           'type as original'), value, state)

            return value

    return _ValidForkType

def ValidPerms(type_='repo'):

    if type_ == 'group':

        EMPTY_PERM = 'group.none'

    elif type_ == 'repo':

        EMPTY_PERM = 'repository.none'

    class _ValidPerms(formencode.validators.FancyValidator):

        messages = {

            'perm_new_member_name':

                _('This username or users group name is not valid')

        }

        def to_python(self, value, state):

            perms_update = []

            perms_new = []

            # build a list of permission to update and new permission to create

            for k, v in value.items():

                # means new added member to permissions

                if k.startswith('perm_new_member'):

                    new_perm = value.get('perm_new_member', False)

                    new_member = value.get('perm_new_member_name', False)

                    new_type = value.get('perm_new_member_type')

                    if new_member and new_perm:

                        if (new_member, new_perm, new_type) not in perms_new:

                            perms_new.append((new_member, new_perm, new_type))

                elif k.startswith('u_perm_') or k.startswith('g_perm_'):

                    member = k[7:]

                    t = {'u': 'user',

                         'g': 'users_group'

                    }[k[0]]

                    if member == 'default':

                        if value.get('private'):