.. _changelog:

=========

Changelog

=========

1.3.4 (**2012-XX-XX**)

----------------------

:status: in-progress

:branch: beta

news

++++

- Whoosh logging is now controlled by the .ini files logging setup

- added clone-url into edit form on /settings page

- added help text into repo add/edit forms

- created rcextensions module with additional mappings (ref #322) and

  post push/pull/create repo hooks callbacks

- implemented #377 Users view for his own permissions on account page

- #399 added inheritance of permissions for users group on repos groups

- #401 repository group is automatically pre-selected when adding repos 

  inside a repository group

- added alternative HTTP 403 response when client failed to authenticate. Helps 

  solving issues with Mercurial and LDAP

- #402 removed group prefix from repository name when listing repositories 

  inside a group

- added gravatars into permission view and permissions autocomplete

fixes

+++++

- fixed #390 cache invalidation problems on repos inside group

- fixed #385 clone by ID url was loosing proxy prefix in URL

- fixed some unicode problems with waitress

- fixed issue with escaping < and > in changeset commits

- fixed error occurring during recursive group creation in API 

  create_repo function

- fixed #393 py2.5 fixes for routes url generator

- fixed #397 Private repository groups shows up before login

- fixed #396 fixed problems with revoking users in nested groups

1.3.3 (**2012-03-02**)

----------------------

news

++++

fixes

+++++

- fixed some python2.5 compatibility issues 

- fixed issues with removed repos was accidentally added as groups, after

  full rescan of paths

- fixes #376 Cannot edit user (using container auth)

- fixes #378 Invalid image urls on changeset screen with proxy-prefix 

  configuration

- fixed initial sorting of repos inside repo group

- fixes issue when user tried to resubmit same permission into user/user_groups

- bumped beaker version that fixes #375 leap error bug

- fixed raw_changeset for git. It was generated with hg patch headers

- fixed vcs issue with last_changeset for filenodes

- fixed missing commit after hook delete

- fixed #372 issues with git operation detection that caused a security issue 

  for git repos

1.3.2 (**2012-02-28**)

----------------------

news

++++

fixes

+++++

    Utilities library for RhodeCode

    :created_on: Apr 18, 2010

    :author: marcink

    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import re

import logging

import datetime

import traceback

import paste

import beaker

import tarfile

import shutil

from os.path import abspath

from os.path import dirname as dn, join as jn

from paste.script.command import Command, BadCommand

from mercurial import ui, config

from webhelpers.text import collapse, remove_formatting, strip_tags

from rhodecode.lib.vcs import get_backend

from rhodecode.lib.vcs.backends.base import BaseChangeset

from rhodecode.lib.vcs.utils.lazy import LazyProperty

from rhodecode.lib.vcs.utils.helpers import get_scm

from rhodecode.lib.vcs.exceptions import VCSError

from rhodecode.lib.caching_query import FromCache

from rhodecode.model import meta

from rhodecode.model.db import Repository, User, RhodeCodeUi, \

from rhodecode.model.meta import Session

from rhodecode.model.repos_group import ReposGroupModel

from rhodecode.lib.utils2 import safe_str, safe_unicode

from rhodecode.lib.vcs.utils.fakemod import create_module

log = logging.getLogger(__name__)

REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')

def recursive_replace(str_, replace=' '):

    """

    Recursive replace of given sign to just one instance

    :param str_: given string

    :param replace: char to find and replace multiple instances

    Examples::

    >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')

    'Mighty-Mighty-Bo-sstones'

    """

    if str_.find(replace * 2) == -1:

        return str_

    else:

        str_ = str_.replace(replace * 2, replace)

        return recursive_replace(str_, replace)

def repo_name_slug(value):

    """

    Return slug of name of repository

    This function is called on each creation/modification

    of repository to prevent bad names in repo

    """

    slug = remove_formatting(value)

    slug = strip_tags(slug)

    for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:

        slug = slug.replace(c, '-')

    slug = recursive_replace(slug, '-')

    slug = collapse(slug, '-')

    return slug

def get_repo_slug(request):

    _repo = request.environ['pylons.routes_dict'].get('repo_name')

                                                                   group_name))

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            sa.add(group)

            rgm._create_default_perms(group)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    """

    from rhodecode.model.repo import RepoModel

    sa = meta.Session

    rm = RepoModel()

    user = sa.query(User).filter(User.admin == True).first()

    if user is None:

        raise Exception('Missing administrative account !')

    added = []

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        if not rm.get_by_repo_name(name, cache=False):

            log.info('repository %s not found creating default' % name)

            added.append(name)

            form_data = {

             'repo_name': name,

             'repo_name_full': name,

             'repo_type': repo.alias,

             'description': repo.description \

                if repo.description != 'unknown' else '%s repository' % name,

             'private': False,

             'group_id': getattr(group, 'group_id', None)

            }

            rm.create(form_data, user, just_db=True)

    sa.commit()

    removed = []

    if remove_obsolete:

        #remove from database those repositories that are not in the filesystem

        for repo in sa.query(Repository).all():

            if repo.repo_name not in initial_repo_list.keys():

                removed.append(repo.repo_name)

                sa.delete(repo)

                sa.commit()

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

    import rhodecode

    from rhodecode.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = rhodecode.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...' % rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:

            log.debug('settings custom INDEX_EXTENSIONS')

    stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)

    commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data

    commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data

    languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data

    repository = relationship('Repository', single_parent=True)

class UserFollowing(Base, BaseModel):

    __tablename__ = 'user_followings'

    __table_args__ = (

        UniqueConstraint('user_id', 'follows_repository_id'),

        UniqueConstraint('user_id', 'follows_user_id'),

        {'extend_existing': True}

    )

    user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)

    follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)

    follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)

    follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)

    user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')

    follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')

    follows_repository = relationship('Repository', order_by='Repository.repo_name')

    @classmethod

    def get_repo_followers(cls, repo_id):

        return cls.query().filter(cls.follows_repo_id == repo_id)

class CacheInvalidation(Base, BaseModel):

    __tablename__ = 'cache_invalidation'

    __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing': True})

    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)

    cache_key = Column("cache_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_args = Column("cache_args", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)

    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)

    def __init__(self, cache_key, cache_args=''):

        self.cache_key = cache_key

        self.cache_args = cache_args

        self.cache_active = False

    def __repr__(self):

        return "<%s('%s:%s')>" % (self.__class__.__name__,

                                  self.cache_id, self.cache_key)

    @classmethod

    def _get_key(cls, key):

        """

        :param key:

        """

        import rhodecode

        prefix = ''

        iid = rhodecode.CONFIG.get('instance_id')

        if iid:

            prefix = iid

    @classmethod

    def get_by_key(cls, key):

        return cls.query().filter(cls.cache_key == key).scalar()

    @classmethod

    def invalidate(cls, key):

        """

        Returns Invalidation object if this given key should be invalidated

        None otherwise. `cache_active = False` means that this cache

        state is not valid and needs to be invalidated

        :param key:

        """

    @classmethod

    def set_invalidate(cls, key):

        """

        Mark this Cache key for invalidation

        :param key:

        """

        if inv_obj:

            inv_obj.cache_active = False

            Session.add(inv_obj)

            Session.commit()

        except Exception:

            log.error(traceback.format_exc())

            Session.rollback()

    @classmethod

    def set_valid(cls, key):

        """

        Mark this cache key as active and currently cached

        :param key:

        """

        inv_obj = cls.get_by_key(key)

        inv_obj.cache_active = True

        Session.add(inv_obj)

        Session.commit()

class ChangesetComment(Base, BaseModel):

    __tablename__ = 'changeset_comments'

    __table_args__ = ({'extend_existing': True},)

    comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True)

    repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)

    revision = Column('revision', String(40), nullable=False)

    line_no = Column('line_no', Unicode(10), nullable=True)

    f_path = Column('f_path', Unicode(1000), nullable=True)

    user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)

    text = Column('text', Unicode(25000), nullable=False)

    modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now)

    author = relationship('User', lazy='joined')

    repo = relationship('Repository')

    @classmethod

    def get_users(cls, revision):

        """

        Returns user associated with this changesetComment. ie those

        who actually commented

        :param cls:

        :param revision:

        """

        return Session.query(User)\

                .filter(cls.revision == revision)\

                .join(ChangesetComment.author).all()

            # since this is internal storage separator for rhodecode

            name = Repository.url_sep().join(name.split(os.sep))

            try:

                if name in repos:

                    raise RepositoryError('Duplicate repository name %s '

                                          'found in %s' % (name, path))

                else:

                    klass = get_backend(path[0])

                    if path[0] == 'hg' and path[0] in BACKENDS.keys():

                        repos[name] = klass(safe_str(path[1]), baseui=baseui)

                    if path[0] == 'git' and path[0] in BACKENDS.keys():

                        repos[name] = klass(path[1])

            except OSError:

                continue

        return repos

    def get_repos(self, all_repos=None, sort_key=None):

        """

        Get all repos from db and for each repo create it's

        backend instance and fill that backed with information from database

        :param all_repos: list of repository names as strings

            give specific repositories list, good for filtering

        """

        if all_repos is None:

            all_repos = self.sa.query(Repository)\

                        .filter(Repository.group_id == None)\

                        .order_by(Repository.repo_name).all()

        repo_iter = CachedRepoList(all_repos, repos_path=self.repos_path,

                                   order_by=sort_key)

        return repo_iter

    def get_repos_groups(self, all_groups=None):

        if all_groups is None:

            all_groups = RepoGroup.query()\

                .filter(RepoGroup.group_parent_id == None).all()

        group_iter = GroupList(all_groups)

        return group_iter

    def mark_for_invalidation(self, repo_name):

        further global cache invalidation

        :param repo_name: this repo that should invalidation take place

        """

        CacheInvalidation.set_invalidate(repo_name)

    def toggle_following_repo(self, follow_repo_id, user_id):

        f = self.sa.query(UserFollowing)\

            .filter(UserFollowing.follows_repo_id == follow_repo_id)\

            .filter(UserFollowing.user_id == user_id).scalar()

        if f is not None:

            try:

                self.sa.delete(f)

                action_logger(UserTemp(user_id),

                              'stopped_following_repo',

                              RepoTemp(follow_repo_id))

                return

            except:

                log.error(traceback.format_exc())

                raise

        try:

            f = UserFollowing()

            f.user_id = user_id

            f.follows_repo_id = follow_repo_id

            self.sa.add(f)

            action_logger(UserTemp(user_id),

                          'started_following_repo',

                          RepoTemp(follow_repo_id))

        except:

            log.error(traceback.format_exc())

            raise

    def toggle_following_user(self, follow_user_id, user_id):

        f = self.sa.query(UserFollowing)\

            .filter(UserFollowing.follows_user_id == follow_user_id)\

            .filter(UserFollowing.user_id == user_id).scalar()

        if f is not None:

            try:

                self.sa.delete(f)

                return

            except:

                log.error(traceback.format_exc())

                raise

        try:

            f = UserFollowing()

            f.user_id = user_id

            f.follows_user_id = follow_user_id