.. _changelog:

Changelog

=========

1.0.0 (**2010-10-xx**)

----------------------

- security bugfix simplehg wasn't checking for permissions on commands

  other than pull or push.

- fixed doubled messages after push or pull in admin journal

- templating and css corrections, fixed repo switcher on chrome,updated titles

- admin menu accessible from options menu on repository view

- permissions cached queries

1.0.0rc4  (**2010-10-12**)

--------------------------

- fixed python2.5 missing simplejson imports (thanks to Jens Bäckman)

- removed cache_manager settings from sqlalchemy meta

- added sqlalchemy cache settings to ini files

- validated password length and added second try of failure on paster setup-app

- fixed setup database destroy prompt even when there was no db

1.0.0rc3 (**2010-10-11**)

-------------------------

- fixed i18n during installation.

# RhodeCode, a web based repository management based on pylons

# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>

# 

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

"""

Created on April 9, 2010

RhodeCode, a web based repository management based on pylons

versioning implementation: http://semver.org/

@author: marcink

"""

__version__ = '.'.join((str(each) for each in VERSION[:4]))

def get_version():

    """

    Returns shorter version (digit parts only) as string.

    """

    return '.'.join((str(each) for each in VERSION[:3]))

    config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)

    config['routes.map'] = make_map(config)

    config['pylons.app_globals'] = app_globals.Globals(config)

    config['pylons.h'] = rhodecode.lib.helpers

    # Setup cache object as early as possible

    import pylons

    pylons.cache._push_object(config['pylons.app_globals'].cache)

    # Create the Mako TemplateLookup, with the default auto-escaping

    config['pylons.app_globals'].mako_lookup = TemplateLookup(

        directories=paths['templates'],

        error_handler=handle_mako_error,

        module_directory=os.path.join(app_conf['cache_dir'], 'templates'),

        input_encoding='utf-8', default_filters=['escape'],

        imports=['from webhelpers.html import escape'])

    #sets the c attribute access when don't existing attribute are accessed

    config['pylons.strict_tmpl_context'] = True

    test = os.path.split(config['__file__'])[-1] == 'test.ini'

    if test:

        from rhodecode.lib.utils import create_test_env, create_test_index

        create_test_env('/tmp', config)

    #MULTIPLE DB configs

    # Setup the SQLAlchemy database engine

    if config['debug'] and not test:

        #use query time debugging.

        from rhodecode.lib.timerproxy import TimerProxy

        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.',

                                                            proxy=TimerProxy())

    else:

        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')

    init_model(sa_engine_db1)

    #init baseui

    config['pylons.app_globals'].baseui = make_ui('db')

    repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial))

    set_available_permissions(config)

    set_base_path(config)

    set_rhodecode_config(config)

    # CONFIGURATION OPTIONS HERE (note: all config options will override

    # any Pylons config options)

    return config

class Globals(object):

    """Globals acts as a container for objects available throughout the

    life of the application

    """

    def __init__(self, config):

        """One instance of Globals is created during application

        initialization and is available during requests via the

        'app_globals' variable

        """

        self.cache = CacheManager(**parse_cache_config_options(config))

        self.available_permissions = None   # propagated after init_model

        self.baseui = None                  # propagated after init_model        

    @LazyProperty

    def paths(self):

        if self.baseui:

            return self.baseui.configitems('paths')

    @LazyProperty

    def base_path(self):

        if self.baseui:

from celery.decorators import task

from operator import itemgetter

from pylons.i18n.translation import _

from rhodecode.lib.celerylib import run_task, locked_task

from rhodecode.lib.helpers import person

from rhodecode.lib.smtp_mailer import SmtpMailer

from rhodecode.lib.utils import OrderedDict

from time import mktime

from vcs.backends.hg import MercurialRepository

import traceback

try:

    import json

except ImportError:

    #python 2.5 compatibility

    import simplejson as json

try:

    from celeryconfig import PYLONS_CONFIG as config

    celery_on = True

except ImportError:

    #if celeryconfig is not present let's just load our pylons

    #config instead

    from pylons import config

    celery_on = False

__all__ = ['whoosh_index', 'get_commits_stats',

           'reset_user_password', 'send_email']

def get_session():

    if celery_on:

        from sqlalchemy import engine_from_config

        from sqlalchemy.orm import sessionmaker, scoped_session

        if each.ui_section == 'hooks':

            v = each.ui_active

        settings[each.ui_section + '_' + k] = v

    return settings

@task

@locked_task

def whoosh_index(repo_location, full_index):

    log = whoosh_index.get_logger()

    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

    WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index)

@task

@locked_task

def get_commits_stats(repo_name, ts_min_y, ts_max_y):

    from rhodecode.model.db import Statistics, Repository

    log = get_commits_stats.get_logger()

    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty

    commits_by_day_author_aggregate = {}

    commits_by_day_aggregate = {}

    skip_date_limit = True

    parse_limit = 250 #limit for single task changeset parsing optimal for

    last_rev = 0

    last_cs = None

    timegetter = itemgetter('time')

    sa = get_session()

    dbrepo = sa.query(Repository)\

        .filter(Repository.repo_name == repo_name).scalar()

    cur_stats = sa.query(Statistics)\

        .filter(Statistics.repository == dbrepo).scalar()

    if cur_stats:

        last_rev = cur_stats.stat_on_revision

    if not repo.revisions:

        return True

    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:

        #pass silently without any work if we're not on first revision or current

        #state of parsing revision(from db marker) is the last revision

        return True

    if cur_stats:

@task

def create_repo_fork(form_data, cur_user):

    import os

    from rhodecode.model.repo import RepoModel

    repo_model = RepoModel(get_session())

    repo_model.create(form_data, cur_user, just_db=True, fork=True)

    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')

    repo_path = os.path.join(repos_path, form_data['repo_name'])

    repo_fork_path = os.path.join(repos_path, form_data['fork_name'])

    MercurialRepository(str(repo_fork_path), True, clone_url=str(repo_path))

def __get_codes_stats(repo_name):

    LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx',

    'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el',

    'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml',

    'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst',

    's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']

    tip = repo.get_changeset()

    code_stats = {}

    def aggregate(cs):

        for f in cs[2]:

            k = f.mimetype

            if f.extension in LANGUAGES_EXTENSIONS:

                if code_stats.has_key(k):

                    code_stats[k] += 1

                else:

                    code_stats[k] = 1

    map(aggregate, tip.walk('/'))

    return code_stats or {}

        web1 = RhodeCodeUi()

        web1.ui_section = 'web'

        web1.ui_key = 'push_ssl'

        web1.ui_value = 'false'

        web2 = RhodeCodeUi()

        web2.ui_section = 'web'

        web2.ui_key = 'allow_archive'

        web2.ui_value = 'gz zip bz2'

        web3 = RhodeCodeUi()

        web3.ui_section = 'web'

        web3.ui_key = 'allow_push'

        web3.ui_value = '*'

        web4 = RhodeCodeUi()

        web4.ui_section = 'web'

        web4.ui_key = 'baseurl'

        web4.ui_value = '/'

        paths = RhodeCodeUi()

        paths.ui_section = 'paths'

        paths.ui_key = '/'

        hgsettings1 = RhodeCodeSettings()

        hgsettings1.app_settings_name = 'realm'

        hgsettings1.app_settings_value = 'RhodeCode authentication'

        hgsettings2 = RhodeCodeSettings()

        hgsettings2.app_settings_name = 'title'

        hgsettings2.app_settings_value = 'RhodeCode'

        try:

            self.sa.add(hooks1)

            self.sa.add(hooks2)

            self.sa.add(web1)

            self.sa.add(web2)

            self.sa.add(web3)

            self.sa.add(web4)

            self.sa.add(paths)

            self.sa.add(hgsettings1)

            self.sa.add(hgsettings2)

            self.sa.commit()

        except:

            self.sa.rollback()

def get_changeset_safe(repo, rev):

    from vcs.backends.base import BaseRepository

    from vcs.exceptions import RepositoryError

    if not isinstance(repo, BaseRepository):

        raise Exception('You must pass an Repository '

                        'object as first argument got %s', type(repo))

    try:

        cs = repo.get_changeset(rev)

    except RepositoryError:

        from rhodecode.lib.utils import EmptyChangeset

        cs = EmptyChangeset()

    return cs

flash = _Flash()

#===============================================================================

# MERCURIAL FILTERS available via h.

#===============================================================================

from mercurial import util

from mercurial.templatefilters import age as _age, person as _person

capitalize = lambda x: x.capitalize()

date = lambda x: util.datestr(x)

email = util.email

email_or_none = lambda x: util.email(x) if util.email(x) != x else None

person = lambda x: _person(x)

hgdate = lambda  x: "%d %d" % x

isodate = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M %1%2')

isodatesec = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2')

localdate = lambda  x: (x[0], util.makedate()[1])

rfc3339date = lambda  x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2")

time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2")

#===============================================================================

# PERMS

#===============================================================================

from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \

HasRepoPermissionAny, HasRepoPermissionAll

#===============================================================================

# GRAVATAR URL

#===============================================================================

import hashlib

import urllib

from pylons import request

def gravatar_url(email_address, size=30):

    ssl_enabled = 'https' == request.environ.get('HTTP_X_URL_SCHEME')

    default = 'identicon'

    baseurl_nossl = "http://www.gravatar.com/avatar/"

    baseurl_ssl = "https://secure.gravatar.com/avatar/"

    baseurl = baseurl_ssl if ssl_enabled else baseurl_nossl

from os.path import dirname as dn, join as jn

from rhodecode.config.environment import load_environment

from rhodecode.model.hg import HgModel

from shutil import rmtree

from webhelpers.html.builder import escape

from vcs.utils.lazy import LazyProperty

from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.fields import TEXT, ID, STORED, Schema, FieldType

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter   

import traceback

#LOCATION WE KEEP THE INDEX

IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')

#EXTENSIONS WE WANT TO INDEX CONTENT OFF

INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',

                    'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',

                    'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',

                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',

                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',

                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',

                    'yaws']

#CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(owner=TEXT(),

                repository=TEXT(stored=True),

                path=TEXT(stored=True),

                content=FieldType(format=Characters(ANALYZER),

                             scorable=True, stored=True),

                modtime=STORED(), extension=TEXT(stored=True))

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') 

FRAGMENTER = SimpleFragmenter(200)

class ResultWrapper(object):

    def __init__(self, search_type, searcher, matcher, highlight_items):

        self.search_type = search_type

        self.searcher = searcher

        self.matcher = matcher

        self.highlight_items = highlight_items

        self.fragment_size = 200 / 2

    @LazyProperty

    def doc_ids(self):

        docs_id = []

        while self.matcher.is_active():

            docnum = self.matcher.id()

            chunks = [offsets for offsets in self.get_chunks()]

            docs_id.append([docnum, chunks])

            self.matcher.next()

        return docs_id   

    def __str__(self):

        return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))

    def __repr__(self):

        return self.__str__()

        return slice   

    def get_full_content(self, docid):

        res = self.searcher.stored_fields(docid[0])

        f_path = res['path'][res['path'].find(res['repository']) \

                             + len(res['repository']):].lstrip('/')

        content_short = self.get_short_content(res, docid[1])

        res.update({'content_short':content_short,

                    'content_short_hl':self.highlight(content_short),

                    'f_path':f_path})

        return res        

    def get_short_content(self, res, chunks):

        return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])

    def get_chunks(self):

        """

        Smart function that implements chunking the content

        but not overlap chunks so it doesn't highlight the same

        close occurrences twice.

        """

        memory = [(0, 0)]

        for span in self.matcher.spans():

            start = span.startchar or 0

            end = span.endchar or 0

            start_offseted = max(0, start - self.fragment_size)

            end_offseted = end + self.fragment_size

            if start_offseted < memory[-1][1]:

                start_offseted = memory[-1][1]

            memory.append((start_offseted, end_offseted,))    

            yield (start_offseted, end_offseted,)  

    def highlight(self, content, top=5):

        if self.search_type != 'content':

            return ''

        hl = highlight(escape(content),

                 self.highlight_items,

                 analyzer=ANALYZER,

                 fragmenter=FRAGMENTER,

                 formatter=FORMATTER,

                 top=top)

        return hl 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

"""

Created on Jan 26, 2010

@author: marcink

A deamon will read from task table and run tasks

"""

import sys

import os

from os.path import dirname as dn

from os.path import join as jn

#to get the rhodecode import

project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))

sys.path.append(project_path)

from rhodecode.model.hg import HgModel

from rhodecode.lib.helpers import safe_unicode

from whoosh.index import create_in, open_dir

from shutil import rmtree

from time import mktime

from vcs.exceptions import ChangesetError, RepositoryError

import logging

log = logging.getLogger('whooshIndexer')

# create logger

log.setLevel(logging.DEBUG)

log.propagate = False

# create console handler and set level to debug

ch = logging.StreamHandler()

ch.setLevel(logging.DEBUG)

# create formatter

formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

# add formatter to ch

ch.setFormatter(formatter)

# add ch to logger

log.addHandler(ch)

class WhooshIndexingDaemon(object):

    """

    Deamon for atomic jobs

    """

        self.indexname = indexname

        self.repo_location = repo_location

        self.initial = False

            log.info('Cannot run incremental index since it does not'

                     ' yet exist running full build')

            self.initial = True

    def get_paths(self, repo):

        """

        recursive walk in root dir and return a set of all path in that dir

        based on repository walk function

        """

        index_paths_ = set()

        try:

                for f in files:

                    index_paths_.add(jn(repo.path, f.path))

                for dir in dirs:

                    for f in files:

                        index_paths_.add(jn(repo.path, f.path))

        except RepositoryError:

            pass

        return index_paths_        

    def get_node(self, repo, path):

        n_path = path[len(repo.path) + 1:]

        node = repo.get_changeset().get_node(n_path)

        return node

    def get_node_mtime(self, node):

        return mktime(node.last_changeset.date.timetuple())

    def add_doc(self, writer, path, repo):

        """Adding doc to writer"""

        node = self.get_node(repo, path)

        #we just index the content of chosen files

        if node.extension in INDEX_EXTENSIONS:

            log.debug('    >> %s [WITH CONTENT]' % path)

            u_content = node.content

        else:

            log.debug('    >> %s' % path)

            #just index file name without it's content

            u_content = u''

        writer.add_document(owner=unicode(repo.contact),

                        repository=safe_unicode(repo.name),

                        path=safe_unicode(path),

                        content=u_content,

                        modtime=self.get_node_mtime(node),

                        extension=node.extension)             

    def build_index(self):

            log.debug('removing previous index')

        writer = idx.writer()

        for cnt, repo in enumerate(self.repo_paths.values()):

            log.debug('building index @ %s' % repo.path)

            for idx_path in self.get_paths(repo):

                self.add_doc(writer, idx_path, repo)

        log.debug('>> COMMITING CHANGES <<')

        writer.commit(merge=True)

        log.debug('>>> FINISHED BUILDING INDEX <<<')

    def update_index(self):

        log.debug('STARTING INCREMENTAL INDEXING UPDATE')

        # The set of all paths in the index

        indexed_paths = set()

        # The set of all paths we need to re-index

        to_index = set()

        reader = idx.reader()

        writer = idx.writer()

        # Loop over the stored fields in the index

        for fields in reader.all_stored_fields():

            indexed_path = fields['path']

            indexed_paths.add(indexed_path)

            repo = self.repo_paths[fields['repository']]

            try:

                node = self.get_node(repo, indexed_path)

            except ChangesetError:

                # This file was deleted since it was indexed

                log.debug('removing from index %s' % indexed_path)

                writer.delete_by_term('path', indexed_path)

            else:

                # Check if this file was changed since it was indexed

                    writer.delete_by_term('path', indexed_path)

                    to_index.add(indexed_path)

        # Loop over the files in the filesystem

        # Assume we have a function that gathers the filenames of the

        # documents to be indexed

        for repo in self.repo_paths.values():

            for path in self.get_paths(repo):

                if path in to_index or path not in indexed_paths:

                    # This is either a file that's changed, or a new file

                    # that wasn't indexed before. So index it!

                    self.add_doc(writer, path, repo)

                    log.debug('re indexing %s' % path)

        log.debug('>> COMMITING CHANGES <<')

        writer.commit(merge=True)

        log.debug('>>> FINISHED REBUILDING INDEX <<<')

    def run(self, full_index=False):

        """Run daemon"""

        if full_index or self.initial:

            self.build_index()

        else:

            self.update_index()