################################################################################

################################################################################

# RhodeCode - Pylons environment configuration                                 #

#                                                                              # 

# The %(here)s variable will be replaced with the parent directory of this file#

################################################################################

[DEFAULT]

debug = true

################################################################################

## Uncomment and replace with the address which should receive                ## 

## any error reports after application crash                                  ##

## Additionally those settings will be used by RhodeCode mailing system       ##

################################################################################

#email_to = admin@localhost

#error_email_from = paste_error@localhost

#app_email_from = rhodecode-noreply@localhost

#error_message =

#smtp_server = mail.server.com

#smtp_username = 

#smtp_password = 

#smtp_port = 

#smtp_use_tls = false

#smtp_use_ssl = true

[server:main]

##nr of threads to spawn

threadpool_workers = 5

##max request before thread respawn

threadpool_max_requests = 6

##option to use threads of process

use_threadpool = false

use = egg:Paste#http

host = 0.0.0.0

port = 5000

[app:main]

use = egg:rhodecode

full_stack = true

static_files = true

lang=en

cache_dir = %(here)s/data

index_dir = %(here)s/data/index

cut_off_limit = 256000

####################################

###        CELERY CONFIG        ####

####################################

use_celery = false

broker.host = localhost

broker.vhost = rabbitmqhost

broker.port = 5672

broker.user = rabbitmq

broker.password = qweqwe

celery.imports = rhodecode.lib.celerylib.tasks

celery.result.backend = amqp

celery.result.dburi = amqp://

celery.result.serialier = json

#celery.send.task.error.emails = true

#celery.amqp.task.result.expires = 18000

celeryd.concurrency = 2

#celeryd.log.file = celeryd.log

celeryd.log.level = debug

celeryd.max.tasks.per.child = 3

#tasks will never be sent to the queue, but executed locally instead.

celery.always.eager = false

####################################

###         BEAKER CACHE        ####

####################################

beaker.cache.data_dir=/%(here)s/data/cache/data

beaker.cache.lock_dir=/%(here)s/data/cache/lock

beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long

beaker.cache.super_short_term.type=memory

beaker.cache.super_short_term.expire=10

beaker.cache.short_term.type=memory

beaker.cache.short_term.expire=60

beaker.cache.long_term.type=memory

beaker.cache.long_term.expire=36000

beaker.cache.sql_cache_short.type=memory

beaker.cache.sql_cache_short.expire=10

beaker.cache.sql_cache_med.type=memory

.. _changelog:

Changelog

=========

1.1.0 (**2010-12-18**)

----------------------

news

++++

- rewrite of internals for vcs >=0.1.10

- uses mercurial 1.7 with dotencode disabled for maintaining compatibility 

  with older clients

- anonymous access, authentication via ldap

- performance upgrade for cached repos list - each repository has it's own 

  cache that's invalidated when needed.

- performance upgrades on repositories with large amount of commits (20K+)

- main page quick filter for filtering repositories

- user dashboards with ability to follow chosen repositories actions

- sends email to admin on new user registration

- added cache/statistics reset options into repository settings

- more detailed action logger (based on hooks) with pushed changesets lists

  and options to disable those hooks from admin panel

- introduced new enhanced changelog for merges that shows more accurate results

- new improved and faster code stats (based on pygments lexers mapping tables, 

  showing up to 10 trending sources for each repository. Additionally stats

  can be disabled in repository settings.

- gui optimizations, fixed application width to 1024px

- added cut off (for large files/changesets) limit into config files

- whoosh, celeryd, upgrade moved to paster command

- other than sqlite database backends can be used

fixes

+++++

- fixes #61 forked repo was showing only after cache expired

- fixes #76 no confirmation on user deletes

- fixes #66 Name field misspelled

- fixes #72 block user removal when he owns repositories

- fixes #69 added password confirmation fields

- fixes #87 RhodeCode crashes occasionally on updating repository owner

- fixes #82 broken annotations on files with more than 1 blank line at the end

- a lot of fixes and tweaks for file browser

- fixed detached session issues

- fixed when user had no repos he would see all repos listed in my account

- fixed ui() instance bug when global hgrc settings was loaded for server 

  instance and all hgrc options were merged with our db ui() object

- numerous small bugfixes

(special thanks for TkSoh for detailed feedback)

Here's a typical ldap setup::

 Enable ldap  = checked                 #controls if ldap access is enabled

 Host         = host.domain.org         #actual ldap server to connect

 Port         = 389 or 689 for ldaps    #ldap server ports

 Enable LDAPS = unchecked               #enable disable ldaps

 Account      = <account>               #access for ldap server(if required)

 Password     = <password>              #password for ldap server(if required)

 Base DN      = uid=%(user)s,CN=users,DC=host,DC=domain,DC=org

`Account` and `Password` are optional, and used for two-phase ldap 

authentication so those are credentials to access Your ldap, if it doesn't 

support anonymous search/user lookups. 

Base DN must have %(user)s template inside, it's a placer where Your uid used

to login would go, it allows admins to specify not standard schema for uid 

variable

If all data are entered correctly, and `python-ldap` is properly installed

Users should be granted to access RhodeCode wit ldap accounts. When 

logging at the first time an special ldap account is created inside RhodeCode, 

so You can control over permissions even on ldap users. If such user exists 

already in RhodeCode database ldap user with the same username would be not 

able to access RhodeCode.

If You have problems with ldap access and believe You entered correct 

information check out the RhodeCode logs,any error messages sent from 

ldap will be saved there.

Setting Up Celery

-----------------

Since version 1.1 celery is configured by the rhodecode ini configuration files

simply set use_celery=true in the ini file then add / change the configuration 

variables inside the ini file.

Remember that the ini files uses format with '.' not with '_' like celery

so for example setting `BROKER_HOST` in celery means setting `broker.host` in

the config file.

In order to make start using celery run::

 paster celeryd <configfile.ini>

Nginx virtual host example

--------------------------

Sample config for nginx using proxy::

 server {

    listen          80;

    server_name     hg.myserver.com;

    access_log      /var/log/nginx/rhodecode.access.log;

    error_log       /var/log/nginx/rhodecode.error.log;

    location / {

            root /var/www/rhodecode/rhodecode/public/;

            if (!-f $request_filename){

                proxy_pass      http://127.0.0.1:5000;

            }

            #this is important for https !!!

            proxy_set_header X-Url-Scheme $scheme;

            include         /etc/nginx/proxy.conf;  

    }

 }  

Here's the proxy.conf. It's tuned so it'll not timeout on long

pushes and also on large pushes::

    proxy_redirect              off;

    proxy_set_header            Host $host;

    proxy_set_header            X-Host $http_host;

    proxy_set_header            X-Real-IP $remote_addr;

    proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;

    proxy_set_header            Proxy-host $proxy_host;

    client_max_body_size        400m;

    client_body_buffer_size     128k;

    proxy_buffering             off;

    proxy_connect_timeout       3600;

    proxy_send_timeout          3600;

    proxy_read_timeout          3600;

    proxy_buffer_size           8k;

    proxy_buffers               8 32k;

    proxy_busy_buffers_size     64k;

    proxy_temp_file_write_size  64k;

Also when using root path with nginx You might set the static files to false

in production.ini file::

  [app:main]

    use = egg:rhodecode

    full_stack = true

    static_files = false

    lang=en

    cache_dir = %(here)s/data

To not have the statics served by the application. And improve speed.

http://wiki.pylonshq.com/display/pylonscookbook/Apache+as+a+reverse+proxy+for+Pylons

Apache's example FCGI config

----------------------------

TODO !

Other configuration files

-------------------------

Some extra configuration files and examples can be found here:

http://hg.python-works.com/rhodecode/files/tip/init.d

and also an celeryconfig file can be use from here:

http://hg.python-works.com/rhodecode/files/tip/celeryconfig.py

Troubleshooting

---------------

- missing static files ?

 - make sure either to set the `static_files = true` in the .ini file or

   double check the root path for Your http setup. It should point to 

   for example:

   /home/my-virtual-python/lib/python2.6/site-packages/rhodecode/public

- can't install celery/rabbitmq

 - don't worry RhodeCode works without them too. No extra setup required

- long lasting push timeouts ?

 - make sure You set a longer timeouts in Your proxy/fcgi settings, timeouts

   are caused by https server and not RhodeCode

- large pushes timeouts ?

 - make sure You set a proper max_body_size for the http server

.. _virtualenv: http://pypi.python.org/pypi/virtualenv

.. _python: http://www.python.org/

.. _mercurial: http://mercurial.selenic.com/

.. _celery: http://celeryproject.org/

.. _rabbitmq: http://www.rabbitmq.com/

################################################################################

################################################################################

# rhodecode - Pylons environment configuration                                 #

#                                                                              # 

# The %(here)s variable will be replaced with the parent directory of this file#

################################################################################

[DEFAULT]

debug = true

################################################################################

## Uncomment and replace with the address which should receive                ## 

## any error reports after application crash                                  ##

## Additionally those settings will be used by rhodecode mailing system       ##

################################################################################

#email_to = admin@localhost

#error_email_from = paste_error@localhost

#app_email_from = rhodecode-noreply@localhost

#error_message =

#smtp_server = mail.server.com

#smtp_username = 

#smtp_password = 

#smtp_port = 

#smtp_use_tls = false

#smtp_use_ssl = true

[server:main]

##nr of threads to spawn

threadpool_workers = 5

##max request before thread respawn

threadpool_max_requests = 2

##option to use threads of process

use_threadpool = true

use = egg:Paste#http

host = 127.0.0.1

port = 8001

[app:main]

use = egg:rhodecode

full_stack = true

static_files = false

lang=en

cache_dir = %(here)s/data

index_dir = %(here)s/data/index

cut_off_limit = 256000

####################################

###        CELERY CONFIG        ####

####################################

use_celery = false

broker.host = localhost

broker.vhost = rabbitmqhost

broker.port = 5672

broker.user = rabbitmq

broker.password = qweqwe

celery.imports = rhodecode.lib.celerylib.tasks

celery.result.backend = amqp

celery.result.dburi = amqp://

celery.result.serialier = json

#celery.send.task.error.emails = true

#celery.amqp.task.result.expires = 18000

celeryd.concurrency = 2

#celeryd.log.file = celeryd.log

celeryd.log.level = debug

celeryd.max.tasks.per.child = 3

#tasks will never be sent to the queue, but executed locally instead.

celery.always.eager = false

####################################

###         BEAKER CACHE        ####

####################################

beaker.cache.data_dir=/%(here)s/data/cache/data

beaker.cache.lock_dir=/%(here)s/data/cache/lock

beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long

beaker.cache.super_short_term.type=memory

beaker.cache.super_short_term.expire=10

beaker.cache.short_term.type=memory

beaker.cache.short_term.expire=60

beaker.cache.long_term.type=memory

beaker.cache.long_term.expire=36000

beaker.cache.sql_cache_short.type=memory

beaker.cache.sql_cache_short.expire=5

################################################################################

# RhodeCode - Pylons environment configuration                                 #

#                                                                              # 

# The %(here)s variable will be replaced with the parent directory of this file#

################################################################################

[DEFAULT]

debug = true

################################################################################

## Uncomment and replace with the address which should receive                ## 

## any error reports after application crash                                  ##

## Additionally those settings will be used by RhodeCode mailing system       ##

################################################################################

#email_to = admin@localhost

#error_email_from = paste_error@localhost

#app_email_from = rhodecode-noreply@localhost

#error_message =

#smtp_server = mail.server.com

#smtp_username = 

#smtp_password = 

#smtp_port = 

#smtp_use_tls = false

#smtp_use_ssl = true

[server:main]

##nr of threads to spawn

threadpool_workers = 5

##max request before thread respawn

threadpool_max_requests = 10

##option to use threads of process

use_threadpool = true

use = egg:Paste#http

host = 127.0.0.1

port = 5000

[app:main]

use = egg:rhodecode

full_stack = true

static_files = true

lang=en

cache_dir = %(here)s/data

index_dir = %(here)s/data/index

app_instance_uuid = ${app_instance_uuid}

cut_off_limit = 256000

####################################

###        CELERY CONFIG        ####

####################################

use_celery = false

broker.host = localhost

broker.vhost = rabbitmqhost

broker.port = 5672

broker.user = rabbitmq

broker.password = qweqwe

celery.imports = rhodecode.lib.celerylib.tasks

celery.result.backend = amqp

celery.result.dburi = amqp://

celery.result.serialier = json

#celery.send.task.error.emails = true

#celery.amqp.task.result.expires = 18000

celeryd.concurrency = 2

#celeryd.log.file = celeryd.log

celeryd.log.level = debug

celeryd.max.tasks.per.child = 3

#tasks will never be sent to the queue, but executed locally instead.

celery.always.eager = false

####################################

###         BEAKER CACHE        ####

####################################

beaker.cache.data_dir=/%(here)s/data/cache/data

beaker.cache.lock_dir=/%(here)s/data/cache/lock

beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long

beaker.cache.super_short_term.type=memory

beaker.cache.super_short_term.expire=10

beaker.cache.short_term.type=memory

beaker.cache.short_term.expire=60

beaker.cache.long_term.type=memory

beaker.cache.long_term.expire=36000

beaker.cache.sql_cache_short.type=memory

beaker.cache.sql_cache_short.expire=10

beaker.cache.sql_cache_med.type=memory

# -*- coding: utf-8 -*-

"""

    rhodecode.controllers.files

    ~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Files controller for RhodeCode

    :created_on: Apr 21, 2010

    :author: marcink

    :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>    

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

import tempfile

import logging

import rhodecode.lib.helpers as h

from mercurial import archival

from pylons import request, response, session, tmpl_context as c, url

from pylons.i18n.translation import _

from pylons.controllers.util import redirect

from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

from rhodecode.lib.base import BaseController, render

from rhodecode.lib.utils import EmptyChangeset

from rhodecode.model.scm import ScmModel

from vcs.nodes import FileNode

from vcs.utils import diffs as differ

log = logging.getLogger(__name__)

class FilesController(BaseController):

    @LoginRequired()

    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',

                                   'repository.admin')

    def __before__(self):

        super(FilesController, self).__before__()

        c.cut_off_limit = self.cut_off_limit

    def index(self, repo_name, revision, f_path):

        hg_model = ScmModel()

        c.repo = hg_model.get_repo(c.repo_name)

        revision = request.POST.get('at_rev', None) or revision

        def get_next_rev(cur):

            max_rev = len(c.repo.revisions) - 1

            r = cur + 1

            if r > max_rev:

                r = max_rev

            return r

        def get_prev_rev(cur):

            r = cur - 1

            return r

        c.f_path = f_path

        try:

            c.changeset = c.repo.get_changeset(revision)

            cur_rev = c.changeset.revision

            prev_rev = c.repo.get_changeset(get_prev_rev(cur_rev)).raw_id

            next_rev = c.repo.get_changeset(get_next_rev(cur_rev)).raw_id

            c.url_prev = url('files_home', repo_name=c.repo_name,

                             revision=prev_rev, f_path=f_path)

            c.url_next = url('files_home', repo_name=c.repo_name,

                         revision=next_rev, f_path=f_path)

            try:

                c.files_list = c.changeset.get_node(f_path)

                c.file_history = self._get_history(c.repo, c.files_list, f_path)

            except RepositoryError, e:

                h.flash(str(e), category='warning')

                redirect(h.url('files_home', repo_name=repo_name, revision=revision))

        except RepositoryError, e:

            h.flash(str(e), category='warning')

            redirect(h.url('files_home', repo_name=repo_name, revision='tip'))

        return render('files/files.html')

    def rawfile(self, repo_name, revision, f_path):

        hg_model = ScmModel()

        c.repo = hg_model.get_repo(c.repo_name)

        file_node = c.repo.get_changeset(revision).get_node(f_path)

        response.content_type = file_node.mimetype

        response.content_disposition = 'attachment; filename=%s' \

                                                    % f_path.split('/')[-1]

        return file_node.content

    def raw(self, repo_name, revision, f_path):

        hg_model = ScmModel()

        c.repo = hg_model.get_repo(c.repo_name)

        file_node = c.repo.get_changeset(revision).get_node(f_path)

        response.content_type = 'text/plain'

        return file_node.content

    def annotate(self, repo_name, revision, f_path):

        hg_model = ScmModel()

        c.repo = hg_model.get_repo(c.repo_name)

        try:

            c.cs = c.repo.get_changeset(revision)

            c.file = c.cs.get_node(f_path)

        except RepositoryError, e:

            h.flash(str(e), category='warning')

            redirect(h.url('files_home', repo_name=repo_name, revision=revision))

        c.file_history = self._get_history(c.repo, c.file, f_path)

        c.f_path = f_path

        return render('files/files_annotate.html')

    def archivefile(self, repo_name, revision, fileformat):

        archive_specs = {

          '.tar.bz2': ('application/x-tar', 'tbz2'),

          '.tar.gz': ('application/x-tar', 'tgz'),

          '.zip': ('application/zip', 'zip'),

        }

        self.indexname = indexname

        self.index_location = index_location

        if not index_location:

            raise Exception('You have to provide index location')

        self.repo_location = repo_location

        if not repo_location:

            raise Exception('You have to provide repositories location')

        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location, None)

        self.initial = False

        if not os.path.isdir(self.index_location):

            os.makedirs(self.index_location)

            log.info('Cannot run incremental index since it does not'

                     ' yet exist running full build')

            self.initial = True

    def get_paths(self, repo):

        """recursive walk in root dir and return a set of all path in that dir

        based on repository walk function

        """

        index_paths_ = set()

        try:

            for topnode, dirs, files in repo.walk('/', 'tip'):

                for f in files:

                    index_paths_.add(jn(repo.path, f.path))

                for dir in dirs:

                    for f in files:

                        index_paths_.add(jn(repo.path, f.path))

        except RepositoryError:

            pass

        return index_paths_

    def get_node(self, repo, path):

        n_path = path[len(repo.path) + 1:]

        node = repo.get_changeset().get_node(n_path)

        return node

    def get_node_mtime(self, node):

        return mktime(node.last_changeset.date.timetuple())

    def add_doc(self, writer, path, repo):

        """Adding doc to writer this function itself fetches data from

        the instance of vcs backend"""

        node = self.get_node(repo, path)

            u_content = node.content

        else:

            log.debug('    >> %s' % path)

            #just index file name without it's content

            u_content = u''

        writer.add_document(owner=unicode(repo.contact),

                        repository=safe_unicode(repo.name),

                        path=safe_unicode(path),

                        content=u_content,

                        modtime=self.get_node_mtime(node),

                        extension=node.extension)

    def build_index(self):

        if os.path.exists(self.index_location):

            log.debug('removing previous index')

            rmtree(self.index_location)

        if not os.path.exists(self.index_location):

            os.mkdir(self.index_location)

        idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)

        writer = idx.writer()

        for cnt, repo in enumerate(self.repo_paths.values()):

            log.debug('building index @ %s' % repo.path)

            for idx_path in self.get_paths(repo):

                self.add_doc(writer, idx_path, repo)

        log.debug('>> COMMITING CHANGES <<')

        writer.commit(merge=True)

        log.debug('>>> FINISHED BUILDING INDEX <<<')

    def update_index(self):

        log.debug('STARTING INCREMENTAL INDEXING UPDATE')

        idx = open_dir(self.index_location, indexname=self.indexname)

        # The set of all paths in the index

        indexed_paths = set()

        # The set of all paths we need to re-index

        to_index = set()

        reader = idx.reader()

        writer = idx.writer()

        # Loop over the stored fields in the index

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.