Changeset - 6afa528ee30e
[Not reviewed]
default
0 7 0
Mads Kiilerich - 10 years ago 2016-03-14 16:17:46
madski@unity3d.com
db: get rid of vcs_full_cache - it should always be used

It might make tests less deterministic, but it makes sure that we test what we actually use.
7 files changed with 6 insertions and 30 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -106,195 +106,192 @@ max_request_body_size = 107374182400
 
## run with uwsgi --ini-paste-logged <inifile.ini>
 
#[uwsgi]
 
#socket = /tmp/uwsgi.sock
 
#master = true
 
#http = 127.0.0.1:5000
 

	
 
## set as deamon and redirect all output to file
 
#daemonize = ./uwsgi_kallithea.log
 

	
 
## master process PID
 
#pidfile = ./uwsgi_kallithea.pid
 

	
 
## stats server with workers statistics, use uwsgitop
 
## for monitoring, `uwsgitop 127.0.0.1:1717`
 
#stats = 127.0.0.1:1717
 
#memory-report = true
 

	
 
## log 5XX errors
 
#log-5xx = true
 

	
 
## Set the socket listen queue size.
 
#listen = 256
 

	
 
## Gracefully Reload workers after the specified amount of managed requests
 
## (avoid memory leaks).
 
#max-requests = 1000
 

	
 
## enable large buffers
 
#buffer-size = 65535
 

	
 
## socket and http timeouts ##
 
#http-timeout = 3600
 
#socket-timeout = 3600
 

	
 
## Log requests slower than the specified number of milliseconds.
 
#log-slow = 10
 

	
 
## Exit if no app can be loaded.
 
#need-app = true
 

	
 
## Set lazy mode (load apps in workers instead of master).
 
#lazy = true
 

	
 
## scaling ##
 
## set cheaper algorithm to use, if not set default will be used
 
#cheaper-algo = spare
 

	
 
## minimum number of workers to keep at all times
 
#cheaper = 1
 

	
 
## number of workers to spawn at startup
 
#cheaper-initial = 1
 

	
 
## maximum number of workers that can be spawned
 
#workers = 4
 

	
 
## how many workers should be spawned at a time
 
#cheaper-step = 1
 

	
 
## COMMON ##
 
#host = 127.0.0.1
 
host = 0.0.0.0
 
port = 5000
 

	
 
## middleware for hosting the WSGI application under a URL prefix
 
#[filter:proxy-prefix]
 
#use = egg:PasteDeploy#prefix
 
#prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:kallithea
 
## enable proxy prefix middleware
 
#filter-with = proxy-prefix
 

	
 
full_stack = true
 
static_files = true
 
## Available Languages:
 
## cs de fr hu ja nl_BE pl pt_BR ru sk zh_CN zh_TW
 
lang =
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 

	
 
## perform a full repository scan on each server start, this should be
 
## set to false after first startup, to allow faster server restarts.
 
#initial_repo_scan = false
 
initial_repo_scan = true
 

	
 
## uncomment and set this path to use archive download cache
 
archive_cache_dir = %(here)s/tarballcache
 

	
 
## change this to unique ID for security
 
app_instance_uuid = development-not-secret
 

	
 
## cut off limit for large diffs (size in bytes)
 
cut_off_limit = 256000
 

	
 
## use cache version of scm repo everywhere
 
vcs_full_cache = true
 

	
 
## force https in Kallithea, fixes https redirects, assumes it's always https
 
force_https = false
 

	
 
## use Strict-Transport-Security headers
 
use_htsts = false
 

	
 
## number of commits stats will parse on each iteration
 
commit_parse_limit = 25
 

	
 
## path to git executable
 
git_path = git
 

	
 
## git rev filter option, --all is the default filter, if you need to
 
## hide all refs in changelog switch this to --branches --tags
 
#git_rev_filter = --branches --tags
 

	
 
## RSS feed options
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 
## options for showing and identifying changesets
 
show_sha_length = 12
 
show_revision_number = false
 

	
 
## gist URL alias, used to create nicer urls for gist. This should be an
 
## url that does rewrites to _admin/gists/<gistid>.
 
## example: http://gist.example.com/{gistid}. Empty means use the internal
 
## Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid>
 
gist_alias_url =
 

	
 
## white list of API enabled controllers. This allows to add list of
 
## controllers to which access will be enabled by api_key. eg: to enable
 
## api access to raw_files put `FilesController:raw`, to enable access to patches
 
## add `ChangesetController:changeset_patch`. This list should be "," separated
 
## Syntax is <ControllerClass>:<function>. Check debug logs for generated names
 
## Recommended settings below are commented out:
 
api_access_controllers_whitelist =
 
#    ChangesetController:changeset_patch,
 
#    ChangesetController:changeset_raw,
 
#    FilesController:raw,
 
#    FilesController:archivefile
 

	
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## issue tracker for Kallithea (leave blank to disable, absent for default)
 
#bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://issues.example.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
 
issue_prefix = #
 

	
 
## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 
## multiple patterns, to other issues server, wiki or others
 
## below an example how to create a wiki pattern
 
# wiki-some-id -> https://wiki.example.com/some-id
 

	
 
#issue_pat_wiki = (?:wiki-)(.+)
 
#issue_server_link_wiki = https://wiki.example.com/{id}
 
#issue_prefix_wiki = WIKI-
 

	
 
## alternative return HTTP header for failed authentication. Default HTTP
 
## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
## extra extensions for indexing, space separated and without the leading '.'.
 
# index.extensions =
docs/usage/performance.rst
Show inline comments
 
.. _performance:
 

	
 
================================
 
Optimizing Kallithea performance
 
================================
 

	
 
When serving a large amount of big repositories, Kallithea can start
 
performing slower than expected. Because of the demanding nature of handling large
 
amounts of data from version control systems, here are some tips on how to get
 
the best performance.
 

	
 
* Kallithea is often I/O bound, and hence a fast disk (SSD/SAN) is
 
Follow these few steps to improve performance of Kallithea system.
 

	
 
1.  Kallithea is often I/O bound, and hence a fast disk (SSD/SAN) is
 
  usually more important than a fast CPU.
 

	
 
* Sluggish loading of the front page can easily be fixed by grouping repositories or by
 
  increasing cache size (see below). This includes using the lightweight dashboard
 
  option and ``vcs_full_cache`` setting in .ini file.
 

	
 
Follow these few steps to improve performance of Kallithea system.
 

	
 
1. Increase cache
 
2. Increase cache
 

	
 
    Tweak beaker cache settings in the ini file. The actual effect of that
 
    is questionable.
 

	
 
2. Switch from SQLite to PostgreSQL or MySQL
 
3. Switch from SQLite to PostgreSQL or MySQL
 

	
 
    SQLite is a good option when having a small load on the system. But due to
 
    locking issues with SQLite, it is not recommended to use it for larger
 
    deployments. Switching to MySQL or PostgreSQL will result in an immediate
 
    performance increase. A tool like SQLAlchemyGrate_ can be used for
 
    migrating to another database platform.
 

	
 
3. Scale Kallithea horizontally
 
4. Scale Kallithea horizontally
 

	
 
    Scaling horizontally can give huge performance benefits when dealing with
 
    large amounts of traffic (many users, CI servers, etc.). Kallithea can be
 
    scaled horizontally on one (recommended) or multiple machines. In order
 
    to scale horizontally you need to do the following:
 

	
 
    - Each instance's ``data`` storage needs to be configured to be stored on a
 
      shared disk storage, preferably together with repositories. This ``data``
 
      dir contains template caches, sessions, whoosh index and is used for
 
      task locking (so it is safe across multiple instances). Set the
 
      ``cache_dir``, ``index_dir``, ``beaker.cache.data_dir``, ``beaker.cache.lock_dir``
 
      variables in each .ini file to a shared location across Kallithea instances
 
    - If celery is used each instance should run a separate Celery instance, but
 
      the message broker should be common to all of them (e.g.,  one
 
      shared RabbitMQ server)
 
    - Load balance using round robin or IP hash, recommended is writing LB rules
 
      that will separate regular user traffic from automated processes like CI
 
      servers or build bots.
 

	
 

	
 
.. _SQLAlchemyGrate: https://github.com/shazow/sqlalchemygrate
kallithea/bin/template.ini.mako
Show inline comments
 
@@ -103,195 +103,192 @@ timeout = 3600
 
<%text>## UWSGI ##</%text>
 
<%text>## run with uwsgi --ini-paste-logged <inifile.ini></%text>
 
[uwsgi]
 
socket = /tmp/uwsgi.sock
 
master = true
 
http = 127.0.0.1:5000
 

	
 
<%text>## set as deamon and redirect all output to file</%text>
 
#daemonize = ./uwsgi_kallithea.log
 

	
 
<%text>## master process PID</%text>
 
pidfile = ./uwsgi_kallithea.pid
 

	
 
<%text>## stats server with workers statistics, use uwsgitop</%text>
 
<%text>## for monitoring, `uwsgitop 127.0.0.1:1717`</%text>
 
stats = 127.0.0.1:1717
 
memory-report = true
 

	
 
<%text>## log 5XX errors</%text>
 
log-5xx = true
 

	
 
<%text>## Set the socket listen queue size.</%text>
 
listen = 256
 

	
 
<%text>## Gracefully Reload workers after the specified amount of managed requests</%text>
 
<%text>## (avoid memory leaks).</%text>
 
max-requests = 1000
 

	
 
<%text>## enable large buffers</%text>
 
buffer-size = 65535
 

	
 
<%text>## socket and http timeouts ##</%text>
 
http-timeout = 3600
 
socket-timeout = 3600
 

	
 
<%text>## Log requests slower than the specified number of milliseconds.</%text>
 
log-slow = 10
 

	
 
<%text>## Exit if no app can be loaded.</%text>
 
need-app = true
 

	
 
<%text>## Set lazy mode (load apps in workers instead of master).</%text>
 
lazy = true
 

	
 
<%text>## scaling ##</%text>
 
<%text>## set cheaper algorithm to use, if not set default will be used</%text>
 
cheaper-algo = spare
 

	
 
<%text>## minimum number of workers to keep at all times</%text>
 
cheaper = 1
 

	
 
<%text>## number of workers to spawn at startup</%text>
 
cheaper-initial = 1
 

	
 
<%text>## maximum number of workers that can be spawned</%text>
 
workers = 4
 

	
 
<%text>## how many workers should be spawned at a time</%text>
 
cheaper-step = 1
 

	
 
%endif
 
<%text>## COMMON ##</%text>
 
host = ${host}
 
port = ${port}
 

	
 
<%text>## middleware for hosting the WSGI application under a URL prefix</%text>
 
#[filter:proxy-prefix]
 
#use = egg:PasteDeploy#prefix
 
#prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:kallithea
 
<%text>## enable proxy prefix middleware</%text>
 
#filter-with = proxy-prefix
 

	
 
full_stack = true
 
static_files = true
 
<%text>## Available Languages:</%text>
 
<%text>## cs de fr hu ja nl_BE pl pt_BR ru sk zh_CN zh_TW</%text>
 
lang =
 
cache_dir = ${here}/data
 
index_dir = ${here}/data/index
 

	
 
<%text>## perform a full repository scan on each server start, this should be</%text>
 
<%text>## set to false after first startup, to allow faster server restarts.</%text>
 
initial_repo_scan = false
 

	
 
<%text>## uncomment and set this path to use archive download cache</%text>
 
archive_cache_dir = ${here}/tarballcache
 

	
 
<%text>## change this to unique ID for security</%text>
 
app_instance_uuid = ${uuid()}
 

	
 
<%text>## cut off limit for large diffs (size in bytes)</%text>
 
cut_off_limit = 256000
 

	
 
<%text>## use cache version of scm repo everywhere</%text>
 
vcs_full_cache = true
 

	
 
<%text>## force https in Kallithea, fixes https redirects, assumes it's always https</%text>
 
force_https = false
 

	
 
<%text>## use Strict-Transport-Security headers</%text>
 
use_htsts = false
 

	
 
<%text>## number of commits stats will parse on each iteration</%text>
 
commit_parse_limit = 25
 

	
 
<%text>## path to git executable</%text>
 
git_path = git
 

	
 
<%text>## git rev filter option, --all is the default filter, if you need to</%text>
 
<%text>## hide all refs in changelog switch this to --branches --tags</%text>
 
#git_rev_filter = --branches --tags
 

	
 
<%text>## RSS feed options</%text>
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 
<%text>## options for showing and identifying changesets</%text>
 
show_sha_length = 12
 
show_revision_number = false
 

	
 
<%text>## gist URL alias, used to create nicer urls for gist. This should be an</%text>
 
<%text>## url that does rewrites to _admin/gists/<gistid>.</%text>
 
<%text>## example: http://gist.example.com/{gistid}. Empty means use the internal</%text>
 
<%text>## Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid></%text>
 
gist_alias_url =
 

	
 
<%text>## white list of API enabled controllers. This allows to add list of</%text>
 
<%text>## controllers to which access will be enabled by api_key. eg: to enable</%text>
 
<%text>## api access to raw_files put `FilesController:raw`, to enable access to patches</%text>
 
<%text>## add `ChangesetController:changeset_patch`. This list should be "," separated</%text>
 
<%text>## Syntax is <ControllerClass>:<function>. Check debug logs for generated names</%text>
 
<%text>## Recommended settings below are commented out:</%text>
 
api_access_controllers_whitelist =
 
#    ChangesetController:changeset_patch,
 
#    ChangesetController:changeset_raw,
 
#    FilesController:raw,
 
#    FilesController:archivefile
 

	
 
<%text>## default encoding used to convert from and to unicode</%text>
 
<%text>## can be also a comma seperated list of encoding in case of mixed encodings</%text>
 
default_encoding = utf8
 

	
 
<%text>## issue tracker for Kallithea (leave blank to disable, absent for default)</%text>
 
#bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 

	
 
<%text>## issue tracking mapping for commits messages</%text>
 
<%text>## comment out issue_pat, issue_server, issue_prefix to enable</%text>
 

	
 
<%text>## pattern to get the issues from commit messages</%text>
 
<%text>## default one used here is #<numbers> with a regex passive group for `#`</%text>
 
<%text>## {id} will be all groups matched from this pattern</%text>
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
<%text>## server url to the issue, each {id} will be replaced with match</%text>
 
<%text>## fetched from the regex and {repo} is replaced with full repository name</%text>
 
<%text>## including groups {repo_name} is replaced with just name of repo</%text>
 

	
 
issue_server_link = https://issues.example.com/{repo}/issue/{id}
 

	
 
<%text>## prefix to add to link to indicate it's an url</%text>
 
<%text>## #314 will be replaced by <issue_prefix><id></%text>
 

	
 
issue_prefix = #
 

	
 
<%text>## issue_pat, issue_server_link, issue_prefix can have suffixes to specify</%text>
 
<%text>## multiple patterns, to other issues server, wiki or others</%text>
 
<%text>## below an example how to create a wiki pattern</%text>
 
# wiki-some-id -> https://wiki.example.com/some-id
 

	
 
#issue_pat_wiki = (?:wiki-)(.+)
 
#issue_server_link_wiki = https://wiki.example.com/{id}
 
#issue_prefix_wiki = WIKI-
 

	
 
<%text>## alternative return HTTP header for failed authentication. Default HTTP</%text>
 
<%text>## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with</%text>
 
<%text>## handling that. Set this variable to 403 to return HTTPForbidden</%text>
 
auth_ret_code =
 

	
 
<%text>## locking return code. When repository is locked return this HTTP code. 2XX</%text>
 
<%text>## codes don't break the transactions while 4XX codes do</%text>
 
lock_ret_code = 423
 

	
 
<%text>## allows to change the repository location in settings page</%text>
 
allow_repo_location_change = True
 

	
 
<%text>## allows to setup custom hooks in settings page</%text>
 
allow_custom_hooks_settings = True
 

	
 
<%text>## extra extensions for indexing, space separated and without the leading '.'.</%text>
 
# index.extensions =
kallithea/config/deployment.ini_tmpl
Show inline comments
 
@@ -99,195 +99,192 @@ max_request_body_size = 107374182400
 

	
 
## UWSGI ##
 
## run with uwsgi --ini-paste-logged <inifile.ini>
 
#[uwsgi]
 
#socket = /tmp/uwsgi.sock
 
#master = true
 
#http = 127.0.0.1:5000
 

	
 
## set as deamon and redirect all output to file
 
#daemonize = ./uwsgi_kallithea.log
 

	
 
## master process PID
 
#pidfile = ./uwsgi_kallithea.pid
 

	
 
## stats server with workers statistics, use uwsgitop
 
## for monitoring, `uwsgitop 127.0.0.1:1717`
 
#stats = 127.0.0.1:1717
 
#memory-report = true
 

	
 
## log 5XX errors
 
#log-5xx = true
 

	
 
## Set the socket listen queue size.
 
#listen = 256
 

	
 
## Gracefully Reload workers after the specified amount of managed requests
 
## (avoid memory leaks).
 
#max-requests = 1000
 

	
 
## enable large buffers
 
#buffer-size = 65535
 

	
 
## socket and http timeouts ##
 
#http-timeout = 3600
 
#socket-timeout = 3600
 

	
 
## Log requests slower than the specified number of milliseconds.
 
#log-slow = 10
 

	
 
## Exit if no app can be loaded.
 
#need-app = true
 

	
 
## Set lazy mode (load apps in workers instead of master).
 
#lazy = true
 

	
 
## scaling ##
 
## set cheaper algorithm to use, if not set default will be used
 
#cheaper-algo = spare
 

	
 
## minimum number of workers to keep at all times
 
#cheaper = 1
 

	
 
## number of workers to spawn at startup
 
#cheaper-initial = 1
 

	
 
## maximum number of workers that can be spawned
 
#workers = 4
 

	
 
## how many workers should be spawned at a time
 
#cheaper-step = 1
 

	
 
## COMMON ##
 
host = 127.0.0.1
 
port = 5000
 

	
 
## middleware for hosting the WSGI application under a URL prefix
 
#[filter:proxy-prefix]
 
#use = egg:PasteDeploy#prefix
 
#prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:kallithea
 
## enable proxy prefix middleware
 
#filter-with = proxy-prefix
 

	
 
full_stack = true
 
static_files = true
 
## Available Languages:
 
## cs de fr hu ja nl_BE pl pt_BR ru sk zh_CN zh_TW
 
lang =
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 

	
 
## perform a full repository scan on each server start, this should be
 
## set to false after first startup, to allow faster server restarts.
 
initial_repo_scan = false
 

	
 
## uncomment and set this path to use archive download cache
 
archive_cache_dir = %(here)s/tarballcache
 

	
 
## change this to unique ID for security
 
app_instance_uuid = ${app_instance_uuid}
 

	
 
## cut off limit for large diffs (size in bytes)
 
cut_off_limit = 256000
 

	
 
## use cache version of scm repo everywhere
 
vcs_full_cache = true
 

	
 
## force https in Kallithea, fixes https redirects, assumes it's always https
 
force_https = false
 

	
 
## use Strict-Transport-Security headers
 
use_htsts = false
 

	
 
## number of commits stats will parse on each iteration
 
commit_parse_limit = 25
 

	
 
## path to git executable
 
git_path = git
 

	
 
## git rev filter option, --all is the default filter, if you need to
 
## hide all refs in changelog switch this to --branches --tags
 
#git_rev_filter = --branches --tags
 

	
 
## RSS feed options
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 
## options for showing and identifying changesets
 
show_sha_length = 12
 
show_revision_number = false
 

	
 
## gist URL alias, used to create nicer urls for gist. This should be an
 
## url that does rewrites to _admin/gists/<gistid>.
 
## example: http://gist.example.com/{gistid}. Empty means use the internal
 
## Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid>
 
gist_alias_url =
 

	
 
## white list of API enabled controllers. This allows to add list of
 
## controllers to which access will be enabled by api_key. eg: to enable
 
## api access to raw_files put `FilesController:raw`, to enable access to patches
 
## add `ChangesetController:changeset_patch`. This list should be "," separated
 
## Syntax is <ControllerClass>:<function>. Check debug logs for generated names
 
## Recommended settings below are commented out:
 
api_access_controllers_whitelist =
 
#    ChangesetController:changeset_patch,
 
#    ChangesetController:changeset_raw,
 
#    FilesController:raw,
 
#    FilesController:archivefile
 

	
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## issue tracker for Kallithea (leave blank to disable, absent for default)
 
#bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://issues.example.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
 
issue_prefix = #
 

	
 
## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 
## multiple patterns, to other issues server, wiki or others
 
## below an example how to create a wiki pattern
 
# wiki-some-id -> https://wiki.example.com/some-id
 

	
 
#issue_pat_wiki = (?:wiki-)(.+)
 
#issue_server_link_wiki = https://wiki.example.com/{id}
 
#issue_prefix_wiki = WIKI-
 

	
 
## alternative return HTTP header for failed authentication. Default HTTP
 
## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
## extra extensions for indexing, space separated and without the leading '.'.
 
# index.extensions =
kallithea/model/db.py
Show inline comments
 
@@ -1346,197 +1346,193 @@ class Repository(Base, BaseModel):
 
        if cs_cache is None:
 
            cs_cache = EmptyChangeset()
 
            # use no-cache version here
 
            scm_repo = self.scm_instance_no_cache()
 
            if scm_repo:
 
                cs_cache = scm_repo.get_changeset()
 

	
 
        if isinstance(cs_cache, BaseChangeset):
 
            cs_cache = cs_cache.__json__()
 

	
 
        if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):
 
            _default = datetime.datetime.fromtimestamp(0)
 
            last_change = cs_cache.get('date') or _default
 
            log.debug('updated repo %s with new cs cache %s',
 
                      self.repo_name, cs_cache)
 
            self.updated_on = last_change
 
            self.changeset_cache = cs_cache
 
            Session().add(self)
 
            Session().commit()
 
        else:
 
            log.debug('changeset_cache for %s already up to date with %s',
 
                      self.repo_name, cs_cache['raw_id'])
 

	
 
    @property
 
    def tip(self):
 
        return self.get_changeset('tip')
 

	
 
    @property
 
    def author(self):
 
        return self.tip.author
 

	
 
    @property
 
    def last_change(self):
 
        return self.scm_instance.last_change
 

	
 
    def get_comments(self, revisions=None):
 
        """
 
        Returns comments for this repository grouped by revisions
 

	
 
        :param revisions: filter query by revisions only
 
        """
 
        cmts = ChangesetComment.query() \
 
            .filter(ChangesetComment.repo == self)
 
        if revisions is not None:
 
            if not revisions:
 
                return {} # don't use sql 'in' on empty set
 
            cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
 
        grouped = collections.defaultdict(list)
 
        for cmt in cmts.all():
 
            grouped[cmt.revision].append(cmt)
 
        return grouped
 

	
 
    def statuses(self, revisions):
 
        """
 
        Returns statuses for this repository.
 
        PRs without any votes do _not_ show up as unreviewed.
 

	
 
        :param revisions: list of revisions to get statuses for
 
        """
 
        if not revisions:
 
            return {}
 

	
 
        statuses = ChangesetStatus.query() \
 
            .filter(ChangesetStatus.repo == self) \
 
            .filter(ChangesetStatus.version == 0) \
 
            .filter(ChangesetStatus.revision.in_(revisions))
 

	
 
        grouped = {}
 
        for stat in statuses.all():
 
            pr_id = pr_nice_id = pr_repo = None
 
            if stat.pull_request:
 
                pr_id = stat.pull_request.pull_request_id
 
                pr_nice_id = PullRequest.make_nice_id(pr_id)
 
                pr_repo = stat.pull_request.other_repo.repo_name
 
            grouped[stat.revision] = [str(stat.status), stat.status_lbl,
 
                                      pr_id, pr_repo, pr_nice_id,
 
                                      stat.author]
 
        return grouped
 

	
 
    def _repo_size(self):
 
        from kallithea.lib import helpers as h
 
        log.debug('calculating repository size...')
 
        return h.format_byte_size(self.scm_instance.size)
 

	
 
    #==========================================================================
 
    # SCM CACHE INSTANCE
 
    #==========================================================================
 

	
 
    def set_invalidate(self):
 
        """
 
        Mark caches of this repo as invalid.
 
        """
 
        CacheInvalidation.set_invalidate(self.repo_name)
 

	
 
    @property
 
    def scm_instance(self):
 
        import kallithea
 
        full_cache = str2bool(kallithea.CONFIG.get('vcs_full_cache'))
 
        if full_cache:
 
            return self.scm_instance_cached()
 
        return self.__get_instance()
 

	
 
    def scm_instance_cached(self, valid_cache_keys=None):
 
        @cache_region('long_term', 'scm_instance_cached')
 
        def _c(repo_name): # repo_name is just for the cache key
 
            log.debug('Creating new %s scm_instance and populating cache', repo_name)
 
            return self.scm_instance_no_cache()
 
        rn = self.repo_name
 

	
 
        valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
 
        if not valid:
 
            log.debug('Cache for %s invalidated, getting new object', rn)
 
            region_invalidate(_c, None, 'scm_instance_cached', rn)
 
        else:
 
            log.debug('Trying to get scm_instance of %s from cache', rn)
 
        return _c(rn)
 

	
 
    def scm_instance_no_cache(self):
 
        repo_full_path = safe_str(self.repo_full_path)
 
        alias = get_scm(repo_full_path)[0]
 
        log.debug('Creating instance of %s repository from %s',
 
                  alias, self.repo_full_path)
 
        backend = get_backend(alias)
 

	
 
        if alias == 'hg':
 
            repo = backend(repo_full_path, create=False,
 
                           baseui=self._ui)
 
        else:
 
            repo = backend(repo_full_path, create=False)
 

	
 
        return repo
 

	
 
    def __json__(self):
 
        return dict(landing_rev = self.landing_rev)
 

	
 
class RepoGroup(Base, BaseModel):
 
    __tablename__ = 'groups'
 
    __table_args__ = (
 
        UniqueConstraint('group_name', 'group_parent_id'),
 
        CheckConstraint('group_id != group_parent_id'),
 
        _table_args_default_dict,
 
    )
 
    __mapper_args__ = {'order_by': 'group_name'}
 

	
 
    SEP = ' &raquo; '
 

	
 
    group_id = Column(Integer(), unique=True, primary_key=True)
 
    group_name = Column(Unicode(255), nullable=False, unique=True)
 
    group_parent_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True)
 
    group_description = Column(Unicode(10000), nullable=False)
 
    enable_locking = Column(Boolean(), nullable=False, default=False)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 

	
 
    repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
 
    users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
 
    parent_group = relationship('RepoGroup', remote_side=group_id)
 
    user = relationship('User')
 

	
 
    def __init__(self, group_name='', parent_group=None):
 
        self.group_name = group_name
 
        self.parent_group = parent_group
 

	
 
    def __unicode__(self):
 
        return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,
 
                                      self.group_name)
 

	
 
    @classmethod
 
    def _generate_choice(cls, repo_group):
 
        """Return tuple with group_id and name as html literal"""
 
        from webhelpers.html import literal
 
        if repo_group is None:
 
            return (-1, u'-- %s --' % _('top level'))
 
        return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))
 

	
 
    @classmethod
 
    def groups_choices(cls, groups):
 
        """Return tuples with group_id and name as html literal."""
 
        return sorted((cls._generate_choice(g) for g in groups),
 
                      key=lambda c: c[1].split(cls.SEP))
 

	
 
    @classmethod
 
    def url_sep(cls):
 
        return URL_SEP
 

	
 
    @classmethod
 
    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
 
        if case_insensitive:
 
            gr = cls.query() \
 
                .filter(func.lower(cls.group_name) == func.lower(group_name))
 
        else:
 
            gr = cls.query() \
 
                .filter(cls.group_name == group_name)
 
        if cache:
 
            gr = gr.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_group_%s" % _hash_key(group_name)
kallithea/tests/test.ini
Show inline comments
 
################################################################################
 
################################################################################
 
# Kallithea - config for tests:                                                #
 
# initial_repo_scan = true                                                     #
 
# vcs_full_cache = false                                                       #
 
# sqlalchemy and kallithea_test.sqlite                                         #
 
# custom logging                                                               #
 
#                                                                              #
 
# The %(here)s variable will be replaced with the parent directory of this file#
 
################################################################################
 
################################################################################
 

	
 
[DEFAULT]
 
debug = true
 
pdebug = false
 

	
 
################################################################################
 
## Email settings                                                             ##
 
##                                                                            ##
 
## Refer to the documentation ("Email settings") for more details.            ##
 
##                                                                            ##
 
## It is recommended to use a valid sender address that passes access         ##
 
## validation and spam filtering in mail servers.                             ##
 
################################################################################
 

	
 
## 'From' header for application emails. You can optionally add a name.
 
## Default:
 
#app_email_from = Kallithea
 
## Examples:
 
#app_email_from = Kallithea <kallithea-noreply@example.com>
 
#app_email_from = kallithea-noreply@example.com
 

	
 
## Subject prefix for application emails.
 
## A space between this prefix and the real subject is automatically added.
 
## Default:
 
#email_prefix =
 
## Example:
 
#email_prefix = [Kallithea]
 

	
 
## Recipients for error emails and fallback recipients of application mails.
 
## Multiple addresses can be specified, space-separated.
 
## Only addresses are allowed, do not add any name part.
 
## Default:
 
#email_to =
 
## Examples:
 
#email_to = admin@example.com
 
#email_to = admin@example.com another_admin@example.com
 

	
 
## 'From' header for error emails. You can optionally add a name.
 
## Default:
 
#error_email_from = pylons@yourapp.com
 
## Examples:
 
#error_email_from = Kallithea Errors <kallithea-noreply@example.com>
 
#error_email_from = paste_error@example.com
 

	
 
## SMTP server settings
 
## Only smtp_server is mandatory. All other settings take the specified default
 
## values.
 
#smtp_server = smtp.example.com
 
#smtp_username =
 
#smtp_password =
 
#smtp_port = 25
 
#smtp_use_tls = false
 
#smtp_use_ssl = false
 
## SMTP authentication parameters to use (e.g. LOGIN PLAIN CRAM-MD5, etc.).
 
## If empty, use any of the authentication parameters supported by the server.
 
#smtp_auth =
 

	
 
[server:main]
 
## PASTE ##
 
#use = egg:Paste#http
 
## nr of worker threads to spawn
 
#threadpool_workers = 5
 
## max request before thread respawn
 
#threadpool_max_requests = 10
 
## option to use threads of process
 
#use_threadpool = true
 

	
 
## WAITRESS ##
 
use = egg:waitress#main
 
## number of worker threads
 
threads = 5
 
## MAX BODY SIZE 100GB
 
max_request_body_size = 107374182400
 
## use poll instead of select, fixes fd limits, may not work on old
 
## windows systems.
 
#asyncore_use_poll = True
 

	
 
## GUNICORN ##
 
#use = egg:gunicorn#main
 
## number of process workers. You must set `instance_id = *` when this option
 
## is set to more than one worker
 
#workers = 1
 
## process name
 
#proc_name = kallithea
 
## type of worker class, one of sync, eventlet, gevent, tornado
 
## recommended for bigger setup is using of of other than sync one
 
#worker_class = sync
 
#max_requests = 1000
 
## ammount of time a worker can handle request before it gets killed and
 
## restarted
 
@@ -105,196 +104,192 @@ max_request_body_size = 107374182400
 
## run with uwsgi --ini-paste-logged <inifile.ini>
 
#[uwsgi]
 
#socket = /tmp/uwsgi.sock
 
#master = true
 
#http = 127.0.0.1:5000
 

	
 
## set as deamon and redirect all output to file
 
#daemonize = ./uwsgi_kallithea.log
 

	
 
## master process PID
 
#pidfile = ./uwsgi_kallithea.pid
 

	
 
## stats server with workers statistics, use uwsgitop
 
## for monitoring, `uwsgitop 127.0.0.1:1717`
 
#stats = 127.0.0.1:1717
 
#memory-report = true
 

	
 
## log 5XX errors
 
#log-5xx = true
 

	
 
## Set the socket listen queue size.
 
#listen = 256
 

	
 
## Gracefully Reload workers after the specified amount of managed requests
 
## (avoid memory leaks).
 
#max-requests = 1000
 

	
 
## enable large buffers
 
#buffer-size = 65535
 

	
 
## socket and http timeouts ##
 
#http-timeout = 3600
 
#socket-timeout = 3600
 

	
 
## Log requests slower than the specified number of milliseconds.
 
#log-slow = 10
 

	
 
## Exit if no app can be loaded.
 
#need-app = true
 

	
 
## Set lazy mode (load apps in workers instead of master).
 
#lazy = true
 

	
 
## scaling ##
 
## set cheaper algorithm to use, if not set default will be used
 
#cheaper-algo = spare
 

	
 
## minimum number of workers to keep at all times
 
#cheaper = 1
 

	
 
## number of workers to spawn at startup
 
#cheaper-initial = 1
 

	
 
## maximum number of workers that can be spawned
 
#workers = 4
 

	
 
## how many workers should be spawned at a time
 
#cheaper-step = 1
 

	
 
## COMMON ##
 
host = 127.0.0.1
 
#port = 5000
 
port = 4999
 

	
 
## middleware for hosting the WSGI application under a URL prefix
 
#[filter:proxy-prefix]
 
#use = egg:PasteDeploy#prefix
 
#prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:kallithea
 
## enable proxy prefix middleware
 
#filter-with = proxy-prefix
 

	
 
full_stack = true
 
static_files = true
 
## Available Languages:
 
## cs de fr hu ja nl_BE pl pt_BR ru sk zh_CN zh_TW
 
lang =
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 

	
 
## perform a full repository scan on each server start, this should be
 
## set to false after first startup, to allow faster server restarts.
 
#initial_repo_scan = false
 
initial_repo_scan = true
 

	
 
## uncomment and set this path to use archive download cache
 
archive_cache_dir = %(here)s/tarballcache
 

	
 
## change this to unique ID for security
 
app_instance_uuid = test
 

	
 
## cut off limit for large diffs (size in bytes)
 
cut_off_limit = 256000
 

	
 
## use cache version of scm repo everywhere
 
#vcs_full_cache = true
 
vcs_full_cache = false
 

	
 
## force https in Kallithea, fixes https redirects, assumes it's always https
 
force_https = false
 

	
 
## use Strict-Transport-Security headers
 
use_htsts = false
 

	
 
## number of commits stats will parse on each iteration
 
commit_parse_limit = 25
 

	
 
## path to git executable
 
git_path = git
 

	
 
## git rev filter option, --all is the default filter, if you need to
 
## hide all refs in changelog switch this to --branches --tags
 
#git_rev_filter = --branches --tags
 

	
 
## RSS feed options
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 
## options for showing and identifying changesets
 
show_sha_length = 12
 
#show_revision_number = false
 
show_revision_number = true
 

	
 
## gist URL alias, used to create nicer urls for gist. This should be an
 
## url that does rewrites to _admin/gists/<gistid>.
 
## example: http://gist.example.com/{gistid}. Empty means use the internal
 
## Kallithea url, ie. http[s]://kallithea.example.com/_admin/gists/<gistid>
 
gist_alias_url =
 

	
 
## white list of API enabled controllers. This allows to add list of
 
## controllers to which access will be enabled by api_key. eg: to enable
 
## api access to raw_files put `FilesController:raw`, to enable access to patches
 
## add `ChangesetController:changeset_patch`. This list should be "," separated
 
## Syntax is <ControllerClass>:<function>. Check debug logs for generated names
 
## Recommended settings below are commented out:
 
api_access_controllers_whitelist =
 
#    ChangesetController:changeset_patch,
 
#    ChangesetController:changeset_raw,
 
#    FilesController:raw,
 
#    FilesController:archivefile
 

	
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## issue tracker for Kallithea (leave blank to disable, absent for default)
 
#bugtracker = https://bitbucket.org/conservancy/kallithea/issues
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://issues.example.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
 
issue_prefix = #
 

	
 
## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 
## multiple patterns, to other issues server, wiki or others
 
## below an example how to create a wiki pattern
 
# wiki-some-id -> https://wiki.example.com/some-id
 

	
 
#issue_pat_wiki = (?:wiki-)(.+)
 
#issue_server_link_wiki = https://wiki.example.com/{id}
 
#issue_prefix_wiki = WIKI-
 

	
 
## alternative return HTTP header for failed authentication. Default HTTP
 
## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
## extra extensions for indexing, space separated and without the leading '.'.
scripts/generate-ini.py
Show inline comments
 
#!/usr/bin/env python2
 
"""
 
Based on kallithea/bin/template.ini.mako, generate
 
  kallithea/config/deployment.ini_tmpl
 
  development.ini
 
  kallithea/tests/test.ini
 
"""
 

	
 
import re
 

	
 
makofile = 'kallithea/bin/template.ini.mako'
 

	
 
# the mako conditionals used in all other ini files and templates
 
selected_mako_conditionals = set([
 
    "database_engine == 'sqlite'",
 
    "http_server == 'waitress'",
 
    "error_aggregation_service == 'errormator'",
 
    "error_aggregation_service == 'sentry'",
 
])
 

	
 
# the mako variables used in all other ini files and templates
 
mako_variable_values = {
 
    'host': '127.0.0.1',
 
    'port': '5000',
 
    'here': '%(here)s',
 
    'uuid()': '${app_instance_uuid}',
 
}
 

	
 
# files to be generated from the mako template
 
ini_files = [
 
    ('kallithea/config/deployment.ini_tmpl',
 
        '''
 
        Kallithea - Example config
 

	
 
        The %(here)s variable will be replaced with the parent directory of this file
 
        ''',
 
        {}, # exactly the same settings as template.ini.mako
 
    ),
 
    ('kallithea/tests/test.ini',
 
        '''
 
        Kallithea - config for tests:
 
        initial_repo_scan = true
 
        vcs_full_cache = false
 
        sqlalchemy and kallithea_test.sqlite
 
        custom logging
 

	
 
        The %(here)s variable will be replaced with the parent directory of this file
 
        ''',
 
        {
 
            '[server:main]': {
 
                'port': '4999',
 
            },
 
            '[app:main]': {
 
                'initial_repo_scan': 'true',
 
                'app_instance_uuid': 'test',
 
                'vcs_full_cache': 'false',
 
                'show_revision_number': 'true',
 
                'beaker.cache.sql_cache_short.expire': '1',
 
                'beaker.session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
 
                'sqlalchemy.db1.url': 'sqlite:///%(here)s/kallithea_test.sqlite',
 
            },
 
            '[logger_root]': {
 
                'level': 'DEBUG',
 
            },
 
            '[logger_sqlalchemy]': {
 
                'level': 'ERROR',
 
                'handlers': 'console',
 
            },
 
            '[handler_console]': {
 
                'level': 'NOTSET',
 
            },
 
        },
 
    ),
 
    ('development.ini',
 
        '''
 
        Kallithea - Development config:
 
        listening on *:5000
 
        sqlite and kallithea.db
 
        initial_repo_scan = true
 
        set debug = true
 
        verbose and colorful logging
 

	
 
        The %(here)s variable will be replaced with the parent directory of this file
 
        ''',
 
        {
 
            '[server:main]': {
 
                'host': '0.0.0.0',
 
            },
 
            '[app:main]': {
 
                'initial_repo_scan': 'true',
 
                'set debug': 'true',
 
                'app_instance_uuid': 'development-not-secret',
 
                'beaker.session.secret': 'development-not-secret',
 
            },
 
            '[handler_console]': {
 
                'level': 'DEBUG',
 
                'formatter': 'color_formatter',
 
            },
 
            '[handler_console_sql]': {
 
                'level': 'DEBUG',
 
                'formatter': 'color_formatter_sql',
 
            },
 
        },
 
    ),
 
]
 

	
 

	
 
def main():
 
    # make sure all mako lines starting with '#' (the '##' comments) are marked up as <text>
 
    print 'reading:', makofile
 
    mako_org = file(makofile).read()
 
    mako_no_text_markup = re.sub(r'</?%text>', '', mako_org)
 
    mako_marked_up = re.sub(r'\n(##.*)', r'\n<%text>\1</%text>', mako_no_text_markup, flags=re.MULTILINE)
 
    if mako_marked_up != mako_org:
 
        print 'writing:', makofile
 
        file(makofile, 'w').write(mako_marked_up)
 

	
 
    # select the right mako conditionals for the other less sophisticated formats
 
    def sub_conditionals(m):
 
        """given a %if...%endif match, replace with just the selected
 
        conditional sections enabled and the rest as comments
 
        """
 
        conditional_lines = m.group(1)
 
        def sub_conditional(m):
 
            """given a conditional and the corresponding lines, return them raw
 
            or commented out, based on whether conditional is selected
 
            """
 
            criteria, lines = m.groups()
 
            if criteria not in selected_mako_conditionals:
 
                lines = '\n'.join((l if not l or l.startswith('#') else '#' + l) for l in lines.split('\n'))
 
            return lines
 
        conditional_lines = re.sub(r'^%(?:el)?if (.*):\n((?:^[^%\n].*\n|\n)*)',
 
            sub_conditional, conditional_lines, flags=re.MULTILINE)
 
        return conditional_lines
 
    mako_no_conditionals = re.sub(r'^(%if .*\n(?:[^%\n].*\n|%elif .*\n|\n)*)%endif\n',
 
        sub_conditionals, mako_no_text_markup, flags=re.MULTILINE)
 

	
 
    # expand mako variables
 
    def pyrepl(m):
 
        return mako_variable_values.get(m.group(1), m.group(0))
 
    mako_no_variables = re.sub(r'\${([^}]*)}', pyrepl, mako_no_conditionals)
 

	
 
    # remove utf-8 coding header
 
    base_ini = re.sub(r'^## -\*- coding: utf-8 -\*-\n', '', mako_no_variables)
 

	
 
    # create ini files
 
    for fn, desc, settings in ini_files:
 
        print 'updating:', fn
 
        ini_lines = re.sub(
 
            '# Kallithea - config file generated with kallithea-config *#\n',
 
            ''.join('# %-77s#\n' % l.strip() for l in desc.strip().split('\n')),
 
            base_ini)
0 comments (0 inline, 0 general)