Changeset - 4610a39d3be9
[Not reviewed]
default
0 5 0
Mads Kiilerich - 10 years ago 2015-08-17 01:11:42
madski@unity3d.com
cleanup: kill unused beaker cache regions
5 files changed with 6 insertions and 63 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -291,122 +291,110 @@ instance_id =
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
####################################
 
###        CELERY CONFIG        ####
 
####################################
 

	
 
use_celery = false
 
broker.host = localhost
 
broker.vhost = rabbitmqhost
 
broker.port = 5672
 
broker.user = rabbitmq
 
broker.password = qweqwe
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 

	
 
celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = DEBUG
 
celeryd.max.tasks.per.child = 1
 

	
 
## tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type = memory
 
beaker.cache.super_short_term.expire = 10
 
beaker.cache.super_short_term.key_length = 256
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type = memory
 
beaker.cache.sql_cache_med.expire = 360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type = file
 
beaker.cache.sql_cache_long.expire = 3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are
 
## dbm, file, memcached, database, and memory.
 
## The storage uses the Container API
 
## that is also used by the cache system.
 

	
 
## db session ##
 
#beaker.session.type = ext:database
 
#beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 
#beaker.session.table_name = db_session
 

	
 
## encrypted cookie client side session, good for many instances ##
 
#beaker.session.type = cookie
 

	
 
## file based cookies (default) ##
 
#beaker.session.type = file
 

	
 
## beaker.session.key should be unique for a given host, even when running
 
## on different ports. Otherwise, cookie sessions will be shared and messed up.
 
beaker.session.key = kallithea
 
beaker.session.secret = development-not-secret
 

	
 
## Secure encrypted cookie. Requires AES and AES python libraries
 
## you must disable beaker.session.secret to use this
 
#beaker.session.encrypt_key = <key_for_encryption>
 
#beaker.session.validate_key = <validation_key>
 

	
 
## sets session as invalid if it haven't been accessed for given amount of time
 
beaker.session.timeout = 2592000
 
beaker.session.httponly = true
 
#beaker.session.cookie_path = /<your-prefix>
 

	
 
## uncomment for https secure cookie
 
beaker.session.secure = false
 

	
 
## auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
## default cookie expiration time in seconds `true` expire at browser close ##
 
#beaker.session.cookie_expires = 3600
 

	
 
############################
 
## ERROR HANDLING SYSTEMS ##
 
############################
 

	
 
####################
docs/usage/performance.rst
Show inline comments
 
.. _performance:
 

	
 
================================
 
Optimizing Kallithea Performance
 
================================
 

	
 
When serving a large amount of big repositories, Kallithea can start
 
performing slower than expected. Because of the demanding nature of handling large
 
amounts of data from version control systems, here are some tips on how to get
 
the best performance.
 

	
 
* Kallithea will perform better on machines with faster disks (SSD/SAN). It's
 
  more important to have a faster disk than a faster CPU.
 

	
 
* Slowness on initial page can be easily fixed by grouping repositories, and/or
 
  increasing cache size (see below). This includes using the lightweight dashboard
 
  option and ``vcs_full_cache`` setting in .ini file
 

	
 

	
 
Follow these few steps to improve performance of Kallithea system.
 

	
 

	
 
1. Increase cache
 

	
 
    In the .ini file::
 

	
 
     beaker.cache.sql_cache_long.expire=3600 <-- set this to higher number
 

	
 
    This option affects the cache expiration time for the main
 
    page. Having several hundreds of repositories on main page can
 
    sometimes make the system behave slowly when the cache expires for
 
    all of them. Increasing the ``expire`` option to a day (86400) or a
 
    week (604800) will improve general response times for the main
 
    page. Kallithea has an intelligent cache expiration system and it
 
    will expire the cache for repositories that have been changed.
 
    Tweak beaker cache settings in the ini file. That actual effect of that
 
    is questionable.
 

	
 
2. Switch from sqlite to postgres or mysql
 

	
 
    sqlite is a good option when having a small load on the system. But due to
 
    locking issues with sqlite, it is not recommended to use it for larger
 
    deployments. Switching to mysql or postgres will result in an immediate
 
    performance increase. A tool like SQLAlchemyGrate_ can be used for
 
    migrating to another database platform.
 

	
 
3. Scale Kallithea horizontally
 

	
 
    Scaling horizontally can give huge performance increases when dealing with
 
    large traffic (large amount of users, CI servers etc). Kallithea can be
 
    scaled horizontally on one (recommended) or multiple machines. In order
 
    to scale horizontally you need to do the following:
 

	
 
    - Each instance needs its own .ini file and unique ``instance_id`` set.
 
    - Each instance's ``data`` storage needs to be configured to be stored on a
 
      shared disk storage, preferably together with repositories. This ``data``
 
      dir contains template caches, sessions, whoosh index and is used for
 
      task locking (so it is safe across multiple instances). Set the
 
      ``cache_dir``, ``index_dir``, ``beaker.cache.data_dir``, ``beaker.cache.lock_dir``
 
      variables in each .ini file to a shared location across Kallithea instances
 
    - If celery is used each instance should run a separate Celery instance, but
 
      the message broker should be common to all of them (e.g.,  one
 
      shared RabbitMQ server)
 
    - Load balance using round robin or IP hash, recommended is writing LB rules
 
      that will separate regular user traffic from automated processes like CI
 
      servers or build bots.
 

	
 
.. _SQLAlchemyGrate: https://github.com/shazow/sqlalchemygrate
kallithea/bin/template.ini.mako
Show inline comments
 
@@ -289,122 +289,110 @@ instance_id =
 
<%text>## handling that. Set this variable to 403 to return HTTPForbidden</%text>
 
auth_ret_code =
 

	
 
<%text>## locking return code. When repository is locked return this HTTP code. 2XX</%text>
 
<%text>## codes don't break the transactions while 4XX codes do</%text>
 
lock_ret_code = 423
 

	
 
<%text>## allows to change the repository location in settings page</%text>
 
allow_repo_location_change = True
 

	
 
<%text>## allows to setup custom hooks in settings page</%text>
 
allow_custom_hooks_settings = True
 

	
 
<%text>####################################</%text>
 
<%text>###        CELERY CONFIG        ####</%text>
 
<%text>####################################</%text>
 

	
 
use_celery = false
 
broker.host = localhost
 
broker.vhost = rabbitmqhost
 
broker.port = 5672
 
broker.user = rabbitmq
 
broker.password = qweqwe
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 

	
 
celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = DEBUG
 
celeryd.max.tasks.per.child = 1
 

	
 
<%text>## tasks will never be sent to the queue, but executed locally instead.</%text>
 
celery.always.eager = false
 

	
 
<%text>####################################</%text>
 
<%text>###         BEAKER CACHE        ####</%text>
 
<%text>####################################</%text>
 

	
 
beaker.cache.data_dir = ${here}/data/cache/data
 
beaker.cache.lock_dir = ${here}/data/cache/lock
 

	
 
beaker.cache.regions = super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type = memory
 
beaker.cache.super_short_term.expire = 10
 
beaker.cache.super_short_term.key_length = 256
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type = memory
 
beaker.cache.sql_cache_med.expire = 360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type = file
 
beaker.cache.sql_cache_long.expire = 3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
<%text>####################################</%text>
 
<%text>###       BEAKER SESSION        ####</%text>
 
<%text>####################################</%text>
 
<%text>## Type of storage used for the session, current types are</%text>
 
<%text>## dbm, file, memcached, database, and memory.</%text>
 
<%text>## The storage uses the Container API</%text>
 
<%text>## that is also used by the cache system.</%text>
 

	
 
<%text>## db session ##</%text>
 
#beaker.session.type = ext:database
 
#beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 
#beaker.session.table_name = db_session
 

	
 
<%text>## encrypted cookie client side session, good for many instances ##</%text>
 
#beaker.session.type = cookie
 

	
 
<%text>## file based cookies (default) ##</%text>
 
#beaker.session.type = file
 

	
 
<%text>## beaker.session.key should be unique for a given host, even when running</%text>
 
<%text>## on different ports. Otherwise, cookie sessions will be shared and messed up.</%text>
 
beaker.session.key = kallithea
 
beaker.session.secret = ${uuid()}
 

	
 
<%text>## Secure encrypted cookie. Requires AES and AES python libraries</%text>
 
<%text>## you must disable beaker.session.secret to use this</%text>
 
#beaker.session.encrypt_key = <key_for_encryption>
 
#beaker.session.validate_key = <validation_key>
 

	
 
<%text>## sets session as invalid if it haven't been accessed for given amount of time</%text>
 
beaker.session.timeout = 2592000
 
beaker.session.httponly = true
 
#beaker.session.cookie_path = /<your-prefix>
 

	
 
<%text>## uncomment for https secure cookie</%text>
 
beaker.session.secure = false
 

	
 
<%text>## auto save the session to not to use .save()</%text>
 
beaker.session.auto = False
 

	
 
<%text>## default cookie expiration time in seconds `true` expire at browser close ##</%text>
 
#beaker.session.cookie_expires = 3600
 

	
 
%if error_aggregation_service == 'errormator':
 
<%text>############################</%text>
 
<%text>## ERROR HANDLING SYSTEMS ##</%text>
 
<%text>############################</%text>
 

	
kallithea/config/deployment.ini_tmpl
Show inline comments
 
@@ -285,122 +285,110 @@ instance_id =
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
####################################
 
###        CELERY CONFIG        ####
 
####################################
 

	
 
use_celery = false
 
broker.host = localhost
 
broker.vhost = rabbitmqhost
 
broker.port = 5672
 
broker.user = rabbitmq
 
broker.password = qweqwe
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 

	
 
celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = DEBUG
 
celeryd.max.tasks.per.child = 1
 

	
 
## tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type = memory
 
beaker.cache.super_short_term.expire = 10
 
beaker.cache.super_short_term.key_length = 256
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type = memory
 
beaker.cache.sql_cache_med.expire = 360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type = file
 
beaker.cache.sql_cache_long.expire = 3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are
 
## dbm, file, memcached, database, and memory.
 
## The storage uses the Container API
 
## that is also used by the cache system.
 

	
 
## db session ##
 
#beaker.session.type = ext:database
 
#beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 
#beaker.session.table_name = db_session
 

	
 
## encrypted cookie client side session, good for many instances ##
 
#beaker.session.type = cookie
 

	
 
## file based cookies (default) ##
 
#beaker.session.type = file
 

	
 
## beaker.session.key should be unique for a given host, even when running
 
## on different ports. Otherwise, cookie sessions will be shared and messed up.
 
beaker.session.key = kallithea
 
beaker.session.secret = ${app_instance_uuid}
 

	
 
## Secure encrypted cookie. Requires AES and AES python libraries
 
## you must disable beaker.session.secret to use this
 
#beaker.session.encrypt_key = <key_for_encryption>
 
#beaker.session.validate_key = <validation_key>
 

	
 
## sets session as invalid if it haven't been accessed for given amount of time
 
beaker.session.timeout = 2592000
 
beaker.session.httponly = true
 
#beaker.session.cookie_path = /<your-prefix>
 

	
 
## uncomment for https secure cookie
 
beaker.session.secure = false
 

	
 
## auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
## default cookie expiration time in seconds `true` expire at browser close ##
 
#beaker.session.cookie_expires = 3600
 

	
 
############################
 
## ERROR HANDLING SYSTEMS ##
 
############################
 

	
 
####################
test.ini
Show inline comments
 
@@ -291,122 +291,110 @@ instance_id =
 
## handling that. Set this variable to 403 to return HTTPForbidden
 
auth_ret_code =
 

	
 
## locking return code. When repository is locked return this HTTP code. 2XX
 
## codes don't break the transactions while 4XX codes do
 
lock_ret_code = 423
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
####################################
 
###        CELERY CONFIG        ####
 
####################################
 

	
 
use_celery = false
 
broker.host = localhost
 
broker.vhost = rabbitmqhost
 
broker.port = 5672
 
broker.user = rabbitmq
 
broker.password = qweqwe
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 

	
 
celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = DEBUG
 
celeryd.max.tasks.per.child = 1
 

	
 
## tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type = memory
 
beaker.cache.super_short_term.expire = 10
 
beaker.cache.super_short_term.key_length = 256
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 1
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type = memory
 
beaker.cache.sql_cache_med.expire = 360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type = file
 
beaker.cache.sql_cache_long.expire = 3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are
 
## dbm, file, memcached, database, and memory.
 
## The storage uses the Container API
 
## that is also used by the cache system.
 

	
 
## db session ##
 
#beaker.session.type = ext:database
 
#beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 
#beaker.session.table_name = db_session
 

	
 
## encrypted cookie client side session, good for many instances ##
 
#beaker.session.type = cookie
 

	
 
## file based cookies (default) ##
 
#beaker.session.type = file
 

	
 
## beaker.session.key should be unique for a given host, even when running
 
## on different ports. Otherwise, cookie sessions will be shared and messed up.
 
beaker.session.key = kallithea
 
beaker.session.secret = {74e0cd75-b339-478b-b129-07dd221def1f}
 

	
 
## Secure encrypted cookie. Requires AES and AES python libraries
 
## you must disable beaker.session.secret to use this
 
#beaker.session.encrypt_key = <key_for_encryption>
 
#beaker.session.validate_key = <validation_key>
 

	
 
## sets session as invalid if it haven't been accessed for given amount of time
 
beaker.session.timeout = 2592000
 
beaker.session.httponly = true
 
#beaker.session.cookie_path = /<your-prefix>
 

	
 
## uncomment for https secure cookie
 
beaker.session.secure = false
 

	
 
## auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
## default cookie expiration time in seconds `true` expire at browser close ##
 
#beaker.session.cookie_expires = 3600
 

	
 
############################
 
## ERROR HANDLING SYSTEMS ##
 
############################
 

	
 
####################
0 comments (0 inline, 0 general)