Changeset - 716e53c085ff
Parent rev.
Child rev.
[Not reviewed]
stable
0 6 0
Mads Kiilerich - 9 years ago 2016-09-06 00:51:18
madski@unity3d.com
Grafted from: d6942b2b421c
config: clarify that we only recommend and support single threaded operation

Sad, but true. Especially because we reuse Repository instances between
threads.
6 files changed with 35 insertions and 18 deletions:
development.ini
3
3
docs/setup.rst
2
3
docs/usage/performance.rst
21
3
kallithea/bin/template.ini.mako
3
3
kallithea/config/deployment.ini_tmpl
3
3
kallithea/tests/test.ini
3
3
0 comments (0 inline, 0 general)
development.ini
➞
Show inline comments
 
@@ -62,34 +62,34 @@ pdebug = false
 
#smtp_password =
 
#smtp_port = 25
 
#smtp_use_tls = false
 
#smtp_use_ssl = false
 
## SMTP authentication parameters to use (e.g. LOGIN PLAIN CRAM-MD5, etc.).
 
## If empty, use any of the authentication parameters supported by the server.
 
#smtp_auth =
 

			




	
	
	
		
 
[server:main]

			




	
	
	
		
 
## PASTE ##

			




	
	
	
		
 
#use = egg:Paste#http

			




	
	
	
		
 
## nr of worker threads to spawn

			




	
	
	
		
 
#threadpool_workers = 5

			




	
	
	
		
 
#threadpool_workers = 1

			




	
	
	
		
 
## max request before thread respawn

			




	
	
	
		
 
#threadpool_max_requests = 10

			




	
	
	
		
 
#threadpool_max_requests = 100

			




	
	
	
		
 
## option to use threads of process

			




	
	
	
		
 
#use_threadpool = true

			




	
	
	
		
 

			




	
	
	
		
 
## WAITRESS ##

			




	
	
	
		
 
use = egg:waitress#main

			




	
	
	
		
 
## number of worker threads

			




	
	
	
		
 
threads = 5

			




	
	
	
		
 
threads = 1

			




	
	
	
		
 
## MAX BODY SIZE 100GB

			




	
	
	
		
 
max_request_body_size = 107374182400

			




	
	
	
		
 
## use poll instead of select, fixes fd limits, may not work on old

			




	
	
	
		
 
## windows systems.

			




	
	
	
		
 
#asyncore_use_poll = True

			




	
	
	
		
 

			




	
	
	
		
 
## GUNICORN ##

			




	
	
	
		
 
#use = egg:gunicorn#main

			




	
	
	
		
 
## number of process workers. You must set `instance_id = *` when this option

			




	
	
	
		
 
## is set to more than one worker

			




	
	
	
		
 
#workers = 1

			




	
	
	
		
 
## process name

			




        

    


    
    

    

        

                

                    docs/setup.rst
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -726,36 +726,35 @@ that, you'll need to:

			




	
	
	
		
 

			




	
	
	
		
 
- Create a wsgi dispatch script, like the one below. Make sure you

			




	
	
	
		
 
  check that the paths correctly point to where you installed Kallithea

			




	
	
	
		
 
  and its Python Virtual Environment.

			




	
	
	
		
 
- Enable the ``WSGIScriptAlias`` directive for the WSGI dispatch script,

			




	
	
	
		
 
  as in the following example. Once again, check the paths are

			




	
	
	
		
 
  correctly specified.

			




	
	
	
		
 

			




	
	
	
		
 
Here is a sample excerpt from an Apache Virtual Host configuration file:

			




	
	
	
		
 

			




	
	
	
		
 
.. code-block:: apache

			




	
	
	
		
 

			




	
	
	
		
 
    WSGIDaemonProcess kallithea \

			




	
	
	
		
 
        threads=4 \

			




	
	
	
		
 
    WSGIDaemonProcess kallithea processes=5 threads=1 maximum-requests=100 \

			




	
	
	
		
 
        python-home=/srv/kallithea/venv

			




	
	
	
		
 
    WSGIProcessGroup kallithea

			




	
	
	
		
 
    WSGIScriptAlias / /srv/kallithea/dispatch.wsgi

			




	
	
	
		
 
    WSGIPassAuthorization On

			




	
	
	
		
 

			




	
	
	
		
 
Or if using a dispatcher WSGI script with proper virtualenv activation:

			




	
	
	
		
 

			




	
	
	
		
 
.. code-block:: apache

			




	
	
	
		
 

			




	
	
	
		
 
    WSGIDaemonProcess kallithea threads=4

			




	
	
	
		
 
    WSGIDaemonProcess kallithea processes=5 threads=1 maximum-requests=100

			




	
	
	
		
 
    WSGIProcessGroup kallithea

			




	
	
	
		
 
    WSGIScriptAlias / /srv/kallithea/dispatch.wsgi

			




	
	
	
		
 
    WSGIPassAuthorization On

			




	
	
	
		
 

			




	
	
	
		
 
Apache will by default run as a special Apache user, on Linux systems

			




	
	
	
		
 
usually ``www-data`` or ``apache``. If you need to have the repositories

			




	
	
	
		
 
directory owned by a different user, use the user and group options to

			




	
	
	
		
 
WSGIDaemonProcess to set the name of the user and group.

			




	
	
	
		
 

			




	
	
	
		
 
.. note::

			




	
	
	
		
 
   If running Kallithea in multiprocess mode,

			




	
	
	
		
 
   make sure you set ``instance_id = *`` in the configuration so each process

			




        

    


    
    

    

        

                

                    docs/usage/performance.rst
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -26,31 +26,49 @@ Follow these few steps to improve perfor

			




	
	
	
		
 
2. Switch from SQLite to PostgreSQL or MySQL

			




	
	
	
		
 

			




	
	
	
		
 
    SQLite is a good option when having a small load on the system. But due to

			




	
	
	
		
 
    locking issues with SQLite, it is not recommended to use it for larger

			




	
	
	
		
 
    deployments. Switching to MySQL or PostgreSQL will result in an immediate

			




	
	
	
		
 
    performance increase. A tool like SQLAlchemyGrate_ can be used for

			




	
	
	
		
 
    migrating to another database platform.

			




	
	
	
		
 

			




	
	
	
		
 
3. Scale Kallithea horizontally

			




	
	
	
		
 

			




	
	
	
		
 
    Scaling horizontally can give huge performance benefits when dealing with

			




	
	
	
		
 
    large amounts of traffic (many users, CI servers, etc.). Kallithea can be

			




	
	
	
		
 
    scaled horizontally on one (recommended) or multiple machines. In order

			




	
	
	
		
 
    to scale horizontally you need to do the following:

			




	
	
	
		
 
    scaled horizontally on one (recommended) or multiple machines.

			




	
	
	
		
 

			




	
	
	
		
 
    It is generally possible to run WSGI applications multithreaded, so that

			




	
	
	
		
 
    several HTTP requests are served from the same Python process at once. That

			




	
	
	
		
 
    can in principle give better utilization of internal caches and less

			




	
	
	
		
 
    process overhead.

			




	
	
	
		
 

			




	
	
	
		
 
    One danger of running multithreaded is that program execution becomes much

			




	
	
	
		
 
    more complex; programs must be written to consider all combinations of

			




	
	
	
		
 
    events and problems might depend on timing and be impossible to reproduce.

			




	
	
	
		
 

			




	
	
	
		
 
    Kallithea can't promise to be thread-safe, just like the embedded Mercurial

			




	
	
	
		
 
    backend doesn't make any strong promises when used as Kallithea uses it.

			




	
	
	
		
 
    Instead, we recommend scaling by using multiple server processes.

			




	
	
	
		
 

			




	
	
	
		
 
    Web servers with multiple worker processes (such as ``mod_wsgi`` with the

			




	
	
	
		
 
    ``WSGIDaemonProcess`` ``processes`` parameter) will work out of the box.

			




	
	
	
		
 

			




	
	
	
		
 
    In order to scale horizontally on multiple machines, you need to do the

			




	
	
	
		
 
    following:

			




	
	
	
		
 

			




	
	
	
		
 
    - Each instance needs its own .ini file and unique ``instance_id`` set.

			




	
	
	
		
 
    - Each instance's ``data`` storage needs to be configured to be stored on a

			




	
	
	
		
 
      shared disk storage, preferably together with repositories. This ``data``

			




	
	
	
		
 
      dir contains template caches, sessions, whoosh index and is used for

			




	
	
	
		
 
      task locking (so it is safe across multiple instances). Set the

			




	
	
	
		
 
      ``cache_dir``, ``index_dir``, ``beaker.cache.data_dir``, ``beaker.cache.lock_dir``

			




	
	
	
		
 
      variables in each .ini file to a shared location across Kallithea instances

			




	
	
	
		
 
    - If celery is used each instance should run a separate Celery instance, but

			




	
	
	
		
 
    - If using several Celery instances,

			




	
	
	
		
 
      the message broker should be common to all of them (e.g.,  one

			




	
	
	
		
 
      shared RabbitMQ server)

			




	
	
	
		
 
    - Load balance using round robin or IP hash, recommended is writing LB rules

			




	
	
	
		
 
      that will separate regular user traffic from automated processes like CI

			




	
	
	
		
 
      servers or build bots.

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
.. _SQLAlchemyGrate: https://github.com/shazow/sqlalchemygrate

			




        

    


    
    

    

        

                

                    kallithea/bin/template.ini.mako
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -57,35 +57,35 @@ pdebug = false

			




	
	
	
		
 
#smtp_port = 25

			




	
	
	
		
 
#smtp_use_tls = false

			




	
	
	
		
 
#smtp_use_ssl = false

			




	
	
	
		
 
<%text>## SMTP authentication parameters to use (e.g. LOGIN PLAIN CRAM-MD5, etc.).</%text>

			




	
	
	
		
 
<%text>## If empty, use any of the authentication parameters supported by the server.</%text>

			




	
	
	
		
 
#smtp_auth =

			




	
	
	
		
 

			




	
	
	
		
 
[server:main]

			




	
	
	
		
 
%if http_server == 'paste':

			




	
	
	
		
 
<%text>## PASTE ##</%text>

			




	
	
	
		
 
use = egg:Paste#http

			




	
	
	
		
 
<%text>## nr of worker threads to spawn</%text>

			




	
	
	
		
 
threadpool_workers = 5

			




	
	
	
		
 
threadpool_workers = 1

			




	
	
	
		
 
<%text>## max request before thread respawn</%text>

			




	
	
	
		
 
threadpool_max_requests = 10

			




	
	
	
		
 
threadpool_max_requests = 100

			




	
	
	
		
 
<%text>## option to use threads of process</%text>

			




	
	
	
		
 
use_threadpool = true

			




	
	
	
		
 

			




	
	
	
		
 
%elif http_server == 'waitress':

			




	
	
	
		
 
<%text>## WAITRESS ##</%text>

			




	
	
	
		
 
use = egg:waitress#main

			




	
	
	
		
 
<%text>## number of worker threads</%text>

			




	
	
	
		
 
threads = 5

			




	
	
	
		
 
threads = 1

			




	
	
	
		
 
<%text>## MAX BODY SIZE 100GB</%text>

			




	
	
	
		
 
max_request_body_size = 107374182400

			




	
	
	
		
 
<%text>## use poll instead of select, fixes fd limits, may not work on old</%text>

			




	
	
	
		
 
<%text>## windows systems.</%text>

			




	
	
	
		
 
#asyncore_use_poll = True

			




	
	
	
		
 

			




	
	
	
		
 
%elif http_server == 'gunicorn':

			




	
	
	
		
 
<%text>## GUNICORN ##</%text>

			




	
	
	
		
 
use = egg:gunicorn#main

			




	
	
	
		
 
<%text>## number of process workers. You must set `instance_id = *` when this option</%text>

			




	
	
	
		
 
<%text>## is set to more than one worker</%text>

			




	
	
	
		
 
workers = 1

			




        

    


    
    

    

        

                

                    kallithea/config/deployment.ini_tmpl
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -57,34 +57,34 @@ pdebug = false

			




	
	
	
		
 
#smtp_password =

			




	
	
	
		
 
#smtp_port = 25

			




	
	
	
		
 
#smtp_use_tls = false

			




	
	
	
		
 
#smtp_use_ssl = false

			




	
	
	
		
 
## SMTP authentication parameters to use (e.g. LOGIN PLAIN CRAM-MD5, etc.).

			




	
	
	
		
 
## If empty, use any of the authentication parameters supported by the server.

			




	
	
	
		
 
#smtp_auth =

			




	
	
	
		
 

			




	
	
	
		
 
[server:main]

			




	
	
	
		
 
## PASTE ##

			




	
	
	
		
 
#use = egg:Paste#http

			




	
	
	
		
 
## nr of worker threads to spawn

			




	
	
	
		
 
#threadpool_workers = 5

			




	
	
	
		
 
#threadpool_workers = 1

			




	
	
	
		
 
## max request before thread respawn

			




	
	
	
		
 
#threadpool_max_requests = 10

			




	
	
	
		
 
#threadpool_max_requests = 100

			




	
	
	
		
 
## option to use threads of process

			




	
	
	
		
 
#use_threadpool = true

			




	
	
	
		
 

			




	
	
	
		
 
## WAITRESS ##

			




	
	
	
		
 
use = egg:waitress#main

			




	
	
	
		
 
## number of worker threads

			




	
	
	
		
 
threads = 5

			




	
	
	
		
 
threads = 1

			




	
	
	
		
 
## MAX BODY SIZE 100GB

			




	
	
	
		
 
max_request_body_size = 107374182400

			




	
	
	
		
 
## use poll instead of select, fixes fd limits, may not work on old

			




	
	
	
		
 
## windows systems.

			




	
	
	
		
 
#asyncore_use_poll = True

			




	
	
	
		
 

			




	
	
	
		
 
## GUNICORN ##

			




	
	
	
		
 
#use = egg:gunicorn#main

			




	
	
	
		
 
## number of process workers. You must set `instance_id = *` when this option

			




	
	
	
		
 
## is set to more than one worker

			




	
	
	
		
 
#workers = 1

			




	
	
	
		
 
## process name

			




        

    


    
    

    

        

                

                    kallithea/tests/test.ini
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -61,34 +61,34 @@ pdebug = false

			




	
	
	
		
 
#smtp_password =

			




	
	
	
		
 
#smtp_port = 25

			




	
	
	
		
 
#smtp_use_tls = false

			




	
	
	
		
 
#smtp_use_ssl = false

			




	
	
	
		
 
## SMTP authentication parameters to use (e.g. LOGIN PLAIN CRAM-MD5, etc.).

			




	
	
	
		
 
## If empty, use any of the authentication parameters supported by the server.

			




	
	
	
		
 
#smtp_auth =

			




	
	
	
		
 

			




	
	
	
		
 
[server:main]

			




	
	
	
		
 
## PASTE ##

			




	
	
	
		
 
#use = egg:Paste#http

			




	
	
	
		
 
## nr of worker threads to spawn

			




	
	
	
		
 
#threadpool_workers = 5

			




	
	
	
		
 
#threadpool_workers = 1

			




	
	
	
		
 
## max request before thread respawn

			




	
	
	
		
 
#threadpool_max_requests = 10

			




	
	
	
		
 
#threadpool_max_requests = 100

			




	
	
	
		
 
## option to use threads of process

			




	
	
	
		
 
#use_threadpool = true

			




	
	
	
		
 

			




	
	
	
		
 
## WAITRESS ##

			




	
	
	
		
 
use = egg:waitress#main

			




	
	
	
		
 
## number of worker threads

			




	
	
	
		
 
threads = 5

			




	
	
	
		
 
threads = 1

			




	
	
	
		
 
## MAX BODY SIZE 100GB

			




	
	
	
		
 
max_request_body_size = 107374182400

			




	
	
	
		
 
## use poll instead of select, fixes fd limits, may not work on old

			




	
	
	
		
 
## windows systems.

			




	
	
	
		
 
#asyncore_use_poll = True

			




	
	
	
		
 

			




	
	
	
		
 
## GUNICORN ##

			




	
	
	
		
 
#use = egg:gunicorn#main

			




	
	
	
		
 
## number of process workers. You must set `instance_id = *` when this option

			




	
	
	
		
 
## is set to more than one worker

			




	
	
	
		
 
#workers = 1

			




	
	
	
		
 
## process name

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2025 by various authors & licensed under GPLv3.