Changeset - 66670aff96c5
[Not reviewed]
default
0 3 0
Mads Kiilerich - 6 years ago 2020-02-18 17:03:09
mads@kiilerich.com
Grafted from: 462c8245e9e4
celery: set default config values in code and remove them from the generated .ini

It is hard to imagine any reason the user should change celery.imports . And if
it ever should change, we want it controlled in code - not left stale in user
controlled config files.

Everybody sould just use .json and there is no reason anybody should specify
that in the .ini ... and it will be the default in Celery 4.
3 files changed with 7 insertions and 14 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -210,101 +210,97 @@ issue_sub =
 
auth_ret_code =
 

	
 
## allows to change the repository location in settings page
 
allow_repo_location_change = True
 

	
 
## allows to setup custom hooks in settings page
 
allow_custom_hooks_settings = True
 

	
 
## extra extensions for indexing, space separated and without the leading '.'.
 
# index.extensions =
 
#    gemfile
 
#    lock
 

	
 
## extra filenames for indexing, space separated
 
# index.filenames =
 
#    .dockerignore
 
#    .editorconfig
 
#    INSTALL
 
#    CHANGELOG
 

	
 
####################################
 
###           SSH CONFIG        ####
 
####################################
 

	
 
## SSH is disabled by default, until an Administrator decides to enable it.
 
ssh_enabled = false
 

	
 
## File where users' SSH keys will be stored *if* ssh_enabled is true.
 
#ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 

	
 
## Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.
 
#kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 

	
 
## Locale to be used in the ssh-serve command.
 
## This is needed because an SSH client may try to use its own locale
 
## settings, which may not be available on the server.
 
## See `locale -a` for valid values on this system.
 
#ssh_locale = C.UTF-8
 

	
 
####################################
 
###        CELERY CONFIG        ####
 
####################################
 

	
 
use_celery = false
 

	
 
## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 
broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 
celery.accept.content = json
 
celery.result.backend = amqp://
 
celery.result.serializer = json
 
celery.task.serializer = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
celeryd.max.tasks.per.child = 1
 

	
 
## If true, tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 

	
 
## Name of session cookie. Should be unique for a given host and path, even when running
 
## on different ports. Otherwise, cookie sessions will be shared and messed up.
 
session.key = kallithea
 
## Sessions should always only be accessible by the browser, not directly by JavaScript.
 
session.httponly = true
 
## Session lifetime. 2592000 seconds is 30 days.
 
session.timeout = 2592000
 

	
 
## Server secret used with HMAC to ensure integrity of cookies.
 
#session.secret = VERY-SECRET
 
session.secret = development-not-secret
 
## Further, encrypt the data with AES.
 
#session.encrypt_key = <key_for_encryption>
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the Kallithea ini file but must be converted to an
 
entirely different format before Celery can use it.
 

	
 
We read the configuration from tg.config at module import time. This module can
 
thus not be imported in global scope but must be imported on demand in function
 
scope after tg.config has been initialized.
 

	
 
To make sure that the config really has been initialized, we check one of the
 
mandatory settings.
 
"""
 

	
 
import celery
 
import tg
 

	
 

	
 
class CeleryConfig(object):
 
    CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
 
    CELERY_ACCEPT_CONTENT = ['json']
 
    CELERY_RESULT_SERIALIZER = 'json'
 
    CELERY_TASK_SERIALIZER = 'json'
 

	
 

	
 
def celery_config(config):
 
    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
 

	
 
    # Verify .ini file configuration has been loaded
 
    assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
 

	
 
    class CeleryConfig(object):
 
        pass
 

	
 
    celery_config = CeleryConfig()
 

	
 
    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
 
    LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()
 

	
 
    for config_key, config_value in sorted(config.items()):
 
        celery_key = config_key.replace('.', '_').upper()
 
        if celery_key.split('_', 1)[0] not in PREFIXES:
 
            continue
 
        if not isinstance(config_value, str):
 
            continue
 
        if celery_key in LIST_PARAMS:
 
            celery_value = config_value.split()
 
        elif config_value.isdigit():
 
            celery_value = int(config_value)
 
        elif config_value.lower() in ['true', 'false']:
 
            celery_value = config_value.lower() == 'true'
 
        else:
 
            celery_value = config_value
 
        setattr(celery_config, celery_key, celery_value)
 
    return celery_config
 

	
 

	
 
def make_app():
 
    """Create celery app from the TurboGears configuration file"""
 
    app = celery.Celery()
 
    app.config_from_object(celery_config(tg.config))
 
    return app
kallithea/lib/paster_commands/template.ini.mako
Show inline comments
 
@@ -316,101 +316,97 @@ allow_custom_hooks_settings = True
 
# index.extensions =
 
#    gemfile
 
#    lock
 

	
 
<%text>## extra filenames for indexing, space separated</%text>
 
# index.filenames =
 
#    .dockerignore
 
#    .editorconfig
 
#    INSTALL
 
#    CHANGELOG
 

	
 
<%text>####################################</%text>
 
<%text>###           SSH CONFIG        ####</%text>
 
<%text>####################################</%text>
 

	
 
<%text>## SSH is disabled by default, until an Administrator decides to enable it.</%text>
 
ssh_enabled = false
 

	
 
<%text>## File where users' SSH keys will be stored *if* ssh_enabled is true.</%text>
 
#ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 
%if user_home_path:
 
ssh_authorized_keys = ${user_home_path}/.ssh/authorized_keys
 
%endif
 

	
 
<%text>## Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.</%text>
 
#kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 
%if kallithea_cli_path:
 
kallithea_cli_path = ${kallithea_cli_path}
 
%endif
 

	
 
<%text>## Locale to be used in the ssh-serve command.</%text>
 
<%text>## This is needed because an SSH client may try to use its own locale</%text>
 
<%text>## settings, which may not be available on the server.</%text>
 
<%text>## See `locale -a` for valid values on this system.</%text>
 
#ssh_locale = C.UTF-8
 
%if ssh_locale:
 
ssh_locale = ${ssh_locale}
 
%endif
 

	
 
<%text>####################################</%text>
 
<%text>###        CELERY CONFIG        ####</%text>
 
<%text>####################################</%text>
 

	
 
use_celery = false
 

	
 
<%text>## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:</%text>
 
broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 

	
 
celery.imports = kallithea.lib.celerylib.tasks
 
celery.accept.content = json
 
celery.result.backend = amqp://
 
celery.result.serializer = json
 
celery.task.serializer = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
celeryd.max.tasks.per.child = 1
 

	
 
<%text>## If true, tasks will never be sent to the queue, but executed locally instead.</%text>
 
celery.always.eager = false
 

	
 
<%text>####################################</%text>
 
<%text>###         BEAKER CACHE        ####</%text>
 
<%text>####################################</%text>
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
<%text>####################################</%text>
 
<%text>###       BEAKER SESSION        ####</%text>
 
<%text>####################################</%text>
 

	
 
<%text>## Name of session cookie. Should be unique for a given host and path, even when running</%text>
 
<%text>## on different ports. Otherwise, cookie sessions will be shared and messed up.</%text>
 
session.key = kallithea
 
<%text>## Sessions should always only be accessible by the browser, not directly by JavaScript.</%text>
 
session.httponly = true
 
<%text>## Session lifetime. 2592000 seconds is 30 days.</%text>
 
session.timeout = 2592000
 

	
 
<%text>## Server secret used with HMAC to ensure integrity of cookies.</%text>
 
session.secret = ${uuid()}
 
<%text>## Further, encrypt the data with AES.</%text>
 
#session.encrypt_key = <key_for_encryption>
 
#session.validate_key = <validation_key>
0 comments (0 inline, 0 general)