Changeset - e432a6a7dd7c
[Not reviewed]
default
0 2 0
Mads Kiilerich - 6 years ago 2020-02-24 18:37:31
mads@kiilerich.com
celery: move initialization of global kallithea.CELERY_EAGER to the place where we read celery configuration

Prepare for backwards compatibility after the name change with Celery 4.
2 files changed with 7 insertions and 3 deletions:
0 comments (0 inline, 0 general)
kallithea/config/app_cfg.py
Show inline comments
 
@@ -140,49 +140,48 @@ def setup_configuration(app):
 
        dbconf = config['sqlalchemy.url']
 
        alembic_cfg = alembic.config.Config()
 
        alembic_cfg.set_main_option('script_location', 'kallithea:alembic')
 
        alembic_cfg.set_main_option('sqlalchemy.url', dbconf)
 
        script_dir = ScriptDirectory.from_config(alembic_cfg)
 
        available_heads = sorted(script_dir.get_heads())
 

	
 
        engine = create_engine(dbconf)
 
        with engine.connect() as conn:
 
            context = MigrationContext.configure(conn)
 
            current_heads = sorted(str(s) for s in context.get_current_heads())
 
        if current_heads != available_heads:
 
            log.error('Failed to run Kallithea:\n\n'
 
                      'The database version does not match the Kallithea version.\n'
 
                      'Please read the documentation on how to upgrade or downgrade the database.\n'
 
                      'Current database version id(s): %s\n'
 
                      'Expected database version id(s): %s\n'
 
                      'If you are a developer and you know what you are doing, you can add `ignore_alembic_revision = True` '
 
                      'to your .ini file to skip the check.\n' % (' '.join(current_heads), ' '.join(available_heads)))
 
            sys.exit(1)
 

	
 
    # store some globals into kallithea
 
    if str2bool(config.get('use_celery')):
 
        kallithea.CELERY_APP = celerypylons.make_app()
 
    kallithea.CELERY_EAGER = str2bool(config.get('celery.always.eager'))
 
    kallithea.CONFIG = config
 

	
 
    load_rcextensions(root_path=config['here'])
 

	
 
    repos_path = safe_str(make_ui().configitems(b'paths')[0][1])
 
    config['base_path'] = repos_path
 
    set_app_settings(config)
 

	
 
    instance_id = kallithea.CONFIG.get('instance_id', '*')
 
    if instance_id == '*':
 
        instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
 
        kallithea.CONFIG['instance_id'] = instance_id
 

	
 
    # update kallithea.CONFIG with the meanwhile changed 'config'
 
    kallithea.CONFIG.update(config)
 

	
 
    # configure vcs and indexer libraries (they are supposed to be independent
 
    # as much as possible and thus avoid importing tg.config or
 
    # kallithea.CONFIG).
 
    set_vcs_config(kallithea.CONFIG)
 
    set_indexer_config(kallithea.CONFIG)
 

	
 
    check_git_version()
 

	
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the Kallithea ini file but must be converted to an
 
entirely different format before Celery can use it.
 

	
 
We read the configuration from tg.config at module import time. This module can
 
thus not be imported in global scope but must be imported on demand in function
 
scope after tg.config has been initialized.
 

	
 
To make sure that the config really has been initialized, we check one of the
 
mandatory settings.
 
"""
 

	
 
import logging
 

	
 
import celery
 
import tg
 

	
 
import kallithea
 

	
 

	
 
class CeleryConfig(object):
 
    CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
 
    CELERY_ACCEPT_CONTENT = ['json']
 
    CELERY_RESULT_SERIALIZER = 'json'
 
    CELERY_TASK_SERIALIZER = 'json'
 
    CELERY_ALWAYS_EAGER = False
 

	
 

	
 
desupported = set([
 
    'celery.result.dburi',
 
    'celery.result.serialier',
 
    'celery.send.task.error.emails',
 
])
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def celery_config(config):
 
def make_celery_config(config):
 
    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
 

	
 
    celery_config = CeleryConfig()
 

	
 
    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
 
    LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split()
 

	
 
    for config_key, config_value in sorted(config.items()):
 
        if config_key in desupported and config_value:
 
            log.error('Celery configuration setting %r is no longer supported', config_key)
 
        celery_key = config_key.replace('.', '_').upper()
 
        if celery_key.split('_', 1)[0] not in PREFIXES:
 
            continue
 
        if not isinstance(config_value, str):
 
            continue
 
        if celery_key in LIST_PARAMS:
 
            celery_value = config_value.split()
 
        elif config_value.isdigit():
 
            celery_value = int(config_value)
 
        elif config_value.lower() in ['true', 'false']:
 
            celery_value = config_value.lower() == 'true'
 
        else:
 
            celery_value = config_value
 
        setattr(celery_config, celery_key, celery_value)
 
    return celery_config
 

	
 

	
 
def make_app():
 
    """Create celery app from the TurboGears configuration file"""
 
    app = celery.Celery()
 
    app.config_from_object(celery_config(tg.config))
 
    celery_config = make_celery_config(tg.config)
 
    kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER
 
    app.config_from_object(celery_config)
 
    return app
0 comments (0 inline, 0 general)