Changeset - f8f50d3b6512
[Not reviewed]
default
0 5 0
Mads Kiilerich - 6 years ago 2020-02-20 02:03:24
mads@kiilerich.com
celery: upgrade to Celery 4

Celery 3 doesn't support Python 3.7 or later. This upgrade is thus essential
for full Python 3 support. But note that
https://docs.celeryproject.org/en/4.4.0/faq.html#does-celery-support-windows
says "No".

The names of config settings changed in Celery 3 to 4, as described on
https://docs.celeryproject.org/en/3.0/whatsnew-4.0.html#lowercase-setting-names .

Celery 4 config settings can now be specified in Kallithea .ini files by prefixing
with `celery.` - for example as `celery.broker_url`.

Remain backwards compatible for the usual settings, and map old names to the
new names.
5 files changed with 34 insertions and 17 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -229,48 +229,49 @@ allow_custom_hooks_settings = True
 

	
 
####################################
 
###           SSH CONFIG        ####
 
####################################
 

	
 
## SSH is disabled by default, until an Administrator decides to enable it.
 
ssh_enabled = false
 

	
 
## File where users' SSH keys will be stored *if* ssh_enabled is true.
 
#ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 

	
 
## Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.
 
#kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 

	
 
## Locale to be used in the ssh-serve command.
 
## This is needed because an SSH client may try to use its own locale
 
## settings, which may not be available on the server.
 
## See `locale -a` for valid values on this system.
 
#ssh_locale = C.UTF-8
 

	
 
####################################
 
###        CELERY CONFIG        ####
 
####################################
 

	
 
## Note: Celery doesn't support Windows.
 
use_celery = false
 

	
 
## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 
broker.url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 

	
 
celery.result.backend = db+sqlite:///celery-results.db
 

	
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
celeryd.max.tasks.per.child = 1
 

	
 
## If true, tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
docs/setup.rst
Show inline comments
 
@@ -311,53 +311,53 @@ when there are encode/decode errors.
 
The Mercurial encoding is configurable as ``hgencoding``. It is similar to
 
setting the ``HGENCODING`` environment variable, but will override it.
 

	
 

	
 
Celery configuration
 
--------------------
 

	
 
Kallithea can use the distributed task queue system Celery_ to run tasks like
 
cloning repositories or sending emails.
 

	
 
Kallithea will in most setups work perfectly fine out of the box (without
 
Celery), executing all tasks in the web server process. Some tasks can however
 
take some time to run and it can be better to run such tasks asynchronously in
 
a separate process so the web server can focus on serving web requests.
 

	
 
For installation and configuration of Celery, see the `Celery documentation`_.
 
Note that Celery requires a message broker service like RabbitMQ_ (recommended)
 
or Redis_.
 

	
 
The use of Celery is configured in the Kallithea ini configuration file.
 
To enable it, simply set::
 

	
 
  use_celery = true
 

	
 
and add or change the ``celery.*`` and ``broker.*`` configuration variables.
 
and add or change the ``celery.*`` configuration variables.
 

	
 
Remember that the ini files use the format with '.' and not with '_' like
 
Celery. So for example setting `BROKER_HOST` in Celery means setting
 
`broker.host` in the configuration file.
 
Configuration settings are prefixed with 'celery.', so for example setting
 
`broker_url` in Celery means setting `celery.broker_url` in the configuration
 
file.
 

	
 
To start the Celery process, run::
 

	
 
  kallithea-cli celery-run -c my.ini
 

	
 
Extra options to the Celery worker can be passed after ``--`` - see ``-- -h``
 
for more info.
 

	
 
.. note::
 
   Make sure you run this command from the same virtualenv, and with the same
 
   user that Kallithea runs.
 

	
 

	
 
HTTPS support
 
-------------
 

	
 
Kallithea will by default generate URLs based on the WSGI environment.
 

	
 
Alternatively, you can use some special configuration settings to control
 
directly which scheme/protocol Kallithea will use when generating URLs:
 

	
 
- With ``https_fixup = true``, the scheme will be taken from the
 
  ``X-Url-Scheme``, ``X-Forwarded-Scheme`` or ``X-Forwarded-Proto`` HTTP header
 
  (default ``http``).
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
@@ -2,76 +2,91 @@
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the Kallithea ini file but must be converted to an
 
entirely different format before Celery can use it.
 

	
 
We read the configuration from tg.config at module import time. This module can
 
thus not be imported in global scope but must be imported on demand in function
 
scope after tg.config has been initialized.
 

	
 
To make sure that the config really has been initialized, we check one of the
 
mandatory settings.
 
"""
 

	
 
import logging
 

	
 
import celery
 
import tg
 

	
 
import kallithea
 

	
 

	
 
class CeleryConfig(object):
 
    CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
 
    CELERY_ACCEPT_CONTENT = ['json']
 
    CELERY_RESULT_SERIALIZER = 'json'
 
    CELERY_TASK_SERIALIZER = 'json'
 
    CELERY_ALWAYS_EAGER = False
 
    imports = ['kallithea.lib.celerylib.tasks']
 
    task_always_eager = False
 

	
 
# map from Kallithea .ini Celery 3 config names to Celery 4 config names
 
celery3_compat = {
 
    'broker.url': 'broker_url',
 
    'celery.accept.content': 'accept_content',
 
    'celery.always.eager': 'task_always_eager',
 
    'celery.amqp.task.result.expires': 'result_expires',
 
    'celeryd.concurrency': 'worker_concurrency',
 
    'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
 
    #'celery.imports' ends up unchanged
 
    'celery.result.backend': 'result_backend',
 
    'celery.result.serializer': 'result_serializer',
 
    'celery.task.serializer': 'task_serializer',
 
}
 

	
 
list_config_names = """imports accept_content""".split()
 

	
 

	
 
desupported = set([
 
    'celery.result.dburi',
 
    'celery.result.serialier',
 
    'celery.send.task.error.emails',
 
])
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def make_celery_config(config):
 
    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
 

	
 
    celery_config = CeleryConfig()
 

	
 
    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
 
    LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split()
 

	
 
    for config_key, config_value in sorted(config.items()):
 
        if config_key in desupported and config_value:
 
            log.error('Celery configuration setting %r is no longer supported', config_key)
 
        celery_key = config_key.replace('.', '_').upper()
 
        if celery_key.split('_', 1)[0] not in PREFIXES:
 
        celery_key = celery3_compat.get(config_key)
 
        parts = config_key.split('.', 1)
 
        if celery_key:  # explicit Celery 3 backwards compatibility
 
            pass
 
        elif parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
 
            celery_key = parts[1]
 
        else:
 
            continue
 
        if not isinstance(config_value, str):
 
            continue
 
        if celery_key in LIST_PARAMS:
 
        if celery_key in list_config_names:
 
            celery_value = config_value.split()
 
        elif config_value.isdigit():
 
            celery_value = int(config_value)
 
        elif config_value.lower() in ['true', 'false']:
 
            celery_value = config_value.lower() == 'true'
 
        else:
 
            celery_value = config_value
 
        setattr(celery_config, celery_key, celery_value)
 
    return celery_config
 

	
 

	
 
def make_app():
 
    """Create celery app from the TurboGears configuration file"""
 
    app = celery.Celery()
 
    celery_config = make_celery_config(tg.config)
 
    kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER
 
    kallithea.CELERY_EAGER = celery_config.task_always_eager
 
    app.config_from_object(celery_config)
 
    return app
kallithea/lib/paster_commands/template.ini.mako
Show inline comments
 
@@ -335,48 +335,49 @@ ssh_enabled = false
 
#ssh_authorized_keys = /home/kallithea/.ssh/authorized_keys
 
%if user_home_path:
 
ssh_authorized_keys = ${user_home_path}/.ssh/authorized_keys
 
%endif
 

	
 
<%text>## Path to be used in ssh_authorized_keys file to invoke kallithea-cli with ssh-serve.</%text>
 
#kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 
%if kallithea_cli_path:
 
kallithea_cli_path = ${kallithea_cli_path}
 
%endif
 

	
 
<%text>## Locale to be used in the ssh-serve command.</%text>
 
<%text>## This is needed because an SSH client may try to use its own locale</%text>
 
<%text>## settings, which may not be available on the server.</%text>
 
<%text>## See `locale -a` for valid values on this system.</%text>
 
#ssh_locale = C.UTF-8
 
%if ssh_locale:
 
ssh_locale = ${ssh_locale}
 
%endif
 

	
 
<%text>####################################</%text>
 
<%text>###        CELERY CONFIG        ####</%text>
 
<%text>####################################</%text>
 

	
 
<%text>## Note: Celery doesn't support Windows.</%text>
 
use_celery = false
 

	
 
<%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':</%text>
 
broker.url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 

	
 
celery.result.backend = db+sqlite:///celery-results.db
 

	
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
celeryd.max.tasks.per.child = 1
 

	
 
<%text>## If true, tasks will never be sent to the queue, but executed locally instead.</%text>
 
celery.always.eager = false
 

	
 
<%text>####################################</%text>
 
<%text>###         BEAKER CACHE        ####</%text>
 
<%text>####################################</%text>
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short
 

	
setup.py
Show inline comments
 
@@ -33,49 +33,49 @@ callback = lambda V: ('.'.join(map(str, 
 
__version__ = _get_meta_var('VERSION', _metadata, callback)
 
__license__ = _get_meta_var('__license__', _metadata)
 
__author__ = _get_meta_var('__author__', _metadata)
 
__url__ = _get_meta_var('__url__', _metadata)
 
# defines current platform
 
__platform__ = platform.system()
 

	
 
is_windows = __platform__ in ['Windows']
 

	
 
requirements = [
 
    "alembic >= 1.0.10, < 1.5",
 
    "gearbox >= 0.1.0, < 1",
 
    "waitress >= 0.8.8, < 1.5",
 
    "WebOb >= 1.8, < 1.9",
 
    "backlash >= 0.1.2, < 1",
 
    "TurboGears2 >= 2.4, < 2.5",
 
    "tgext.routes >= 0.2.0, < 1",
 
    "Beaker >= 1.10.1, < 2",
 
    "WebHelpers2 >= 2.0, < 2.1",
 
    "FormEncode >= 1.3.1, < 1.4",
 
    "SQLAlchemy >= 1.2.9, < 1.4",
 
    "Mako >= 0.9.1, < 1.2",
 
    "Pygments >= 2.2.0, < 2.6",
 
    "Whoosh >= 2.7.1, < 2.8",
 
    "celery >= 3.1, < 4.0", # TODO: celery 4 doesn't work
 
    "celery >= 4.3, < 4.5",
 
    "Babel >= 1.3, < 2.9",
 
    "python-dateutil >= 2.1.0, < 2.9",
 
    "Markdown >= 2.2.1, < 3.2",
 
    "docutils >= 0.11, < 0.17",
 
    "URLObject >= 2.3.4, < 2.5",
 
    "Routes >= 2.0, < 2.5",
 
    "dulwich >= 0.19.0, < 0.20",
 
    "mercurial >= 5.2, < 5.4",
 
    "decorator >= 4.2.1, < 4.5",
 
    "Paste >= 2.0.3, < 3.4",
 
    "bleach >= 3.0, < 3.2",
 
    "Click >= 7.0, < 8",
 
    "ipaddr >= 2.2.0, < 2.3",
 
    "paginate >= 0.5, < 0.6",
 
    "paginate_sqlalchemy >= 0.3.0, < 0.4",
 
]
 

	
 
if not is_windows:
 
    requirements.append("bcrypt >= 3.1.0, < 3.2")
 

	
 
dependency_links = [
 
]
 

	
 
classifiers = [
0 comments (0 inline, 0 general)