kallithea Changeset - f8f50d3b6512

Changeset - f8f50d3b6512

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Mads Kiilerich - 6 years ago 2020-02-20 02:03:24
mads@kiilerich.com

celery: upgrade to Celery 4

Celery 3 doesn't support Python 3.7 or later. This upgrade is thus essential
for full Python 3 support. But note that
https://docs.celeryproject.org/en/4.4.0/faq.html#does-celery-support-windows
says "No".

The names of config settings changed in Celery 3 to 4, as described on
https://docs.celeryproject.org/en/3.0/whatsnew-4.0.html#lowercase-setting-names .

Celery 4 config settings can now be specified in Kallithea .ini files by prefixing
with `celery.` - for example as `celery.broker_url`.

Remain backwards compatible for the usual settings, and map old names to the
new names.

5 files changed with 34 insertions and 17 deletions:

development.ini

docs/setup.rst

kallithea/lib/celerypylons/__init__.py

kallithea/lib/paster_commands/template.ini.mako

setup.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -250,6 +250,7 @@ ssh_enabled = false @@
 ###        CELERY CONFIG        ####
 ####################################
 ## Note: Celery doesn't support Windows.
 use_celery = false
 ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':

docs/setup.rst

➞

Show inline comments

@@ @@ -332,11 +332,11 @@ To enable it, simply set:: @@
   use_celery = true
-and add or change the ``celery.*`` and ``broker.*`` configuration variables.
 and add or change the ``celery.*`` configuration variables.
 Remember that the ini files use the format with '.' and not with '_' like
 Celery. So for example setting `BROKER_HOST` in Celery means setting
 `broker.host` in the configuration file.
 Configuration settings are prefixed with 'celery.', so for example setting
 `broker_url` in Celery means setting `celery.broker_url` in the configuration
 file.
 To start the Celery process, run::

kallithea/lib/celerypylons/__init__.py

➞

Show inline comments

@@ @@ -23,11 +23,24 @@ import kallithea @@
 class CeleryConfig(object):
     CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
     CELERY_ACCEPT_CONTENT = ['json']
     CELERY_RESULT_SERIALIZER = 'json'
     CELERY_TASK_SERIALIZER = 'json'
     CELERY_ALWAYS_EAGER = False
     imports = ['kallithea.lib.celerylib.tasks']
     task_always_eager = False
 # map from Kallithea .ini Celery 3 config names to Celery 4 config names
 celery3_compat = {
     'broker.url': 'broker_url',
     'celery.accept.content': 'accept_content',
     'celery.always.eager': 'task_always_eager',
     'celery.amqp.task.result.expires': 'result_expires',
     'celeryd.concurrency': 'worker_concurrency',
     'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
     #'celery.imports' ends up unchanged
     'celery.result.backend': 'result_backend',
     'celery.result.serializer': 'result_serializer',
     'celery.task.serializer': 'task_serializer',
+}
 list_config_names = """imports accept_content""".split()
 desupported = set([
@@ @@ -45,18 +58,20 @@ def make_celery_config(config): @@
     celery_config = CeleryConfig()
     PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
     LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split()
     for config_key, config_value in sorted(config.items()):
         if config_key in desupported and config_value:
             log.error('Celery configuration setting %r is no longer supported', config_key)
         celery_key = config_key.replace('.', '_').upper()
         if celery_key.split('_', 1)[0] not in PREFIXES:
         celery_key = celery3_compat.get(config_key)
         parts = config_key.split('.', 1)
         if celery_key:  # explicit Celery 3 backwards compatibility
             pass
         elif parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
             celery_key = parts[1]
         else:
             continue
         if not isinstance(config_value, str):
             continue
-        if celery_key in LIST_PARAMS:
+        if celery_key in list_config_names:
             celery_value = config_value.split()
         elif config_value.isdigit():
             celery_value = int(config_value)
@@ @@ -72,6 +87,6 @@ def make_app(): @@
     """Create celery app from the TurboGears configuration file"""
     app = celery.Celery()
     celery_config = make_celery_config(tg.config)
-    kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER
+    kallithea.CELERY_EAGER = celery_config.task_always_eager
     app.config_from_object(celery_config)
     return app

kallithea/lib/paster_commands/template.ini.mako

➞

Show inline comments

@@ @@ -356,6 +356,7 @@ ssh_locale = ${ssh_locale} @@
 <%text>###        CELERY CONFIG        ####</%text>
 <%text>####################################</%text>
 <%text>## Note: Celery doesn't support Windows.</%text>
 use_celery = false
 <%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':</%text>

setup.py

➞

Show inline comments

@@ @@ -54,7 +54,7 @@ requirements = [ @@
     "Mako >= 0.9.1, < 1.2",
     "Pygments >= 2.2.0, < 2.6",
     "Whoosh >= 2.7.1, < 2.8",
-    "celery >= 3.1, < 4.0", # TODO: celery 4 doesn't work
+    "celery >= 4.3, < 4.5",
     "Babel >= 1.3, < 2.9",
     "python-dateutil >= 2.1.0, < 2.9",
     "Markdown >= 2.2.1, < 3.2",

0 comments (0 inline, 0 general)