kallithea Changeset - f8f50d3b6512

Changeset - f8f50d3b6512

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Mads Kiilerich - 6 years ago 2020-02-20 02:03:24
mads@kiilerich.com

celery: upgrade to Celery 4

Celery 3 doesn't support Python 3.7 or later. This upgrade is thus essential
for full Python 3 support. But note that
https://docs.celeryproject.org/en/4.4.0/faq.html#does-celery-support-windows
says "No".

The names of config settings changed in Celery 3 to 4, as described on
https://docs.celeryproject.org/en/3.0/whatsnew-4.0.html#lowercase-setting-names .

Celery 4 config settings can now be specified in Kallithea .ini files by prefixing
with `celery.` - for example as `celery.broker_url`.

Remain backwards compatible for the usual settings, and map old names to the
new names.

5 files changed with 34 insertions and 17 deletions:

development.ini

docs/setup.rst

kallithea/lib/celerypylons/__init__.py

kallithea/lib/paster_commands/template.ini.mako

setup.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -241,24 +241,25 @@ ssh_enabled = false @@
 #kallithea_cli_path = /srv/kallithea/venv/bin/kallithea-cli
 ## Locale to be used in the ssh-serve command.
 ## This is needed because an SSH client may try to use its own locale
 ## settings, which may not be available on the server.
 ## See `locale -a` for valid values on this system.
 #ssh_locale = C.UTF-8
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 ## Note: Celery doesn't support Windows.
 use_celery = false
 ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 broker.url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 celery.result.backend = db+sqlite:///celery-results.db
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1

docs/setup.rst

➞

Show inline comments

@@ @@ -323,29 +323,29 @@ Celery), executing all tasks in the web @@
 take some time to run and it can be better to run such tasks asynchronously in
 a separate process so the web server can focus on serving web requests.
 For installation and configuration of Celery, see the `Celery documentation`_.
 Note that Celery requires a message broker service like RabbitMQ_ (recommended)
 or Redis_.
 The use of Celery is configured in the Kallithea ini configuration file.
 To enable it, simply set::
   use_celery = true
-and add or change the ``celery.*`` and ``broker.*`` configuration variables.
 and add or change the ``celery.*`` configuration variables.
 Remember that the ini files use the format with '.' and not with '_' like
 Celery. So for example setting `BROKER_HOST` in Celery means setting
 `broker.host` in the configuration file.
 Configuration settings are prefixed with 'celery.', so for example setting
 `broker_url` in Celery means setting `celery.broker_url` in the configuration
 file.
 To start the Celery process, run::
   kallithea-cli celery-run -c my.ini
 Extra options to the Celery worker can be passed after ``--`` - see ``-- -h``
 for more info.
 .. note::
    Make sure you run this command from the same virtualenv, and with the same
    user that Kallithea runs.

kallithea/lib/celerypylons/__init__.py

➞

Show inline comments

@@ @@ -14,64 +14,79 @@ To make sure that the config really has @@
 mandatory settings.
 """
 import logging
 import celery
 import tg
 import kallithea
 class CeleryConfig(object):
     CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks']
     CELERY_ACCEPT_CONTENT = ['json']
     CELERY_RESULT_SERIALIZER = 'json'
     CELERY_TASK_SERIALIZER = 'json'
     CELERY_ALWAYS_EAGER = False
     imports = ['kallithea.lib.celerylib.tasks']
     task_always_eager = False
 # map from Kallithea .ini Celery 3 config names to Celery 4 config names
 celery3_compat = {
     'broker.url': 'broker_url',
     'celery.accept.content': 'accept_content',
     'celery.always.eager': 'task_always_eager',
     'celery.amqp.task.result.expires': 'result_expires',
     'celeryd.concurrency': 'worker_concurrency',
     'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
     #'celery.imports' ends up unchanged
     'celery.result.backend': 'result_backend',
     'celery.result.serializer': 'result_serializer',
     'celery.task.serializer': 'task_serializer',
+}
 list_config_names = """imports accept_content""".split()
 desupported = set([
     'celery.result.dburi',
     'celery.result.serialier',
     'celery.send.task.error.emails',
 ])
 log = logging.getLogger(__name__)
 def make_celery_config(config):
     """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
     celery_config = CeleryConfig()
     PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
     LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split()
     for config_key, config_value in sorted(config.items()):
         if config_key in desupported and config_value:
             log.error('Celery configuration setting %r is no longer supported', config_key)
         celery_key = config_key.replace('.', '_').upper()
         if celery_key.split('_', 1)[0] not in PREFIXES:
         celery_key = celery3_compat.get(config_key)
         parts = config_key.split('.', 1)
         if celery_key:  # explicit Celery 3 backwards compatibility
             pass
         elif parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
             celery_key = parts[1]
         else:
             continue
         if not isinstance(config_value, str):
             continue
-        if celery_key in LIST_PARAMS:
+        if celery_key in list_config_names:
             celery_value = config_value.split()
         elif config_value.isdigit():
             celery_value = int(config_value)
         elif config_value.lower() in ['true', 'false']:
             celery_value = config_value.lower() == 'true'
         else:
             celery_value = config_value
         setattr(celery_config, celery_key, celery_value)
     return celery_config
 def make_app():
     """Create celery app from the TurboGears configuration file"""
     app = celery.Celery()
     celery_config = make_celery_config(tg.config)
-    kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER
+    kallithea.CELERY_EAGER = celery_config.task_always_eager
     app.config_from_object(celery_config)
     return app

kallithea/lib/paster_commands/template.ini.mako

➞

Show inline comments

@@ @@ -347,24 +347,25 @@ kallithea_cli_path = ${kallithea_cli_pat @@
 <%text>## This is needed because an SSH client may try to use its own locale</%text>
 <%text>## settings, which may not be available on the server.</%text>
 <%text>## See `locale -a` for valid values on this system.</%text>
 #ssh_locale = C.UTF-8
 %if ssh_locale:
 ssh_locale = ${ssh_locale}
 %endif
 <%text>####################################</%text>
 <%text>###        CELERY CONFIG        ####</%text>
 <%text>####################################</%text>
 <%text>## Note: Celery doesn't support Windows.</%text>
 use_celery = false
 <%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':</%text>
 broker.url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 celery.result.backend = db+sqlite:///celery-results.db
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1

setup.py

➞

Show inline comments

@@ @@ -45,25 +45,25 @@ requirements = [ @@
     "waitress >= 0.8.8, < 1.5",
     "WebOb >= 1.8, < 1.9",
     "backlash >= 0.1.2, < 1",
     "TurboGears2 >= 2.4, < 2.5",
     "tgext.routes >= 0.2.0, < 1",
     "Beaker >= 1.10.1, < 2",
     "WebHelpers2 >= 2.0, < 2.1",
     "FormEncode >= 1.3.1, < 1.4",
     "SQLAlchemy >= 1.2.9, < 1.4",
     "Mako >= 0.9.1, < 1.2",
     "Pygments >= 2.2.0, < 2.6",
     "Whoosh >= 2.7.1, < 2.8",
-    "celery >= 3.1, < 4.0", # TODO: celery 4 doesn't work
+    "celery >= 4.3, < 4.5",
     "Babel >= 1.3, < 2.9",
     "python-dateutil >= 2.1.0, < 2.9",
     "Markdown >= 2.2.1, < 3.2",
     "docutils >= 0.11, < 0.17",
     "URLObject >= 2.3.4, < 2.5",
     "Routes >= 2.0, < 2.5",
     "dulwich >= 0.19.0, < 0.20",
     "mercurial >= 5.2, < 5.4",
     "decorator >= 4.2.1, < 4.5",
     "Paste >= 2.0.3, < 3.4",
     "bleach >= 3.0, < 3.2",
     "Click >= 7.0, < 8",

0 comments (0 inline, 0 general)