diff --git a/kallithea/config/environment.py b/kallithea/config/environment.py --- a/kallithea/config/environment.py +++ b/kallithea/config/environment.py @@ -25,9 +25,6 @@ import mako.lookup import beaker import formencode -# don't remove this import it does magic for celery -from kallithea.lib import celerypylons - import kallithea.lib.app_globals as app_globals from kallithea.config.routing import make_map diff --git a/kallithea/controllers/admin/repos.py b/kallithea/controllers/admin/repos.py --- a/kallithea/controllers/admin/repos.py +++ b/kallithea/controllers/admin/repos.py @@ -189,9 +189,9 @@ class ReposController(BaseRepoController if task_id and task_id not in ['None']: from kallithea import CELERY_ON - from celery.result import AsyncResult + from kallithea.lib import celerypylons if CELERY_ON: - task = AsyncResult(task_id) + task = celerypylons.result.AsyncResult(task_id) if task.failed(): raise HTTPInternalServerError(task.traceback) diff --git a/kallithea/lib/celerylib/__init__.py b/kallithea/lib/celerylib/__init__.py --- a/kallithea/lib/celerylib/__init__.py +++ b/kallithea/lib/celerylib/__init__.py @@ -72,8 +72,8 @@ def task(f_org): finally: log.info('executed %s task', f_org.__name__) f_async.__name__ = f_org.__name__ - import celery.task - runner = celery.task.task(ignore_result=True)(f_async) + from kallithea.lib import celerypylons + runner = celerypylons.task(ignore_result=True)(f_async) def f_wrapped(*args, **kwargs): t = runner.apply_async(args=args, kwargs=kwargs) log.info('executing task %s in async mode - id %s', f_org, t.task_id) diff --git a/kallithea/lib/celerypylons/__init__.py b/kallithea/lib/celerypylons/__init__.py --- a/kallithea/lib/celerypylons/__init__.py +++ b/kallithea/lib/celerypylons/__init__.py @@ -1,19 +1,36 @@ # -*- coding: utf-8 -*- """ -Automatically sets the environment variable `CELERY_LOADER` to -`celerypylons.loader:PylonsLoader`. This ensures the loader is -specified when accessing the rest of this package, and allows celery -to be installed in a webapp just by importing celerypylons:: +Kallithea wrapper of Celery + +The Celery configuration is in the normal Pylons ini file. We thus have to set +the `CELERY_LOADER` environment variable to point at a custom "loader" that can +read it. That environment variable must be set *before* importing celery. To +ensure that, we wrap celery in this module. - import celerypylons +Also, the loader depends on Pylons being configured to it can read the Celery +configuration out of it. To make sure that really is the case and give an early +warning, we check one of the mandatory settings. +This module must thus not be imported in global scope but must be imported on +demand in function scope. """ import os import warnings +# Verify Pylons configuration has been loaded +from pylons import config +assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded' + +# Prepare environment to point at Kallithea Pylons loader CELERYPYLONS_LOADER = 'kallithea.lib.celerypylons.loader.PylonsLoader' if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER: warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.") os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER + +# Import (and expose) celery, thus immediately triggering use of the custom Pylons loader +import celery.app as app +import celery.result as result +from celery.task import task +from celery.bin import camqadm, celerybeat, celeryd, celeryev diff --git a/kallithea/lib/celerypylons/commands.py b/kallithea/lib/celerypylons/commands.py --- a/kallithea/lib/celerypylons/commands.py +++ b/kallithea/lib/celerypylons/commands.py @@ -3,8 +3,7 @@ import kallithea from kallithea.lib.paster_commands.common import BasePasterCommand from kallithea.lib.utils import Command, load_rcextensions -from celery.app import app_or_default -from celery.bin import camqadm, celerybeat, celeryd, celeryev + from kallithea.lib.utils2 import str2bool @@ -26,12 +25,13 @@ class CeleryCommand(BasePasterCommand): allow options/arguments to be passed through to the underlying celery command. """ - - cmd = self.celery_command(app_or_default()) + from kallithea.lib import celerypylons + cmd = self.celery_command(celerypylons.app.app_or_default()) for x in cmd.get_options(): self.parser.add_option(x) def command(self): + from kallithea.lib import celerypylons from pylons import config try: CELERY_ON = str2bool(config['app_conf'].get('use_celery')) @@ -43,7 +43,7 @@ class CeleryCommand(BasePasterCommand): 'file before running celeryd') kallithea.CELERY_ON = CELERY_ON load_rcextensions(config['here']) - cmd = self.celery_command(app_or_default()) + cmd = self.celery_command(celerypylons.app.app_or_default()) return cmd.run(**vars(self.options)) @@ -58,7 +58,7 @@ class CeleryDaemonCommand(CeleryCommand) description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celeryd.WorkerCommand + celery_command = celerypylons.celeryd.WorkerCommand class CeleryBeatCommand(CeleryCommand): @@ -72,7 +72,7 @@ class CeleryBeatCommand(CeleryCommand): description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celerybeat.BeatCommand + celery_command = celerypylons.celerybeat.BeatCommand class CAMQPAdminCommand(CeleryCommand): @@ -85,7 +85,7 @@ class CAMQPAdminCommand(CeleryCommand): description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = camqadm.AMQPAdminCommand + celery_command = celerypylons.camqadm.AMQPAdminCommand class CeleryEventCommand(CeleryCommand): @@ -98,4 +98,4 @@ class CeleryEventCommand(CeleryCommand): description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celeryev.EvCommand + celery_command = celerypylons.celeryev.EvCommand diff --git a/kallithea/lib/paster_commands/common.py b/kallithea/lib/paster_commands/common.py --- a/kallithea/lib/paster_commands/common.py +++ b/kallithea/lib/paster_commands/common.py @@ -50,12 +50,6 @@ def ask_ok(prompt, retries=4, complaint= class BasePasterCommand(Command): """ Abstract Base Class for paster commands. - - The celery commands are somewhat aggressive about loading - celery.conf, and since our module sets the `CELERY_LOADER` - environment variable to our loader, we have to bootstrap a bit and - make sure we've had a chance to load the pylons config off of the - command line, otherwise everything fails. """ min_args = 1 min_args_error = "Please provide a paster config file as an argument."