Changeset - 4f2e231df222
[Not reviewed]
default
0 3 0
Mads Kiilerich - 9 years ago 2016-09-06 00:51:18
madski@unity3d.com
celery: upgrade to Celery 3

Celery 3 is more than 2 and is the future and should be more stable than Celery 2.

There might be problems like
PRECONDITION_FAILED - cannot redeclare exchange 'celeryresults' in vhost 'kallithea' with different type, durable, internal or autodelete value
when celery is upgraded and it tries to upgrade existing tasks.

One way to avoid that problem:
- make sure celery has run to completion and there are no important pending tasks,
- delete the old vhost,
- upgrade celery,
- create new vhost
- set permissions for vhost
- profit
3 files changed with 4 insertions and 4 deletions:
0 comments (0 inline, 0 general)
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the normal Pylons ini file. We thus have to set
 
the `CELERY_LOADER` environment variable to point at a custom "loader" that can
 
read it. That environment variable must be set *before* importing celery. To
 
ensure that, we wrap celery in this module.
 

	
 
Also, the loader depends on Pylons being configured to it can read the Celery
 
configuration out of it. To make sure that really is the case and give an early
 
warning, we check one of the mandatory settings.
 

	
 
This module must thus not be imported in global scope but must be imported on
 
demand in function scope.
 
"""
 

	
 
import os
 
import warnings
 

	
 
# Verify Pylons configuration has been loaded
 
from pylons import config
 
assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
 

	
 
# Prepare environment to point at Kallithea Pylons loader
 
CELERYPYLONS_LOADER = 'kallithea.lib.celerypylons.loader.PylonsLoader'
 
if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER:
 
    warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.")
 
os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER
 

	
 
# Import (and expose) celery, thus immediately triggering use of the custom Pylons loader
 
import celery.app as app
 
import celery.result as result
 
from celery.task import task
 
from celery.bin import celeryd
 
from celery.bin import worker
kallithea/lib/paster_commands/celeryd.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
import kallithea
 
from kallithea.lib.paster_commands.common import BasePasterCommand
 
from kallithea.lib.utils import load_rcextensions
 
from kallithea.lib.utils2 import str2bool
 

	
 
__all__ = ['Command']
 

	
 

	
 
class Command(BasePasterCommand):
 
    """Start the celery worker
 

	
 
    Starts the celery worker that uses a paste.deploy configuration
 
    file.
 
    """
 

	
 
    usage = 'CONFIG_FILE [celeryd options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 
    group_name = "Kallithea"
 

	
 
    parser = BasePasterCommand.standard_parser(quiet=True)
 

	
 
    def update_parser(self):
 
        from kallithea.lib import celerypylons
 
        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
 
        cmd = celerypylons.worker.worker(celerypylons.app.app_or_default())
 
        for x in cmd.get_options():
 
            self.parser.add_option(x)
 

	
 
    def command(self):
 
        from kallithea.lib import celerypylons
 
        from pylons import config
 
        try:
 
            CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
        except KeyError:
 
            CELERY_ON = False
 

	
 
        if not CELERY_ON:
 
            raise Exception('Please set use_celery = true in .ini config '
 
                            'file before running celeryd')
 
        kallithea.CELERY_ON = CELERY_ON
 

	
 
        load_rcextensions(config['here'])
 
        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
 
        cmd = celerypylons.worker.worker(celerypylons.app.app_or_default())
 
        return cmd.run(**vars(self.options))
setup.py
Show inline comments
 
@@ -2,97 +2,97 @@
 
# -*- coding: utf-8 -*-
 
import os
 
import sys
 
import platform
 

	
 
if sys.version_info < (2, 6) or sys.version_info >= (3,):
 
    raise Exception('Kallithea requires python 2.6 or 2.7')
 

	
 

	
 
here = os.path.abspath(os.path.dirname(__file__))
 

	
 

	
 
def _get_meta_var(name, data, callback_handler=None):
 
    import re
 
    matches = re.compile(r'(?:%s)\s*=\s*(.*)' % name).search(data)
 
    if matches:
 
        if not callable(callback_handler):
 
            callback_handler = lambda v: v
 

	
 
        return callback_handler(eval(matches.groups()[0]))
 

	
 
_meta = open(os.path.join(here, 'kallithea', '__init__.py'), 'rb')
 
_metadata = _meta.read()
 
_meta.close()
 

	
 
callback = lambda V: ('.'.join(map(str, V[:3])) + '.'.join(V[3:]))
 
__version__ = _get_meta_var('VERSION', _metadata, callback)
 
__license__ = _get_meta_var('__license__', _metadata)
 
__author__ = _get_meta_var('__author__', _metadata)
 
__url__ = _get_meta_var('__url__', _metadata)
 
# defines current platform
 
__platform__ = platform.system()
 

	
 
is_windows = __platform__ in ['Windows']
 

	
 
requirements = [
 
    "alembic>=0.8.0,<0.9",
 
    "waitress>=0.8.8,<1.0",
 
    "webob>=1.0.8,<=1.1.1",
 
    "Pylons>=1.0.0,<=1.0.2",
 
    "WebTest<2.0", # make the Pylons dependency pick a version that supports our WebOb version
 
    "Beaker==1.6.4",
 
    "WebHelpers==1.3",
 
    "formencode>=1.2.4,<=1.2.6",
 
    "SQLAlchemy>=1.0,<1.1",
 
    "Mako>=0.9.0,<=1.0.0",
 
    "pygments>=1.5",
 
    "whoosh>=2.4.0,<=2.5.7",
 
    "celery>=2.2.5,<2.3",
 
    "celery>=3.1,<3.2",
 
    "babel>=0.9.6,<=1.3",
 
    "python-dateutil>=1.5.0,<2.0.0",
 
    "markdown==2.2.1",
 
    "docutils>=0.8.1,<=0.11",
 
    "URLObject==2.3.4",
 
    "Routes==1.13",
 
    "dulwich>=0.14.1",
 
    "mercurial>=2.9,<4.0",
 
]
 

	
 
if sys.version_info < (2, 7):
 
    requirements.append("importlib==1.0.1")
 
    requirements.append("argparse")
 

	
 
if not is_windows:
 
    requirements.append("bcrypt>=2.0.0")
 

	
 
dependency_links = [
 
]
 

	
 
classifiers = [
 
    'Development Status :: 4 - Beta',
 
    'Environment :: Web Environment',
 
    'Framework :: Pylons',
 
    'Intended Audience :: Developers',
 
    'License :: OSI Approved :: GNU General Public License (GPL)',
 
    'Operating System :: OS Independent',
 
    'Programming Language :: Python',
 
    'Programming Language :: Python :: 2.6',
 
    'Programming Language :: Python :: 2.7',
 
    'Topic :: Software Development :: Version Control',
 
]
 

	
 

	
 
# additional files from project that goes somewhere in the filesystem
 
# relative to sys.prefix
 
data_files = []
 

	
 
description = ('Kallithea is a fast and powerful management tool '
 
               'for Mercurial and Git with a built in push/pull server, '
 
               'full text search and code-review.')
 

	
 
keywords = ' '.join([
 
    'kallithea', 'mercurial', 'git', 'code review',
 
    'repo groups', 'ldap', 'repository management', 'hgweb replacement',
 
    'hgwebdir', 'gitweb replacement', 'serving hgweb',
 
])
 

	
0 comments (0 inline, 0 general)