Changeset - cf73bd884a53
[Not reviewed]
default
1 3 1
Mads Kiilerich - 9 years ago 2016-09-06 00:51:18
madski@unity3d.com
celeryd: move the celeryd paster command to the place where paster commands live

Also, partial implementations of other Celery functionality is dropped.
4 files changed with 16 insertions and 73 deletions:
0 comments (0 inline, 0 general)
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the normal Pylons ini file. We thus have to set
 
the `CELERY_LOADER` environment variable to point at a custom "loader" that can
 
read it. That environment variable must be set *before* importing celery. To
 
ensure that, we wrap celery in this module.
 

	
 
Also, the loader depends on Pylons being configured to it can read the Celery
 
configuration out of it. To make sure that really is the case and give an early
 
warning, we check one of the mandatory settings.
 

	
 
This module must thus not be imported in global scope but must be imported on
 
demand in function scope.
 
"""
 

	
 
import os
 
import warnings
 

	
 
# Verify Pylons configuration has been loaded
 
from pylons import config
 
assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
 

	
 
# Prepare environment to point at Kallithea Pylons loader
 
CELERYPYLONS_LOADER = 'kallithea.lib.celerypylons.loader.PylonsLoader'
 
if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER:
 
    warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.")
 
os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER
 

	
 
# Import (and expose) celery, thus immediately triggering use of the custom Pylons loader
 
import celery.app as app
 
import celery.result as result
 
from celery.task import task
 
from celery.bin import camqadm, celerybeat, celeryd, celeryev
 
from celery.bin import celeryd
kallithea/lib/paster_commands/celeryd.py
Show inline comments
 
file renamed from kallithea/lib/celerypylons/commands.py to kallithea/lib/paster_commands/celeryd.py
 
# -*- coding: utf-8 -*-
 

	
 
import kallithea
 
from kallithea.lib.paster_commands.common import BasePasterCommand
 
from kallithea.lib.utils import Command, load_rcextensions
 

	
 

	
 
from kallithea.lib.utils import load_rcextensions
 
from kallithea.lib.utils2 import str2bool
 

	
 
__all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
 
           'CAMQPAdminCommand', 'CeleryEventCommand']
 
__all__ = ['Command']
 

	
 

	
 
class CeleryCommand(BasePasterCommand):
 
    """Abstract class implements run methods needed for celery
 
class Command(BasePasterCommand):
 
    """Start the celery worker
 

	
 
    Starts the celery worker that uses a paste.deploy configuration
 
    file.
 
    """
 

	
 
    usage = 'CONFIG_FILE [celeryd options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 
    group_name = "Kallithea"
 

	
 
    parser = BasePasterCommand.standard_parser(quiet=True)
 

	
 
    def update_parser(self):
 
        """
 
        Abstract method.  Allows for the class's parser to be updated
 
        before the superclass's `run` method is called.  Necessary to
 
        allow options/arguments to be passed through to the underlying
 
        celery command.
 
        """
 
        from kallithea.lib import celerypylons
 
        cmd = self.celery_command(celerypylons.app.app_or_default())
 
        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
 
        for x in cmd.get_options():
 
            self.parser.add_option(x)
 

	
 
    def command(self):
 
        from kallithea.lib import celerypylons
 
        from pylons import config
 
        try:
 
            CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
        except KeyError:
 
            CELERY_ON = False
 

	
 
        if not CELERY_ON:
 
            raise Exception('Please set use_celery = true in .ini config '
 
                            'file before running celeryd')
 
        kallithea.CELERY_ON = CELERY_ON
 

	
 
        load_rcextensions(config['here'])
 
        cmd = self.celery_command(celerypylons.app.app_or_default())
 
        cmd = celerypylons.celeryd.WorkerCommand(celerypylons.app.app_or_default())
 
        return cmd.run(**vars(self.options))
 

	
 

	
 
class CeleryDaemonCommand(CeleryCommand):
 
    """Start the celery worker
 

	
 
    Starts the celery worker that uses a paste.deploy configuration
 
    file.
 
    """
 
    usage = 'CONFIG_FILE [celeryd options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 

	
 
    parser = Command.standard_parser(quiet=True)
 
    celery_command = celerypylons.celeryd.WorkerCommand
 

	
 

	
 
class CeleryBeatCommand(CeleryCommand):
 
    """Start the celery beat server
 

	
 
    Starts the celery beat server using a paste.deploy configuration
 
    file.
 
    """
 
    usage = 'CONFIG_FILE [celerybeat options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 

	
 
    parser = Command.standard_parser(quiet=True)
 
    celery_command = celerypylons.celerybeat.BeatCommand
 

	
 

	
 
class CAMQPAdminCommand(CeleryCommand):
 
    """CAMQP Admin
 

	
 
    CAMQP celery admin tool.
 
    """
 
    usage = 'CONFIG_FILE [camqadm options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 

	
 
    parser = Command.standard_parser(quiet=True)
 
    celery_command = celerypylons.camqadm.AMQPAdminCommand
 

	
 

	
 
class CeleryEventCommand(CeleryCommand):
 
    """Celery event command.
 

	
 
    Capture celery events.
 
    """
 
    usage = 'CONFIG_FILE [celeryev options...]'
 
    summary = __doc__.splitlines()[0]
 
    description = "".join(__doc__.splitlines()[2:])
 

	
 
    parser = Command.standard_parser(quiet=True)
 
    celery_command = celerypylons.celeryev.EvCommand
kallithea/lib/utils.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.lib.utils
 
~~~~~~~~~~~~~~~~~~~
 

	
 
Utilities library for Kallithea
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 18, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import os
 
import re
 
import logging
 
import datetime
 
import traceback
 
import paste
 
import beaker
 
import tarfile
 
import shutil
 
import decorator
 
import warnings
 
from os.path import abspath
 
from os.path import dirname
 

	
 
from paste.script.command import Command, BadCommand
 

	
 
from webhelpers.text import collapse, remove_formatting, strip_tags
 
from beaker.cache import _cache_decorate
 

	
 
from kallithea.lib.vcs.utils.hgcompat import ui, config
 
from kallithea.lib.vcs.utils.helpers import get_scm
 
from kallithea.lib.vcs.exceptions import VCSError
 

	
 
from kallithea.model import meta
 
from kallithea.model.db import Repository, User, Ui, \
 
    UserLog, RepoGroup, Setting, UserGroup
 
from kallithea.model.meta import Session
 
from kallithea.model.repo_group import RepoGroupModel
 
from kallithea.lib.utils2 import safe_str, safe_unicode, get_current_authuser
 
from kallithea.lib.vcs.utils.fakemod import create_module
 

	
 
log = logging.getLogger(__name__)
 

	
 
REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 

	
 

	
 
def recursive_replace(str_, replace=' '):
 
    """
 
    Recursive replace of given sign to just one instance
 

	
 
    :param str_: given string
 
    :param replace: char to find and replace multiple instances
 

	
 
    Examples::
 
    >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
 
    'Mighty-Mighty-Bo-sstones'
 
    """
 

	
 
    if str_.find(replace * 2) == -1:
 
        return str_
 
    else:
 
        str_ = str_.replace(replace * 2, replace)
 
        return recursive_replace(str_, replace)
 

	
 

	
 
def repo_name_slug(value):
 
    """
 
    Return slug of name of repository
 
    This function is called on each creation/modification
 
    of repository to prevent bad names in repo
 
    """
 

	
 
    slug = remove_formatting(value)
 
    slug = strip_tags(slug)
setup.py
Show inline comments
 
@@ -121,52 +121,52 @@ sdist.sdist = sdist_new
 
packages = setuptools.find_packages(exclude=['ez_setup'])
 

	
 
setuptools.setup(
 
    name='Kallithea',
 
    version=__version__,
 
    description=description,
 
    long_description=long_description,
 
    keywords=keywords,
 
    license=__license__,
 
    author=__author__,
 
    author_email='kallithea@sfconservancy.org',
 
    dependency_links=dependency_links,
 
    url=__url__,
 
    install_requires=requirements,
 
    classifiers=classifiers,
 
    setup_requires=['PasteScript>=1.6.3'],
 
    data_files=data_files,
 
    packages=packages,
 
    include_package_data=True,
 
    message_extractors={'kallithea': [
 
            ('**.py', 'python', None),
 
            ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
 
            ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
 
            ('public/**', 'ignore', None)]},
 
    zip_safe=False,
 
    paster_plugins=['PasteScript', 'Pylons'],
 
    entry_points="""
 
    [console_scripts]
 
    kallithea-api =    kallithea.bin.kallithea_api:main
 
    kallithea-gist =   kallithea.bin.kallithea_gist:main
 
    kallithea-config = kallithea.bin.kallithea_config:main
 

	
 
    [paste.app_factory]
 
    main = kallithea.config.middleware:make_app
 

	
 
    [paste.app_install]
 
    main = pylons.util:PylonsInstaller
 

	
 
    [paste.global_paster_command]
 
    setup-db=kallithea.lib.paster_commands.setup_db:Command
 
    cleanup-repos=kallithea.lib.paster_commands.cleanup:Command
 
    update-repoinfo=kallithea.lib.paster_commands.update_repoinfo:Command
 
    make-rcext=kallithea.lib.paster_commands.make_rcextensions:Command
 
    repo-scan=kallithea.lib.paster_commands.repo_scan:Command
 
    cache-keys=kallithea.lib.paster_commands.cache_keys:Command
 
    ishell=kallithea.lib.paster_commands.ishell:Command
 
    make-index=kallithea.lib.paster_commands.make_index:Command
 
    upgrade-db=kallithea.lib.dbmigrate:UpgradeDb
 
    celeryd=kallithea.lib.celerypylons.commands:CeleryDaemonCommand
 
    celeryd=kallithea.lib.paster_commands.celeryd:Command
 
    install-iis=kallithea.lib.paster_commands.install_iis:Command
 
    """,
 
)
0 comments (0 inline, 0 general)