kallithea Changeset - 277427ac29a9

Changeset - 277427ac29a9

Parent rev.

Child rev.

[Not reviewed]

beta

0 4 0

Marcin Kuzminski - 15 years ago 2010-11-28 05:31:05
marcin@python-works.com

complete rewrite of paster commands,
used fancy celery-pylons baseCommand to implement better support for paster commands
fixed make-index, and paster celeryd. Moved config's add_cache to utils

4 files changed with 131 insertions and 116 deletions:

rhodecode/lib/celerylib/tasks.py

rhodecode/lib/celerypylons/commands.py

rhodecode/lib/indexers/__init__.py

rhodecode/lib/utils.py

0 comments (0 inline, 0 general)

rhodecode/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -2,7 +2,6 @@ from celery.decorators import task @@
 import os
 import traceback
 import beaker
 from time import mktime
 from operator import itemgetter
@@ @@ -12,7 +11,7 @@ from pylons.i18n.translation import _ @@
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict
+from rhodecode.lib.utils import OrderedDict, add_cache
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import RhodeCodeUi
@@ @@ -21,29 +20,6 @@ from vcs.backends import get_repo @@
 from sqlalchemy import engine_from_config
 #set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions':None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
 ))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 add_cache(config)
 try:

rhodecode/lib/celerypylons/commands.py

➞

Show inline comments

 import os
 from paste.script.command import Command, BadCommand
 import paste.deploy
 from pylons import config
 from rhodecode.lib.utils import BasePasterCommand, Command
 __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
            'CAMQPAdminCommand', 'CeleryEventCommand']
 class CeleryCommand(Command):
     """
     Abstract Base Class for celery commands.
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
     def run(self, args):
         """
         Overrides Command.run
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(CeleryCommand, self).run(args[1:])
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + path_to_ini_file)
         config.init_app(conf.global_conf, conf.local_conf)
 class CeleryDaemonCommand(CeleryCommand):
 class CeleryDaemonCommand(BasePasterCommand):
     """Start the celery worker
     Starts the celery worker that uses a paste.deploy configuration
@@ @@ -80,7 +27,7 @@ class CeleryDaemonCommand(CeleryCommand) @@
         return celeryd.WorkerCommand().run(**vars(self.options))
-class CeleryBeatCommand(CeleryCommand):
+class CeleryBeatCommand(BasePasterCommand):
     """Start the celery beat server
     Starts the celery beat server using a paste.deploy configuration
@@ @@ -101,7 +48,7 @@ class CeleryBeatCommand(CeleryCommand): @@
         from celery.bin import celerybeat
         return celerybeat.BeatCommand(**vars(self.options))
-class CAMQPAdminCommand(CeleryCommand):
+class CAMQPAdminCommand(BasePasterCommand):
     """CAMQP Admin
     CAMQP celery admin tool.
@@ @@ -122,7 +69,7 @@ class CAMQPAdminCommand(CeleryCommand): @@
         return camqadm.camqadm(*self.args, **vars(self.options))
-class CeleryEventCommand(CeleryCommand):
+class CeleryEventCommand(BasePasterCommand):
     """Celery event commandd.
     Capture celery events.

rhodecode/lib/indexers/__init__.py

➞

Show inline comments

 import os
 import sys
 import traceback
 from os.path import dirname as dn, join as jn
 #to get the rhodecode import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 from rhodecode.model import init_model
 from rhodecode.model.scm import ScmModel
 from rhodecode.config.environment import load_environment
 from rhodecode.model.scm import ScmModel
 from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 from sqlalchemy import engine_from_config
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
 import traceback
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
@@ @@ -45,10 +50,8 @@ IDX_NAME = 'HG_INDEX' @@
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = SimpleFragmenter(200)
 from paste.script import command
 import ConfigParser
-class MakeIndex(command.Command):
+class MakeIndex(BasePasterCommand):
     max_args = 1
     min_args = 1
@@ @@ -57,26 +60,16 @@ class MakeIndex(command.Command): @@
     summary = "Creates index for full text search given configuration file"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = command.Command.standard_parser(verbose=True)
     parser.add_option('--repo-location',
                       action='store',
                       dest='repo_location',
                       help="Specifies repositories location to index REQUIRED",
+                      )
     parser.add_option('-f',
                       action='store_true',
                       dest='full_index',
                       help="Specifies that index should be made full i.e"
                             " destroy old and build from scratch",
                       default=False)
     parser = Command.standard_parser(verbose=True)
     def command(self):
         config_name = self.args[0]
         p = config_name.split('/')
         root = '.' if len(p) == 1 else '/'.join(p[:-1])
         config = ConfigParser.ConfigParser({'here':root})
         config.read(config_name)
         index_location = dict(config.items('app:main'))['index_dir']
         from pylons import config
         add_cache(config)
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
         index_location = config['index_dir']
         repo_location = self.options.repo_location
         #======================================================================
@@ @@ -93,6 +86,18 @@ class MakeIndex(command.Command): @@
         except LockHeld:
             sys.exit(1)
     def update_parser(self):
         self.parser.add_option('--repo-location',
                           action='store',
                           dest='repo_location',
                           help="Specifies repositories location to index REQUIRED",
+                          )
         self.parser.add_option('-f',
                           action='store_true',
                           dest='full_index',
                           help="Specifies that index should be made full i.e"
                                 " destroy old and build from scratch",
                           default=False)
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -36,7 +36,10 @@ from UserDict import DictMixin @@
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 from paste.script import command
 import paste
 import beaker
 from paste.script.command import Command, BadCommand
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
@@ @@ -416,6 +419,31 @@ class OrderedDict(dict, DictMixin): @@
         return not self == other
 #set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions':None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
 ))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 #===============================================================================
 # TEST FUNCTIONS AND CREATORS
 #===============================================================================
@@ @@ -498,7 +526,66 @@ def create_test_env(repos_test_path, con @@
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 class UpgradeDb(command.Command):
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
     def run(self, args):
         """
         Overrides Command.run
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
         path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)
 class UpgradeDb(BasePasterCommand):
     """Command used for paster to upgrade our database to newer version
     """
@@ @@ -509,16 +596,16 @@ class UpgradeDb(command.Command): @@
     summary = "Upgrades current db to newer version given configuration file"
     group_name = "RhodeCode"
-    parser = command.Command.standard_parser(verbose=True)
     parser = Command.standard_parser(verbose=True)
     parser.add_option('--sql',
     def command(self):
         from pylons import config
         raise NotImplementedError('Not implemented yet')
     def update_parser(self):
         self.parser.add_option('--sql',
                       action='store_true',
                       dest='just_sql',
                       help="Prints upgrade sql for further investigation",
                       default=False)
     def command(self):
         config_name = self.args[0]
         p = config_name.split('/')
         root = '.' if len(p) == 1 else '/'.join(p[:-1])
         config = ConfigParser.ConfigParser({'here':root})
         config.read(config_name)

0 comments (0 inline, 0 general)