kallithea Changeset - fb0c3af6031b

Changeset - fb0c3af6031b

Parent rev.

Child rev.

[Not reviewed]

celery

1 6 1

Marcin Kuzminski - 15 years ago 2010-09-23 01:08:33
marcin@python-works.com

Implemented locking for task, to prevent for running the same tasks,
moved out pidlock library. Added dirsize display

7 files changed with 60 insertions and 20 deletions:

pylons_app/lib/celerylib/__init__.py

pylons_app/lib/celerylib/tasks.py

pylons_app/lib/indexers/__init__.py

pylons_app/lib/indexers/daemon.py

pylons_app/lib/pidlock.py

pylons_app/lib/utils.py

pylons_app/templates/files/files_browser.html

0 comments (0 inline, 0 general)

pylons_app/lib/celerylib/__init__.py

➞

Show inline comments

 from pylons_app.lib.pidlock import DaemonLock, LockHeld
 from vcs.utils.lazy import LazyProperty
 from decorator import decorator
 import logging
 import os
 import sys
 import traceback
+from hashlib import md5
 log = logging.getLogger(__name__)
 class ResultWrapper(object):
@@ @@ -20,6 +22,7 @@ def run_task(task, *args, **kwargs): @@
         log.info('running task %s', t.task_id)
         return t
     except Exception, e:
         print e
         if e.errno == 111:
             log.debug('Unnable to connect. Sync execution')
         else:
@@ @@ -27,3 +30,37 @@ def run_task(task, *args, **kwargs): @@
         #pure sync version
         return ResultWrapper(task(*args, **kwargs))
 class LockTask(object):
     """LockTask decorator"""
     def __init__(self, func):
         self.func = func
     def __call__(self, func):
         return decorator(self.__wrapper, func)
     def __wrapper(self, func, *fargs, **fkwargs):
         params = []
         params.extend(fargs)
         params.extend(fkwargs.values())
         lockkey = 'task_%s' % \
            md5(str(self.func) + '-' + '-'.join(map(str, params))).hexdigest()
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(lockkey)
             return func(*fargs, **fkwargs)
             l.release()
         except LockHeld:
             log.info('LockHeld')
             return 'Task with key %s already running' % lockkey

pylons_app/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -2,7 +2,7 @@ from celery.decorators import task @@
 from celery.task.sets import subtask
 from celeryconfig import PYLONS_CONFIG as config
 from pylons.i18n.translation import _
 from pylons_app.lib.celerylib import run_task
+from pylons_app.lib.celerylib import run_task, LockTask
 from pylons_app.lib.helpers import person
 from pylons_app.lib.smtp_mailer import SmtpMailer
 from pylons_app.lib.utils import OrderedDict
@@ @@ -68,7 +68,7 @@ def get_hg_ui_settings(): @@
 @task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
-    from pylons_app.lib.indexers import DaemonLock
+    from pylons_app.lib.pidlock import DaemonLock
     from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld
     try:
         l = DaemonLock()
@@ @@ -80,7 +80,9 @@ def whoosh_index(repo_location, full_ind @@
         log.info('LockHeld')
         return 'LockHeld'
 @task
 @LockTask('get_commits_stats')
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
@@ @@ -92,7 +94,7 @@ def get_commits_stats(repo_name, ts_min_ @@
     repo = MercurialRepository(repos_path + repo_name)
     skip_date_limit = True
-    parse_limit = 500 #limit for single task changeset parsing
+    parse_limit = 350 #limit for single task changeset parsing
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
@@ @@ -206,6 +208,8 @@ def get_commits_stats(repo_name, ts_min_ @@
         sa.rollback()
         return False
     run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
     return True
 @task

pylons_app/lib/indexers/__init__.py

➞

Show inline comments

 from os.path import dirname as dn, join as jn
 from pidlock import LockHeld, DaemonLock
 from pylons_app.config.environment import load_environment
 from pylons_app.model.hg_model import HgModel
 from shutil import rmtree

pylons_app/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -32,7 +32,7 @@ from os.path import join as jn @@
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 from pidlock import LockHeld, DaemonLock
+from pylons_app.lib.pidlock import LockHeld, DaemonLock
 from pylons_app.model.hg_model import HgModel
 from pylons_app.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir

pylons_app/lib/pidlock.py

➞

Show inline comments

@@ @@ -6,7 +6,7 @@ class LockHeld(Exception):pass @@
 class DaemonLock(object):
-    '''daemon locking
+    """daemon locking
     USAGE:
     try:
         l = lock()
@@ @@ -14,7 +14,7 @@ class DaemonLock(object): @@
         l.release()
     except LockHeld:
         sys.exit(1)
-    '''
+    """
     def __init__(self, file=None, callbackfn=None,
                  desc='daemon lock', debug=False):
@@ @@ -40,9 +40,9 @@ class DaemonLock(object): @@
     def lock(self):
-        '''
+        """
         locking function, if lock is present it will raise LockHeld exception
-        '''
+        """
         lockname = '%s' % (os.getpid())
         self.trylock()
@@ @@ -75,9 +75,9 @@ class DaemonLock(object): @@
     def release(self):
-        '''
+        """
         releases the pid by removing the pidfile
-        '''
+        """
         if self.callbackfn:
             #execute callback function on release
             if self.debug:
@@ @@ -94,11 +94,11 @@ class DaemonLock(object): @@
             pass
     def makelock(self, lockname, pidfile):
-        '''
+        """
         this function will make an actual lock
         @param lockname: acctual pid of file
         @param pidfile: the file to write the pid in
-        '''
+        """
         if self.debug:
             print 'creating a file %s and pid: %s' % (pidfile, lockname)
         pidfile = open(self.pidfile, "wb")

pylons_app/lib/utils.py

➞

Show inline comments

@@ @@ -374,9 +374,8 @@ def create_test_index(repo_location, ful @@
     @param repo_location:
     @param full_index:
     """
     from pylons_app.lib.indexers import daemon
     from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon
-    from pylons_app.lib.indexers.pidlock import DaemonLock, LockHeld
     from pylons_app.lib.pidlock import DaemonLock, LockHeld
     from pylons_app.lib.indexers import IDX_LOCATION
     import shutil

pylons_app/templates/files/files_browser.html

➞

Show inline comments

@@ @@ -29,26 +29,27 @@ @@
 		                 <th>${_('Last commiter')}</th>
 		             </tr>
 		         </thead>
           		% if c.files_list.parent:
 		         	<tr class="parity0">
 		          		<td>
 		          		% if c.files_list.parent:
 		          			${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")}
 		          		%endif
 		          		</td>
 		          		<td></td>
 		          		<td></td>
 		          		<td></td>
 		          		<td></td>
 	          		<td></td>
 		         	</tr>
           		%endif
 		    %for cnt,node in enumerate(c.files_list,1):
 				<tr class="parity${cnt%2}">
 		             <td>
 						${h.link_to(node.name,h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=node.path),class_=file_class(node))}
 		             </td>
 		             <td>
 		                %if node.is_file():
 		             		${h.format_byte_size(node.size,binary=True)}
 		             	%endif
 		             </td>
 		             <td>
 		              %if node.is_file():

0 comments (0 inline, 0 general)