kallithea Changeset - d5efb83590ef

Changeset - d5efb83590ef

Parent rev.

Child rev.

[Not reviewed]

default

0 7 0

Marcin Kuzminski - 15 years ago 2010-09-27 02:17:03
marcin@python-works.com

fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions!
added tredning languages stats

7 files changed with 165 insertions and 31 deletions:

celeryconfig.py

pylons_app/controllers/summary.py

pylons_app/lib/celerylib/__init__.py

pylons_app/lib/celerylib/tasks.py

pylons_app/lib/pidlock.py

pylons_app/public/css/style.css

pylons_app/templates/summary/summary.html

0 comments (0 inline, 0 general)

celeryconfig.py

➞

Show inline comments

@@ @@ -36,7 +36,7 @@ BROKER_PASSWORD = "qweqwe" @@
 CELERYD_CONCURRENCY = 2
 # CELERYD_LOG_FILE = "celeryd.log"
 CELERYD_LOG_LEVEL = "DEBUG"
-CELERYD_MAX_TASKS_PER_CHILD = 1
+CELERYD_MAX_TASKS_PER_CHILD = 3
 #Tasks will never be sent to the queue, but executed locally instead.
 CELERY_ALWAYS_EAGER = False

pylons_app/controllers/summary.py

➞

Show inline comments

@@ @@ -35,7 +35,7 @@ from datetime import datetime, timedelta @@
 from time import mktime
 import calendar
 import logging
+import json
 log = logging.getLogger(__name__)
 class SummaryController(BaseController):
@@ @@ -79,18 +79,25 @@ class SummaryController(BaseController): @@
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == c.repo_info.dbrepo)\
             .scalar()
         if stats:
         if stats and stats.languages:
             lang_stats = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
             c.trending_languages = json.dumps(OrderedDict(
                                        sorted(lang_stats.items(), reverse=True,
                                             key=lambda k: k[1])[:2]
+                                        )
+                                    )
             print c.trending_languages
         else:
             import json
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
             c.trending_languages = json.dumps({})
         return render('summary/summary.html')

pylons_app/lib/celerylib/__init__.py

➞

Show inline comments

@@ @@ -42,7 +42,7 @@ def locked_task(func): @@
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(lockkey)
-            return func(*fargs, **fkwargs)
             func(*fargs, **fkwargs)
             l.release()
         except LockHeld:
             log.info('LockHeld')

pylons_app/lib/celerylib/tasks.py

➞

Show inline comments

 from celery.decorators import task
 from celery.task.sets import subtask
 from celeryconfig import PYLONS_CONFIG as config
 from operator import itemgetter
 from pylons.i18n.translation import _
 from pylons_app.lib.celerylib import run_task, locked_task
 from pylons_app.lib.helpers import person
 from pylons_app.lib.smtp_mailer import SmtpMailer
 from pylons_app.lib.utils import OrderedDict
-from operator import itemgetter
+from time import mktime
 from vcs.backends.hg import MercurialRepository
-from time import mktime
+import json
 import traceback
 import json
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
@@ @@ -75,10 +75,10 @@ def whoosh_index(repo_location, full_ind @@
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from pylons_app.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
     author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
     from pylons_app.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
     repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
@@ @@ -99,7 +99,7 @@ def get_commits_stats(repo_name, ts_min_ @@
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     if last_rev == repo.revisions[-1]:
+    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work
         return True
@@ @@ -109,6 +109,7 @@ def get_commits_stats(repo_name, ts_min_ @@
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
     log.debug('starting parsing %s', parse_limit)
     for cnt, rev in enumerate(repo.revisions[last_rev:]):
         last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
@@ @@ -187,9 +188,16 @@ def get_commits_stats(repo_name, ts_min_ @@
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
     log.debug('last revison %s', last_rev)
     leftovers = len(repo.revisions[last_rev:])
     log.debug('revisions to parse %s', leftovers)
     if last_rev == 0 or leftovers < parse_limit:
         stats.languages = json.dumps(__get_codes_stats(repo_name))
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
     stats.languages = json.dumps({'_TOTAL_':0, '':0})
     try:
         sa.add(stats)
@@ @@ -198,7 +206,7 @@ def get_commits_stats(repo_name, ts_min_ @@
         log.error(traceback.format_exc())
         sa.rollback()
         return False
+    if len(repo.revisions) > 1:
     run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
     return True
@@ @@ -259,3 +267,31 @@ def send_email(recipients, subject, body @@
         log.error(traceback.format_exc())
         return False
     return True
 def __get_codes_stats(repo_name):
     LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
                     'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl',
                     'h', 'java', 'js', 'jsp', 'jspx', 'lisp',
                     'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
                     'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh',
                     'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
                     'yaws']
     repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
     repo = MercurialRepository(repos_path + repo_name)
     code_stats = {}
     for topnode, dirs, files in repo.walk('/', 'tip'):
         for f in files:
             k = f.mimetype
             if f.extension in LANGUAGES_EXTENSIONS:
                 if code_stats.has_key(k):
                     code_stats[k] += 1
                 else:
                     code_stats[k] = 1
     return code_stats or {}

pylons_app/lib/pidlock.py

➞

Show inline comments

 import os, time
 import sys
 from warnings import warn
 from multiprocessing.util import Finalize
 import errno
 class LockHeld(Exception):pass
@@ @@ -27,55 +29,67 @@ class DaemonLock(object): @@
         self.held = False
         #run the lock automatically !
         self.lock()
     def __del__(self):
         if self.held:
         self._finalize = Finalize(self, DaemonLock._on_finalize,
                                     args=(self, debug), exitpriority=10)
 #            warn("use lock.release instead of del lock",
 #                    category = DeprecationWarning,
 #                    stacklevel = 2)
             # ensure the lock will be removed
             self.release()
     @staticmethod
     def _on_finalize(lock, debug):
         if lock.held:
             if debug:
                 print 'leck held finilazing and running lock.release()'
             lock.release()
     def lock(self):
         """locking function, if lock is present it will raise LockHeld exception
         """
         lockname = '%s' % (os.getpid())
         if self.debug:
             print 'running lock'
         self.trylock()
         self.makelock(lockname, self.pidfile)
         return True
     def trylock(self):
         running_pid = False
         if self.debug:
             print 'checking for already running process'
         try:
             pidfile = open(self.pidfile, "r")
             pidfile.seek(0)
             running_pid = pidfile.readline()
             running_pid = int(pidfile.readline())
             pidfile.close()
             if self.debug:
                 print 'lock file present running_pid: %s, checking for execution'\
                 % running_pid
             # Now we check the PID from lock file matches to the current
             # process PID
             if running_pid:
                 if os.path.exists("/proc/%s" % running_pid):
                         print "You already have an instance of the program running"
                         print "It is running as process %s" % running_pid
                         raise LockHeld
                 else:
                 try:
                     os.kill(running_pid, 0)
                 except OSError, exc:
                     if exc.errno in (errno.ESRCH, errno.EPERM):
                         print "Lock File is there but the program is not running"
                         print "Removing lock file for the: %s" % running_pid
                         self.release()
                     raise
                 else:
                     print "You already have an instance of the program running"
                     print "It is running as process %s" % running_pid
                     raise LockHeld()
         except IOError, e:
             if e.errno != 2:
                 raise
     def release(self):
         """releases the pid by removing the pidfile
         """
         if self.debug:
             print 'trying to release the pidlock'
         if self.callbackfn:
             #execute callback function on release
             if self.debug:

pylons_app/public/css/style.css

➞

Show inline comments

@@ @@ -2963,6 +2963,28 @@ div.form div.fields div.buttons input @@
 /* -----------------------------------------------------------
 	SUMMARY
 ----------------------------------------------------------- */
 .trending_language_tbl, .trending_language_tbl td {
 	margin:  0px !important;
 	padding: 0px !important;
 	border: 0 !important;
+}
 .trending_language{
 	-moz-border-radius-bottomright:4px;
 	-moz-border-radius-topright:4px;
 	border-bottom-right-radius: 4px 4px;
 	border-top-right-radius: 4px 4px;
 	background-color:#336699;
 	color:#FFFFFF;
 	display:block;
 	min-width:20px;
 	max-width:400px;
 	padding:3px;
 	text-decoration:none;
 	height: 10px;
 	margin-bottom: 4px;
 	margin-left: 5px;
+}
 #clone_url{
 	border: none;

pylons_app/templates/summary/summary.html

➞

Show inline comments

@@ @@ -93,6 +93,61 @@ E.onDOMReady(function(e){ @@
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Trending languages')}:</label>
 			  </div>
 			  <div class="input-short">
 			    <div id="lang_stats">
 			    </div>
 			  	<script type="text/javascript">
 			  		var data = ${c.trending_languages|n};
 			  		var total = 0;
 			  		var no_data = true;
 			  		for (k in data){
 			  		    total += data[k];
 			  		    no_data = false;
+			  		}
 					var tbl = document.createElement('table');
 					tbl.setAttribute('class','trending_language_tbl');
 			  		for (k in data){
 				  		var tr = document.createElement('tr');
 				  		var percentage = Math.round((data[k]/total*100),2);
 						var value = data[k];
 				  		var td1 = document.createElement('td');
 				  		td1.width=150;
 				  		var trending_language_label = document.createElement('div');
 				  		trending_language_label.innerHTML = k;
 				  		td1.appendChild(trending_language_label);
 				  		var td2 = document.createElement('td');
 			  		    var trending_language = document.createElement('div');
 			  		    trending_language.title = k;
 			  		    trending_language.innerHTML = "<b>"+value+" ${_('files')} - "+percentage+" %</b>";
 			  		    trending_language.setAttribute("class", 'trending_language');
 			  		    trending_language.style.width=percentage+"%";
 						td2.appendChild(trending_language);
 						tr.appendChild(td1);
 						tr.appendChild(td2);
 			  		    tbl.appendChild(tr);
  				  		//YAHOO.util.Dom.get('lang_stats').appendChild(trending_language_label);
+			  		}
 			  		if(no_data){
 			  			var tr = document.createElement('tr');
 			  			var td1 = document.createElement('td');
 			  			td1.innerHTML = "${_('No data loaded yet...')}";
 			  			tr.appendChild(td1);
 			  			tbl.appendChild(tr);
+					}
 			  		YAHOO.util.Dom.get('lang_stats').appendChild(tbl);
 			  	</script>
 			  </div>
 			 </div>
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Download')}:</label>
 			  </div>
 			  <div class="input-short">

0 comments (0 inline, 0 general)