Changeset - d5efb83590ef
[Not reviewed]
default
0 7 0
Marcin Kuzminski - 15 years ago 2010-09-27 02:17:03
marcin@python-works.com
fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions!
added tredning languages stats
7 files changed with 165 insertions and 31 deletions:
0 comments (0 inline, 0 general)
celeryconfig.py
Show inline comments
 
@@ -33,13 +33,13 @@ BROKER_PASSWORD = "qweqwe"
 
## but if mostly spending CPU, try to keep it close to the
 
## number of CPUs on your machine. If not set, the number of CPUs/cores
 
## available will be used.
 
CELERYD_CONCURRENCY = 2
 
# CELERYD_LOG_FILE = "celeryd.log"
 
CELERYD_LOG_LEVEL = "DEBUG"
 
CELERYD_MAX_TASKS_PER_CHILD = 1
 
CELERYD_MAX_TASKS_PER_CHILD = 3
 

	
 
#Tasks will never be sent to the queue, but executed locally instead.
 
CELERY_ALWAYS_EAGER = False
 

	
 
#===============================================================================
 
# EMAIL SETTINGS
pylons_app/controllers/summary.py
Show inline comments
 
@@ -32,13 +32,13 @@ from webhelpers.paginate import Page
 
from pylons_app.lib.celerylib import run_task
 
from pylons_app.lib.celerylib.tasks import get_commits_stats
 
from datetime import datetime, timedelta
 
from time import mktime
 
import calendar
 
import logging
 

	
 
import json
 
log = logging.getLogger(__name__)
 

	
 
class SummaryController(BaseController):
 
    
 
    @LoginRequired()
 
    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
 
@@ -76,21 +76,28 @@ class SummaryController(BaseController):
 
        ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
 
            
 
        run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y)
 
        c.ts_min = ts_min_m
 
        c.ts_max = ts_max_y
 
        
 
        
 
        stats = self.sa.query(Statistics)\
 
            .filter(Statistics.repository == c.repo_info.dbrepo)\
 
            .scalar()
 

	
 
        if stats:
 
        
 
        if stats and stats.languages:
 
            lang_stats = json.loads(stats.languages)
 
            c.commit_data = stats.commit_activity
 
            c.overview_data = stats.commit_activity_combined
 
            c.trending_languages = json.dumps(OrderedDict(
 
                                       sorted(lang_stats.items(), reverse=True,
 
                                            key=lambda k: k[1])[:2]
 
                                        )
 
                                    )
 
            print c.trending_languages
 
        else:
 
            import json
 
            c.commit_data = json.dumps({})
 
            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
 
            c.trending_languages = json.dumps({})
 
        
 
        return render('summary/summary.html')
 

	
pylons_app/lib/celerylib/__init__.py
Show inline comments
 
@@ -39,13 +39,13 @@ def locked_task(func):
 
        lockkey = 'task_%s' % \
 
            md5(str(func.__name__) + '-' + \
 
                '-'.join(map(str, params))).hexdigest()
 
        log.info('running task with lockkey %s', lockkey)
 
        try:
 
            l = DaemonLock(lockkey)
 
            return func(*fargs, **fkwargs)
 
            func(*fargs, **fkwargs)
 
            l.release()
 
        except LockHeld:
 
            log.info('LockHeld')
 
            return 'Task with key %s already running' % lockkey   
 

	
 
    return decorator(__wrapper, func)      
pylons_app/lib/celerylib/tasks.py
Show inline comments
 
from celery.decorators import task
 
from celery.task.sets import subtask
 
from celeryconfig import PYLONS_CONFIG as config
 
from operator import itemgetter
 
from pylons.i18n.translation import _
 
from pylons_app.lib.celerylib import run_task, locked_task
 
from pylons_app.lib.helpers import person
 
from pylons_app.lib.smtp_mailer import SmtpMailer
 
from pylons_app.lib.utils import OrderedDict
 
from operator import itemgetter
 
from time import mktime
 
from vcs.backends.hg import MercurialRepository
 
from time import mktime
 
import json
 
import traceback
 
import json
 

	
 
__all__ = ['whoosh_index', 'get_commits_stats',
 
           'reset_user_password', 'send_email']
 

	
 
def get_session():
 
    from sqlalchemy import engine_from_config
 
@@ -72,16 +72,16 @@ def whoosh_index(repo_location, full_ind
 
    from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon
 
    WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index)
 

	
 
@task
 
@locked_task
 
def get_commits_stats(repo_name, ts_min_y, ts_max_y):
 
    from pylons_app.model.db import Statistics, Repository
 
    log = get_commits_stats.get_logger()
 
    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
 
        
 
    from pylons_app.model.db import Statistics, Repository
 
    log = get_commits_stats.get_logger()
 
    commits_by_day_author_aggregate = {}
 
    commits_by_day_aggregate = {}
 
    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
 
    repo = MercurialRepository(repos_path + repo_name)
 

	
 
    skip_date_limit = True
 
@@ -96,22 +96,23 @@ def get_commits_stats(repo_name, ts_min_
 
        .filter(Repository.repo_name == repo_name).scalar()
 
    cur_stats = sa.query(Statistics)\
 
        .filter(Statistics.repository == dbrepo).scalar()
 
    if cur_stats:
 
        last_rev = cur_stats.stat_on_revision
 
    
 
    if last_rev == repo.revisions[-1]:
 
    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
 
        #pass silently without any work
 
        return True
 
    
 
    if cur_stats:
 
        commits_by_day_aggregate = OrderedDict(
 
                                       json.loads(
 
                                        cur_stats.commit_activity_combined))
 
        commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
 
    
 
    log.debug('starting parsing %s', parse_limit)
 
    for cnt, rev in enumerate(repo.revisions[last_rev:]):
 
        last_cs = cs = repo.get_changeset(rev)
 
        k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
 
                          cs.date.timetuple()[2])
 
        timetupple = [int(x) for x in k.split('-')]
 
        timetupple.extend([0 for _ in xrange(6)])
 
@@ -184,24 +185,31 @@ def get_commits_stats(repo_name, ts_min_
 
            "schema":["commits"],
 
        }
 

	
 
    stats = cur_stats if cur_stats else Statistics()
 
    stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
 
    stats.commit_activity_combined = json.dumps(overview_data)
 

	
 
    log.debug('last revison %s', last_rev)
 
    leftovers = len(repo.revisions[last_rev:])
 
    log.debug('revisions to parse %s', leftovers)
 
    
 
    if last_rev == 0 or leftovers < parse_limit:    
 
        stats.languages = json.dumps(__get_codes_stats(repo_name))
 
        
 
    stats.repository = dbrepo
 
    stats.stat_on_revision = last_cs.revision
 
    stats.languages = json.dumps({'_TOTAL_':0, '':0})
 
    
 
    try:
 
        sa.add(stats)
 
        sa.commit()    
 
    except:
 
        log.error(traceback.format_exc())
 
        sa.rollback()
 
        return False
 
    
 
    if len(repo.revisions) > 1:
 
    run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
 
                            
 
    return True
 

	
 
@task
 
def reset_user_password(user_email):
 
@@ -256,6 +264,34 @@ def send_email(recipients, subject, body
 
        m.send(recipients, subject, body)  
 
    except:
 
        log.error('Mail sending failed')
 
        log.error(traceback.format_exc())
 
        return False
 
    return True
 

	
 
def __get_codes_stats(repo_name):
 
    LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
 
                    'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl',
 
                    'h', 'java', 'js', 'jsp', 'jspx', 'lisp',
 
                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
 
                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh',
 
                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
 
                    'yaws']
 
    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
 
    repo = MercurialRepository(repos_path + repo_name)
 

	
 
    code_stats = {}
 
    for topnode, dirs, files in repo.walk('/', 'tip'):
 
        for f in files:
 
            k = f.mimetype
 
            if f.extension in LANGUAGES_EXTENSIONS:
 
                if code_stats.has_key(k):
 
                    code_stats[k] += 1
 
                else:
 
                    code_stats[k] = 1
 
                    
 
    return code_stats or {}
 

	
 

	
 
            
 

	
 

	
pylons_app/lib/pidlock.py
Show inline comments
 
import os, time
 
import sys
 
from warnings import warn
 
from multiprocessing.util import Finalize
 
import errno
 

	
 
class LockHeld(Exception):pass
 

	
 

	
 
class DaemonLock(object):
 
    """daemon locking
 
@@ -24,61 +26,73 @@ class DaemonLock(object):
 
        self.callbackfn = callbackfn
 
        self.desc = desc
 
        self.debug = debug
 
        self.held = False
 
        #run the lock automatically !
 
        self.lock()
 

	
 
    def __del__(self):
 
        if self.held:
 
        self._finalize = Finalize(self, DaemonLock._on_finalize,
 
                                    args=(self, debug), exitpriority=10)
 

	
 
#            warn("use lock.release instead of del lock",
 
#                    category = DeprecationWarning,
 
#                    stacklevel = 2)
 

	
 
            # ensure the lock will be removed
 
            self.release()
 
    @staticmethod
 
    def _on_finalize(lock, debug):
 
        if lock.held:
 
            if debug:
 
                print 'leck held finilazing and running lock.release()'
 
            lock.release()
 

	
 

	
 
    def lock(self):
 
        """locking function, if lock is present it will raise LockHeld exception
 
        """
 
        lockname = '%s' % (os.getpid())
 

	
 
        if self.debug:
 
            print 'running lock'
 
        self.trylock()
 
        self.makelock(lockname, self.pidfile)
 
        return True
 

	
 
    def trylock(self):
 
        running_pid = False
 
        if self.debug:
 
            print 'checking for already running process'
 
        try:
 
            pidfile = open(self.pidfile, "r")
 
            pidfile.seek(0)
 
            running_pid = pidfile.readline()
 
            running_pid = int(pidfile.readline())
 
            
 
            pidfile.close()
 
            
 
            if self.debug:
 
                print 'lock file present running_pid: %s, checking for execution'\
 
                % running_pid
 
            # Now we check the PID from lock file matches to the current
 
            # process PID
 
            if running_pid:
 
                if os.path.exists("/proc/%s" % running_pid):
 
                        print "You already have an instance of the program running"
 
                        print "It is running as process %s" % running_pid
 
                        raise LockHeld
 
                else:
 
                try:
 
                    os.kill(running_pid, 0)
 
                except OSError, exc:
 
                    if exc.errno in (errno.ESRCH, errno.EPERM):
 
                        print "Lock File is there but the program is not running"
 
                        print "Removing lock file for the: %s" % running_pid
 
                        self.release()
 
                    raise
 
                else:
 
                    print "You already have an instance of the program running"
 
                    print "It is running as process %s" % running_pid                    
 
                    raise LockHeld()
 
                         
 
        except IOError, e:
 
            if e.errno != 2:
 
                raise
 

	
 

	
 
    def release(self):
 
        """releases the pid by removing the pidfile
 
        """
 
        if self.debug:
 
            print 'trying to release the pidlock'
 
            
 
        if self.callbackfn:
 
            #execute callback function on release
 
            if self.debug:
 
                print 'executing callback function %s' % self.callbackfn
 
            self.callbackfn()
 
        try:
pylons_app/public/css/style.css
Show inline comments
 
@@ -2960,12 +2960,34 @@ div.form div.fields div.buttons input
 
	
 
}
 
 
/* -----------------------------------------------------------
 
	SUMMARY
 
----------------------------------------------------------- */
 
.trending_language_tbl, .trending_language_tbl td {
 
	margin:  0px !important;
 
	padding: 0px !important;
 
	border: 0 !important;
 
 
}
 
.trending_language{
 
	-moz-border-radius-bottomright:4px;
 
	-moz-border-radius-topright:4px;
 
	border-bottom-right-radius: 4px 4px;
 
	border-top-right-radius: 4px 4px;
 
	background-color:#336699;
 
	color:#FFFFFF;
 
	display:block;
 
	min-width:20px;
 
	max-width:400px;
 
	padding:3px;
 
	text-decoration:none;
 
	height: 10px;
 
	margin-bottom: 4px;
 
	margin-left: 5px;
 
}
 
 
#clone_url{
 
	border: none;
 
}
 
/* -----------------------------------------------------------
 
    FILES
pylons_app/templates/summary/summary.html
Show inline comments
 
@@ -90,12 +90,67 @@ E.onDOMReady(function(e){
 
			      <input type="text" id="clone_url" readonly="readonly" value="hg clone ${c.clone_repo_url}" size="70"/>
 
			  </div>
 
			 </div>
 
			
 
			 <div class="field">
 
			  <div class="label">
 
			      <label>${_('Trending languages')}:</label>
 
			  </div>
 
			  <div class="input-short">
 
			    <div id="lang_stats">
 
			    
 
			    </div> 			  
 
			  	<script type="text/javascript">
 
			  		var data = ${c.trending_languages|n};
 
			  		var total = 0;
 
			  		var no_data = true;
 
			  		for (k in data){
 
			  		    total += data[k];
 
			  		    no_data = false;
 
			  		} 
 
					var tbl = document.createElement('table');
 
					tbl.setAttribute('class','trending_language_tbl');
 
			  		for (k in data){
 
				  		var tr = document.createElement('tr');
 
				  		var percentage = Math.round((data[k]/total*100),2);
 
						var value = data[k];
 
				  		var td1 = document.createElement('td');
 
				  		td1.width=150;
 
				  		var trending_language_label = document.createElement('div');
 
				  		trending_language_label.innerHTML = k;
 
				  		td1.appendChild(trending_language_label);
 

	
 
				  		var td2 = document.createElement('td');
 
			  		    var trending_language = document.createElement('div');
 
			  		    trending_language.title = k;
 
			  		    trending_language.innerHTML = "<b>"+value+" ${_('files')} - "+percentage+" %</b>";
 
			  		    trending_language.setAttribute("class", 'trending_language');
 
			  		    trending_language.style.width=percentage+"%";
 
						td2.appendChild(trending_language);
 
						
 
						tr.appendChild(td1);
 
						tr.appendChild(td2);
 
			  		    tbl.appendChild(tr);
 
 				  		//YAHOO.util.Dom.get('lang_stats').appendChild(trending_language_label);
 
			  		    
 
			  		}
 
			  		if(no_data){
 
			  			var tr = document.createElement('tr');
 
			  			var td1 = document.createElement('td');
 
			  			td1.innerHTML = "${_('No data loaded yet...')}";
 
			  			tr.appendChild(td1);
 
			  			tbl.appendChild(tr);
 
					}
 
			  		YAHOO.util.Dom.get('lang_stats').appendChild(tbl);
 
			  	</script>
 
 
 
			  </div>
 
			 </div>
 
			 			
 
			 <div class="field">
 
			  <div class="label">
 
			      <label>${_('Download')}:</label>
 
			  </div>
 
			  <div class="input-short">
 
		        %for cnt,archive in enumerate(c.repo_info._get_archives()):
 
		             %if cnt >=1:
 
		             |
0 comments (0 inline, 0 general)