# HG changeset patch # User Marcin Kuzminski # Date 2011-04-11 22:24:04 # Node ID 0eceb478c720a543372bf1ed9ac2a8f9353f6f1d # Parent e82d6aaa245629fa6cfe667301fc17c359daf605 fixed issue #165 trending source files are now stored in cache as ext only, and translated to description only when displaying, so future changes of mappings will take affect right away. Also changes the way how map is generating, ie. autogenerate from pygments lexers, and than complete with user defined mappings. diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py --- a/rhodecode/controllers/summary.py +++ b/rhodecode/controllers/summary.py @@ -41,7 +41,8 @@ from rhodecode.lib.base import BaseRepoC from rhodecode.lib.utils import OrderedDict, EmptyChangeset from rhodecode.lib.celerylib import run_task -from rhodecode.lib.celerylib.tasks import get_commits_stats +from rhodecode.lib.celerylib.tasks import get_commits_stats, \ + LANGUAGES_EXTENSIONS_MAP from rhodecode.lib.helpers import RepoPage try: @@ -131,8 +132,14 @@ class SummaryController(BaseRepoControll lang_stats = json.loads(stats.languages) c.commit_data = stats.commit_activity c.overview_data = stats.commit_activity_combined + + lang_stats = [(x, {"count":y, + "desc":LANGUAGES_EXTENSIONS_MAP.get(x)}) + for x, y in lang_stats.items()] + print lang_stats + c.trending_languages = json.dumps(OrderedDict( - sorted(lang_stats.items(), reverse=True, + sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10] ) ) diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -31,6 +31,8 @@ import logging from time import mktime from operator import itemgetter +from pygments import lexers +from string import lower from pylons import config from pylons.i18n.translation import _ @@ -60,6 +62,43 @@ __all__ = ['whoosh_index', 'get_commits_ CELERY_ON = str2bool(config['app_conf'].get('use_celery')) +LANGUAGES_EXTENSIONS_MAP = {} + + +def __clean(s): + + s = s.lstrip('*') + s = s.lstrip('.') + + if s.find('[') != -1: + exts = [] + start, stop = s.find('['), s.find(']') + + for suffix in s[start + 1:stop]: + exts.append(s[:s.find('[')] + suffix) + return map(lower, exts) + else: + return map(lower, [s]) + +for lx, t in sorted(lexers.LEXERS.items()): + m = map(__clean, t[-2]) + if m: + m = reduce(lambda x, y: x + y, m) + for ext in m: + desc = lx.replace('Lexer', '') + if ext in LANGUAGES_EXTENSIONS_MAP: + if desc not in LANGUAGES_EXTENSIONS_MAP[ext]: + LANGUAGES_EXTENSIONS_MAP[ext].append(desc) + else: + LANGUAGES_EXTENSIONS_MAP[ext] = [desc] + +#Additional mappings that are not present in the pygments lexers +# NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP +ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} + +LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) + + def get_session(): if CELERY_ON: engine = engine_from_config(config, 'sqlalchemy.db1.') @@ -67,11 +106,13 @@ def get_session(): sa = meta.Session() return sa + def get_repos_path(): sa = get_session() q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() return q.ui_value + @task(ignore_result=True) @locked_task def whoosh_index(repo_location, full_index): @@ -82,6 +123,7 @@ def whoosh_index(repo_location, full_ind repo_location=repo_location, sa=get_session())\ .run(full_index=full_index) + @task(ignore_result=True) @locked_task def get_commits_stats(repo_name, ts_min_y, ts_max_y): @@ -93,9 +135,9 @@ def get_commits_stats(repo_name, ts_min_ from rhodecode.model.db import Statistics, Repository #for js data compatibilty - author_key_cleaner = lambda k: person(k).replace('"', "") + akc = lambda k: person(k).replace('"', "") - commits_by_day_author_aggregate = {} + co_day_auth_aggr = {} commits_by_day_aggregate = {} repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) @@ -130,7 +172,7 @@ def get_commits_stats(repo_name, ts_min_ commits_by_day_aggregate = OrderedDict( json.loads( cur_stats.commit_activity_combined)) - commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) + co_day_auth_aggr = json.loads(cur_stats.commit_activity) log.debug('starting parsing %s', parse_limit) lmktime = mktime @@ -138,22 +180,21 @@ def get_commits_stats(repo_name, ts_min_ last_rev = last_rev + 1 if last_rev > 0 else last_rev for cs in repo[last_rev:last_rev + parse_limit]: - last_cs = cs #remember last parsed changeset + last_cs = cs # remember last parsed changeset k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) - if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)): + if akc(cs.author) in co_day_auth_aggr: try: - l = [timegetter(x) for x in commits_by_day_author_aggregate\ - [author_key_cleaner(cs.author)]['data']] + l = [timegetter(x) for x in + co_day_auth_aggr[akc(cs.author)]['data']] time_pos = l.index(k) except ValueError: time_pos = False if time_pos >= 0 and time_pos is not False: - datadict = commits_by_day_author_aggregate\ - [author_key_cleaner(cs.author)]['data'][time_pos] + datadict = co_day_auth_aggr[akc(cs.author)]['data'][time_pos] datadict["commits"] += 1 datadict["added"] += len(cs.added) @@ -163,44 +204,44 @@ def get_commits_stats(repo_name, ts_min_ else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: - datadict = {"time":k, - "commits":1, - "added":len(cs.added), - "changed":len(cs.changed), - "removed":len(cs.removed), + datadict = {"time": k, + "commits": 1, + "added": len(cs.added), + "changed": len(cs.changed), + "removed": len(cs.removed), } - commits_by_day_author_aggregate\ - [author_key_cleaner(cs.author)]['data'].append(datadict) + co_day_auth_aggr[akc(cs.author)]['data']\ + .append(datadict) else: if k >= ts_min_y and k <= ts_max_y or skip_date_limit: - commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = { - "label":author_key_cleaner(cs.author), - "data":[{"time":k, + co_day_auth_aggr[akc(cs.author)] = { + "label": akc(cs.author), + "data": [{"time":k, "commits":1, "added":len(cs.added), "changed":len(cs.changed), "removed":len(cs.removed), }], - "schema":["commits"], + "schema": ["commits"], } #gather all data by day - if commits_by_day_aggregate.has_key(k): + if k in commits_by_day_aggregate: commits_by_day_aggregate[k] += 1 else: commits_by_day_aggregate[k] = 1 overview_data = sorted(commits_by_day_aggregate.items(), key=itemgetter(0)) - if not commits_by_day_author_aggregate: - commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = { - "label":author_key_cleaner(repo.contact), - "data":[0, 1], - "schema":["commits"], + if not co_day_auth_aggr: + co_day_auth_aggr[akc(repo.contact)] = { + "label": akc(repo.contact), + "data": [0, 1], + "schema": ["commits"], } stats = cur_stats if cur_stats else Statistics() - stats.commit_activity = json.dumps(commits_by_day_author_aggregate) + stats.commit_activity = json.dumps(co_day_auth_aggr) stats.commit_activity_combined = json.dumps(overview_data) log.debug('last revison %s', last_rev) @@ -225,6 +266,7 @@ def get_commits_stats(repo_name, ts_min_ return True + @task(ignore_result=True) def reset_user_password(user_email): try: @@ -259,13 +301,13 @@ def reset_user_password(user_email): 'Your new rhodecode password:%s' % (new_passwd)) log.info('send new password mail to %s', user_email) - except: log.error('Failed to update user password') log.error(traceback.format_exc()) return True + @task(ignore_result=True) def send_email(recipients, subject, body): """ @@ -306,6 +348,7 @@ def send_email(recipients, subject, body return False return True + @task(ignore_result=True) def create_repo_fork(form_data, cur_user): try: @@ -328,60 +371,8 @@ def create_repo_fork(form_data, cur_user backend = get_backend(alias) backend(str(repo_fork_path), create=True, src_url=str(repo_path)) + def __get_codes_stats(repo_name): - LANGUAGES_EXTENSIONS_MAP = {'scm': 'Scheme', 'asmx': 'VbNetAspx', 'Rout': - 'RConsole', 'rest': 'Rst', 'abap': 'ABAP', 'go': 'Go', 'phtml': 'HtmlPhp', - 'ns2': 'Newspeak', 'xml': 'EvoqueXml', 'sh-session': 'BashSession', 'ads': - 'Ada', 'clj': 'Clojure', 'll': 'Llvm', 'ebuild': 'Bash', 'adb': 'Ada', - 'ada': 'Ada', 'c++-objdump': 'CppObjdump', 'aspx': - 'VbNetAspx', 'ksh': 'Bash', 'coffee': 'CoffeeScript', 'vert': 'GLShader', - 'Makefile.*': 'Makefile', 'di': 'D', 'dpatch': 'DarcsPatch', 'rake': - 'Ruby', 'moo': 'MOOCode', 'erl-sh': 'ErlangShell', 'geo': 'GLShader', - 'pov': 'Povray', 'bas': 'VbNet', 'bat': 'Batch', 'd': 'D', 'lisp': - 'CommonLisp', 'h': 'C', 'rbx': 'Ruby', 'tcl': 'Tcl', 'c++': 'Cpp', 'md': - 'MiniD', '.vimrc': 'Vim', 'xsd': 'Xml', 'ml': 'Ocaml', 'el': 'CommonLisp', - 'befunge': 'Befunge', 'xsl': 'Xslt', 'pyx': 'Cython', 'cfm': - 'ColdfusionHtml', 'evoque': 'Evoque', 'cfg': 'Ini', 'htm': 'Html', - 'Makefile': 'Makefile', 'cfc': 'ColdfusionHtml', 'tex': 'Tex', 'cs': - 'CSharp', 'mxml': 'Mxml', 'patch': 'Diff', 'apache.conf': 'ApacheConf', - 'scala': 'Scala', 'applescript': 'AppleScript', 'GNUmakefile': 'Makefile', - 'c-objdump': 'CObjdump', 'lua': 'Lua', 'apache2.conf': 'ApacheConf', 'rb': - 'Ruby', 'gemspec': 'Ruby', 'rl': 'RagelObjectiveC', 'vala': 'Vala', 'tmpl': - 'Cheetah', 'bf': 'Brainfuck', 'plt': 'Gnuplot', 'G': 'AntlrRuby', 'xslt': - 'Xslt', 'flxh': 'Felix', 'asax': 'VbNetAspx', 'Rakefile': 'Ruby', 'S': 'S', - 'wsdl': 'Xml', 'js': 'Javascript', 'autodelegate': 'Myghty', 'properties': - 'Ini', 'bash': 'Bash', 'c': 'C', 'g': 'AntlrRuby', 'r3': 'Rebol', 's': - 'Gas', 'ashx': 'VbNetAspx', 'cxx': 'Cpp', 'boo': 'Boo', 'prolog': 'Prolog', - 'sqlite3-console': 'SqliteConsole', 'cl': 'CommonLisp', 'cc': 'Cpp', 'pot': - 'Gettext', 'vim': 'Vim', 'pxi': 'Cython', 'yaml': 'Yaml', 'SConstruct': - 'Python', 'diff': 'Diff', 'txt': 'Text', 'cw': 'Redcode', 'pxd': 'Cython', - 'plot': 'Gnuplot', 'java': 'Java', 'hrl': 'Erlang', 'py': 'Python', - 'makefile': 'Makefile', 'squid.conf': 'SquidConf', 'asm': 'Nasm', 'toc': - 'Tex', 'kid': 'Genshi', 'rhtml': 'Rhtml', 'po': 'Gettext', 'pl': 'Prolog', - 'pm': 'Perl', 'hx': 'Haxe', 'ascx': 'VbNetAspx', 'ooc': 'Ooc', 'asy': - 'Asymptote', 'hs': 'Haskell', 'SConscript': 'Python', 'pytb': - 'PythonTraceback', 'myt': 'Myghty', 'hh': 'Cpp', 'R': 'S', 'aux': 'Tex', - 'rst': 'Rst', 'cpp-objdump': 'CppObjdump', 'lgt': 'Logtalk', 'rss': 'Xml', - 'flx': 'Felix', 'b': 'Brainfuck', 'f': 'Fortran', 'rbw': 'Ruby', - '.htaccess': 'ApacheConf', 'cxx-objdump': 'CppObjdump', 'j': 'ObjectiveJ', - 'mll': 'Ocaml', 'yml': 'Yaml', 'mu': 'MuPAD', 'r': 'Rebol', 'ASM': 'Nasm', - 'erl': 'Erlang', 'mly': 'Ocaml', 'mo': 'Modelica', 'def': 'Modula2', 'ini': - 'Ini', 'control': 'DebianControl', 'vb': 'VbNet', 'vapi': 'Vala', 'pro': - 'Prolog', 'spt': 'Cheetah', 'mli': 'Ocaml', 'as': 'ActionScript3', 'cmd': - 'Batch', 'cpp': 'Cpp', 'io': 'Io', 'tac': 'Python', 'haml': 'Haml', 'rkt': - 'Racket', 'st':'Smalltalk', 'inc': 'Povray', 'pas': 'Delphi', 'cmake': - 'CMake', 'csh':'Tcsh', 'hpp': 'Cpp', 'feature': 'Gherkin', 'html': 'Html', - 'php':'Php', 'php3':'Php', 'php4':'Php', 'php5':'Php', 'xhtml': 'Html', - 'hxx': 'Cpp', 'eclass': 'Bash', 'css': 'Css', - 'frag': 'GLShader', 'd-objdump': 'DObjdump', 'weechatlog': 'IrcLogs', - 'tcsh': 'Tcsh', 'objdump': 'Objdump', 'pyw': 'Python', 'h++': 'Cpp', - 'py3tb': 'Python3Traceback', 'jsp': 'Jsp', 'sql': 'Sql', 'mak': 'Makefile', - 'php': 'Php', 'mao': 'Mako', 'man': 'Groff', 'dylan': 'Dylan', 'sass': - 'Sass', 'cfml': 'ColdfusionHtml', 'darcspatch': 'DarcsPatch', 'tpl': - 'Smarty', 'm': 'ObjectiveC', 'f90': 'Fortran', 'mod': 'Modula2', 'sh': - 'Bash', 'lhs': 'LiterateHaskell', 'sources.list': 'SourcesList', 'axd': - 'VbNetAspx', 'sc': 'Python'} - repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) repo = get_repo(p) @@ -390,14 +381,12 @@ def __get_codes_stats(repo_name): def aggregate(cs): for f in cs[2]: - ext = f.extension - key = LANGUAGES_EXTENSIONS_MAP.get(ext, ext) - key = key or ext + ext = lower(f.extension) if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary: - if code_stats.has_key(key): - code_stats[key] += 1 + if ext in code_stats: + code_stats[ext] += 1 else: - code_stats[key] = 1 + code_stats[ext] = 1 map(aggregate, tip.walk('/')) diff --git a/rhodecode/templates/summary/summary.html b/rhodecode/templates/summary/summary.html --- a/rhodecode/templates/summary/summary.html +++ b/rhodecode/templates/summary/summary.html @@ -197,26 +197,26 @@ var total = 0; var no_data = true; for (k in data){ - total += data[k]; + total += data[k].count; no_data = false; } var tbl = document.createElement('table'); tbl.setAttribute('class','trending_language_tbl'); - var cnt =0; + var cnt = 0; for (k in data){ - cnt+=1; + cnt += 1; var hide = cnt>2; var tr = document.createElement('tr'); if (hide){ tr.setAttribute('style','display:none'); tr.setAttribute('class','stats_hidden'); } - var percentage = Math.round((data[k]/total*100),2); - var value = data[k]; + var percentage = Math.round((data[k].count/total*100),2); + var value = data[k].count; var td1 = document.createElement('td'); - td1.width=150; + td1.width = 150; var trending_language_label = document.createElement('div'); - trending_language_label.innerHTML = k; + trending_language_label.innerHTML = data[k].desc+" ("+k+")"; td1.appendChild(trending_language_label); var td2 = document.createElement('td'); @@ -226,7 +226,7 @@ trending_language.title = k+" "+nr_files; - if (percentage>20){ + if (percentage>22){ trending_language.innerHTML = ""+percentage+"% "+nr_files+ ""; } else{ @@ -245,7 +245,7 @@ var td=document.createElement('td'); lnk = document.createElement('a'); lnk.href='#'; - lnk.innerHTML = "${_("show more")}"; + lnk.innerHTML = "${_('show more')}"; lnk.id='code_stats_show_more'; td.appendChild(lnk); show_more.appendChild(td);