kallithea Changeset - 82662f9faaf4

Changeset - 82662f9faaf4

Parent rev.

Child rev.

[Not reviewed]

default

0 8 0

Mads Kiilerich - 9 years ago 2016-09-06 00:51:18
madski@unity3d.com

celeryd: annotate tasks so they can be run directly without run_task

This also makes the system less forgiving about celery configuration problems
and thus easier to debug. I like that.

8 files changed with 56 insertions and 60 deletions:

kallithea/controllers/admin/settings.py

kallithea/controllers/summary.py

kallithea/lib/celerylib/__init__.py

kallithea/lib/celerylib/tasks.py

kallithea/model/notification.py

kallithea/model/repo.py

kallithea/model/user.py

kallithea/tests/models/test_notifications.py

0 comments (0 inline, 0 general)

kallithea/controllers/admin/settings.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.admin.settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 settings controller for Kallithea admin
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jul 14, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import traceback
 import formencode
 from formencode import htmlfill
 from pylons import request, tmpl_context as c, url, config
 from pylons.i18n.translation import _
 from webob.exc import HTTPFound
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import LoginRequired, HasPermissionAnyDecorator
 from kallithea.lib.base import BaseController, render
-from kallithea.lib.celerylib import tasks, run_task
 from kallithea.lib.celerylib import tasks
 from kallithea.lib.exceptions import HgsubversionImportError
 from kallithea.lib.utils import repo2db_mapper, set_app_settings
 from kallithea.model.db import Ui, Repository, Setting
 from kallithea.model.forms import ApplicationSettingsForm, \
     ApplicationUiSettingsForm, ApplicationVisualisationForm
 from kallithea.model.scm import ScmModel
 from kallithea.model.notification import EmailNotificationModel
 from kallithea.model.meta import Session
 from kallithea.lib.utils2 import str2bool, safe_unicode
 log = logging.getLogger(__name__)
 class SettingsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('setting', 'settings', controller='admin/settings',
     #         path_prefix='/admin', name_prefix='admin_')
     @LoginRequired()
     def __before__(self):
         super(SettingsController, self).__before__()
     def _get_hg_ui_settings(self):
         ret = Ui.query().all()
         settings = {}
         for each in ret:
             k = each.ui_section + '_' + each.ui_key
             v = each.ui_value
             if k == 'paths_/':
                 k = 'paths_root_path'
             k = k.replace('.', '_')
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
             settings[k] = v
         return settings
     @HasPermissionAnyDecorator('hg.admin')
     def settings_vcs(self):
         c.active = 'vcs'
         if request.POST:
             application_form = ApplicationUiSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                      render('admin/settings/settings.html'),
                      defaults=errors.value,
                      errors=errors.error_dict or {},
                      prefix_error=False,
                      encoding="UTF-8",
                      force_defaults=False)
             try:
                 if c.visual.allow_repo_location_change:
                     sett = Ui.get_by_key('paths', '/')
                     sett.ui_value = form_result['paths_root_path']
                 #HOOKS
                 sett = Ui.get_by_key('hooks', Ui.HOOK_UPDATE)
                 sett.ui_active = form_result['hooks_changegroup_update']
                 sett = Ui.get_by_key('hooks', Ui.HOOK_REPO_SIZE)
                 sett.ui_active = form_result['hooks_changegroup_repo_size']
                 sett = Ui.get_by_key('hooks', Ui.HOOK_PUSH)
                 sett.ui_active = form_result['hooks_changegroup_push_logger']
                 sett = Ui.get_by_key('hooks', Ui.HOOK_PULL)
                 sett.ui_active = form_result['hooks_outgoing_pull_logger']
                 ## EXTENSIONS
                 sett = Ui.get_or_create('extensions', 'largefiles')
                 sett.ui_active = form_result['extensions_largefiles']
                 sett = Ui.get_or_create('extensions', 'hgsubversion')
                 sett.ui_active = form_result['extensions_hgsubversion']
                 if sett.ui_active:
                     try:
                         import hgsubversion  # pragma: no cover
                     except ImportError:
                         raise HgsubversionImportError
 #                sett = Ui.get_or_create('extensions', 'hggit')
 #                sett.ui_active = form_result['extensions_hggit']
                 Session().commit()
                 h.flash(_('Updated VCS settings'), category='success')
             except HgsubversionImportError:
                 log.error(traceback.format_exc())
                 h.flash(_('Unable to activate hgsubversion support. '
                           'The "hgsubversion" library is missing'),
                         category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred while updating '
                           'application settings'), category='error')
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_mapping(self):
         c.active = 'mapping'
         if request.POST:
             rm_obsolete = request.POST.get('destroy', False)
             install_git_hooks = request.POST.get('hooks', False)
             overwrite_git_hooks = request.POST.get('hooks_overwrite', False);
             invalidate_cache = request.POST.get('invalidate', False)
             log.debug('rescanning repo location with destroy obsolete=%s, '
                       'install git hooks=%s and '
                       'overwrite git hooks=%s' % (rm_obsolete, install_git_hooks, overwrite_git_hooks))
             if invalidate_cache:
                 log.debug('invalidating all repositories cache')
                 for repo in Repository.get_all():
                     ScmModel().mark_for_invalidation(repo.repo_name)
             filesystem_repos = ScmModel().repo_scan()
             added, removed = repo2db_mapper(filesystem_repos, rm_obsolete,
                                             install_git_hooks=install_git_hooks,
                                             user=c.authuser.username,
                                             overwrite_git_hooks=overwrite_git_hooks)
             h.flash(h.literal(_('Repositories successfully rescanned. Added: %s. Removed: %s.') %
                 (', '.join(h.link_to(safe_unicode(repo_name), h.url('summary_home', repo_name=repo_name))
                  for repo_name in added) or '-',
                  ', '.join(h.escape(safe_unicode(repo_name)) for repo_name in removed) or '-')),
                 category='success')
             raise HTTPFound(location=url('admin_settings_mapping'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_global(self):
         c.active = 'global'
         if request.POST:
             application_form = ApplicationSettingsForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('admin/settings/settings.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             try:
                 sett1 = Setting.create_or_update('title',
                                             form_result['title'])
                 Session().add(sett1)
                 sett2 = Setting.create_or_update('realm',
                                             form_result['realm'])
                 Session().add(sett2)
                 sett3 = Setting.create_or_update('ga_code',
                                             form_result['ga_code'])
                 Session().add(sett3)
                 sett4 = Setting.create_or_update('captcha_public_key',
                                     form_result['captcha_public_key'])
                 Session().add(sett4)
                 sett5 = Setting.create_or_update('captcha_private_key',
                                     form_result['captcha_private_key'])
                 Session().add(sett5)
                 Session().commit()
                 set_app_settings(config)
                 h.flash(_('Updated application settings'), category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred while updating '
                           'application settings'),
                           category='error')
             raise HTTPFound(location=url('admin_settings_global'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_visual(self):
         c.active = 'visual'
         if request.POST:
             application_form = ApplicationVisualisationForm()()
             try:
                 form_result = application_form.to_python(dict(request.POST))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('admin/settings/settings.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             try:
                 settings = [
                     ('show_public_icon', 'show_public_icon', 'bool'),
                     ('show_private_icon', 'show_private_icon', 'bool'),
                     ('stylify_metatags', 'stylify_metatags', 'bool'),
                     ('repository_fields', 'repository_fields', 'bool'),
                     ('dashboard_items', 'dashboard_items', 'int'),
                     ('admin_grid_items', 'admin_grid_items', 'int'),
                     ('show_version', 'show_version', 'bool'),
                     ('use_gravatar', 'use_gravatar', 'bool'),
                     ('gravatar_url', 'gravatar_url', 'unicode'),
                     ('clone_uri_tmpl', 'clone_uri_tmpl', 'unicode'),
+                ]
                 for setting, form_key, type_ in settings:
                     sett = Setting.create_or_update(setting,
                                         form_result[form_key], type_)
                     Session().add(sett)
                 Session().commit()
                 set_app_settings(config)
                 h.flash(_('Updated visualisation settings'),
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during updating '
                           'visualisation settings'),
                         category='error')
             raise HTTPFound(location=url('admin_settings_visual'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_email(self):
         c.active = 'email'
         if request.POST:
             test_email = request.POST.get('test_email')
             test_email_subj = 'Kallithea test email'
             test_body = ('Kallithea Email test, '
                                'Kallithea version: %s' % c.kallithea_version)
             if not test_email:
                 h.flash(_('Please enter email address'), category='error')
                 raise HTTPFound(location=url('admin_settings_email'))
             test_email_txt_body = EmailNotificationModel() \
                 .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
                                 'txt', body=test_body)
             test_email_html_body = EmailNotificationModel() \
                 .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
                                 'html', body=test_body)
             recipients = [test_email] if test_email else None
             run_task(tasks.send_email, recipients, test_email_subj,
                      test_email_txt_body, test_email_html_body)
             tasks.send_email(recipients, test_email_subj,
                              test_email_txt_body, test_email_html_body)
             h.flash(_('Send email task created'), category='success')
             raise HTTPFound(location=url('admin_settings_email'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         import kallithea
         c.ini = kallithea.CONFIG
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_hooks(self):
         c.active = 'hooks'
         if request.POST:
             if c.visual.allow_custom_hooks_settings:
                 ui_key = request.POST.get('new_hook_ui_key')
                 ui_value = request.POST.get('new_hook_ui_value')
                 hook_id = request.POST.get('hook_id')
                 try:
                     ui_key = ui_key and ui_key.strip()
                     if ui_value and ui_key:
                         Ui.create_or_update_hook(ui_key, ui_value)
                         h.flash(_('Added new hook'), category='success')
                     elif hook_id:
                         Ui.delete(hook_id)
                         Session().commit()
                     # check for edits
                     update = False
                     _d = request.POST.dict_of_lists()
                     for k, v in zip(_d.get('hook_ui_key', []),
                                     _d.get('hook_ui_value_new', [])):
                         Ui.create_or_update_hook(k, v)
                         update = True
                     if update:
                         h.flash(_('Updated hooks'), category='success')
                     Session().commit()
                 except Exception:
                     log.error(traceback.format_exc())
                     h.flash(_('Error occurred during hook creation'),
                             category='error')
                 raise HTTPFound(location=url('admin_settings_hooks'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         c.hooks = Ui.get_builtin_hooks()
         c.custom_hooks = Ui.get_custom_hooks()
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_search(self):
         c.active = 'search'
         if request.POST:
             repo_location = self._get_hg_ui_settings()['paths_root_path']
             full_index = request.POST.get('full_index', False)
-            run_task(tasks.whoosh_index, repo_location, full_index)
+            tasks.whoosh_index(repo_location, full_index)
             h.flash(_('Whoosh reindex task scheduled'), category='success')
             raise HTTPFound(location=url('admin_settings_search'))
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_system(self):
         c.active = 'system'
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         import kallithea
         c.ini = kallithea.CONFIG
         c.update_url = defaults.get('update_url')
         server_info = Setting.get_server_info()
         for key, val in server_info.iteritems():
             setattr(c, key, val)
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
     @HasPermissionAnyDecorator('hg.admin')
     def settings_system_update(self):
         import json
         import urllib2
         from kallithea.lib.verlib import NormalizedVersion
         from kallithea import __version__
         defaults = Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())
         _update_url = defaults.get('update_url', '')
         _update_url = "" # FIXME: disabled
         _err = lambda s: '<div style="color:#ff8888; padding:4px 0px">%s</div>' % (s)
         try:
             import kallithea
             ver = kallithea.__version__
             log.debug('Checking for upgrade on `%s` server', _update_url)
             opener = urllib2.build_opener()
             opener.addheaders = [('User-agent', 'Kallithea-SCM/%s' % ver)]
             response = opener.open(_update_url)
             response_data = response.read()
             data = json.loads(response_data)
         except urllib2.URLError as e:
             log.error(traceback.format_exc())
             return _err('Failed to contact upgrade server: %r' % e)
         except ValueError as e:
             log.error(traceback.format_exc())
             return _err('Bad data sent from update server')
         latest = data['versions'][0]
         c.update_url = _update_url
         c.latest_data = latest
         c.latest_ver = latest['version']
         c.cur_ver = __version__
         c.should_upgrade = False
         if NormalizedVersion(c.latest_ver) > NormalizedVersion(c.cur_ver):
             c.should_upgrade = True
         c.important_notices = latest['general']
         return render('admin/settings/settings_system_update.html'),

kallithea/controllers/summary.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.summary
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import traceback
 import calendar
 import logging
 from time import mktime
 from datetime import timedelta, date
 from pylons import tmpl_context as c, request
 from pylons.i18n.translation import _
 from webob.exc import HTTPBadRequest
 from beaker.cache import cache_region, region_invalidate
 from kallithea.lib.compat import product
 from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeDoesNotExistError
 from kallithea.config.conf import ALL_READMES, ALL_EXTS, LANGUAGES_EXTENSIONS_MAP
 from kallithea.model.db import Statistics, CacheInvalidation, User
 from kallithea.lib.utils import jsonify
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator, \
     NotAnonymous
 from kallithea.lib.base import BaseRepoController, render
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.markup_renderer import MarkupRenderer
 from kallithea.lib.celerylib import run_task
 from kallithea.lib.celerylib.tasks import get_commits_stats
 from kallithea.lib.compat import json
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.controllers.changelog import _load_changelog_summary
 log = logging.getLogger(__name__)
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 class SummaryController(BaseRepoController):
     def __before__(self):
         super(SummaryController, self).__before__()
     def __get_readme_data(self, db_repo):
         repo_name = db_repo.repo_name
         log.debug('Looking for README file')
         @cache_region('long_term', '_get_readme_from_cache')
         def _get_readme_from_cache(key, kind):
             readme_data = None
             readme_file = None
             try:
                 # gets the landing revision! or tip if fails
                 cs = db_repo.get_landing_changeset()
                 if isinstance(cs, EmptyChangeset):
                     raise EmptyRepositoryError()
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         if not isinstance(readme, FileNode):
                             continue
                         readme_file = f
                         log.debug('Found README file `%s` rendering...',
                                   readme_file)
                         readme_data = renderer.render(readme.content,
                                                       filename=f)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 log.error(traceback.format_exc())
                 pass
             except EmptyRepositoryError:
                 pass
             return readme_data, readme_file
         kind = 'README'
         valid = CacheInvalidation.test_and_set_valid(repo_name, kind)
         if not valid:
             region_invalidate(_get_readme_from_cache, None, '_get_readme_from_cache', repo_name, kind)
         return _get_readme_from_cache(repo_name, kind)
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name):
         _load_changelog_summary()
         if self.authuser.is_default_user:
             username = ''
         else:
             username = safe_str(self.authuser.username)
         _def_clone_uri = _def_clone_uri_by_id = c.clone_uri_tmpl
         if '{repo}' in _def_clone_uri:
             _def_clone_uri_by_id = _def_clone_uri.replace('{repo}', '_{repoid}')
         elif '{repoid}' in _def_clone_uri:
             _def_clone_uri_by_id = _def_clone_uri.replace('_{repoid}', '{repo}')
         c.clone_repo_url = c.db_repo.clone_url(user=username,
                                                 uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = c.db_repo.clone_url(user=username,
                                                 uri_tmpl=_def_clone_uri_by_id)
         if c.db_repo.enable_statistics:
             c.show_stats = True
         else:
             c.show_stats = False
         stats = self.sa.query(Statistics) \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
             c.trending_languages = json.dumps(
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
+            )
         else:
             c.no_data = True
             c.trending_languages = json.dumps([])
         c.enable_downloads = c.db_repo.enable_downloads
         c.readme_data, c.readme_file = \
             self.__get_readme_data(c.db_repo)
         return render('summary/summary.html')
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def repo_size(self, repo_name):
         if request.is_xhr:
             return c.db_repo._repo_size()
         else:
             raise HTTPBadRequest()
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def statistics(self, repo_name):
         if c.db_repo.enable_statistics:
             c.show_stats = True
             c.no_data_msg = _('No data ready yet')
         else:
             c.show_stats = False
             c.no_data_msg = _('Statistics are disabled for this repository')
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
         stats = self.sa.query(Statistics) \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
             c.trending_languages = json.dumps(
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
+            )
             last_rev = stats.stat_on_revision + 1
             c.repo_last_rev = c.db_repo_scm_instance.count() \
                 if c.db_repo_scm_instance.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = json.dumps({})
             c.no_data = True
         recurse_limit = 500  # don't recurse more than 500 times when parsing
         run_task(get_commits_stats, c.db_repo.repo_name, ts_min_y,
                  ts_max_y, recurse_limit)
         get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)
         return render('summary/statistics.html')

kallithea/lib/celerylib/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.celerylib
 ~~~~~~~~~~~~~~~~~~~~~~~
 celery libs for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Nov 27, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import socket
 import traceback
 import logging
 from pylons import config
 from hashlib import md5
 from decorator import decorator
 from kallithea import CELERY_ON, CELERY_EAGER
 from kallithea.lib.utils2 import str2bool, safe_str
 from kallithea.lib.pidlock import DaemonLock, LockHeld
 from kallithea.model import init_model
 from kallithea.model import meta
 from sqlalchemy import engine_from_config
 log = logging.getLogger(__name__)
 class FakeTask(object):
     """Fake a sync result to make it look like a finished task"""
     def __init__(self, result):
         self.result = result
     def failed(self):
         return False
     traceback = None # if failed
     task_id = None
 def run_task(task, *args, **kwargs):
     global CELERY_ON
     if CELERY_ON:
         try:
             t = task.apply_async(args=args, kwargs=kwargs)
             log.info('running task %s:%s', t.task_id, task)
             return t
 def task(f_org):
     """Wrapper of celery.task.task, running async if CELERY_ON
     """
         except socket.error as e:
             if isinstance(e, IOError) and e.errno == 111:
                 log.debug('Unable to connect to celeryd. Sync execution')
                 CELERY_ON = False
             else:
                 log.error(traceback.format_exc())
         except KeyError as e:
                 log.debug('Unable to connect to celeryd. Sync execution')
         except Exception as e:
             log.error(traceback.format_exc())
     if CELERY_ON:
         def f_async(*args, **kwargs):
             log.info('executing %s task', f_org.__name__)
             try:
                 f_org(*args, **kwargs)
             finally:
                 log.info('executed %s task', f_org.__name__)
         f_async.__name__ = f_org.__name__
         import celery.task
         runner = celery.task.task(ignore_result=True)(f_async)
         def f_wrapped(*args, **kwargs):
             t = runner.apply_async(args=args, kwargs=kwargs)
             log.info('executing task %s in async mode - id %s', f_org, t.task_id)
             return t
     else:
         def f_wrapped(*args, **kwargs):
             log.info('executing task %s in sync', f_org.__name__)
             try:
                 result = f_org(*args, **kwargs)
             except Exception as e:
                 log.error('exception executing sync task %s in sync', f_org.__name__, e)
                 raise # TODO: return this in FakeTask as with async tasks?
             return FakeTask(result)
     log.debug('executing task %s in sync mode', task)
     try:
         result = task(*args, **kwargs)
     except Exception as e:
         log.error('exception running sync task %s: %s', task, e)
         raise # TODO: return this in FakeTask as with async tasks?
     return FakeTask(result)
     return f_wrapped
 def __get_lockkey(func, *fargs, **fkwargs):
     params = list(fargs)
     params.extend(['%s-%s' % ar for ar in fkwargs.items()])
     func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)
     lockkey = 'task_%s.lock' % \
         md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest()
     return lockkey
 def locked_task(func):
     def __wrapper(func, *fargs, **fkwargs):
         lockkey = __get_lockkey(func, *fargs, **fkwargs)
         lockkey_path = config['app_conf']['cache_dir']
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(file_=os.path.join(lockkey_path, lockkey))
             ret = func(*fargs, **fkwargs)
             l.release()
             return ret
         except LockHeld:
             log.info('LockHeld')
             return 'Task with key %s already running' % lockkey
     return decorator(__wrapper, func)
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session()
     return sa
 def dbsession(func):
     def __wrapper(func, *fargs, **fkwargs):
         try:
             ret = func(*fargs, **fkwargs)
             return ret
         finally:
             if CELERY_ON and not CELERY_EAGER:
                 meta.Session.remove()
     return decorator(__wrapper, func)

kallithea/lib/celerylib/tasks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.celerylib.tasks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Kallithea task modules, containing all task that suppose to be run
 by celery daemon
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Oct 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from celery.task import task
 import os
 import traceback
 import logging
 import rfc822
 from time import mktime
 from operator import itemgetter
 from string import lower
 from pylons import config
 from kallithea import CELERY_ON
 from kallithea.lib.celerylib import run_task, locked_task, dbsession, \
 from kallithea.lib import celerylib
 from kallithea.lib.celerylib import locked_task, dbsession, \
     str2bool, __get_lockkey, LockHeld, DaemonLock, get_session
 from kallithea.lib.helpers import person
 from kallithea.lib.rcmail.smtp_mailer import SmtpMailer
 from kallithea.lib.utils import setup_cache_regions, action_logger
 from kallithea.lib.vcs.utils import author_email
 from kallithea.lib.compat import json, OrderedDict
 from kallithea.lib.hooks import log_create_repository
 from kallithea.model.db import Statistics, Repository, User
 setup_cache_regions(config)  # pragma: no cover
 __all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
 log = logging.getLogger(__name__)
-@task(ignore_result=True)
+@celerylib.task
 @locked_task
 @dbsession
 def whoosh_index(repo_location, full_index):
     from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
     DBS = get_session()
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=DBS) \
                          .run(full_index=full_index)
-@task(ignore_result=True)
+@celerylib.task
 @dbsession
 def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
     DBS = get_session()
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = config['app_conf']['cache_dir']
     log.info('running task with lockkey %s', lockkey)
     try:
         lock = l = DaemonLock(file_=os.path.join(lockkey_path, lockkey))
         # for js data compatibility cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repo = Repository.get_by_repo_name(repo_name)
         if repo is None:
             return True
         repo = repo.scm_instance
         repo_size = repo.count()
         # return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
         skip_date_limit = True
         parse_limit = int(config['app_conf'].get('commit_parse_limit'))
         last_rev = None
         last_cs = None
         timegetter = itemgetter('time')
         dbrepo = DBS.query(Repository) \
             .filter(Repository.repo_name == repo_name).scalar()
         cur_stats = DBS.query(Statistics) \
             .filter(Statistics.repository == dbrepo).scalar()
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             # pass silently without any work if we're not on first revision or
             # current state of parsing revision(from db marker) is the
             # last revision
             lock.release()
             return True
         if cur_stats:
             commits_by_day_aggregate = OrderedDict(json.loads(
                                         cur_stats.commit_activity_combined))
             co_day_auth_aggr = json.loads(cur_stats.commit_activity)
         log.debug('starting parsing %s', parse_limit)
         lmktime = mktime
         last_rev = last_rev + 1 if last_rev >= 0 else 0
         log.debug('Getting revisions from %s to %s',
              last_rev, last_rev + parse_limit
+        )
         for cs in repo[last_rev:last_rev + parse_limit]:
             log.debug('parsing %s', cs)
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
             if akc(cs.author) in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[akc(cs.author)]['data']]
                     time_pos = l.index(k)
                 except ValueError:
                     time_pos = None
                 if time_pos >= 0 and time_pos is not None:
                     datadict = \
                         co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
                     datadict["commits"] += 1
                     datadict["added"] += len(cs.added)
                     datadict["changed"] += len(cs.changed)
                     datadict["removed"] += len(cs.removed)
                 else:
                     if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                         datadict = {"time": k,
                                     "commits": 1,
                                     "added": len(cs.added),
                                     "changed": len(cs.changed),
                                     "removed": len(cs.removed),
+                                   }
                         co_day_auth_aggr[akc(cs.author)]['data'] \
                             .append(datadict)
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                     co_day_auth_aggr[akc(cs.author)] = {
                                         "label": akc(cs.author),
                                         "data": [{"time":k,
                                                  "commits":1,
                                                  "added":len(cs.added),
                                                  "changed":len(cs.changed),
                                                  "removed":len(cs.removed),
                                                  }],
                                         "schema": ["commits"],
+                                        }
             #gather all data by day
             if k in commits_by_day_aggregate:
                 commits_by_day_aggregate[k] += 1
             else:
                 commits_by_day_aggregate[k] = 1
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
+            }
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
         log.debug('last revision %s', last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s', leftovers)
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
             stats.languages = json.dumps(__get_codes_stats(repo_name))
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             DBS.add(stats)
             DBS.commit()
         except:
             log.error(traceback.format_exc())
             DBS.rollback()
             lock.release()
             return False
         # final release
         lock.release()
         # execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
             recurse_limit -= 1
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y,
                      recurse_limit)
         if recurse_limit <= 0:
             log.debug('Breaking recursive mode due to reach of recurse limit')
         return True
             get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)
         elif recurse_limit <= 0:
             log.debug('Not recursing - limit has been reached')
         else:
             log.debug('Not recursing')
     except LockHeld:
         log.info('Task with key %s already running', lockkey)
         return 'Task with key %s already running' % lockkey
-@task(ignore_result=True)
+@celerylib.task
 @dbsession
 def send_email(recipients, subject, body='', html_body='', headers=None, author=None):
     """
     Sends an email with defined parameters from the .ini files.
     :param recipients: list of recipients, if this is None, the defined email
         address from field 'email_to' and all admins is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     :param headers: dictionary of prepopulated e-mail headers
     :param author: User object of the author of this mail, if known and relevant
     """
     assert isinstance(recipients, list), recipients
     if headers is None:
         headers = {}
     else:
         # do not modify the original headers object passed by the caller
         headers = headers.copy()
     email_config = config
     email_prefix = email_config.get('email_prefix', '')
     if email_prefix:
         subject = "%s %s" % (email_prefix, subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         recipients = [u.email for u in User.query()
                       .filter(User.admin == True).all()]
         if email_config.get('email_to') is not None:
             recipients += [email_config.get('email_to')]
         # If there are still no recipients, there are no admins and no address
         # configured in email_to, so return.
         if not recipients:
             log.error("No recipients specified and no fallback available.")
             return False
         log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
     # SMTP sender
     envelope_from = email_config.get('app_email_from', 'Kallithea')
     # 'From' header
     if author is not None:
         # set From header based on author but with a generic e-mail address
         # In case app_email_from is in "Some Name <e-mail>" format, we first
         # extract the e-mail address.
         envelope_addr = author_email(envelope_from)
         headers['From'] = '"%s" <%s>' % (
             rfc822.quote('%s (no-reply)' % author.full_name_or_username),
             envelope_addr)
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(email_config.get('debug'))
     smtp_auth = email_config.get('smtp_auth')
     logmsg = ("Mail details:\n"
               "recipients: %s\n"
               "headers: %s\n"
               "subject: %s\n"
               "body:\n%s\n"
               "html:\n%s\n"
               % (' '.join(recipients), headers, subject, body, html_body))
     if mail_server:
         log.debug("Sending e-mail. " + logmsg)
     else:
         log.error("SMTP mail server not configured - cannot send e-mail.")
         log.warning(logmsg)
         return False
     try:
         m = SmtpMailer(envelope_from, user, passwd, mail_server, smtp_auth,
                        mail_port, ssl, tls, debug=debug)
         m.send(recipients, subject, body, html_body, headers=headers)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
-@task(ignore_result=False)
+@celerylib.task
 @dbsession
 def create_repo(form_data, cur_user):
     from kallithea.model.repo import RepoModel
     from kallithea.model.user import UserModel
     from kallithea.model.db import Setting
     DBS = get_session()
     cur_user = UserModel(DBS)._get_user(cur_user)
     owner = cur_user
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = form_data['repo_group']
     landing_rev = form_data['repo_landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
     # repo creation defaults, private and repo_type are filled in form
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     try:
         repo = RepoModel(DBS)._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
+        )
         action_logger(cur_user, 'user_created_repo',
                       form_data['repo_name_full'], '', DBS)
         DBS.commit()
         # now create this repo on Filesystem
         RepoModel(DBS)._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel(DBS)._get_repo_group(repo_group),
             clone_uri=clone_uri,
+        )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(repo.get_dict(), created_by=owner.username)
         # update repo changeset caches initially
         repo.update_changeset_cache()
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         DBS.commit()
     except Exception as e:
         log.warning('Exception %s occurred when forking repository, '
                     'doing cleanup...' % e)
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             DBS.commit()
             RepoModel(DBS)._delete_filesystem_repo(repo)
         raise
     return True
-@task(ignore_result=False)
+@celerylib.task
 @dbsession
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using interval VCS methods
     :param form_data:
     :param cur_user:
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.user import UserModel
     DBS = get_session()
     base_path = Repository.base_path()
     cur_user = UserModel(DBS)._get_user(cur_user)
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     owner = cur_user
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = form_data['repo_group']
     landing_rev = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     try:
         fork_of = RepoModel(DBS)._get_repo(form_data.get('fork_parent_id'))
         RepoModel(DBS)._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=form_data['description'],
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
+        )
         action_logger(cur_user, 'user_forked_repo:%s' % repo_name_full,
                       fork_of.repo_name, '', DBS)
         DBS.commit()
         update_after_clone = form_data['update_after_clone'] # FIXME - unused!
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
         # now create this repo on Filesystem
         RepoModel(DBS)._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel(DBS)._get_repo_group(repo_group),
             clone_uri=source_repo_path,
+        )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(repo.get_dict(), created_by=owner.username)
         # update repo changeset caches initially
         repo.update_changeset_cache()
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         DBS.commit()
     except Exception as e:
         log.warning('Exception %s occurred when forking repository, '
                     'doing cleanup...' % e)
         #rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             DBS.commit()
             RepoModel(DBS)._delete_filesystem_repo(repo)
         raise
     return True
 def __get_codes_stats(repo_name):
     from kallithea.config.conf import LANGUAGES_EXTENSIONS_MAP
     repo = Repository.get_by_repo_name(repo_name).scm_instance
     tip = repo.get_changeset()
     code_stats = {}
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if ext in code_stats:
                     code_stats[ext] += 1
                 else:
                     code_stats[ext] = 1
     map(aggregate, tip.walk('/'))
     return code_stats or {}

kallithea/model/notification.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.notification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Model for notifications
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Nov 20, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import traceback
 from pylons import tmpl_context as c
 from pylons.i18n.translation import _
 from sqlalchemy.orm import joinedload, subqueryload
 import kallithea
 from kallithea.lib import helpers as h
 from kallithea.lib.utils2 import safe_unicode
 from kallithea.model import BaseModel
 from kallithea.model.db import Notification, User, UserNotification
 from kallithea.model.meta import Session
 log = logging.getLogger(__name__)
 class NotificationModel(BaseModel):
     def create(self, created_by, subject, body, recipients=None,
                type_=Notification.TYPE_MESSAGE, with_email=True,
                email_kwargs=None, repo_name=None):
         """
         Creates notification of given type
         :param created_by: int, str or User instance. User who created this
             notification
         :param subject:
         :param body:
         :param recipients: list of int, str or User objects, when None
             is given send to all admins
         :param type_: type of notification
         :param with_email: send email with this notification
         :param email_kwargs: additional dict to pass as args to email template
         """
-        from kallithea.lib.celerylib import tasks, run_task
         from kallithea.lib.celerylib import tasks
         email_kwargs = email_kwargs or {}
         if recipients and not getattr(recipients, '__iter__', False):
             raise Exception('recipients must be a list or iterable')
         created_by_obj = self._get_user(created_by)
         recipients_objs = []
         if recipients:
             for u in recipients:
                 obj = self._get_user(u)
                 if obj is not None:
                     recipients_objs.append(obj)
                 else:
                     # TODO: inform user that requested operation couldn't be completed
                     log.error('cannot email unknown user %r', u)
             recipients_objs = set(recipients_objs)
             log.debug('sending notifications %s to %s',
                 type_, recipients_objs
+            )
         elif recipients is None:
             # empty recipients means to all admins
             recipients_objs = User.query().filter(User.admin == True).all()
             log.debug('sending notifications %s to admins: %s',
                 type_, recipients_objs
+            )
         #else: silently skip notification mails?
         # TODO: inform user who are notified
         notif = Notification.create(
             created_by=created_by_obj, subject=subject,
             body=body, recipients=recipients_objs, type_=type_
+        )
         if not with_email:
             return notif
         #don't send email to person who created this comment
         rec_objs = set(recipients_objs).difference(set([created_by_obj]))
         headers = None
         if 'threading' in email_kwargs:
             headers = {'References': ' '.join('<%s>' % x for x in email_kwargs['threading'])}
         # send email with notification to all other participants
         for rec in rec_objs:
             ## this is passed into template
             html_kwargs = {
                       'subject': subject,
                       'body': h.render_w_mentions(body, repo_name),
                       'when': h.fmt_date(notif.created_on),
                       'user': notif.created_by_user.username,
+                      }
             txt_kwargs = {
                       'subject': subject,
                       'body': body,
                       'when': h.fmt_date(notif.created_on),
                       'user': notif.created_by_user.username,
+                      }
             html_kwargs.update(email_kwargs)
             txt_kwargs.update(email_kwargs)
             email_subject = EmailNotificationModel() \
                                 .get_email_description(type_, **txt_kwargs)
             email_txt_body = EmailNotificationModel() \
                                 .get_email_tmpl(type_, 'txt', **txt_kwargs)
             email_html_body = EmailNotificationModel() \
                                 .get_email_tmpl(type_, 'html', **html_kwargs)
-            run_task(tasks.send_email, [rec.email], email_subject, email_txt_body,
+            tasks.send_email([rec.email], email_subject, email_txt_body,
                      email_html_body, headers, author=created_by_obj)
         return notif
     def delete(self, user, notification):
         # we don't want to remove actual notification just the assignment
         try:
             notification = Notification.guess_instance(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query() \
                         .filter(UserNotification.user == user) \
                         .filter(UserNotification.notification
                                 == notification) \
                         .one()
                 Session().delete(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
     def query_for_user(self, user, filter_=None):
         """
         Get notifications for given user, filter them if filter dict is given
         :param user:
         :param filter:
         """
         user = self._get_user(user)
         q = UserNotification.query() \
             .filter(UserNotification.user == user) \
             .join((Notification, UserNotification.notification_id ==
                                  Notification.notification_id)) \
             .options(joinedload('notification')) \
             .options(subqueryload('notification.created_by_user')) \
             .order_by(Notification.created_on.desc())
         if filter_:
             q = q.filter(Notification.type_.in_(filter_))
         return q
     def mark_read(self, user, notification):
         try:
             notification = Notification.guess_instance(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query() \
                         .filter(UserNotification.user == user) \
                         .filter(UserNotification.notification
                                 == notification) \
                         .one()
                 obj.read = True
                 Session().add(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
     def mark_all_read_for_user(self, user, filter_=None):
         user = self._get_user(user)
         q = UserNotification.query() \
             .filter(UserNotification.user == user) \
             .filter(UserNotification.read == False) \
             .join((Notification, UserNotification.notification_id ==
                                  Notification.notification_id))
         if filter_:
             q = q.filter(Notification.type_.in_(filter_))
         # this is a little inefficient but sqlalchemy doesn't support
         # update on joined tables :(
         for obj in q.all():
             obj.read = True
             Session().add(obj)
     def get_unread_cnt_for_user(self, user):
         user = self._get_user(user)
         return UserNotification.query() \
                 .filter(UserNotification.read == False) \
                 .filter(UserNotification.user == user).count()
     def get_unread_for_user(self, user):
         user = self._get_user(user)
         return [x.notification for x in UserNotification.query() \
                 .filter(UserNotification.read == False) \
                 .filter(UserNotification.user == user).all()]
     def get_user_notification(self, user, notification):
         user = self._get_user(user)
         notification = Notification.guess_instance(notification)
         return UserNotification.query() \
             .filter(UserNotification.notification == notification) \
             .filter(UserNotification.user == user).scalar()
     def make_description(self, notification, show_age=True):
         """
         Creates a human readable description based on properties
         of notification object
         """
         #alias
         _n = notification
         if show_age:
             return {
                     _n.TYPE_CHANGESET_COMMENT: _('%(user)s commented on changeset %(age)s'),
                     _n.TYPE_MESSAGE: _('%(user)s sent message %(age)s'),
                     _n.TYPE_MENTION: _('%(user)s mentioned you %(age)s'),
                     _n.TYPE_REGISTRATION: _('%(user)s registered in Kallithea %(age)s'),
                     _n.TYPE_PULL_REQUEST: _('%(user)s opened new pull request %(age)s'),
                     _n.TYPE_PULL_REQUEST_COMMENT: _('%(user)s commented on pull request %(age)s'),
                 }[notification.type_] % dict(
                     user=notification.created_by_user.username,
                     age=h.age(notification.created_on),
+                )
         else:
             return {
                     _n.TYPE_CHANGESET_COMMENT: _('%(user)s commented on changeset at %(when)s'),
                     _n.TYPE_MESSAGE: _('%(user)s sent message at %(when)s'),
                     _n.TYPE_MENTION: _('%(user)s mentioned you at %(when)s'),
                     _n.TYPE_REGISTRATION: _('%(user)s registered in Kallithea at %(when)s'),
                     _n.TYPE_PULL_REQUEST: _('%(user)s opened new pull request at %(when)s'),
                     _n.TYPE_PULL_REQUEST_COMMENT: _('%(user)s commented on pull request at %(when)s'),
                 }[notification.type_] % dict(
                     user=notification.created_by_user.username,
                     when=h.fmt_date(notification.created_on),
+                )
 class EmailNotificationModel(BaseModel):
     TYPE_CHANGESET_COMMENT = Notification.TYPE_CHANGESET_COMMENT
     TYPE_MESSAGE = Notification.TYPE_MESSAGE # only used for testing
     # Notification.TYPE_MENTION is not used
     TYPE_PASSWORD_RESET = 'password_link'
     TYPE_REGISTRATION = Notification.TYPE_REGISTRATION
     TYPE_PULL_REQUEST = Notification.TYPE_PULL_REQUEST
     TYPE_PULL_REQUEST_COMMENT = Notification.TYPE_PULL_REQUEST_COMMENT
     TYPE_DEFAULT = 'default'
     def __init__(self):
         super(EmailNotificationModel, self).__init__()
         self._template_root = kallithea.CONFIG['pylons.paths']['templates'][0]
         self._tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
         self.email_types = {
             self.TYPE_CHANGESET_COMMENT: 'changeset_comment',
             self.TYPE_PASSWORD_RESET: 'password_reset',
             self.TYPE_REGISTRATION: 'registration',
             self.TYPE_DEFAULT: 'default',
             self.TYPE_PULL_REQUEST: 'pull_request',
             self.TYPE_PULL_REQUEST_COMMENT: 'pull_request_comment',
+        }
         self._subj_map = {
             self.TYPE_CHANGESET_COMMENT: _('[Comment] %(repo_name)s changeset %(short_id)s "%(message_short)s" on %(branch)s'),
             self.TYPE_MESSAGE: 'Test Message',
             # self.TYPE_PASSWORD_RESET
             self.TYPE_REGISTRATION: _('New user %(new_username)s registered'),
             # self.TYPE_DEFAULT
             self.TYPE_PULL_REQUEST: _('[Review] %(repo_name)s PR %(pr_nice_id)s "%(pr_title_short)s" from %(pr_source_branch)s by %(pr_owner_username)s'),
             self.TYPE_PULL_REQUEST_COMMENT: _('[Comment] %(repo_name)s PR %(pr_nice_id)s "%(pr_title_short)s" from %(pr_source_branch)s by %(pr_owner_username)s'),
+        }
     def get_email_description(self, type_, **kwargs):
         """
         return subject for email based on given type
         """
         tmpl = self._subj_map[type_]
         try:
             subj = tmpl % kwargs
         except KeyError as e:
             log.error('error generating email subject for %r from %s: %s', type_, ','.join(self._subj_map.keys()), e)
             raise
         l = [safe_unicode(x) for x in [kwargs.get('status_change'), kwargs.get('closing_pr') and _('Closing')] if x]
         if l:
             if subj.startswith('['):
                 subj = '[' + ', '.join(l) + ': ' + subj[1:]
             else:
                 subj = '[' + ', '.join(l) + '] ' + subj
         return subj
     def get_email_tmpl(self, type_, content_type, **kwargs):
         """
         return generated template for email based on given type
         """
         base = 'email_templates/' + self.email_types.get(type_, self.email_types[self.TYPE_DEFAULT]) + '.' + content_type
         email_template = self._tmpl_lookup.get_template(base)
         # translator and helpers inject
         _kwargs = {'_': _,
                    'h': h,
                    'c': c}

kallithea/model/repo.py

➞

Show inline comments

@@ @@ -284,435 +284,435 @@ class RepoModel(BaseModel): @@
         for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                          (1, 'repo_description'), (1, 'repo_enable_locking'),
                          (1, 'repo_landing_rev'), (0, 'clone_uri'),
                          (1, 'repo_private'), (1, 'repo_enable_statistics')]:
             attr = k
             if strip:
                 attr = remove_prefix(k, 'repo_')
             val = defaults[attr]
             if k == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[k] = val
             if k == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
         # fill owner
         if repo_info.user:
             defaults.update({'user': repo_info.user.username})
         else:
             replacement_user = User.query().filter(User.admin ==
                                                    True).first().username
             defaults.update({'user': replacement_user})
         # fill repository users
         for p in repo_info.repo_to_perm:
             defaults.update({'u_perm_%s' % p.user.username:
                                  p.permission.permission_name})
         # fill repository groups
         for p in repo_info.users_group_to_perm:
             defaults.update({'g_perm_%s' % p.users_group.users_group_name:
                                  p.permission.permission_name})
         return defaults
     def update(self, repo, **kwargs):
         try:
             cur_repo = self._get_repo(repo)
             org_repo_name = cur_repo.repo_name
             if 'user' in kwargs:
                 cur_repo.user = User.get_by_username(kwargs['user'])
             if 'repo_group' in kwargs:
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
                 cur_repo.repo_name = cur_repo.get_new_name(cur_repo.just_name)
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
             for k in ['repo_enable_downloads',
                       'repo_description',
                       'repo_enable_locking',
                       'repo_landing_rev',
                       'repo_private',
                       'repo_enable_statistics',
                       ]:
                 if k in kwargs:
                     setattr(cur_repo, remove_prefix(k, 'repo_'), kwargs[k])
             clone_uri = kwargs.get('clone_uri')
             if clone_uri is not None and clone_uri != cur_repo.clone_uri_hidden:
                 cur_repo.clone_uri = clone_uri
             if 'repo_name' in kwargs:
                 cur_repo.repo_name = cur_repo.get_new_name(kwargs['repo_name'])
             #if private flag is set, reset default permission to NONE
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user='default', perm=EMPTY_PERM
+                )
                 #handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX),
                                 kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
                     self.sa.add(ex_field)
             self.sa.add(cur_repo)
             if org_repo_name != cur_repo.repo_name:
                 # rename repository
                 self._rename_filesystem_repo(old=org_repo_name, new=cur_repo.repo_name)
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev='rev:tip', fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_locking=False, enable_downloads=False,
                      copy_group_permissions=False, state=Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state. This should only be
         executed by create() repo, with exception of importing existing repos.
         """
         from kallithea.model.scm import ScmModel
         owner = self._get_user(owner)
         fork_of = self._get_repo(fork_of)
         repo_group = self._get_repo_group(repo_group)
         try:
             repo_name = safe_unicode(repo_name)
             description = safe_unicode(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(self.URL_SEPARATOR)[-1]
             new_repo = Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.user = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_locking = enable_locking
             new_repo.enable_downloads = enable_downloads
             if repo_group:
                 new_repo.enable_locking = repo_group.enable_locking
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
             self.sa.add(new_repo)
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = UserRepoToPerm.query() \
                     .filter(UserRepoToPerm.repository == repo).all()
                 group_perms = UserGroupRepoToPerm.query() \
                     .filter(UserGroupRepoToPerm.repository == repo).all()
                 for perm in user_perms:
                     UserRepoToPerm.create(perm.user, new_repo, perm.permission)
                 for perm in group_perms:
                     UserGroupRepoToPerm.create(perm.users_group, new_repo,
                                                perm.permission)
             elif repo_group and copy_group_permissions:
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == repo_group).all()
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == repo_group).all()
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserRepoToPerm.create(perm.user, new_repo, perm_obj)
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
             else:
                 perm_obj = self._create_default_perms(new_repo, private)
                 self.sa.add(perm_obj)
             # now automatically start following this repository as owner
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id,
                                                     owner.user_id)
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             self.sa.flush()
             return new_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo, form_data, cur_user)
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo(form_data, cur_user)
     def _update_permissions(self, repo, perms_new=None, perms_updates=None,
                             check_perms=True):
         if not perms_new:
             perms_new = []
         if not perms_updates:
             perms_updates = []
         # update permissions
         for member, perm, member_type in perms_updates:
             if member_type == 'user':
                 # this updates existing one
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
+                )
             else:
                 #check if we have permissions to alter this usergroup
                 req_perms = (
                     'usergroup.read', 'usergroup.write', 'usergroup.admin')
                 if not check_perms or HasUserGroupPermissionAny(*req_perms)(
                         member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
+                    )
             # set new permissions
         for member, perm, member_type in perms_new:
             if member_type == 'user':
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
+                )
             else:
                 #check if we have permissions to alter this usergroup
                 req_perms = (
                     'usergroup.read', 'usergroup.write', 'usergroup.admin')
                 if not check_perms or HasUserGroupPermissionAny(*req_perms)(
                         member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
+                    )
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo_fork, form_data, cur_user)
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo_fork(form_data, cur_user)
     def delete(self, repo, forks=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
         repo = self._get_repo(repo)
         if repo is not None:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
                     self.sa.add(r)
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
             old_repo_dict = repo.get_dict()
             try:
                 self.sa.delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 log_delete_repository(old_repo_dict,
                                       deleted_by=cur_user)
             except Exception:
                 log.error(traceback.format_exc())
                 raise
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         permission = self._get_perm(perm)
         # check if we have that permission already
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         return obj
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj is not None:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         return obj
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj is not None:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
         :param repo_name:
         """
         repo = self._get_repo(repo_name)
         try:
             obj = self.sa.query(Statistics) \
                 .filter(Statistics.repository == repo).scalar()
             if obj is not None:
                 self.sa.delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None):
         """
         Makes repository on filesystem. Operation is group aware, meaning that it will create
         a repository within a group, and alter the paths accordingly to the group location.
         :param repo_name:
         :param alias:
         :param parent:
         :param clone_uri:
         :param repo_store_location:
         """
         from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
         from kallithea.model.scm import ScmModel
         if '/' in repo_name:
             raise ValueError('repo_name must not contain groups got `%s`' % repo_name)
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
         log.info('creating repo %s in %s from url: `%s`',
             repo_name, safe_unicode(repo_path),
             obfuscate_url_pw(clone_uri))

kallithea/model/user.py

➞

Show inline comments

@@ @@ -140,387 +140,386 @@ class UserModel(BaseModel): @@
             log.debug('updating user %s', username)
             new_user = user
             edit = True
         try:
             new_user.username = username
             new_user.admin = admin
             new_user.email = email
             new_user.active = active
             new_user.extern_name = safe_str(extern_name) \
                 if extern_name else None
             new_user.extern_type = safe_str(extern_type) \
                 if extern_type else None
             new_user.name = firstname
             new_user.lastname = lastname
             if not edit:
                 new_user.api_key = generate_api_key()
             # set password only if creating an user or password is changed
             password_change = new_user.password and \
                 not check_password(password, new_user.password)
             if not edit or password_change:
                 reason = 'new password' if edit else 'new user'
                 log.debug('Updating password reason=>%s', reason)
                 new_user.password = get_crypt_password(password) \
                     if password else None
             self.sa.add(new_user)
             if not edit:
                 log_create_user(new_user.get_dict(), cur_user)
             return new_user
         except (DatabaseError,):
             log.error(traceback.format_exc())
             raise
     def create_registration(self, form_data):
         from kallithea.model.notification import NotificationModel
         import kallithea.lib.helpers as h
         form_data['admin'] = False
         form_data['extern_name'] = EXTERN_TYPE_INTERNAL
         form_data['extern_type'] = EXTERN_TYPE_INTERNAL
         new_user = self.create(form_data)
         self.sa.add(new_user)
         self.sa.flush()
         # notification to admins
         subject = _('New user registration')
         body = (
             u'New user registration\n'
             '---------------------\n'
             '- Username: {user.username}\n'
             '- Full Name: {user.full_name}\n'
             '- Email: {user.email}\n'
             ).format(user=new_user)
         edit_url = h.canonical_url('edit_user', id=new_user.user_id)
         email_kwargs = {
             'registered_user_url': edit_url,
             'new_username': new_user.username,
             'new_email': new_user.email,
             'new_full_name': new_user.full_name}
         NotificationModel().create(created_by=new_user, subject=subject,
                                    body=body, recipients=None,
                                    type_=Notification.TYPE_REGISTRATION,
                                    email_kwargs=email_kwargs)
     def update(self, user_id, form_data, skip_attrs=None):
         from kallithea.lib.auth import get_crypt_password
         skip_attrs = skip_attrs or []
         user = self.get(user_id, cache=False)
         if user.username == User.DEFAULT_USER:
             raise DefaultUserException(
                             _("You can't edit this user since it's "
                               "crucial for entire application"))
         for k, v in form_data.items():
             if k in skip_attrs:
                 continue
             if k == 'new_password' and v:
                 user.password = get_crypt_password(v)
             else:
                 # old legacy thing orm models store firstname as name,
                 # need proper refactor to username
                 if k == 'firstname':
                     k = 'name'
                 setattr(user, k, v)
         self.sa.add(user)
     def update_user(self, user, **kwargs):
         from kallithea.lib.auth import get_crypt_password
         user = self._get_user(user)
         if user.username == User.DEFAULT_USER:
             raise DefaultUserException(
                 _("You can't edit this user since it's"
                   " crucial for entire application")
+            )
         for k, v in kwargs.items():
             if k == 'password' and v:
                 v = get_crypt_password(v)
             setattr(user, k, v)
         self.sa.add(user)
         return user
     def delete(self, user, cur_user=None):
         if cur_user is None:
             cur_user = getattr(get_current_authuser(), 'username', None)
         user = self._get_user(user)
         if user.username == User.DEFAULT_USER:
             raise DefaultUserException(
                 _("You can't remove this user since it is"
                   " crucial for the entire application"))
         if user.repositories:
             repos = [x.repo_name for x in user.repositories]
             raise UserOwnsReposException(
                 _('User "%s" still owns %s repositories and cannot be '
                   'removed. Switch owners or remove those repositories: %s')
                 % (user.username, len(repos), ', '.join(repos)))
         if user.repo_groups:
             repogroups = [x.group_name for x in user.repo_groups]
             raise UserOwnsReposException(_(
                 'User "%s" still owns %s repository groups and cannot be '
                 'removed. Switch owners or remove those repository groups: %s')
                 % (user.username, len(repogroups), ', '.join(repogroups)))
         if user.user_groups:
             usergroups = [x.users_group_name for x in user.user_groups]
             raise UserOwnsReposException(
                 _('User "%s" still owns %s user groups and cannot be '
                   'removed. Switch owners or remove those user groups: %s')
                 % (user.username, len(usergroups), ', '.join(usergroups)))
         self.sa.delete(user)
         from kallithea.lib.hooks import log_delete_user
         log_delete_user(user.get_dict(), cur_user)
     def can_change_password(self, user):
         from kallithea.lib import auth_modules
         managed_fields = auth_modules.get_managed_fields(user)
         return 'password' not in managed_fields
     def get_reset_password_token(self, user, timestamp, session_id):
         """
         The token is a 40-digit hexstring, calculated as a HMAC-SHA1.
         In a traditional HMAC scenario, an attacker is unable to know or
         influence the secret key, but can know or influence the message
         and token. This scenario is slightly different (in particular
         since the message sender is also the message recipient), but
         sufficiently similar to use an HMAC. Benefits compared to a plain
         SHA1 hash includes resistance against a length extension attack.
         The HMAC key consists of the following values (known only to the
         server and authorized users):
         * per-application secret (the `app_instance_uuid` setting), without
           which an attacker cannot counterfeit tokens
         * hashed user password, invalidating the token upon password change
         The HMAC message consists of the following values (potentially known
         to an attacker):
         * session ID (the anti-CSRF token), requiring an attacker to have
           access to the browser session in which the token was created
         * numeric user ID, limiting the token to a specific user (yet allowing
           users to be renamed)
         * user email address
         * time of token issue (a Unix timestamp, to enable token expiration)
         The key and message values are separated by NUL characters, which are
         guaranteed not to occur in any of the values.
         """
         app_secret = config.get('app_instance_uuid')
         return hmac.HMAC(
             key=u'\0'.join([app_secret, user.password]).encode('utf-8'),
             msg=u'\0'.join([session_id, str(user.user_id), user.email, str(timestamp)]).encode('utf-8'),
             digestmod=hashlib.sha1,
         ).hexdigest()
     def send_reset_password_email(self, data):
         """
         Sends email with a password reset token and link to the password
         reset confirmation page with all information (including the token)
         pre-filled. Also returns URL of that page, only without the token,
         allowing users to copy-paste or manually enter the token from the
         email.
         """
-        from kallithea.lib.celerylib import tasks, run_task
         from kallithea.lib.celerylib import tasks
         from kallithea.model.notification import EmailNotificationModel
         import kallithea.lib.helpers as h
         user_email = data['email']
         user = User.get_by_email(user_email)
         timestamp = int(time.time())
         if user is not None:
             if self.can_change_password(user):
                 log.debug('password reset user %s found', user)
                 token = self.get_reset_password_token(user,
                                                       timestamp,
                                                       h.authentication_token())
                 # URL must be fully qualified; but since the token is locked to
                 # the current browser session, we must provide a URL with the
                 # current scheme and hostname, rather than the canonical_url.
                 link = h.url('reset_password_confirmation', qualified=True,
                              email=user_email,
                              timestamp=timestamp,
                              token=token)
             else:
                 log.debug('password reset user %s found but was managed', user)
                 token = link = None
             reg_type = EmailNotificationModel.TYPE_PASSWORD_RESET
             body = EmailNotificationModel().get_email_tmpl(
                 reg_type, 'txt',
                 user=user.short_contact,
                 reset_token=token,
                 reset_url=link)
             html_body = EmailNotificationModel().get_email_tmpl(
                 reg_type, 'html',
                 user=user.short_contact,
                 reset_token=token,
                 reset_url=link)
             log.debug('sending email')
             run_task(tasks.send_email, [user_email],
                      _("Password reset link"), body, html_body)
             tasks.send_email([user_email], _("Password reset link"), body, html_body)
             log.info('send new password mail to %s', user_email)
         else:
             log.debug("password reset email %s not found", user_email)
         return h.url('reset_password_confirmation',
                      email=user_email,
                      timestamp=timestamp)
     def verify_reset_password_token(self, email, timestamp, token):
-        from kallithea.lib.celerylib import tasks, run_task
         from kallithea.lib.celerylib import tasks
         from kallithea.lib import auth
         import kallithea.lib.helpers as h
         user = User.get_by_email(email)
         if user is None:
             log.debug("user with email %s not found", email)
             return False
         token_age = int(time.time()) - int(timestamp)
         if token_age < 0:
             log.debug('timestamp is from the future')
             return False
         if token_age > UserModel.password_reset_token_lifetime:
             log.debug('password reset token expired')
             return False
         expected_token = self.get_reset_password_token(user,
                                                        timestamp,
                                                        h.authentication_token())
         log.debug('computed password reset token: %s', expected_token)
         log.debug('received password reset token: %s', token)
         return expected_token == token
     def reset_password(self, user_email, new_passwd):
-        from kallithea.lib.celerylib import tasks, run_task
         from kallithea.lib.celerylib import tasks
         from kallithea.lib import auth
         user = User.get_by_email(user_email)
         if user is not None:
             if not self.can_change_password(user):
                 raise Exception('trying to change password for external user')
             user.password = auth.get_crypt_password(new_passwd)
             Session().add(user)
             Session().commit()
             log.info('change password for %s', user_email)
         if new_passwd is None:
             raise Exception('unable to set new password')
-        run_task(tasks.send_email, [user_email],
+        tasks.send_email([user_email],
                  _('Password reset notification'),
                  _('The password to your account %s has been changed using password reset form.') % (user.username,))
         log.info('send password reset mail to %s', user_email)
         return True
     def has_perm(self, user, perm):
         perm = self._get_perm(perm)
         user = self._get_user(user)
         return UserToPerm.query().filter(UserToPerm.user == user) \
             .filter(UserToPerm.permission == perm).scalar() is not None
     def grant_perm(self, user, perm):
         """
         Grant user global permissions
         :param user:
         :param perm:
         """
         user = self._get_user(user)
         perm = self._get_perm(perm)
         # if this permission is already granted skip it
         _perm = UserToPerm.query() \
             .filter(UserToPerm.user == user) \
             .filter(UserToPerm.permission == perm) \
             .scalar()
         if _perm:
             return
         new = UserToPerm()
         new.user = user
         new.permission = perm
         self.sa.add(new)
         return new
     def revoke_perm(self, user, perm):
         """
         Revoke users global permissions
         :param user:
         :param perm:
         """
         user = self._get_user(user)
         perm = self._get_perm(perm)
         UserToPerm.query().filter(
             UserToPerm.user == user,
             UserToPerm.permission == perm,
         ).delete()
     def add_extra_email(self, user, email):
         """
         Adds email address to UserEmailMap
         :param user:
         :param email:
         """
         from kallithea.model import forms
         form = forms.UserExtraEmailForm()()
         data = form.to_python(dict(email=email))
         user = self._get_user(user)
         obj = UserEmailMap()
         obj.user = user
         obj.email = data['email']
         self.sa.add(obj)
         return obj
     def delete_extra_email(self, user, email_id):
         """
         Removes email address from UserEmailMap
         :param user:
         :param email_id:
         """
         user = self._get_user(user)
         obj = UserEmailMap.query().get(email_id)
         if obj is not None:
             self.sa.delete(obj)
     def add_extra_ip(self, user, ip):
         """
         Adds IP address to UserIpMap
         :param user:
         :param ip:
         """
         from kallithea.model import forms
         form = forms.UserExtraIpForm()()
         data = form.to_python(dict(ip=ip))
         user = self._get_user(user)
         obj = UserIpMap()
         obj.user = user
         obj.ip_addr = data['ip']
         self.sa.add(obj)
         return obj
     def delete_extra_ip(self, user, ip_id):
         """
         Removes IP address from UserIpMap
         :param user:
         :param ip_id:
         """
         user = self._get_user(user)
         obj = UserIpMap.query().get(ip_id)
         if obj:
             self.sa.delete(obj)

kallithea/tests/models/test_notifications.py

➞

Show inline comments

@@ @@ -76,209 +76,209 @@ class TestNotifications(TestController): @@
         notification1 = NotificationModel().create(created_by=self.u1,
                                             subject=u'subj', body=u'hi there1',
                                             recipients=[self.u3])
         Session().commit()
         notification2 = NotificationModel().create(created_by=self.u1,
                                             subject=u'subj', body=u'hi there2',
                                             recipients=[self.u3])
         Session().commit()
         u3 = Session().query(User).get(self.u3)
         assert sorted([x.notification for x in u3.notifications]) == sorted([notification2, notification1])
     def test_delete_notifications(self):
         notification = NotificationModel().create(created_by=self.u1,
                                            subject=u'title', body=u'hi there3',
                                     recipients=[self.u3, self.u1, self.u2])
         Session().commit()
         notifications = Notification.query().all()
         assert notification in notifications
         Notification.delete(notification.notification_id)
         Session().commit()
         notifications = Notification.query().all()
         assert not notification in notifications
         un = UserNotification.query().filter(UserNotification.notification
                                              == notification).all()
         assert un == []
     def test_delete_association(self):
         notification = NotificationModel().create(created_by=self.u1,
                                            subject=u'title', body=u'hi there3',
                                     recipients=[self.u3, self.u1, self.u2])
         Session().commit()
         unotification = UserNotification.query() \
                             .filter(UserNotification.notification ==
                                     notification) \
                             .filter(UserNotification.user_id == self.u3) \
                             .scalar()
         assert unotification.user_id == self.u3
         NotificationModel().delete(self.u3,
                                    notification.notification_id)
         Session().commit()
         u3notification = UserNotification.query() \
                             .filter(UserNotification.notification ==
                                     notification) \
                             .filter(UserNotification.user_id == self.u3) \
                             .scalar()
         assert u3notification == None
         # notification object is still there
         assert Notification.query().all() == [notification]
         #u1 and u2 still have assignments
         u1notification = UserNotification.query() \
                             .filter(UserNotification.notification ==
                                     notification) \
                             .filter(UserNotification.user_id == self.u1) \
                             .scalar()
         assert u1notification != None
         u2notification = UserNotification.query() \
                             .filter(UserNotification.notification ==
                                     notification) \
                             .filter(UserNotification.user_id == self.u2) \
                             .scalar()
         assert u2notification != None
     def test_notification_counter(self):
         NotificationModel().create(created_by=self.u1,
                             subject=u'title', body=u'hi there_delete',
                             recipients=[self.u3, self.u1])
         Session().commit()
         assert NotificationModel().get_unread_cnt_for_user(self.u1) == 0
         assert NotificationModel().get_unread_cnt_for_user(self.u2) == 0
         assert NotificationModel().get_unread_cnt_for_user(self.u3) == 1
         notification = NotificationModel().create(created_by=self.u1,
                                            subject=u'title', body=u'hi there3',
                                     recipients=[self.u3, self.u1, self.u2])
         Session().commit()
         assert NotificationModel().get_unread_cnt_for_user(self.u1) == 0
         assert NotificationModel().get_unread_cnt_for_user(self.u2) == 1
         assert NotificationModel().get_unread_cnt_for_user(self.u3) == 2
     @mock.patch.object(h, 'canonical_url', (lambda arg, **kwargs: 'http://%s/?%s' % (arg, '&'.join('%s=%s' % (k, v) for (k, v) in sorted(kwargs.items())))))
     def test_dump_html_mails(self):
         # Exercise all notification types and dump them to one big html file
         l = []
         def send_email(recipients, subject, body='', html_body='', headers=None, author=None):
             l.append('<hr/>\n')
             l.append('<h1>%s</h1>\n' % desc) # desc is from outer scope
             l.append('<pre>\n')
             l.append('From: %s\n' % author.username)
             l.append('To: %s\n' % ' '.join(recipients))
             l.append('Subject: %s\n' % subject)
             l.append('</pre>\n')
             l.append('<hr/>\n')
             l.append('<pre>%s</pre>\n' % body)
             l.append('<hr/>\n')
             l.append(html_body)
             l.append('<hr/>\n')
         with mock.patch.object(kallithea.lib.celerylib.tasks, 'send_email', send_email):
             pr_kwargs = dict(
                 pr_nice_id='#7',
                 pr_title='The Title',
                 pr_title_short='The Title',
                 pr_url='http://pr.org/7',
                 pr_target_repo='http://mainline.com/repo',
                 pr_target_branch='trunk',
                 pr_source_repo='https://dev.org/repo',
                 pr_source_branch='devbranch',
                 pr_owner=User.get(self.u2),
                 pr_owner_username='u2'
+                )
             for type_, body, kwargs in [
                 (Notification.TYPE_CHANGESET_COMMENT,
                  u'This is the new comment.\n\n - and here it ends indented.',
                  dict(
                     short_id='cafe1234',
                     raw_id='cafe1234c0ffeecafe',
                     branch='brunch',
                     cs_comment_user='Opinionated User (jsmith)',
                     cs_comment_url='http://comment.org',
                     is_mention=[False, True],
                     message='This changeset did something clever which is hard to explain',
                     message_short='This changeset did something cl...',
                     status_change=[None, 'Approved'],
                     cs_target_repo='repo_target',
                     cs_url='http://changeset.com',
                     cs_author=User.get(self.u2))),
                 (Notification.TYPE_MESSAGE,
                  u'This is the body of the test message\n - nothing interesting here except indentation.',
                  dict()),
                 #(Notification.TYPE_MENTION, '$body', None), # not used
                 (Notification.TYPE_REGISTRATION,
                  u'Registration body',
                  dict(
                     new_username='newbie',
                     registered_user_url='http://newbie.org',
                     new_email='new@email.com',
                     new_full_name='New Full Name')),
                 (Notification.TYPE_PULL_REQUEST,
                  u'This PR is awesome because it does stuff\n - please approve indented!',
                  dict(
                     pr_user_created='Requesting User (root)', # pr_owner should perhaps be used for @mention in description ...
                     is_mention=[False, True],
                     pr_revisions=[('123abc'*7, "Introduce one and two\n\nand that's it"), ('567fed'*7, 'Make one plus two equal tree')],
                     org_repo_name='repo_org',
                     **pr_kwargs)),
                 (Notification.TYPE_PULL_REQUEST_COMMENT,
                  u'Me too!\n\n - and indented on second line',
                  dict(
                     closing_pr=[False, True],
                     is_mention=[False, True],
                     pr_comment_user='Opinionated User (jsmith)',
                     pr_comment_url='http://pr.org/comment',
                     status_change=[None, 'Under Review'],
                     **pr_kwargs)),
                 ]:
                 kwargs['repo_name'] = u'repo/name'
                 params = [(type_, type_, body, kwargs)]
                 for param_name in ['is_mention', 'status_change', 'closing_pr']: # TODO: inline/general
                     if not isinstance(kwargs.get(param_name), list):
                         continue
                     new_params = []
                     for v in kwargs[param_name]:
                         for desc, type_, body, kwargs in params:
                             kwargs = dict(kwargs)
                             kwargs[param_name] = v
                             new_params.append(('%s, %s=%r' % (desc, param_name, v), type_, body, kwargs))
                     params = new_params
                 for desc, type_, body, kwargs in params:
                     # desc is used as "global" variable
                     notification = NotificationModel().create(created_by=self.u1,
                                                        subject=u'unused', body=body, email_kwargs=kwargs,
                                                        recipients=[self.u2], type_=type_)
             # Email type TYPE_PASSWORD_RESET has no corresponding notification type - test it directly:
             desc = 'TYPE_PASSWORD_RESET'
             kwargs = dict(user='John Doe', reset_token='decbf64715098db5b0bd23eab44bd792670ab746', reset_url='http://reset.com/decbf64715098db5b0bd23eab44bd792670ab746')
-            kallithea.lib.celerylib.run_task(kallithea.lib.celerylib.tasks.send_email, ['john@doe.com'],
+            kallithea.lib.celerylib.tasks.send_email(['john@doe.com'],
                 "Password reset link",
                 EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'txt', **kwargs),
                 EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'html', **kwargs),
                 author=User.get(self.u1))
         out = '<!doctype html>\n<html lang="en">\n<head><title>Notifications</title><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"></head>\n<body>\n%s\n</body>\n</html>\n' % \
             re.sub(r'<(/?(?:!doctype|html|head|title|meta|body)\b[^>]*)>', r'<!--\1-->', ''.join(l))
         outfn = os.path.join(os.path.dirname(__file__), 'test_dump_html_mails.out.html')
         reffn = os.path.join(os.path.dirname(__file__), 'test_dump_html_mails.ref.html')
         with file(outfn, 'w') as f:
             f.write(out)
         with file(reffn) as f:
             ref = f.read()
         assert ref == out # copy test_dump_html_mails.out.html to test_dump_html_mails.ref.html to update expectations
         os.unlink(outfn)

0 comments (0 inline, 0 general)