Changeset - aafca212c8e2
[Not reviewed]
default
0 9 0
Mads Kiilerich - 5 years ago 2020-12-29 22:23:01
mads@kiilerich.com
celery: move send_email task to a better home in notification model

Avoid bundling everything from many different layers in one big task library.

This is more feasible now when we don't need kallithea.CELERY_APP set at import
time.
9 files changed with 152 insertions and 155 deletions:
0 comments (0 inline, 0 general)
kallithea/controllers/admin/settings.py
Show inline comments
 
@@ -44,7 +44,7 @@ from kallithea.lib.utils import repo2db_
 
from kallithea.lib.utils2 import safe_str
 
from kallithea.lib.vcs import VCSError
 
from kallithea.lib.webutils import url
 
from kallithea.model import async_tasks, db, meta
 
from kallithea.model import db, meta, notification
 
from kallithea.model.forms import ApplicationSettingsForm, ApplicationUiSettingsForm, ApplicationVisualisationForm
 
from kallithea.model.notification import EmailNotificationModel
 
from kallithea.model.scm import ScmModel
 
@@ -301,7 +301,7 @@ class SettingsController(base.BaseContro
 

	
 
            recipients = [test_email] if test_email else None
 

	
 
            async_tasks.send_email(recipients, test_email_subj,
 
            notification.send_email(recipients, test_email_subj,
 
                             test_email_txt_body, test_email_html_body)
 

	
 
            webutils.flash(_('Send email task created'), category='success')
kallithea/lib/celery_app.py
Show inline comments
 
@@ -21,6 +21,7 @@ class CeleryConfig(object):
 
    imports = [
 
        'kallithea.lib.indexers.daemon',
 
        'kallithea.model.async_tasks',
 
        'kallithea.model.notification',
 
        'kallithea.model.repo',
 
    ]
 
    task_always_eager = False
kallithea/model/async_tasks.py
Show inline comments
 
@@ -26,11 +26,7 @@ Original author and date, and relevant c
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import email.message
 
import email.utils
 
import os
 
import smtplib
 
import time
 
import traceback
 
from collections import OrderedDict
 
from operator import itemgetter
 
@@ -46,7 +42,7 @@ from kallithea.lib.vcs.utils import auth
 
from kallithea.model import db, meta
 

	
 

	
 
__all__ = ['get_commits_stats', 'send_email']
 
__all__ = ['get_commits_stats']
 

	
 

	
 
log = celery.utils.log.get_task_logger(__name__)
 
@@ -219,120 +215,6 @@ def get_commits_stats(repo_name, ts_min_
 
        log.info('Task with key %s already running', lockkey)
 

	
 

	
 
@celerylib.task
 
def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
    """
 
    Sends an email with defined parameters from the .ini files.
 

	
 
    :param recipients: list of recipients, if this is None, the defined email
 
        address from field 'email_to' and all admins is used instead
 
    :param subject: subject of the mail
 
    :param body: plain text body of the mail
 
    :param html_body: html version of body
 
    :param headers: dictionary of prepopulated e-mail headers
 
    :param from_name: full name to be used as sender of this mail - often a
 
    .full_name_or_username value
 
    """
 
    assert isinstance(recipients, list), recipients
 
    if headers is None:
 
        headers = {}
 
    else:
 
        # do not modify the original headers object passed by the caller
 
        headers = headers.copy()
 

	
 
    email_config = config
 
    email_prefix = email_config.get('email_prefix', '')
 
    if email_prefix:
 
        subject = "%s %s" % (email_prefix, subject)
 

	
 
    if not recipients:
 
        # if recipients are not defined we send to email_config + all admins
 
        recipients = [u.email for u in db.User.query()
 
                      .filter(db.User.admin == True).all()]
 
        if email_config.get('email_to') is not None:
 
            recipients += email_config.get('email_to').split(',')
 

	
 
        # If there are still no recipients, there are no admins and no address
 
        # configured in email_to, so return.
 
        if not recipients:
 
            log.error("No recipients specified and no fallback available.")
 
            return
 

	
 
        log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
 

	
 
    # SMTP sender
 
    app_email_from = email_config.get('app_email_from', 'Kallithea')
 
    # 'From' header
 
    if from_name is not None:
 
        # set From header based on from_name but with a generic e-mail address
 
        # In case app_email_from is in "Some Name <e-mail>" format, we first
 
        # extract the e-mail address.
 
        envelope_addr = author_email(app_email_from)
 
        headers['From'] = '"%s" <%s>' % (
 
            email.utils.quote('%s (no-reply)' % from_name),
 
            envelope_addr)
 

	
 
    smtp_server = email_config.get('smtp_server')
 
    smtp_port = email_config.get('smtp_port')
 
    smtp_use_tls = asbool(email_config.get('smtp_use_tls'))
 
    smtp_use_ssl = asbool(email_config.get('smtp_use_ssl'))
 
    smtp_auth = email_config.get('smtp_auth')  # undocumented - overrule automatic choice of auth mechanism
 
    smtp_username = email_config.get('smtp_username')
 
    smtp_password = email_config.get('smtp_password')
 

	
 
    logmsg = ("Mail details:\n"
 
              "recipients: %s\n"
 
              "headers: %s\n"
 
              "subject: %s\n"
 
              "body:\n%s\n"
 
              "html:\n%s\n"
 
              % (' '.join(recipients), headers, subject, body, html_body))
 

	
 
    if smtp_server:
 
        log.debug("Sending e-mail. " + logmsg)
 
    else:
 
        log.error("SMTP mail server not configured - cannot send e-mail.")
 
        log.warning(logmsg)
 
        return
 

	
 
    msg = email.message.EmailMessage()
 
    msg['Subject'] = subject
 
    msg['From'] = app_email_from  # fallback - might be overridden by a header
 
    msg['To'] = ', '.join(recipients)
 
    msg['Date'] = email.utils.formatdate(time.time())
 

	
 
    for key, value in headers.items():
 
        del msg[key]  # Delete key first to make sure add_header will replace header (if any), no matter the casing
 
        msg.add_header(key, value)
 

	
 
    msg.set_content(body)
 
    msg.add_alternative(html_body, subtype='html')
 

	
 
    try:
 
        if smtp_use_ssl:
 
            smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port)
 
        else:
 
            smtp_serv = smtplib.SMTP(smtp_server, smtp_port)
 

	
 
        if smtp_use_tls:
 
            smtp_serv.starttls()
 

	
 
        if smtp_auth:
 
            smtp_serv.ehlo()  # populate esmtp_features
 
            smtp_serv.esmtp_features["auth"] = smtp_auth
 

	
 
        if smtp_username and smtp_password is not None:
 
            smtp_serv.login(smtp_username, smtp_password)
 

	
 
        smtp_serv.sendmail(app_email_from, recipients, msg.as_string())
 
        smtp_serv.quit()
 

	
 
        log.info('Mail was sent to: %s' % recipients)
 
    except:
 
        log.error('Mail sending failed')
 
        log.error(traceback.format_exc())
 

	
 

	
 
def __get_codes_stats(repo_name):
 
    scm_repo = db.Repository.get_by_repo_name(repo_name).scm_instance
 

	
kallithea/model/notification.py
Show inline comments
 
@@ -27,14 +27,21 @@ Original author and date, and relevant c
 
"""
 

	
 
import datetime
 
import email.message
 
import email.utils
 
import logging
 
import smtplib
 
import time
 
import traceback
 

	
 
from tg import app_globals
 
from tg import app_globals, config
 
from tg import tmpl_context as c
 
from tg.i18n import ugettext as _
 

	
 
from kallithea.lib import webutils
 
from kallithea.model import async_tasks, db
 
from kallithea.lib import celerylib, webutils
 
from kallithea.lib.utils2 import asbool
 
from kallithea.lib.vcs.utils import author_email
 
from kallithea.model import db
 

	
 

	
 
log = logging.getLogger(__name__)
 
@@ -133,7 +140,7 @@ class NotificationModel(object):
 

	
 
        # send email with notification to participants
 
        for rec_mail in sorted(rec_mails):
 
            async_tasks.send_email([rec_mail], email_subject, email_txt_body,
 
            send_email([rec_mail], email_subject, email_txt_body,
 
                     email_html_body, headers,
 
                     from_name=created_by_obj.full_name_or_username)
 

	
 
@@ -228,3 +235,117 @@ class EmailNotificationModel(object):
 

	
 
        log.debug('rendering tmpl %s with kwargs %s', base, _kwargs)
 
        return email_template.render_unicode(**_kwargs)
 

	
 

	
 
@celerylib.task
 
def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
    """
 
    Sends an email with defined parameters from the .ini files.
 

	
 
    :param recipients: list of recipients, if this is None, the defined email
 
        address from field 'email_to' and all admins is used instead
 
    :param subject: subject of the mail
 
    :param body: plain text body of the mail
 
    :param html_body: html version of body
 
    :param headers: dictionary of prepopulated e-mail headers
 
    :param from_name: full name to be used as sender of this mail - often a
 
    .full_name_or_username value
 
    """
 
    assert isinstance(recipients, list), recipients
 
    if headers is None:
 
        headers = {}
 
    else:
 
        # do not modify the original headers object passed by the caller
 
        headers = headers.copy()
 

	
 
    email_config = config
 
    email_prefix = email_config.get('email_prefix', '')
 
    if email_prefix:
 
        subject = "%s %s" % (email_prefix, subject)
 

	
 
    if not recipients:
 
        # if recipients are not defined we send to email_config + all admins
 
        recipients = [u.email for u in db.User.query()
 
                      .filter(db.User.admin == True).all()]
 
        if email_config.get('email_to') is not None:
 
            recipients += email_config.get('email_to').split(',')
 

	
 
        # If there are still no recipients, there are no admins and no address
 
        # configured in email_to, so return.
 
        if not recipients:
 
            log.error("No recipients specified and no fallback available.")
 
            return
 

	
 
        log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
 

	
 
    # SMTP sender
 
    app_email_from = email_config.get('app_email_from', 'Kallithea')
 
    # 'From' header
 
    if from_name is not None:
 
        # set From header based on from_name but with a generic e-mail address
 
        # In case app_email_from is in "Some Name <e-mail>" format, we first
 
        # extract the e-mail address.
 
        envelope_addr = author_email(app_email_from)
 
        headers['From'] = '"%s" <%s>' % (
 
            email.utils.quote('%s (no-reply)' % from_name),
 
            envelope_addr)
 

	
 
    smtp_server = email_config.get('smtp_server')
 
    smtp_port = email_config.get('smtp_port')
 
    smtp_use_tls = asbool(email_config.get('smtp_use_tls'))
 
    smtp_use_ssl = asbool(email_config.get('smtp_use_ssl'))
 
    smtp_auth = email_config.get('smtp_auth')  # undocumented - overrule automatic choice of auth mechanism
 
    smtp_username = email_config.get('smtp_username')
 
    smtp_password = email_config.get('smtp_password')
 

	
 
    logmsg = ("Mail details:\n"
 
              "recipients: %s\n"
 
              "headers: %s\n"
 
              "subject: %s\n"
 
              "body:\n%s\n"
 
              "html:\n%s\n"
 
              % (' '.join(recipients), headers, subject, body, html_body))
 

	
 
    if smtp_server:
 
        log.debug("Sending e-mail. " + logmsg)
 
    else:
 
        log.error("SMTP mail server not configured - cannot send e-mail.")
 
        log.warning(logmsg)
 
        return
 

	
 
    msg = email.message.EmailMessage()
 
    msg['Subject'] = subject
 
    msg['From'] = app_email_from  # fallback - might be overridden by a header
 
    msg['To'] = ', '.join(recipients)
 
    msg['Date'] = email.utils.formatdate(time.time())
 

	
 
    for key, value in headers.items():
 
        del msg[key]  # Delete key first to make sure add_header will replace header (if any), no matter the casing
 
        msg.add_header(key, value)
 

	
 
    msg.set_content(body)
 
    msg.add_alternative(html_body, subtype='html')
 

	
 
    try:
 
        if smtp_use_ssl:
 
            smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port)
 
        else:
 
            smtp_serv = smtplib.SMTP(smtp_server, smtp_port)
 

	
 
        if smtp_use_tls:
 
            smtp_serv.starttls()
 

	
 
        if smtp_auth:
 
            smtp_serv.ehlo()  # populate esmtp_features
 
            smtp_serv.esmtp_features["auth"] = smtp_auth
 

	
 
        if smtp_username and smtp_password is not None:
 
            smtp_serv.login(smtp_username, smtp_password)
 

	
 
        smtp_serv.sendmail(app_email_from, recipients, msg.as_string())
 
        smtp_serv.quit()
 

	
 
        log.info('Mail was sent to: %s' % recipients)
 
    except:
 
        log.error('Mail sending failed')
 
        log.error(traceback.format_exc())
kallithea/model/user.py
Show inline comments
 
@@ -39,7 +39,7 @@ from tg.i18n import ugettext as _
 
from kallithea.lib import hooks, webutils
 
from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException
 
from kallithea.lib.utils2 import check_password, generate_api_key, get_crypt_password, get_current_authuser
 
from kallithea.model import db, forms, meta
 
from kallithea.model import db, forms, meta, notification
 

	
 

	
 
log = logging.getLogger(__name__)
 
@@ -162,8 +162,6 @@ class UserModel(object):
 
            raise
 

	
 
    def create_registration(self, form_data):
 
        from kallithea.model import notification
 

	
 
        form_data['admin'] = False
 
        form_data['extern_type'] = db.User.DEFAULT_AUTH_TYPE
 
        form_data['extern_name'] = ''
 
@@ -298,8 +296,6 @@ class UserModel(object):
 
        allowing users to copy-paste or manually enter the token from the
 
        email.
 
        """
 
        from kallithea.model import async_tasks, notification
 

	
 
        user_email = data['email']
 
        user = db.User.get_by_email(user_email)
 
        timestamp = int(time.time())
 
@@ -331,7 +327,7 @@ class UserModel(object):
 
                reset_token=token,
 
                reset_url=link)
 
            log.debug('sending email')
 
            async_tasks.send_email([user_email], _("Password reset link"), body, html_body)
 
            notification.send_email([user_email], _("Password reset link"), body, html_body)
 
            log.info('send new password mail to %s', user_email)
 
        else:
 
            log.debug("password reset email %s not found", user_email)
 
@@ -364,7 +360,6 @@ class UserModel(object):
 
        return expected_token == token
 

	
 
    def reset_password(self, user_email, new_passwd):
 
        from kallithea.model import async_tasks
 
        user = db.User.get_by_email(user_email)
 
        if user is not None:
 
            if not self.can_change_password(user):
 
@@ -375,7 +370,7 @@ class UserModel(object):
 
        if new_passwd is None:
 
            raise Exception('unable to set new password')
 

	
 
        async_tasks.send_email([user_email],
 
        notification.send_email([user_email],
 
                 _('Password reset notification'),
 
                 _('The password to your account %s has been changed using password reset form.') % (user.username,))
 
        log.info('send password reset mail to %s', user_email)
kallithea/tests/functional/test_login.py
Show inline comments
 
@@ -6,7 +6,7 @@ import urllib.parse
 
import mock
 
from tg.util.webtest import test_context
 

	
 
import kallithea.model.async_tasks
 
import kallithea.model.notification
 
from kallithea.lib import webutils
 
from kallithea.lib.utils2 import check_password, generate_api_key
 
from kallithea.model import db, meta, validators
 
@@ -410,7 +410,7 @@ class TestLoginController(base.TestContr
 
        def mock_send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
            collected.append((recipients, subject, body, html_body))
 

	
 
        with mock.patch.object(kallithea.model.async_tasks, 'send_email', mock_send_email), \
 
        with mock.patch.object(kallithea.model.notification, 'send_email', mock_send_email), \
 
                mock.patch.object(time, 'time', lambda: timestamp):
 
            response = self.app.post(base.url(controller='login',
 
                                         action='password_reset'),
kallithea/tests/models/test_notifications.py
Show inline comments
 
@@ -5,7 +5,6 @@ import mock
 
from tg.util.webtest import test_context
 

	
 
import kallithea.lib.celerylib
 
import kallithea.model.async_tasks
 
from kallithea.lib import webutils
 
from kallithea.model import db, meta
 
from kallithea.model.notification import EmailNotificationModel, NotificationModel
 
@@ -48,7 +47,7 @@ class TestNotifications(base.TestControl
 
                assert body == "hi there"
 
                assert '>hi there<' in html_body
 
                assert from_name == 'u1 u1'
 
            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
 
            with mock.patch.object(kallithea.model.notification, 'send_email', send_email):
 
                NotificationModel().create(created_by=self.u1,
 
                                                   body='hi there',
 
                                                   recipients=usrs)
 
@@ -73,7 +72,7 @@ class TestNotifications(base.TestControl
 
            l.append('<hr/>\n')
 

	
 
        with test_context(self.app):
 
            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
 
            with mock.patch.object(kallithea.model.notification, 'send_email', send_email):
 
                pr_kwargs = dict(
 
                    pr_nice_id='#7',
 
                    pr_title='The Title',
 
@@ -155,7 +154,7 @@ class TestNotifications(base.TestControl
 
                # Email type TYPE_PASSWORD_RESET has no corresponding notification type - test it directly:
 
                desc = 'TYPE_PASSWORD_RESET'
 
                kwargs = dict(user='John Doe', reset_token='decbf64715098db5b0bd23eab44bd792670ab746', reset_url='http://reset.com/decbf64715098db5b0bd23eab44bd792670ab746')
 
                kallithea.model.async_tasks.send_email(['john@doe.com'],
 
                kallithea.model.notification.send_email(['john@doe.com'],
 
                    "Password reset link",
 
                    EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'txt', **kwargs),
 
                    EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'html', **kwargs),
kallithea/tests/other/test_mail.py
Show inline comments
 
@@ -25,7 +25,7 @@ class smtplib_mock(object):
 
        smtplib_mock.lastmsg = msg
 

	
 

	
 
@mock.patch('kallithea.model.async_tasks.smtplib', smtplib_mock)
 
@mock.patch('kallithea.model.notification.smtplib', smtplib_mock)
 
class TestMail(base.TestController):
 

	
 
    def test_send_mail_trivial(self):
 
@@ -40,8 +40,8 @@ class TestMail(base.TestController):
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -64,8 +64,8 @@ class TestMail(base.TestController):
 
            'app_email_from': envelope_from,
 
            'email_to': email_to,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL, email_to])
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -88,8 +88,8 @@ class TestMail(base.TestController):
 
            'app_email_from': envelope_from,
 
            'email_to': email_to,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL] + email_to.split(','))
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -110,8 +110,8 @@ class TestMail(base.TestController):
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL])
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -133,8 +133,8 @@ class TestMail(base.TestController):
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -157,8 +157,8 @@ class TestMail(base.TestController):
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
@@ -181,8 +181,8 @@ class TestMail(base.TestController):
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body,
 
        with mock.patch('kallithea.model.notification.config', config_mock):
 
            kallithea.model.notification.send_email(recipients, subject, body, html_body,
 
                                                     from_name=author.full_name_or_username, headers=headers)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
scripts/deps.py
Show inline comments
 
@@ -158,7 +158,6 @@ known_violations = set([
 
('kallithea.lib.utils', 'kallithea.model'),  # clean up utils
 
('kallithea.lib.utils', 'kallithea.model.db'),
 
('kallithea.lib.utils', 'kallithea.model.scm'),
 
('kallithea.model.async_tasks', 'kallithea.model'),
 
('kallithea.model', 'kallithea.lib.auth'),  # auth.HasXXX
 
('kallithea.model', 'kallithea.lib.auth_modules'),  # validators
 
('kallithea.model', 'kallithea.lib.hooks'),  # clean up hooks
0 comments (0 inline, 0 general)