kallithea Changeset - aafca212c8e2

Changeset - aafca212c8e2

Parent rev.

Child rev.

[Not reviewed]

default

0 9 0

Mads Kiilerich - 5 years ago 2020-12-29 22:23:01
mads@kiilerich.com

celery: move send_email task to a better home in notification model

Avoid bundling everything from many different layers in one big task library.

This is more feasible now when we don't need kallithea.CELERY_APP set at import
time.

9 files changed with 152 insertions and 155 deletions:

kallithea/controllers/admin/settings.py

kallithea/lib/celery_app.py

kallithea/model/async_tasks.py

119

kallithea/model/notification.py

125

kallithea/model/user.py

kallithea/tests/functional/test_login.py

kallithea/tests/models/test_notifications.py

kallithea/tests/other/test_mail.py

scripts/deps.py

0 comments (0 inline, 0 general)

kallithea/controllers/admin/settings.py

➞

Show inline comments

@@ @@ -44,7 +44,7 @@ from kallithea.lib.utils import repo2db_ @@
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.vcs import VCSError
 from kallithea.lib.webutils import url
-from kallithea.model import async_tasks, db, meta
+from kallithea.model import db, meta, notification
 from kallithea.model.forms import ApplicationSettingsForm, ApplicationUiSettingsForm, ApplicationVisualisationForm
 from kallithea.model.notification import EmailNotificationModel
 from kallithea.model.scm import ScmModel
@@ @@ -301,7 +301,7 @@ class SettingsController(base.BaseContro @@
             recipients = [test_email] if test_email else None
-            async_tasks.send_email(recipients, test_email_subj,
+            notification.send_email(recipients, test_email_subj,
                              test_email_txt_body, test_email_html_body)
             webutils.flash(_('Send email task created'), category='success')

kallithea/lib/celery_app.py

➞

Show inline comments

@@ @@ -21,6 +21,7 @@ class CeleryConfig(object): @@
     imports = [
         'kallithea.lib.indexers.daemon',
         'kallithea.model.async_tasks',
         'kallithea.model.notification',
         'kallithea.model.repo',
+    ]
     task_always_eager = False

kallithea/model/async_tasks.py

➞

Show inline comments

@@ @@ -26,11 +26,7 @@ Original author and date, and relevant c @@
 :license: GPLv3, see LICENSE.md for more details.
 """
 import email.message
 import email.utils
 import os
 import smtplib
 import time
 import traceback
 from collections import OrderedDict
 from operator import itemgetter
@@ @@ -46,7 +42,7 @@ from kallithea.lib.vcs.utils import auth @@
 from kallithea.model import db, meta
-__all__ = ['get_commits_stats', 'send_email']
 __all__ = ['get_commits_stats']
 log = celery.utils.log.get_task_logger(__name__)
@@ @@ -219,120 +215,6 @@ def get_commits_stats(repo_name, ts_min_ @@
         log.info('Task with key %s already running', lockkey)
 @celerylib.task
 def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
     """
     Sends an email with defined parameters from the .ini files.
     :param recipients: list of recipients, if this is None, the defined email
         address from field 'email_to' and all admins is used instead
     :param subject: subject of the mail
     :param body: plain text body of the mail
     :param html_body: html version of body
     :param headers: dictionary of prepopulated e-mail headers
     :param from_name: full name to be used as sender of this mail - often a
     .full_name_or_username value
     """
     assert isinstance(recipients, list), recipients
     if headers is None:
         headers = {}
     else:
         # do not modify the original headers object passed by the caller
         headers = headers.copy()
     email_config = config
     email_prefix = email_config.get('email_prefix', '')
     if email_prefix:
         subject = "%s %s" % (email_prefix, subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         recipients = [u.email for u in db.User.query()
                       .filter(db.User.admin == True).all()]
         if email_config.get('email_to') is not None:
             recipients += email_config.get('email_to').split(',')
         # If there are still no recipients, there are no admins and no address
         # configured in email_to, so return.
         if not recipients:
             log.error("No recipients specified and no fallback available.")
             return
         log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
     # SMTP sender
     app_email_from = email_config.get('app_email_from', 'Kallithea')
     # 'From' header
     if from_name is not None:
         # set From header based on from_name but with a generic e-mail address
         # In case app_email_from is in "Some Name <e-mail>" format, we first
         # extract the e-mail address.
         envelope_addr = author_email(app_email_from)
         headers['From'] = '"%s" <%s>' % (
             email.utils.quote('%s (no-reply)' % from_name),
             envelope_addr)
     smtp_server = email_config.get('smtp_server')
     smtp_port = email_config.get('smtp_port')
     smtp_use_tls = asbool(email_config.get('smtp_use_tls'))
     smtp_use_ssl = asbool(email_config.get('smtp_use_ssl'))
     smtp_auth = email_config.get('smtp_auth')  # undocumented - overrule automatic choice of auth mechanism
     smtp_username = email_config.get('smtp_username')
     smtp_password = email_config.get('smtp_password')
     logmsg = ("Mail details:\n"
               "recipients: %s\n"
               "headers: %s\n"
               "subject: %s\n"
               "body:\n%s\n"
               "html:\n%s\n"
               % (' '.join(recipients), headers, subject, body, html_body))
     if smtp_server:
         log.debug("Sending e-mail. " + logmsg)
     else:
         log.error("SMTP mail server not configured - cannot send e-mail.")
         log.warning(logmsg)
         return
     msg = email.message.EmailMessage()
     msg['Subject'] = subject
     msg['From'] = app_email_from  # fallback - might be overridden by a header
     msg['To'] = ', '.join(recipients)
     msg['Date'] = email.utils.formatdate(time.time())
     for key, value in headers.items():
         del msg[key]  # Delete key first to make sure add_header will replace header (if any), no matter the casing
         msg.add_header(key, value)
     msg.set_content(body)
     msg.add_alternative(html_body, subtype='html')
     try:
         if smtp_use_ssl:
             smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port)
         else:
             smtp_serv = smtplib.SMTP(smtp_server, smtp_port)
         if smtp_use_tls:
             smtp_serv.starttls()
         if smtp_auth:
             smtp_serv.ehlo()  # populate esmtp_features
             smtp_serv.esmtp_features["auth"] = smtp_auth
         if smtp_username and smtp_password is not None:
             smtp_serv.login(smtp_username, smtp_password)
         smtp_serv.sendmail(app_email_from, recipients, msg.as_string())
         smtp_serv.quit()
         log.info('Mail was sent to: %s' % recipients)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
 def __get_codes_stats(repo_name):
     scm_repo = db.Repository.get_by_repo_name(repo_name).scm_instance

kallithea/model/notification.py

➞

Show inline comments

@@ @@ -27,14 +27,21 @@ Original author and date, and relevant c @@
 """
 import datetime
 import email.message
 import email.utils
 import logging
 import smtplib
 import time
 import traceback
 from tg import app_globals
+from tg import app_globals, config
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from kallithea.lib import webutils
 from kallithea.model import async_tasks, db
 from kallithea.lib import celerylib, webutils
 from kallithea.lib.utils2 import asbool
 from kallithea.lib.vcs.utils import author_email
 from kallithea.model import db
 log = logging.getLogger(__name__)
@@ @@ -133,7 +140,7 @@ class NotificationModel(object): @@
         # send email with notification to participants
         for rec_mail in sorted(rec_mails):
-            async_tasks.send_email([rec_mail], email_subject, email_txt_body,
             send_email([rec_mail], email_subject, email_txt_body,
                      email_html_body, headers,
                      from_name=created_by_obj.full_name_or_username)
@@ @@ -228,3 +235,117 @@ class EmailNotificationModel(object): @@
         log.debug('rendering tmpl %s with kwargs %s', base, _kwargs)
         return email_template.render_unicode(**_kwargs)
 @celerylib.task
 def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
     """
     Sends an email with defined parameters from the .ini files.
     :param recipients: list of recipients, if this is None, the defined email
         address from field 'email_to' and all admins is used instead
     :param subject: subject of the mail
     :param body: plain text body of the mail
     :param html_body: html version of body
     :param headers: dictionary of prepopulated e-mail headers
     :param from_name: full name to be used as sender of this mail - often a
     .full_name_or_username value
     """
     assert isinstance(recipients, list), recipients
     if headers is None:
         headers = {}
     else:
         # do not modify the original headers object passed by the caller
         headers = headers.copy()
     email_config = config
     email_prefix = email_config.get('email_prefix', '')
     if email_prefix:
         subject = "%s %s" % (email_prefix, subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         recipients = [u.email for u in db.User.query()
                       .filter(db.User.admin == True).all()]
         if email_config.get('email_to') is not None:
             recipients += email_config.get('email_to').split(',')
         # If there are still no recipients, there are no admins and no address
         # configured in email_to, so return.
         if not recipients:
             log.error("No recipients specified and no fallback available.")
             return
         log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
     # SMTP sender
     app_email_from = email_config.get('app_email_from', 'Kallithea')
     # 'From' header
     if from_name is not None:
         # set From header based on from_name but with a generic e-mail address
         # In case app_email_from is in "Some Name <e-mail>" format, we first
         # extract the e-mail address.
         envelope_addr = author_email(app_email_from)
         headers['From'] = '"%s" <%s>' % (
             email.utils.quote('%s (no-reply)' % from_name),
             envelope_addr)
     smtp_server = email_config.get('smtp_server')
     smtp_port = email_config.get('smtp_port')
     smtp_use_tls = asbool(email_config.get('smtp_use_tls'))
     smtp_use_ssl = asbool(email_config.get('smtp_use_ssl'))
     smtp_auth = email_config.get('smtp_auth')  # undocumented - overrule automatic choice of auth mechanism
     smtp_username = email_config.get('smtp_username')
     smtp_password = email_config.get('smtp_password')
     logmsg = ("Mail details:\n"
               "recipients: %s\n"
               "headers: %s\n"
               "subject: %s\n"
               "body:\n%s\n"
               "html:\n%s\n"
               % (' '.join(recipients), headers, subject, body, html_body))
     if smtp_server:
         log.debug("Sending e-mail. " + logmsg)
     else:
         log.error("SMTP mail server not configured - cannot send e-mail.")
         log.warning(logmsg)
         return
     msg = email.message.EmailMessage()
     msg['Subject'] = subject
     msg['From'] = app_email_from  # fallback - might be overridden by a header
     msg['To'] = ', '.join(recipients)
     msg['Date'] = email.utils.formatdate(time.time())
     for key, value in headers.items():
         del msg[key]  # Delete key first to make sure add_header will replace header (if any), no matter the casing
         msg.add_header(key, value)
     msg.set_content(body)
     msg.add_alternative(html_body, subtype='html')
     try:
         if smtp_use_ssl:
             smtp_serv = smtplib.SMTP_SSL(smtp_server, smtp_port)
         else:
             smtp_serv = smtplib.SMTP(smtp_server, smtp_port)
         if smtp_use_tls:
             smtp_serv.starttls()
         if smtp_auth:
             smtp_serv.ehlo()  # populate esmtp_features
             smtp_serv.esmtp_features["auth"] = smtp_auth
         if smtp_username and smtp_password is not None:
             smtp_serv.login(smtp_username, smtp_password)
         smtp_serv.sendmail(app_email_from, recipients, msg.as_string())
         smtp_serv.quit()
         log.info('Mail was sent to: %s' % recipients)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())

kallithea/model/user.py

➞

Show inline comments

@@ @@ -39,7 +39,7 @@ from tg.i18n import ugettext as _ @@
 from kallithea.lib import hooks, webutils
 from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException
 from kallithea.lib.utils2 import check_password, generate_api_key, get_crypt_password, get_current_authuser
 from kallithea.model import db, forms, meta
+from kallithea.model import db, forms, meta, notification
 log = logging.getLogger(__name__)
@@ @@ -162,8 +162,6 @@ class UserModel(object): @@
             raise
     def create_registration(self, form_data):
         from kallithea.model import notification
         form_data['admin'] = False
         form_data['extern_type'] = db.User.DEFAULT_AUTH_TYPE
         form_data['extern_name'] = ''
@@ @@ -298,8 +296,6 @@ class UserModel(object): @@
         allowing users to copy-paste or manually enter the token from the
         email.
         """
         from kallithea.model import async_tasks, notification
         user_email = data['email']
         user = db.User.get_by_email(user_email)
         timestamp = int(time.time())
@@ @@ -331,7 +327,7 @@ class UserModel(object): @@
                 reset_token=token,
                 reset_url=link)
             log.debug('sending email')
-            async_tasks.send_email([user_email], _("Password reset link"), body, html_body)
+            notification.send_email([user_email], _("Password reset link"), body, html_body)
             log.info('send new password mail to %s', user_email)
         else:
             log.debug("password reset email %s not found", user_email)
@@ @@ -364,7 +360,6 @@ class UserModel(object): @@
         return expected_token == token
     def reset_password(self, user_email, new_passwd):
         from kallithea.model import async_tasks
         user = db.User.get_by_email(user_email)
         if user is not None:
             if not self.can_change_password(user):
@@ @@ -375,7 +370,7 @@ class UserModel(object): @@
         if new_passwd is None:
             raise Exception('unable to set new password')
-        async_tasks.send_email([user_email],
+        notification.send_email([user_email],
                  _('Password reset notification'),
                  _('The password to your account %s has been changed using password reset form.') % (user.username,))
         log.info('send password reset mail to %s', user_email)

kallithea/tests/functional/test_login.py

➞

Show inline comments

@@ @@ -6,7 +6,7 @@ import urllib.parse @@
 import mock
 from tg.util.webtest import test_context
-import kallithea.model.async_tasks
+import kallithea.model.notification
 from kallithea.lib import webutils
 from kallithea.lib.utils2 import check_password, generate_api_key
 from kallithea.model import db, meta, validators
@@ @@ -410,7 +410,7 @@ class TestLoginController(base.TestContr @@
         def mock_send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
             collected.append((recipients, subject, body, html_body))
-        with mock.patch.object(kallithea.model.async_tasks, 'send_email', mock_send_email), \
+        with mock.patch.object(kallithea.model.notification, 'send_email', mock_send_email), \
                 mock.patch.object(time, 'time', lambda: timestamp):
             response = self.app.post(base.url(controller='login',
                                          action='password_reset'),

kallithea/tests/models/test_notifications.py

➞

Show inline comments

@@ @@ -5,7 +5,6 @@ import mock @@
 from tg.util.webtest import test_context
 import kallithea.lib.celerylib
 import kallithea.model.async_tasks
 from kallithea.lib import webutils
 from kallithea.model import db, meta
 from kallithea.model.notification import EmailNotificationModel, NotificationModel
@@ @@ -48,7 +47,7 @@ class TestNotifications(base.TestControl @@
                 assert body == "hi there"
                 assert '>hi there<' in html_body
                 assert from_name == 'u1 u1'
-            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
+            with mock.patch.object(kallithea.model.notification, 'send_email', send_email):
                 NotificationModel().create(created_by=self.u1,
                                                    body='hi there',
                                                    recipients=usrs)
@@ @@ -73,7 +72,7 @@ class TestNotifications(base.TestControl @@
             l.append('<hr/>\n')
         with test_context(self.app):
-            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
+            with mock.patch.object(kallithea.model.notification, 'send_email', send_email):
                 pr_kwargs = dict(
                     pr_nice_id='#7',
                     pr_title='The Title',
@@ @@ -155,7 +154,7 @@ class TestNotifications(base.TestControl @@
                 # Email type TYPE_PASSWORD_RESET has no corresponding notification type - test it directly:
                 desc = 'TYPE_PASSWORD_RESET'
                 kwargs = dict(user='John Doe', reset_token='decbf64715098db5b0bd23eab44bd792670ab746', reset_url='http://reset.com/decbf64715098db5b0bd23eab44bd792670ab746')
-                kallithea.model.async_tasks.send_email(['john@doe.com'],
+                kallithea.model.notification.send_email(['john@doe.com'],
                     "Password reset link",
                     EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'txt', **kwargs),
                     EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'html', **kwargs),

kallithea/tests/other/test_mail.py

➞

Show inline comments

@@ @@ -25,7 +25,7 @@ class smtplib_mock(object): @@
         smtplib_mock.lastmsg = msg
-@mock.patch('kallithea.model.async_tasks.smtplib', smtplib_mock)
+@mock.patch('kallithea.model.notification.smtplib', smtplib_mock)
 class TestMail(base.TestController):
     def test_send_mail_trivial(self):
@@ @@ -40,8 +40,8 @@ class TestMail(base.TestController): @@
             'smtp_server': mailserver,
             'app_email_from': envelope_from,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body)
         assert smtplib_mock.lastdest == set(recipients)
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -64,8 +64,8 @@ class TestMail(base.TestController): @@
             'app_email_from': envelope_from,
             'email_to': email_to,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body)
         assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL, email_to])
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -88,8 +88,8 @@ class TestMail(base.TestController): @@
             'app_email_from': envelope_from,
             'email_to': email_to,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body)
         assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL] + email_to.split(','))
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -110,8 +110,8 @@ class TestMail(base.TestController): @@
             'smtp_server': mailserver,
             'app_email_from': envelope_from,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body)
         assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL])
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -133,8 +133,8 @@ class TestMail(base.TestController): @@
             'smtp_server': mailserver,
             'app_email_from': envelope_from,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
         assert smtplib_mock.lastdest == set(recipients)
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -157,8 +157,8 @@ class TestMail(base.TestController): @@
             'smtp_server': mailserver,
             'app_email_from': envelope_from,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
         assert smtplib_mock.lastdest == set(recipients)
         assert smtplib_mock.lastsender == envelope_from
@@ @@ -181,8 +181,8 @@ class TestMail(base.TestController): @@
             'smtp_server': mailserver,
             'app_email_from': envelope_from,
+        }
         with mock.patch('kallithea.model.async_tasks.config', config_mock):
             kallithea.model.async_tasks.send_email(recipients, subject, body, html_body,
         with mock.patch('kallithea.model.notification.config', config_mock):
             kallithea.model.notification.send_email(recipients, subject, body, html_body,
                                                      from_name=author.full_name_or_username, headers=headers)
         assert smtplib_mock.lastdest == set(recipients)

scripts/deps.py

➞

Show inline comments

@@ @@ -158,7 +158,6 @@ known_violations = set([ @@
 ('kallithea.lib.utils', 'kallithea.model'),  # clean up utils
 ('kallithea.lib.utils', 'kallithea.model.db'),
 ('kallithea.lib.utils', 'kallithea.model.scm'),
 ('kallithea.model.async_tasks', 'kallithea.model'),
 ('kallithea.model', 'kallithea.lib.auth'),  # auth.HasXXX
 ('kallithea.model', 'kallithea.lib.auth_modules'),  # validators
 ('kallithea.model', 'kallithea.lib.hooks'),  # clean up hooks

0 comments (0 inline, 0 general)