kallithea Changeset - d3957c90499b

Changeset - d3957c90499b

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Mads Kiilerich - 9 years ago 2016-09-06 00:51:18
madski@unity3d.com

celery: use Celery 3 config settings instead of deprecated

As warned by:
The 'CELERYD_LOG_LEVEL' setting is scheduled for deprecation in version 2.4 and removal in version v4.0. Use the --loglevel argument instead
remove celeryd.log.file and celeryd.log.level from the ini file.
Instead, use:
paster celeryd my.ini --loglevel=DEBUG --logfile=my.log
or, in the future:
gearbox celeryd -c my.ini -- --loglevel=DEBUG --logfile=my.log

As warned by:
The 'BROKER_VHOST' setting is scheduled for deprecation in version 2.5 and removal in version v4.0. Use the BROKER_URL setting instead
The 'BROKER_HOST' setting is scheduled for deprecation in version 2.5 and removal in version v4.0. Use the BROKER_URL setting instead
The 'BROKER_USER' setting is scheduled for deprecation in version 2.5 and removal in version v4.0. Use the BROKER_URL setting instead
The 'BROKER_PASSWORD' setting is scheduled for deprecation in version 2.5 and removal in version v4.0. Use the BROKER_URL setting instead
The 'BROKER_PORT' setting is scheduled for deprecation in version 2.5 and removal in version v4.0. Use the BROKER_URL setting instead
change the .ini template to use:
broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost

As warned by:
Starting from version 3.2 Celery will refuse to accept pickle by default.

The pickle serializer is a security concern as it may give attackers
the ability to execute any command. It's important to secure
your broker from unauthorized access when using pickle, so we think
that enabling pickle should require a deliberate action and not be
the default choice.

If you depend on pickle then you should set a setting to disable this
warning and to be sure that everything will continue working
when you upgrade to Celery 3.2::

CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']

You must only enable the serializers that you will actually use.
change the .ini template to use:
celery.accept.content = pickle

(Note: The warning is there for a reason. It would probably be nice to change
from pickle to something like json. That is left as an exercise.)

5 files changed with 21 insertions and 37 deletions:

development.ini

kallithea/bin/template.ini.mako

kallithea/config/deployment.ini_tmpl

kallithea/lib/celerypylons/loader.py

kallithea/tests/test.ini

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -269,117 +269,113 @@ issue_pat = (?:\s*#)(\d+) @@
 issue_server_link = https://issues.example.com/{repo}/issue/{id}
 ## prefix to add to link to indicate it's an url
 ## #314 will be replaced by <issue_prefix><id>
 issue_prefix = #
 ## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 ## multiple patterns, to other issues server, wiki or others
 ## below an example how to create a wiki pattern
 # wiki-some-id -> https://wiki.example.com/some-id
 #issue_pat_wiki = (?:wiki-)(.+)
 #issue_server_link_wiki = https://wiki.example.com/{id}
 #issue_prefix_wiki = WIKI-
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 ## handling that. Set this variable to 403 to return HTTPForbidden
 auth_ret_code =
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 ## allows to change the repository location in settings page
 allow_repo_location_change = True
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
+celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = DEBUG
 celeryd.max.tasks.per.child = 1
 ## tasks will never be sent to the queue, but executed locally instead.
+## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = development-not-secret
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #beaker.session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.

kallithea/bin/template.ini.mako

➞

Show inline comments

@@ @@ -266,117 +266,113 @@ issue_pat = (?:\s*#)(\d+) @@
 issue_server_link = https://issues.example.com/{repo}/issue/{id}
 <%text>## prefix to add to link to indicate it's an url</%text>
 <%text>## #314 will be replaced by <issue_prefix><id></%text>
 issue_prefix = #
 <%text>## issue_pat, issue_server_link, issue_prefix can have suffixes to specify</%text>
 <%text>## multiple patterns, to other issues server, wiki or others</%text>
 <%text>## below an example how to create a wiki pattern</%text>
 # wiki-some-id -> https://wiki.example.com/some-id
 #issue_pat_wiki = (?:wiki-)(.+)
 #issue_server_link_wiki = https://wiki.example.com/{id}
 #issue_prefix_wiki = WIKI-
 <%text>## alternative return HTTP header for failed authentication. Default HTTP</%text>
 <%text>## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with</%text>
 <%text>## handling that. Set this variable to 403 to return HTTPForbidden</%text>
 auth_ret_code =
 <%text>## locking return code. When repository is locked return this HTTP code. 2XX</%text>
 <%text>## codes don't break the transactions while 4XX codes do</%text>
 lock_ret_code = 423
 <%text>## allows to change the repository location in settings page</%text>
 allow_repo_location_change = True
 <%text>## allows to setup custom hooks in settings page</%text>
 allow_custom_hooks_settings = True
 <%text>## extra extensions for indexing, space separated and without the leading '.'.</%text>
 # index.extensions =
 #    gemfile
 #    lock
 <%text>## extra filenames for indexing, space separated</%text>
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 <%text>####################################</%text>
 <%text>###        CELERY CONFIG        ####</%text>
 <%text>####################################</%text>
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 <%text>## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:</%text>
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
+celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = DEBUG
 celeryd.max.tasks.per.child = 1
 <%text>## tasks will never be sent to the queue, but executed locally instead.</%text>
+<%text>## If true, tasks will never be sent to the queue, but executed locally instead.</%text>
 celery.always.eager = false
 <%text>####################################</%text>
 <%text>###         BEAKER CACHE        ####</%text>
 <%text>####################################</%text>
 beaker.cache.data_dir = ${here}/data/cache/data
 beaker.cache.lock_dir = ${here}/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 <%text>####################################</%text>
 <%text>###       BEAKER SESSION        ####</%text>
 <%text>####################################</%text>
 <%text>## Name of session cookie. Should be unique for a given host and path, even when running</%text>
 <%text>## on different ports. Otherwise, cookie sessions will be shared and messed up.</%text>
 beaker.session.key = kallithea
 <%text>## Sessions should always only be accessible by the browser, not directly by JavaScript.</%text>
 beaker.session.httponly = true
 <%text>## Session lifetime. 2592000 seconds is 30 days.</%text>
 beaker.session.timeout = 2592000
 <%text>## Server secret used with HMAC to ensure integrity of cookies.</%text>
 beaker.session.secret = ${uuid()}
 <%text>## Further, encrypt the data with AES.</%text>
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 <%text>## Type of storage used for the session, current types are</%text>
 <%text>## dbm, file, memcached, database, and memory.</%text>
 <%text>## File system storage of session data. (default)</%text>
 #beaker.session.type = file
 <%text>## Cookie only, store all session data inside the cookie. Requires secure secrets.</%text>

kallithea/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -262,117 +262,113 @@ issue_pat = (?:\s*#)(\d+) @@
 issue_server_link = https://issues.example.com/{repo}/issue/{id}
 ## prefix to add to link to indicate it's an url
 ## #314 will be replaced by <issue_prefix><id>
 issue_prefix = #
 ## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 ## multiple patterns, to other issues server, wiki or others
 ## below an example how to create a wiki pattern
 # wiki-some-id -> https://wiki.example.com/some-id
 #issue_pat_wiki = (?:wiki-)(.+)
 #issue_server_link_wiki = https://wiki.example.com/{id}
 #issue_prefix_wiki = WIKI-
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 ## handling that. Set this variable to 403 to return HTTPForbidden
 auth_ret_code =
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 ## allows to change the repository location in settings page
 allow_repo_location_change = True
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
+celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = DEBUG
 celeryd.max.tasks.per.child = 1
 ## tasks will never be sent to the queue, but executed locally instead.
+## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = ${app_instance_uuid}
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #beaker.session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.

kallithea/lib/celerypylons/loader.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 from celery.loaders.base import BaseLoader
 from pylons import config
 # TODO: drop this mangling and just use a separate celery config section
 to_pylons = lambda x: x.replace('_', '.').lower()
 to_celery = lambda x: x.replace('.', '_').upper()
 LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES""".split()
+LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()
 class PylonsSettingsProxy(object):
     """Pylons Settings Proxy
     Make settings from pylons.config appear the way Celery expects them.
     """
     def __getattr__(self, key):
         try:
             return self[key]
         except KeyError:
             raise AttributeError(key)
     def get(self, key, default=None):
         try:
             return self[key]
         except KeyError:
             return default
     def __getitem__(self, key):
         pylons_key = to_pylons(key)
         value = config[pylons_key]
         if key in LIST_PARAMS:
             return value.split()
         return self.type_converter(value)
     def __contains__(self, key):
         pylons_key = to_pylons(key)
         try:
             config[pylons_key]
         except KeyError:
             return False
         return True
     def __setattr__(self, key, value):
         pylons_key = to_pylons(key)
         config[pylons_key] = value
     def __setitem__(self, key, value):
         self.__setattr__(key, value)
     def type_converter(self, value):
         #cast to int
         if value.isdigit():
             return int(value)
         #cast to bool

kallithea/tests/test.ini

➞

Show inline comments

@@ @@ -271,117 +271,113 @@ issue_pat = (?:\s*#)(\d+) @@
 issue_server_link = https://issues.example.com/{repo}/issue/{id}
 ## prefix to add to link to indicate it's an url
 ## #314 will be replaced by <issue_prefix><id>
 issue_prefix = #
 ## issue_pat, issue_server_link, issue_prefix can have suffixes to specify
 ## multiple patterns, to other issues server, wiki or others
 ## below an example how to create a wiki pattern
 # wiki-some-id -> https://wiki.example.com/some-id
 #issue_pat_wiki = (?:wiki-)(.+)
 #issue_server_link_wiki = https://wiki.example.com/{id}
 #issue_prefix_wiki = WIKI-
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently Mercurial clients have trouble with
 ## handling that. Set this variable to 403 to return HTTPForbidden
 auth_ret_code =
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 ## allows to change the repository location in settings page
 allow_repo_location_change = True
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
+celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = DEBUG
 celeryd.max.tasks.per.child = 1
 ## tasks will never be sent to the queue, but executed locally instead.
+## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 #beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.data_dir = %(here)s/../../data/test/cache/data
 #beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.lock_dir = %(here)s/../../data/test/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 #beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.expire = 1
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = {74e0cd75-b339-478b-b129-07dd221def1f}
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)

0 comments (0 inline, 0 general)