kallithea Changeset - db71228a4c23

Changeset - db71228a4c23

Parent rev.

Child rev.

[Not reviewed]

beta

0 4 0

Marcin Kuzminski - 15 years ago 2011-02-24 23:44:21
marcin@python-works.com

moved statistics parse_limit into .ini files
decreased celery max task per child to 1 for better memory usage
stats task take usage of new slice option of repository

4 files changed with 16 insertions and 11 deletions:

development.ini

production.ini

rhodecode/config/deployment.ini_tmpl

rhodecode/lib/celerylib/tasks.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -42,14 +42,16 @@ port = 5000 @@
 use = egg:rhodecode
 full_stack = true
 static_files = true
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 app_instance_uuid = develop
 cut_off_limit = 256000
 force_https = false
 commit_parse_limit = 25
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
@@ @@ -67,22 +69,23 @@ celery.result.serialier = json @@
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
-celeryd.max.tasks.per.child = 3
+celeryd.max.tasks.per.child = 1
 #tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
 beaker.cache.data_dir=%(here)s/data/cache/data
 beaker.cache.lock_dir=%(here)s/data/cache/lock
 beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.short_term.type=memory

production.ini

➞

Show inline comments

@@ @@ -44,12 +44,13 @@ full_stack = true @@
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 cut_off_limit = 256000
 force_https = false
 commit_parse_limit = 250
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
@@ @@ -67,13 +68,13 @@ celery.result.serialier = json @@
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
-celeryd.max.tasks.per.child = 3
+celeryd.max.tasks.per.child = 1
 #tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####

rhodecode/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -45,12 +45,13 @@ static_files = true @@
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 app_instance_uuid = ${app_instance_uuid}
 cut_off_limit = 256000
 force_https = false
 commit_parse_limit = 50
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
@@ @@ -68,13 +69,13 @@ celery.result.serialier = json @@
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
-celeryd.max.tasks.per.child = 3
+celeryd.max.tasks.per.child = 1
 #tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####

rhodecode/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -101,13 +101,13 @@ def get_commits_stats(repo_name, ts_min_ @@
     commits_by_day_aggregate = {}
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     skip_date_limit = True
-    parse_limit = 250 #limit for single task changeset parsing optimal for
+    parse_limit = int(config['app_conf'].get('commit_parse_limit'))
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
     sa = get_session()
@@ @@ -132,14 +132,15 @@ def get_commits_stats(repo_name, ts_min_ @@
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
     log.debug('starting parsing %s', parse_limit)
     lmktime = mktime
     last_rev = last_rev + 1 if last_rev > 0 else last_rev
     for rev in repo.revisions[last_rev:last_rev + parse_limit]:
         last_cs = cs = repo.get_changeset(rev)
     for cs in repo[last_rev:last_rev + parse_limit]:
         last_cs = cs #remember last parsed changeset
         k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                       cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
@@ @@ -206,16 +207,15 @@ def get_commits_stats(repo_name, ts_min_ @@
     log.debug('revisions to parse %s', leftovers)
     if last_rev == 0 or leftovers < parse_limit:
         log.debug('getting code trending stats')
         stats.languages = json.dumps(__get_codes_stats(repo_name))
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
     try:
         stats.repository = dbrepo
         stats.stat_on_revision = last_cs.revision if last_cs else 0
         sa.add(stats)
         sa.commit()
     except:
         log.error(traceback.format_exc())
         sa.rollback()
         return False

0 comments (0 inline, 0 general)