kallithea Changeset - b9ea10d3e419

Changeset - b9ea10d3e419

Parent rev.

Child rev.

[Not reviewed]

beta

0 5 0

Marcin Kuzminski - 15 years ago 2011-03-28 22:13:21
marcin@python-works.com

timperproxy will just measure the time of queries, and formatting will be handled by the new sql_formatter of queries from sqlalchemy itself. Updated ini files for new way logging

5 files changed with 148 insertions and 100 deletions:

development.ini

production.ini

rhodecode/config/deployment.ini_tmpl

rhodecode/lib/colored_formatter.py

rhodecode/lib/timerproxy.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -98,124 +98,135 @@ beaker.cache.long_term.expire=36000 @@
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## The storage uses the Container API
 ##that is also used by the cache system.
 beaker.session.type = file
 beaker.session.key = rhodecode
 beaker.session.secret = g654dcno0-9873jhgfreyu
 beaker.session.timeout = 36000
 ##auto save the session to not to use .save()
 beaker.session.auto = False
 ##true exire at browser close
 #beaker.session.cookie_expires = 3600
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
 sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.echo = True
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy,beaker,templates
+keys = root, routes, rhodecode, sqlalchemy, beaker, templates
 [handlers]
 keys = console
+keys = console, console_sql
 [formatters]
 keys = generic,color_formatter
+keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 [logger_routes]
 level = DEBUG
-handlers = console
+handlers =
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
-propagate = 0
+propagate = 1
 [logger_beaker]
 level = ERROR
 handlers = console
 level = DEBUG
 handlers =
 qualname = beaker.container
-propagate = 0
+propagate = 1
 [logger_templates]
 level = INFO
-handlers = console
+handlers =
 qualname = pylons.templating
-propagate = 0
+propagate = 1
 [logger_rhodecode]
 level = DEBUG
-handlers = console
+handlers =
 qualname = rhodecode
-propagate = 0
+propagate = 1
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter_sql
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class=rhodecode.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class=rhodecode.lib.colored_formatter.ColorFormatterSql
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
@@ \ No newline at end of file @@

production.ini

➞

Show inline comments

 ################################################################################
 ################################################################################
 # RhodeCode - Pylons environment configuration                                 #
 #                                                                              #
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 [DEFAULT]
 debug = true
 ################################################################################
 ## Uncomment and replace with the address which should receive                ##
 ## any error reports after application crash                                  ##
 ## Additionally those settings will be used by RhodeCode mailing system       ##
 ################################################################################
 #email_to = admin@localhost
 #error_email_from = paste_error@localhost
 #app_email_from = rhodecode-noreply@localhost
 #error_message =
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 ##max request before thread respawn
-threadpool_max_requests = 6
+threadpool_max_requests = 10
 ##option to use threads of process
 use_threadpool = true
 use = egg:Paste#http
 host = 127.0.0.1
 port = 8001
 [app:main]
 use = egg:rhodecode
 full_stack = true
-static_files = false
+static_files = true
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 app_instance_uuid = prod1234
 cut_off_limit = 256000
 force_https = false
 commit_parse_limit = 25
 force_https = false
 commit_parse_limit = 50
 use_gravatar = true
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 celery.imports = rhodecode.lib.celerylib.tasks
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
 celeryd.max.tasks.per.child = 1
 #tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=%(here)s/data/cache/data
 beaker.cache.lock_dir=%(here)s/data/cache/lock
 beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## The storage uses the Container API
 ##that is also used by the cache system.
 beaker.session.type = file
 beaker.session.key = rhodecode
 beaker.session.secret = g654dcno0-9873jhgfreyu
 beaker.session.timeout = 36000
 ##auto save the session to not to use .save()
 beaker.session.auto = False
 ##true exire at browser close
 #beaker.session.cookie_expires = 3600
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
 sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.echo = True
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy,beaker,templates
+keys = root, routes, rhodecode, sqlalchemy, beaker, templates
 [handlers]
 keys = console
+keys = console, console_sql
 [formatters]
 keys = generic,color_formatter
+keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
-level = INFO
+level = NOTSET
 handlers = console
 [logger_routes]
 level = INFO
 handlers = console
 level = DEBUG
 handlers =
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
-propagate = 0
+propagate = 1
 [logger_beaker]
 level = ERROR
 handlers = console
 level = DEBUG
 handlers =
 qualname = beaker.container
-propagate = 0
+propagate = 1
 [logger_templates]
 level = INFO
-handlers = console
+handlers =
 qualname = pylons.templating
-propagate = 0
+propagate = 1
 [logger_rhodecode]
 level = DEBUG
-handlers = console
+handlers =
 qualname = rhodecode
-propagate = 0
+propagate = 1
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
-level = NOTSET
+level = INFO
 formatter = color_formatter
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 formatter = color_formatter_sql
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class=rhodecode.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class=rhodecode.lib.colored_formatter.ColorFormatterSql
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
@@ \ No newline at end of file @@

rhodecode/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -97,124 +97,135 @@ beaker.cache.long_term.expire=36000 @@
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## The storage uses the Container API
 ##that is also used by the cache system.
 beaker.session.type = file
 beaker.session.key = rhodecode
 beaker.session.secret = ${app_instance_secret}
 beaker.session.timeout = 36000
 ##auto save the session to not to use .save()
 beaker.session.auto = False
 ##true exire at browser close
 #beaker.session.cookie_expires = 3600
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.echo = True
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy,beaker,templates
+keys = root, routes, rhodecode, sqlalchemy, beaker, templates
 [handlers]
 keys = console
+keys = console, console_sql
 [formatters]
 keys = generic,color_formatter
+keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
-level = INFO
+level = NOTSET
 handlers = console
 [logger_routes]
 level = INFO
 handlers = console
 level = DEBUG
 handlers =
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
-propagate = 0
+propagate = 1
 [logger_beaker]
 level = ERROR
 handlers = console
 level = DEBUG
 handlers =
 qualname = beaker.container
-propagate = 0
+propagate = 1
 [logger_templates]
 level = INFO
-handlers = console
+handlers =
 qualname = pylons.templating
-propagate = 0
+propagate = 1
 [logger_rhodecode]
 level = DEBUG
-handlers = console
+handlers =
 qualname = rhodecode
-propagate = 0
+propagate = 1
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
-level = NOTSET
+level = INFO
 formatter = color_formatter
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 formatter = color_formatter_sql
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class=rhodecode.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class=rhodecode.lib.colored_formatter.ColorFormatterSql
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
@@ \ No newline at end of file @@

rhodecode/lib/colored_formatter.py

➞

Show inline comments

 import logging
 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
 # Sequences
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[1;%dm"
 BOLD_SEQ = "\033[1m"
 COLORS = {
     'CRITICAL': MAGENTA, # level 50
     'ERROR': RED, # level 40
     'WARNING': CYAN, # level 30
     'INFO': GREEN, # level 20
     'DEBUG': BLUE, # level 10
     'SQL' : YELLOW
+}
 def one_space_trim(s):
     if s.find("  ") == -1:
         return s
     else:
         s = s.replace('  ', ' ')
         return one_space_trim(s)
 def format_sql(sql):
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
         .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
         .replace('FROM', '\n\tFROM')\
         .replace('ORDER BY', '\n\tORDER BY')\
         .replace('LIMIT', '\n\tLIMIT')\
         .replace('WHERE', '\n\tWHERE')\
         .replace('AND', '\n\tAND')\
         .replace('LEFT', '\n\tLEFT')\
         .replace('INNER', '\n\tINNER')\
         .replace('INSERT', '\n\tINSERT')\
         .replace('DELETE', '\n\tDELETE')
     return sql
 class ColorFormatter(logging.Formatter):
     def __init__(self, *args, **kwargs):
         # can't do super(...) here because Formatter is an old school class
         logging.Formatter.__init__(self, *args, **kwargs)
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         def_record = logging.Formatter.format(self, record)
         end = RESET_SEQ
         colored_record = start + def_record + end
         return colored_record
 logging.ColorFormatter = ColorFormatter
 class ColorFormatterSql(logging.Formatter):
     def __init__(self, *args, **kwargs):
         # can't do super(...) here because Formatter is an old school class
         logging.Formatter.__init__(self, *args, **kwargs)
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         start = COLOR_SEQ % (COLORS['SQL'])
         def_record = format_sql(logging.Formatter.format(self, record))
         end = RESET_SEQ
         colored_record = start + def_record + end
         return colored_record

rhodecode/lib/timerproxy.py

➞

Show inline comments

 from sqlalchemy.interfaces import ConnectionProxy
 import time
 from sqlalchemy import log
 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
 def color_sql(sql):
     COLOR_SEQ = "\033[1;%dm"
     COLOR_SQL = YELLOW
     normal = '\x1b[0m'
     return COLOR_SEQ % COLOR_SQL + sql + normal
 def one_space_trim(s):
     if s.find("  ") == -1:
         return s
     else:
         s = s.replace('  ', ' ')
         return one_space_trim(s)
 def format_sql(sql):
     sql = color_sql(sql)
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
         .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
         .replace('FROM', '\n\tFROM')\
         .replace('ORDER BY', '\n\tORDER BY')\
         .replace('LIMIT', '\n\tLIMIT')\
         .replace('WHERE', '\n\tWHERE')\
         .replace('AND', '\n\tAND')\
         .replace('LEFT', '\n\tLEFT')\
         .replace('INNER', '\n\tINNER')\
         .replace('INSERT', '\n\tINSERT')\
         .replace('DELETE', '\n\tDELETE')
     return sql
     return COLOR_SEQ % COLOR_SQL + sql + normal
 class TimerProxy(ConnectionProxy):
     def __init__(self):
         super(TimerProxy, self).__init__()
         self.logging_name = 'timerProxy'
         self.log = log.instance_logger(self, True)
     def cursor_execute(self, execute, cursor, statement, parameters, context, executemany):
     def cursor_execute(self, execute, cursor, statement, parameters,
                        context, executemany):
         now = time.time()
         try:
             self.log.info(">>>>> STARTING QUERY >>>>>")
+            self.log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
             return execute(cursor, statement, parameters, context)
         finally:
             total = time.time() - now
             try:
                 self.log.info(format_sql("Query: %s" % statement % parameters))
             except TypeError:
                 self.log.info(format_sql("Query: %s %s" % (statement, parameters)))
             self.log.info("<<<<< TOTAL TIME: %f <<<<<" % total)
             self.log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))

0 comments (0 inline, 0 general)