kallithea Changeset - 238885eaead4

Changeset - 238885eaead4

Parent rev.

Child rev.

[Not reviewed]

default

0 2 0

Mads Kiilerich - 6 years ago 2020-04-22 20:53:22
mads@kiilerich.com

inifile: new implementation of setting updates to optimize reuse of comments and append location

Code comments should explain the algorithm, and test coverage suggest it works
correctly.

2 files changed with 40 insertions and 20 deletions:

development.ini

kallithea/lib/inifile.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -293,195 +293,194 @@ beaker.cache.long_term_file.key_length = @@
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 #session.secret = VERY-SECRET
 session.secret = development-not-secret
 ## Further, encrypt the data with AES.
 #session.encrypt_key = <key_for_encryption>
 #session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.
 #session.type = cookie
 ## Database storage of session data.
 #session.type = ext:database
 #session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #session.table_name = db_session
 ############################
 ## ERROR HANDLING SYSTEMS ##
 ############################
 ## Propagate email settings to ErrorReporter of TurboGears2
 ## You do not normally need to change these lines
 get trace_errors.smtp_server = smtp_server
 get trace_errors.smtp_port = smtp_port
 get trace_errors.from_address = error_email_from
 get trace_errors.error_email = email_to
 get trace_errors.smtp_username = smtp_username
 get trace_errors.smtp_password = smtp_password
 get trace_errors.smtp_use_tls = smtp_use_tls
 ################################################################################
 ## WARNING: *DEBUG MODE MUST BE OFF IN A PRODUCTION ENVIRONMENT*              ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #debug = false
 debug = true
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 ## SQLITE [default]
 sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 ## see sqlalchemy docs for other backends
 sqlalchemy.pool_recycle = 3600
 ################################
 ### ALEMBIC CONFIGURATION   ####
 ################################
 [alembic]
 script_location = kallithea:alembic
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash
 [handlers]
 keys = console, console_color, console_color_sql, null
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 #handlers = console
 ## For coloring based on log level:
 handlers = console_color
 ## For coloring based on log level:
 #handlers = console_color
 [logger_routes]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 [logger_beaker]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = beaker.container
 [logger_templates]
 #level = WARN
 level = INFO
 handlers =
 qualname = pylons.templating
 [logger_kallithea]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = kallithea
 [logger_tg]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = tg
 [logger_gearbox]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = gearbox
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 ## For coloring based on log level and pretty printing of SQL:
 #level = INFO
 #handlers = console_color_sql
 #propagate = 0
 [logger_whoosh_indexer]
 #level = WARN
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 [logger_werkzeug]
 level = WARN
 handlers =
 qualname = werkzeug
 [logger_backlash]
 level = WARN
 handlers =
 qualname = backlash
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 formatter = generic
 [handler_console_color]
 ## ANSI color coding based on log level
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter
 [handler_console_color_sql]
 ## ANSI color coding and pretty printing of SQL statements
 class = StreamHandler
 args = (sys.stderr,)
 formatter = color_formatter_sql
 [handler_null]
 class = NullHandler
 args = ()
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

kallithea/lib/inifile.py

➞

Show inline comments

@@ @@ -10,190 +10,211 @@ @@
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.inifile
 ~~~~~~~~~~~~~~~~~~~~~
 Handling of .ini files, mainly creating them from Mako templates and adding
 other custom values.
 """
 import logging
 import os
 import re
 import mako.template
 log = logging.getLogger(__name__)
 template_file = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
     'kallithea/lib/paster_commands/template.ini.mako')
 default_variables = {
     'database_engine': 'sqlite',
     'http_server': 'waitress',
     'host': '127.0.0.1',
     'port': '5000',
     'uuid': lambda: 'VERY-SECRET',
+}
 variable_options = {
     'database_engine': ['sqlite', 'postgres', 'mysql'],
     'http_server': ['waitress', 'gearbox', 'gevent', 'gunicorn', 'uwsgi'],
+}
 def expand(template, mako_variable_values, settings):
     """Expand mako template and tweak it.
     Not entirely stable for random templates as input, but good enough for our
     single template.
     >>> template = '''
     ... [first-section]
     ...
     ... variable=${mako_variable}
     ... variable2  =\tvalue after tab
     ... ## This section had some whitespace and stuff
     ...
     ...
     ... # ${mako_function()}
     ... [second-section]
     ... %if conditional_options == 'option-a':
     ... # option a was chosen
     ... %elif conditional_options == 'option-b':
     ... some_variable = "never mind - option-b will not be used anyway ..."
     ... %endif
     ...
     ... [comment-section]
     ... #variable3 = 3.0
     ... #variable4 = 4.0
     ... #variable5 = 5.0
     ... variable5 = 5.1
     ... #variable6 = 6.0
     ... #variable6 = 6.1
     ... #variable7 = 7.0
     ... variable7 = 7.1
     ... '''
     >>> mako_variable_values = {'mako_variable': 'VALUE', 'mako_function': (lambda: 'FUNCTION RESULT'),
     ...                         'conditional_options': 'option-a', 'http_server': 'nc'}
     >>> settings = { # only partially used
     ...     '[first-section]': {'variable2': 'VAL2', 'first_extra': 'EXTRA'},
     ...     '[comment-section]': {'variable3': '3.0', 'variable4': '4.1', 'variable5': '5.2', 'variable6': '6.2', 'variable7': '7.0'},
     ...     '[third-section]': {'third_extra': ' 3'},
     ...     '[fourth-section]': {'fourth_extra': '4', 'fourth': '"four"'},
     ... }
     >>> print(expand(template, mako_variable_values, settings))
     ERROR: http_server is 'nc' - it should be one of 'waitress', 'gearbox', 'gevent', 'gunicorn', 'uwsgi'
     <BLANKLINE>
     [first-section]
     <BLANKLINE>
     variable=VALUE
     #variable2  =    value after tab
     variable2 = VAL2
     <BLANKLINE>
     first_extra = EXTRA
     <BLANKLINE>
     <BLANKLINE>
     # FUNCTION RESULT
     [second-section]
     # option a was chosen
     <BLANKLINE>
     [comment-section]
-    #variable3 = 3.0
     variable3 = 3.0
     #variable4 = 4.0
     variable4 = 4.1
     #variable5 = 5.0
     #variable5 = 5.1
     variable5 = 5.2
     #variable6 = 6.0
     #variable6 = 6.1
     #variable7 = 7.0
     #variable7 = 7.1
     variable6 = 6.2
     variable7 = 7.0
     <BLANKLINE>
     variable3 = 3.0
     variable4 = 4.1
     variable6 = 6.2
     #variable7 = 7.1
     <BLANKLINE>
     [fourth-section]
     fourth = "four"
     fourth_extra = 4
     <BLANKLINE>
     [third-section]
     third_extra =  3
     <BLANKLINE>
     """
     mako_variables = dict(default_variables)
     mako_variables.update(mako_variable_values or {})
     settings = dict((k, dict(v)) for k, v in settings.items()) # deep copy before mutating
     for key, value in mako_variables.items():
         if key in variable_options:
             if value not in variable_options[key]:
                 print('ERROR: %s is %r - it should be one of %s' %
                       (key, value, ', '.join(repr(x) for x in variable_options[key])))
     ini_lines = mako.template.Template(template).render(**mako_variables)
     def process_section(m):
         """process a ini section, replacing values as necessary"""
         sectionname, lines = m.groups()
         if sectionname in settings:
             section_settings = settings.pop(sectionname)
             add_after_key_value = {}  # map key to value it should be added after
             def process_line(m):
                 """process a section line and update value if necessary"""
                 key, value = m.groups()
             # 1st pass:
             # comment out lines with keys that have new values
             # find best line for keeping or un-commenting (because it has the right value) or adding after (because it is the last with other value)
             def comment_out(m):
                 """process a section line if in section_settings and comment out and track in add_after_key_value"""
                 line = m.group(0)
                 if key in section_settings:
                     new_line = '%s = %s' % (key, section_settings.pop(key))
                     if new_line != line:
                         # keep old entry as example - comments might refer to it
                         line = '#%s\n%s' % (line, new_line)
                 return line.rstrip()
                 comment, key, line_value = m.groups()
                 if key not in section_settings:
                     return line
                 new_value = section_settings[key]
                 if line_value == new_value or add_after_key_value.get(key) != new_value:
                     add_after_key_value[key] = line_value
                 if comment:
                     return line
                 return '#' + line
             lines = re.sub(r'^(#)?([^#\n\s]*)[ \t]*=[ \t]*(.*)$', comment_out, lines, flags=re.MULTILINE)
             # process lines that not are comments or empty and look like name=value
             lines = re.sub(r'^([^#\n\s]*)[ \t]*=[ \t]*(.*)$', process_line, lines, flags=re.MULTILINE)
             def add_after_comment(m):
                 """process a section comment line and add new value"""
                 line = m.group(0)
                 key, line_value = m.groups()
                 if key not in section_settings:
                     return line
                 if line_value != add_after_key_value.get(key):
                     return line
                 new_value = section_settings[key]
                 if new_value == line_value:
                     line = line.lstrip('#')
                 else:
                     line += '\n%s = %s' % (key, new_value)
                 section_settings.pop(key)
                 return line
             lines = re.sub(r'^#([^#\n\s]*)[ \t]*=[ \t]*(.*)$', add_after_comment, lines, flags=re.MULTILINE)
             # add unused section settings
             if section_settings:
                 lines += '\n' + ''.join('%s = %s\n' % (key, value) for key, value in sorted(section_settings.items()))
         return sectionname + '\n' + lines
     # process sections until comments before next section or end
     ini_lines = re.sub(r'''^
         (\[.*\])\n
         # after the section name, a number of chunks with:
+        (
             (?:
                 # a number of comments or empty lines
                 (?:[#].*\n|\n)*
                 # one or more non-empty non-comments non-section-start lines
                 (?:[^\n#[].*\n)+
                 # a number of comments - not empty lines
                 (?:[#].*\n)*
             )*
+        )
         ''',
         process_section, ini_lines, flags=re.MULTILINE | re.VERBOSE) \
         + \
         ''.join(
             '\n' + sectionname + '\n' + ''.join('%s = %s\n' % (key, value) for key, value in sorted(section_settings.items()))
             for sectionname, section_settings in sorted(settings.items())
             if section_settings)
     return ini_lines
 def create(dest_file, mako_variable_values, settings):
     """Create an ini file at dest_file"""
     with open(template_file, 'rb') as f:
         template = f.read().decode('utf-8')
     ini_lines = expand(template, mako_variable_values, settings)
     with open(dest_file, 'wb') as f:
         f.write(ini_lines.encode('utf-8'))

0 comments (0 inline, 0 general)