kallithea Changeset - f973b866fffc

Changeset - f973b866fffc

Parent rev.

Child rev.

[Not reviewed]

default

0 15 0

Thomas De Schampheleire - 9 years ago 2016-08-22 17:50:51
thomas.de.schampheleire@gmail.com

Turbogears2 migration: use sqlalchemy.url iso sqlalchemy.db1.url

In Turbogears2, much of the application initialization is handled by the
framework, whereas in Pylons the application was responsible for it.
Initializing SQLAlchemy is one such part of initialization which is handled
by Turbogears2.

Turbogears2 expects the configuration file to refer to the database using
'sqlalchemy.url' rather than the current 'sqlalchemy.db1.url'. While the
exact name is not really important, not following this approach means we'll
need to override the sqlalchemy initialization method.

Therefore, as a preparation to the Turbogears2 migration, already change the
database reference string under Pylons.
When upgrading to a version of Kallithea containing this commit, the .ini
file will manually need to be adapted to remove the .db1 strings.

15 files changed with 34 insertions and 34 deletions:

development.ini

docs/upgrade.rst

kallithea/alembic/env.py

kallithea/bin/template.ini.mako

kallithea/config/deployment.ini_tmpl

kallithea/config/environment.py

kallithea/lib/celerylib/__init__.py

kallithea/lib/hooks.py

kallithea/lib/paster_commands/common.py

kallithea/lib/utils.py

kallithea/tests/scripts/manual_test_concurrency.py

kallithea/tests/test.ini

kallithea/websetup.py

scripts/dbmigrate-test

scripts/generate-ini.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -295,307 +295,307 @@ lock_ret_code = 423 @@
 allow_repo_location_change = True
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
 celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1
 ## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = development-not-secret
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #beaker.session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.
 #beaker.session.type = cookie
 ## Database storage of session data.
 #beaker.session.type = ext:database
 #beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #beaker.session.table_name = db_session
 ############################
 ## ERROR HANDLING SYSTEMS ##
 ############################
 ####################
 ### [errormator] ###
 ####################
 ## Errormator is tailored to work with Kallithea, see
 ## http://errormator.com for details how to obtain an account
 ## you must install python package `errormator_client` to make it work
 ## errormator enabled
 errormator = false
 errormator.server_url = https://api.errormator.com
 errormator.api_key = YOUR_API_KEY
 ## TWEAK AMOUNT OF INFO SENT HERE
 ## enables 404 error logging (default False)
 errormator.report_404 = false
 ## time in seconds after request is considered being slow (default 1)
 errormator.slow_request_time = 1
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 errormator.slow_requests = true
 ## enable hooking to application loggers
 #errormator.logging = true
 ## minimum log level for log capture
 #errormator.logging.level = WARNING
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 errormator.logging_on_error = false
 ## list of additional keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 errormator.environ_keys_whitelist =
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 errormator.request_keys_blacklist =
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: errormator_client.client)
 errormator.log_namespace_blacklist =
 ################
 ### [sentry] ###
 ################
 ## sentry is a alternative open source error aggregator
 ## you must install python packages `sentry` and `raven` to enable
 sentry.dsn = YOUR_DNS
 sentry.servers =
 sentry.name =
 sentry.key =
 sentry.public_key =
 sentry.secret_key =
 sentry.project =
 sentry.site =
 sentry.include_paths =
 sentry.exclude_paths =
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 set debug = true
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 # SQLITE [default]
-sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 # POSTGRESQL
-#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 #sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 # MySQL
-#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 #sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 # see sqlalchemy docs for others
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.echo = false
 sqlalchemy.pool_recycle = 3600
 ################################
 ### ALEMBIC CONFIGURATION   ####
 ################################
 [alembic]
 script_location = kallithea:alembic
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 [handlers]
 keys = console, console_sql
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 [logger_kallithea]
 level = DEBUG
 handlers =
 qualname = kallithea
 propagate = 1
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 #level = INFO
 level = DEBUG
 #formatter = generic
 formatter = color_formatter
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 #level = WARN
 level = DEBUG
 #formatter = generic
 formatter = color_formatter_sql
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

docs/upgrade.rst

➞

Show inline comments

 .. _upgrade:
 ===================
 Upgrading Kallithea
 ===================
 This describes the process for upgrading Kallithea, independently of the
 Kallithea installation method.
 .. note::
     If you are upgrading from a RhodeCode installation, you must first
     install Kallithea 0.3.2 and follow the instructions in the 0.3.2
     README to perform a one-time conversion of the database from
     RhodeCode to Kallithea, before upgrading to the latest version
     of Kallithea.
 . Stop the Kallithea web application
 -------------------------------------
 This step depends entirely on the web server software used to serve
 Kallithea, but in any case, Kallithea should not be running during
 the upgrade.
 .. note::
     If you're using Celery, make sure you stop all instances during the
     upgrade.
 . Create a backup of both database and configuration
 -----------------------------------------------------
 You are of course strongly recommended to make backups regularly, but it
 is *especially* important to make a full database and configuration
 backup before performing a Kallithea upgrade.
 Back up your configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Make a copy of your Kallithea configuration (``.ini``) file.
 If you are using :ref:`rcextensions <customization>`, you should also
 make a copy of the entire ``rcextensions`` directory.
 Back up your database
 ^^^^^^^^^^^^^^^^^^^^^
 If using SQLite, simply make a copy of the Kallithea database (``.db``)
 file.
 If using PostgreSQL, please consult the documentation for the ``pg_dump``
 utility.
 If using MySQL, please consult the documentation for the ``mysqldump``
 utility.
-Look for ``sqlalchemy.db1.url`` in your configuration file to determine
 Look for ``sqlalchemy.url`` in your configuration file to determine
 database type, settings, location, etc.
 . Activate the Kallithea virtual environment (if any)
 ------------------------------------------------------
 Verify that you are using the Python environment that you originally
 installed Kallithea in by running::
     pip freeze
 This will list all packages installed in the current environment. If
 Kallithea isn't listed, activate the correct virtual environment.
 See the appropriate installation page for details.
 . Install new version of Kallithea
 -----------------------------------
 Please refer to the instructions for the installation method you
 originally used to install Kallithea.
 If you originally installed using pip, it is as simple as::
     pip install --upgrade kallithea
 If you originally installed from version control, it is as simple as::
     cd my-kallithea-clone
     hg pull -u
     pip install -e .
 . Upgrade your configuration
 -----------------------------
 Run the following command to upgrade your configuration (``.ini``) file::
     paster make-config Kallithea my.ini
 This will display any changes made by the new version of Kallithea to your
 current configuration, and attempt an automatic merge. It is recommended
 that you check the contents after the merge.
 .. note::
     Please always make sure your ``.ini`` files are up to date. Errors
     can often be caused by missing parameters added in new versions.
 .. _upgrade_db:
 . Upgrade your database
 ------------------------
 .. note::
     If you are *downgrading* Kallithea, you should perform the database
     migration step *before* installing the older version. (That is,
     always perform migrations using the most recent of the two versions
     you're migrating between.)
 First, run the following command to see your current database version::
     alembic -c my.ini current
 Typical output will be something like "9358dc3d6828 (head)", which is
 the current Alembic database "revision ID". Write down the entire output
 for troubleshooting purposes.
 The output will be empty if you're upgrading from Kallithea 0.3.x or
 older. That's expected. If you get an error that the config file was not
 found or has no ``[alembic]`` section, see the next section.
 Next, if you are performing an *upgrade*: Run the following command to
 upgrade your database to the current Kallithea version::
     alembic -c my.ini upgrade head
 If you are performing a *downgrade*: Run the following command to
 downgrade your database to the given version::
     alembic -c my.ini downgrade 0.4
 Alembic will show the necessary migrations (if any) as it executes them.
 If no "ERROR" is displayed, the command was successful.
 Should an error occur, the database may be "stranded" half-way
 through the migration, and you should restore it from backup.
 Enabling old Kallithea config files for Alembic use
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Kallithea configuration files created before the introduction of Alembic
 (i.e. predating Kallithea 0.4) need to be updated for use with Alembic.
 Without this, Alembic will fail with an error like this::
     FAILED: No config file 'my.ini' found, or file has no '[alembic]' section
 If Alembic complains specifically about a missing ``alembic.ini``, it is
 likely because you did not specify a config file using the ``-c`` option.
 On the other hand, if the mentioned config file actually exists, you
 need to append the following lines to it::
     [alembic]
     script_location = kallithea:alembic
 Your config file should now work with Alembic.
 . Rebuild the Whoosh full-text index
 -------------------------------------
 It is recommended that you rebuild the Whoosh index after upgrading since
 new Whoosh versions can introduce incompatible index changes.
 . Start the Kallithea web application
 --------------------------------------
 This step once again depends entirely on the web server software used to
 serve Kallithea.
 Before starting the new version of Kallithea, you may find it helpful to
 clear out your log file so that new errors are readily apparent.
 .. note::
     If you're using Celery, make sure you restart all instances of it after
     upgrade.
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv

kallithea/alembic/env.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # Alembic migration environment (configuration).
 import logging
 from logging.config import fileConfig
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from kallithea.model import db
 # The alembic.config.Config object, which wraps the current .ini file.
 config = context.config
 # Default to use the main Kallithea database string in [app:main].
 # For advanced uses, this can be overridden by specifying an explicit
 # [alembic] sqlalchemy.url.
 database_url = (
     config.get_main_option('sqlalchemy.url') or
-    config.get_section_option('app:main', 'sqlalchemy.db1.url')
     config.get_section_option('app:main', 'sqlalchemy.url')
+)
 # Configure default logging for Alembic. (This can be overriden by the
 # config file, but usually isn't.)
 logging.getLogger('alembic').setLevel(logging.INFO)
 # Setup Python loggers based on the config file provided to the alembic
 # command. If we're being invoked via the Alembic API (presumably for
 # stamping during "paster setup-db"), config_file_name is not available,
 # and loggers are assumed to already have been configured.
 if config.config_file_name:
     fileConfig(config.config_file_name, disable_existing_loggers=False)
 def include_in_autogeneration(object, name, type, reflected, compare_to):
     """Filter changes subject to autogeneration of migrations. """
     # Don't include changes to sqlite_sequence.
     if type == 'table' and name == 'sqlite_sequence':
         return False
     return True
 def run_migrations_offline():
     """Run migrations in 'offline' (--sql) mode.
     This produces an SQL script instead of directly applying the changes.
     Some migrations may not run in offline mode.
     """
     context.configure(
         url=database_url,
         literal_binds=True,
+    )
     with context.begin_transaction():
         context.run_migrations()
 def run_migrations_online():
     """Run migrations in 'online' mode.
     Connects to the database and directly applies the necessary
     migrations.
     """
     cfg = config.get_section(config.config_ini_section)
     cfg['sqlalchemy.url'] = database_url
     connectable = engine_from_config(
         cfg,
         prefix='sqlalchemy.',
         poolclass=pool.NullPool)
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
             # Support autogeneration of migration scripts based on "diff" between
             # current database schema and kallithea.model.db schema.
             target_metadata=db.Base.metadata,
             include_object=include_in_autogeneration,
             render_as_batch=True, # batch mode is needed for SQLite support
+        )
         with context.begin_transaction():
             context.run_migrations()
 if context.is_offline_mode():
     run_migrations_offline()
 else:
     run_migrations_online()

kallithea/bin/template.ini.mako

➞

Show inline comments

@@ @@ -295,306 +295,306 @@ allow_repo_location_change = True @@
 allow_custom_hooks_settings = True
 <%text>## extra extensions for indexing, space separated and without the leading '.'.</%text>
 # index.extensions =
 #    gemfile
 #    lock
 <%text>## extra filenames for indexing, space separated</%text>
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 <%text>####################################</%text>
 <%text>###        CELERY CONFIG        ####</%text>
 <%text>####################################</%text>
 use_celery = false
 <%text>## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:</%text>
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
 celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1
 <%text>## If true, tasks will never be sent to the queue, but executed locally instead.</%text>
 celery.always.eager = false
 <%text>####################################</%text>
 <%text>###         BEAKER CACHE        ####</%text>
 <%text>####################################</%text>
 beaker.cache.data_dir = ${here}/data/cache/data
 beaker.cache.lock_dir = ${here}/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 <%text>####################################</%text>
 <%text>###       BEAKER SESSION        ####</%text>
 <%text>####################################</%text>
 <%text>## Name of session cookie. Should be unique for a given host and path, even when running</%text>
 <%text>## on different ports. Otherwise, cookie sessions will be shared and messed up.</%text>
 beaker.session.key = kallithea
 <%text>## Sessions should always only be accessible by the browser, not directly by JavaScript.</%text>
 beaker.session.httponly = true
 <%text>## Session lifetime. 2592000 seconds is 30 days.</%text>
 beaker.session.timeout = 2592000
 <%text>## Server secret used with HMAC to ensure integrity of cookies.</%text>
 beaker.session.secret = ${uuid()}
 <%text>## Further, encrypt the data with AES.</%text>
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 <%text>## Type of storage used for the session, current types are</%text>
 <%text>## dbm, file, memcached, database, and memory.</%text>
 <%text>## File system storage of session data. (default)</%text>
 #beaker.session.type = file
 <%text>## Cookie only, store all session data inside the cookie. Requires secure secrets.</%text>
 #beaker.session.type = cookie
 <%text>## Database storage of session data.</%text>
 #beaker.session.type = ext:database
 #beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #beaker.session.table_name = db_session
 %if error_aggregation_service == 'errormator':
 <%text>############################</%text>
 <%text>## ERROR HANDLING SYSTEMS ##</%text>
 <%text>############################</%text>
 <%text>####################</%text>
 <%text>### [errormator] ###</%text>
 <%text>####################</%text>
 <%text>## Errormator is tailored to work with Kallithea, see</%text>
 <%text>## http://errormator.com for details how to obtain an account</%text>
 <%text>## you must install python package `errormator_client` to make it work</%text>
 <%text>## errormator enabled</%text>
 errormator = false
 errormator.server_url = https://api.errormator.com
 errormator.api_key = YOUR_API_KEY
 <%text>## TWEAK AMOUNT OF INFO SENT HERE</%text>
 <%text>## enables 404 error logging (default False)</%text>
 errormator.report_404 = false
 <%text>## time in seconds after request is considered being slow (default 1)</%text>
 errormator.slow_request_time = 1
 <%text>## record slow requests in application</%text>
 <%text>## (needs to be enabled for slow datastore recording and time tracking)</%text>
 errormator.slow_requests = true
 <%text>## enable hooking to application loggers</%text>
 #errormator.logging = true
 <%text>## minimum log level for log capture</%text>
 #errormator.logging.level = WARNING
 <%text>## send logs only from erroneous/slow requests</%text>
 <%text>## (saves API quota for intensive logging)</%text>
 errormator.logging_on_error = false
 <%text>## list of additional keywords that should be grabbed from environ object</%text>
 <%text>## can be string with comma separated list of words in lowercase</%text>
 <%text>## (by default client will always send following info:</%text>
 <%text>## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that</%text>
 <%text>## start with HTTP* this list be extended with additional keywords here</%text>
 errormator.environ_keys_whitelist =
 <%text>## list of keywords that should be blanked from request object</%text>
 <%text>## can be string with comma separated list of words in lowercase</%text>
 <%text>## (by default client will always blank keys that contain following words</%text>
 <%text>## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'</%text>
 <%text>## this list be extended with additional keywords set here</%text>
 errormator.request_keys_blacklist =
 <%text>## list of namespaces that should be ignores when gathering log entries</%text>
 <%text>## can be string with comma separated list of namespaces</%text>
 <%text>## (by default the client ignores own entries: errormator_client.client)</%text>
 errormator.log_namespace_blacklist =
 %elif error_aggregation_service == 'sentry':
 <%text>################</%text>
 <%text>### [sentry] ###</%text>
 <%text>################</%text>
 <%text>## sentry is a alternative open source error aggregator</%text>
 <%text>## you must install python packages `sentry` and `raven` to enable</%text>
 sentry.dsn = YOUR_DNS
 sentry.servers =
 sentry.name =
 sentry.key =
 sentry.public_key =
 sentry.secret_key =
 sentry.project =
 sentry.site =
 sentry.include_paths =
 sentry.exclude_paths =
 %endif
 <%text>################################################################################</%text>
 <%text>## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##</%text>
 <%text>## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##</%text>
 <%text>## execute malicious code after an exception is raised.                       ##</%text>
 <%text>################################################################################</%text>
 set debug = false
 <%text>##################################</%text>
 <%text>###       LOGVIEW CONFIG       ###</%text>
 <%text>##################################</%text>
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 <%text>#########################################################</%text>
 <%text>### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###</%text>
 <%text>#########################################################</%text>
 %if database_engine == 'sqlite':
 # SQLITE [default]
-sqlalchemy.db1.url = sqlite:///${here}/kallithea.db?timeout=60
 sqlalchemy.url = sqlite:///${here}/kallithea.db?timeout=60
 %elif database_engine == 'postgres':
 # POSTGRESQL
-sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 %elif database_engine == 'mysql':
 # MySQL
-sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 %endif
 # see sqlalchemy docs for others
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.echo = false
 sqlalchemy.pool_recycle = 3600
 <%text>################################</%text>
 <%text>### ALEMBIC CONFIGURATION   ####</%text>
 <%text>################################</%text>
 [alembic]
 script_location = kallithea:alembic
 <%text>################################</%text>
 <%text>### LOGGING CONFIGURATION   ####</%text>
 <%text>################################</%text>
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 [handlers]
 keys = console, console_sql
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 <%text>#############</%text>
 <%text>## LOGGERS ##</%text>
 <%text>#############</%text>
 [logger_root]
 level = NOTSET
 handlers = console
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 <%text>## "level = DEBUG" logs the route matched and routing variables.</%text>
 propagate = 1
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 [logger_kallithea]
 level = DEBUG
 handlers =
 qualname = kallithea
 propagate = 1
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 <%text>##############</%text>
 <%text>## HANDLERS ##</%text>
 <%text>##############</%text>
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = INFO
 formatter = generic
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 formatter = generic
 <%text>################</%text>
 <%text>## FORMATTERS ##</%text>
 <%text>################</%text>
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

kallithea/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -287,303 +287,303 @@ lock_ret_code = 423 @@
 ## allows to change the repository location in settings page
 allow_repo_location_change = True
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
 celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1
 ## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = ${app_instance_uuid}
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #beaker.session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.
 #beaker.session.type = cookie
 ## Database storage of session data.
 #beaker.session.type = ext:database
 #beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #beaker.session.table_name = db_session
 ############################
 ## ERROR HANDLING SYSTEMS ##
 ############################
 ####################
 ### [errormator] ###
 ####################
 ## Errormator is tailored to work with Kallithea, see
 ## http://errormator.com for details how to obtain an account
 ## you must install python package `errormator_client` to make it work
 ## errormator enabled
 errormator = false
 errormator.server_url = https://api.errormator.com
 errormator.api_key = YOUR_API_KEY
 ## TWEAK AMOUNT OF INFO SENT HERE
 ## enables 404 error logging (default False)
 errormator.report_404 = false
 ## time in seconds after request is considered being slow (default 1)
 errormator.slow_request_time = 1
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 errormator.slow_requests = true
 ## enable hooking to application loggers
 #errormator.logging = true
 ## minimum log level for log capture
 #errormator.logging.level = WARNING
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 errormator.logging_on_error = false
 ## list of additional keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 errormator.environ_keys_whitelist =
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 errormator.request_keys_blacklist =
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: errormator_client.client)
 errormator.log_namespace_blacklist =
 ################
 ### [sentry] ###
 ################
 ## sentry is a alternative open source error aggregator
 ## you must install python packages `sentry` and `raven` to enable
 sentry.dsn = YOUR_DNS
 sentry.servers =
 sentry.name =
 sentry.key =
 sentry.public_key =
 sentry.secret_key =
 sentry.project =
 sentry.site =
 sentry.include_paths =
 sentry.exclude_paths =
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 # SQLITE [default]
-sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 # POSTGRESQL
-#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 #sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 # MySQL
-#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 #sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 # see sqlalchemy docs for others
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.echo = false
 sqlalchemy.pool_recycle = 3600
 ################################
 ### ALEMBIC CONFIGURATION   ####
 ################################
 [alembic]
 script_location = kallithea:alembic
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 [handlers]
 keys = console, console_sql
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 [logger_kallithea]
 level = DEBUG
 handlers =
 qualname = kallithea
 propagate = 1
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = INFO
 formatter = generic
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 formatter = generic
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

kallithea/config/environment.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
     Pylons environment configuration
 """
 import os
 import logging
 import kallithea
 import platform
 import pylons
 import mako.lookup
 import beaker
 import formencode
 import kallithea.lib.app_globals as app_globals
 from kallithea.config.routing import make_map
 from kallithea.lib import helpers
 from kallithea.lib.auth import set_available_permissions
 from kallithea.lib.utils import repo2db_mapper, make_ui, set_app_settings, \
     load_rcextensions, check_git_version, set_vcs_config, set_indexer_config
 from kallithea.lib.utils2 import engine_from_config, str2bool
 from kallithea.lib.db_manage import DbManage
 from kallithea.model import init_model
 from kallithea.model.scm import ScmModel
 log = logging.getLogger(__name__)
 def load_environment(global_conf, app_conf,
                      test_env=None, test_index=None):
     """
     Configure the Pylons environment via the ``pylons.config``
     object
     """
     config = pylons.configuration.PylonsConfig()
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     paths = dict(
         root=root,
         controllers=os.path.join(root, 'controllers'),
         static_files=os.path.join(root, 'public'),
         templates=[os.path.join(root, 'templates')]
+    )
     # Initialize config with the basic options
     config.init_app(global_conf, app_conf, package='kallithea', paths=paths)
     # store some globals into kallithea
     kallithea.CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
     kallithea.CELERY_EAGER = str2bool(config['app_conf'].get('celery.always.eager'))
     config['routes.map'] = make_map(config)
     config['pylons.app_globals'] = app_globals.Globals(config)
     config['pylons.h'] = helpers
     kallithea.CONFIG = config
     load_rcextensions(root_path=config['here'])
     # Setup cache object as early as possible
     pylons.cache._push_object(config['pylons.app_globals'].cache)
     # Create the Mako TemplateLookup, with the default auto-escaping
     config['pylons.app_globals'].mako_lookup = mako.lookup.TemplateLookup(
         directories=paths['templates'],
         strict_undefined=True,
         module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
         input_encoding='utf-8', default_filters=['escape'],
         imports=['from webhelpers.html import escape'])
     # sets the c attribute access when don't existing attribute are accessed
     config['pylons.strict_tmpl_context'] = True
     test = os.path.split(config['__file__'])[-1] == 'test.ini'
     if test:
         if test_env is None:
             test_env = not int(os.environ.get('KALLITHEA_NO_TMP_PATH', 0))
         if test_index is None:
             test_index = not int(os.environ.get('KALLITHEA_WHOOSH_TEST_DISABLE', 0))
         if os.environ.get('TEST_DB'):
             # swap config if we pass enviroment variable
-            config['sqlalchemy.db1.url'] = os.environ.get('TEST_DB')
             config['sqlalchemy.url'] = os.environ.get('TEST_DB')
         from kallithea.lib.utils import create_test_env, create_test_index
         from kallithea.tests import TESTS_TMP_PATH
         #set KALLITHEA_NO_TMP_PATH=1 to disable re-creating the database and
         #test repos
         if test_env:
             create_test_env(TESTS_TMP_PATH, config)
         #set KALLITHEA_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests
         if test_index:
             create_test_index(TESTS_TMP_PATH, config, True)
     # MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
     sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(sa_engine_db1)
     sa_engine = engine_from_config(config, 'sqlalchemy.')
     init_model(sa_engine)
     set_available_permissions(config)
     repos_path = make_ui('db').configitems('paths')[0][1]
     config['base_path'] = repos_path
     set_app_settings(config)
     instance_id = kallithea.CONFIG.get('instance_id', '*')
     if instance_id == '*':
         instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
         kallithea.CONFIG['instance_id'] = instance_id
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
     # store config reference into our module to skip import magic of
     # pylons
     kallithea.CONFIG.update(config)
     set_vcs_config(kallithea.CONFIG)
     set_indexer_config(kallithea.CONFIG)
     #check git version
     check_git_version()
     if str2bool(config.get('initial_repo_scan', True)):
         repo2db_mapper(ScmModel().repo_scan(repos_path),
                        remove_obsolete=False, install_git_hooks=False)
     formencode.api.set_stdtranslation(languages=[config.get('lang')])
     return config

kallithea/lib/celerylib/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.celerylib
 ~~~~~~~~~~~~~~~~~~~~~~~
 celery libs for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Nov 27, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import logging
 from pylons import config
 from hashlib import md5
 from decorator import decorator
 from kallithea import CELERY_ON, CELERY_EAGER
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.pidlock import DaemonLock, LockHeld
 from kallithea.model import init_model
 from kallithea.model import meta
 from sqlalchemy import engine_from_config
 log = logging.getLogger(__name__)
 class FakeTask(object):
     """Fake a sync result to make it look like a finished task"""
     def __init__(self, result):
         self.result = result
     def failed(self):
         return False
     traceback = None # if failed
     task_id = None
 def task(f_org):
     """Wrapper of celery.task.task, running async if CELERY_ON
     """
     if CELERY_ON:
         def f_async(*args, **kwargs):
             log.info('executing %s task', f_org.__name__)
             try:
                 f_org(*args, **kwargs)
             finally:
                 log.info('executed %s task', f_org.__name__)
         f_async.__name__ = f_org.__name__
         from kallithea.lib import celerypylons
         runner = celerypylons.task(ignore_result=True)(f_async)
         def f_wrapped(*args, **kwargs):
             t = runner.apply_async(args=args, kwargs=kwargs)
             log.info('executing task %s in async mode - id %s', f_org, t.task_id)
             return t
     else:
         def f_wrapped(*args, **kwargs):
             log.info('executing task %s in sync', f_org.__name__)
             try:
                 result = f_org(*args, **kwargs)
             except Exception as e:
                 log.error('exception executing sync task %s in sync', f_org.__name__, e)
                 raise # TODO: return this in FakeTask as with async tasks?
             return FakeTask(result)
     return f_wrapped
 def __get_lockkey(func, *fargs, **fkwargs):
     params = list(fargs)
     params.extend(['%s-%s' % ar for ar in fkwargs.items()])
     func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)
     lockkey = 'task_%s.lock' % \
         md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest()
     return lockkey
 def locked_task(func):
     def __wrapper(func, *fargs, **fkwargs):
         lockkey = __get_lockkey(func, *fargs, **fkwargs)
         lockkey_path = config['app_conf']['cache_dir']
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(file_=os.path.join(lockkey_path, lockkey))
             ret = func(*fargs, **fkwargs)
             l.release()
             return ret
         except LockHeld:
             log.info('LockHeld')
             return 'Task with key %s already running' % lockkey
     return decorator(__wrapper, func)
 def get_session():
     if CELERY_ON:
-        engine = engine_from_config(config, 'sqlalchemy.db1.')
         engine = engine_from_config(config, 'sqlalchemy.')
         init_model(engine)
     sa = meta.Session()
     return sa
 def dbsession(func):
     def __wrapper(func, *fargs, **fkwargs):
         try:
             ret = func(*fargs, **fkwargs)
             return ret
         finally:
             if CELERY_ON and not CELERY_EAGER:
                 meta.Session.remove()
     return decorator(__wrapper, func)

kallithea/lib/hooks.py

➞

Show inline comments

@@ @@ -206,261 +206,261 @@ def log_push_action(ui, repo, **kwargs): @@
             #2xx Codes don't raise exceptions
             ui.status(safe_str(_http_ret.title))
     return 0
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook.
     :param repository: dict dump of repository object
     :param created_by: username who created repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PRE_CREATE_USER_HOOK', None)
     if callable(callback):
         allowed, reason = callback(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 def log_create_user(user_dict, created_by, **kwargs):
     """
     Post create user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_USER_HOOK', None)
     if callable(callback):
         return callback(created_by=created_by, **user_dict)
     return 0
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
     Post delete repository Hook.
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def log_delete_user(user_dict, deleted_by, **kwargs):
     """
     Post delete user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_USER_HOOK', None)
     if callable(callback):
         return callback(deleted_by=deleted_by, **user_dict)
     return 0
 def handle_git_pre_receive(repo_path, revs, env):
     return handle_git_receive(repo_path, revs, env, hook_type='pre')
 def handle_git_post_receive(repo_path, revs, env):
     return handle_git_receive(repo_path, revs, env, hook_type='post')
 def handle_git_receive(repo_path, revs, env, hook_type):
     """
     A really hacky method that is run by git post-receive hook and logs
     an push action together with pushed revisions. It's executed by subprocess
     thus needs all info to be able to create a on the fly pylons environment,
     connect to database and run the logging code. Hacky as sh*t but works.
     :param repo_path:
     :param revs:
     :param env:
     """
     from paste.deploy import appconfig
     from sqlalchemy import engine_from_config
     from kallithea.config.environment import load_environment
     from kallithea.model import init_model
     from kallithea.model.db import Ui
     from kallithea.lib.utils import make_ui
     extras = _extract_extras(env)
     repo_path = safe_unicode(repo_path)
     path, ini_name = os.path.split(extras['config'])
     conf = appconfig('config:%s' % ini_name, relative_to=path)
     load_environment(conf.global_conf, conf.local_conf, test_env=False,
                      test_index=False)
-    engine = engine_from_config(conf, 'sqlalchemy.db1.')
     engine = engine_from_config(conf, 'sqlalchemy.')
     init_model(engine)
     baseui = make_ui('db')
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
     repo = Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database'
                       % (safe_str(repo_path)))
     _hooks = dict(baseui.configitems('hooks')) or {}
     if hook_type == 'pre':
         repo = repo.scm_instance
     else:
         #post push shouldn't use the cached instance never
         repo = repo.scm_instance_no_cache()
     if hook_type == 'pre':
         pre_push(baseui, repo)
     # if push hook is enabled via web interface
     elif hook_type == 'post' and _hooks.get(Ui.HOOK_PUSH):
         rev_data = []
         for l in revs:
             old_rev, new_rev, ref = l.strip().split(' ')
             _ref_data = ref.split('/')
             if _ref_data[1] in ['tags', 'heads']:
                 rev_data.append({'old_rev': old_rev,
                                  'new_rev': new_rev,
                                  'ref': ref,
                                  'type': _ref_data[1],
                                  'name': '/'.join(_ref_data[2:])})
         git_revs = []
         for push_ref in rev_data:
             _type = push_ref['type']
             if _type == 'heads':
                 if push_ref['old_rev'] == EmptyChangeset().raw_id:
                     # update the symbolic ref if we push new repo
                     if repo.is_empty():
                         repo._repo.refs.set_symbolic_ref('HEAD',
                                             'refs/heads/%s' % push_ref['name'])
                     cmd = ['for-each-ref', '--format=%(refname)','refs/heads/*']
                     heads = repo.run_git_command(cmd)[0]
                     cmd = ['log', push_ref['new_rev'],
                            '--reverse', '--pretty=format:%H', '--not']
                     heads = heads.replace(push_ref['ref'], '')
                     for l in heads.splitlines():
                         cmd.append(l.strip())
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
                 elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                     #delete branch case
                     git_revs += ['delete_branch=>%s' % push_ref['name']]
                 else:
                     cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
                            '--reverse', '--pretty=format:%H']
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
             elif _type == 'tags':
                 git_revs += ['tag=>%s' % push_ref['name']]
         log_push_action(baseui, repo, _git_revs=git_revs)

kallithea/lib/paster_commands/common.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.common
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Common code for Paster commands.
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import logging
 import paste
 from paste.script.command import Command, BadCommand
 from kallithea.lib.utils import setup_cache_regions
 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print complaint
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
     def run(self, args):
         """
         Overrides Command.run
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
         self.path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)
     def _init_session(self):
         """
         Inits SqlAlchemy Session
         """
         logging.config.fileConfig(self.path_to_ini_file)
         from pylons import config
         from kallithea.model import init_model
         from kallithea.lib.utils2 import engine_from_config
         setup_cache_regions(config)
-        engine = engine_from_config(config, 'sqlalchemy.db1.')
         engine = engine_from_config(config, 'sqlalchemy.')
         init_model(engine)

kallithea/lib/utils.py

➞

Show inline comments

@@ @@ -443,357 +443,357 @@ def map_groups(path): @@
         group_name = u'/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                       lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
         parent = group
     return group
 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
                    install_git_hooks=False, user=None, overwrite_git_hooks=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hooks: if this is True, also check and install git hook
         for a repo if missing
     :param overwrite_git_hooks: if this is True, overwrite any existing git hooks
         that may be encountered (even if user-deployed)
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.scm import ScmModel
     sa = meta.Session()
     repo_model = RepoModel()
     if user is None:
         user = User.get_first_admin()
     added = []
     ##creation defaults
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in Kallithea database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
             new_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
+            )
             sa.commit()
             # we added that repo just now, and make sure it has githook
             # installed, and updated server info
             if new_repo.repo_type == 'git':
                 git_repo = new_repo.scm_instance
                 ScmModel().install_git_hooks(git_repo)
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
             new_repo.update_changeset_cache()
         elif install_git_hooks:
             if db_repo.repo_type == 'git':
                 ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)
     removed = []
     # remove from database those repositories that are not in the filesystem
     unicode_initial_repo_list = set(safe_unicode(name) for name in initial_repo_list)
     for repo in sa.query(Repository).all():
         if repo.repo_name not in unicode_initial_repo_list:
             if remove_obsolete:
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                 except Exception:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
             removed.append(repo.repo_name)
     return added, removed
 def load_rcextensions(root_path):
     import kallithea
     from kallithea.config import conf
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = kallithea.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...', rcext)
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
         #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
         if getattr(EXT, 'INDEX_EXTENSIONS', []):
             log.debug('settings custom INDEX_EXTENSIONS')
             conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
         #ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from kallithea.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it's defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import kallithea
     from pygments import lexers
     #check if we didn't define this extension as other lexer
     if kallithea.EXTENSIONS and extension in kallithea.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = kallithea.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
     :param config: test config
     :param full_index:
     """
     from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
     from kallithea.lib.pidlock import DaemonLock, LockHeld
     repo_location = repo_location
     index_location = os.path.join(config['app_conf']['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
     try:
         l = DaemonLock(file_=os.path.join(dirname(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location) \
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 def create_test_env(repos_test_path, config):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
     from kallithea.lib.db_manage import DbManage
     from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
     # PART ONE create db
-    dbconf = config['sqlalchemy.db1.url']
     dbconf = config['sqlalchemy.url']
     log.debug('making test db %s', dbconf)
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s', repos_test_path)
         os.makedirs(repos_test_path)
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
     idx_path = config['app_conf']['index_dir']
     data_path = config['app_conf']['cache_dir']
     #clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
     #CREATE DEFAULT TEST REPOS
     cur_dir = dirname(dirname(abspath(__file__)))
     tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))
     tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))
     tar.close()
     cur_dir = dirname(dirname(abspath(__file__)))
     tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))
     tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))
     tar.close()
     #LOAD VCS test stuff
     from kallithea.tests.vcs import setup_package
     setup_package()
 def check_git_version():
     """
     Checks what version of git is installed in system, and issues a warning
     if it's too old for Kallithea to work properly.
     """
     from kallithea import BACKENDS
     from kallithea.lib.vcs.backends.git.repository import GitRepository
     from kallithea.lib.vcs.conf import settings
     from distutils.version import StrictVersion
     if 'git' not in BACKENDS:
         return None
     stdout, stderr = GitRepository._run_git_command(['--version'], _bare=True,
                                                     _safe=True)
     m = re.search("\d+.\d+.\d+", stdout)
     if m:
         ver = StrictVersion(m.group(0))
     else:
         ver = StrictVersion('0.0.0')
     req_ver = StrictVersion('1.7.4')
     log.debug('Git executable: "%s" version %s detected: %s',
               settings.GIT_EXECUTABLE_PATH, ver, stdout)
     if stderr:
         log.warning('Error detecting git version: %r', stderr)
     elif ver < req_ver:
         log.warning('Kallithea detected git version %s, which is too old '
                     'for the system to function properly. '
                     'Please upgrade to version %s or later.' % (ver, req_ver))
     return ver
 @decorator.decorator
 def jsonify(func, *args, **kwargs):
     """Action decorator that formats output for JSON
     Given a function that will return content, this decorator will turn
     the result into JSON, with a content-type of 'application/json' and
     output it.
     """
     from pylons.decorators.util import get_pylons
     from kallithea.lib.compat import json
     pylons = get_pylons(args)
     pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
     data = func(*args, **kwargs)
     if isinstance(data, (list, tuple)):
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
     return json.dumps(data, encoding='utf-8')
 #===============================================================================
 # CACHE RELATED METHODS
 #===============================================================================
 # set cache regions for beaker so celery can utilise it
 def setup_cache_regions(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
 ))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 def conditional_cache(region, prefix, condition, func):
     """
     Conditional caching function use like::
         def _c(arg):
             #heavy computation function
             return data
         # depending from condition the compute is wrapped in cache or not
         compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)
         return compute(arg)
     :param region: name of cache region
     :param prefix: cache region prefix
     :param condition: condition for cache to be triggered, and return data cached
     :param func: wrapped heavy function to compute
     """
     wrapped = func
     if condition:
         log.debug('conditional_cache: True, wrapping call of '
                   'func: %s into %s region cache' % (region, func))
         wrapped = _cache_decorate((prefix,), None, None, region)(func)
     return wrapped

kallithea/tests/scripts/manual_test_concurrency.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_concurrency
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Test suite for making push/pull operations
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import sys
 import shutil
 import logging
 from os.path import dirname
 from tempfile import _RandomNameSequence
 from subprocess import Popen, PIPE
 from paste.deploy import appconfig
 from sqlalchemy import engine_from_config
 from kallithea.lib.utils import setup_cache_regions
 from kallithea.model import init_model
 from kallithea.model import meta
 from kallithea.model.db import User, Repository, Ui
 from kallithea.lib.auth import get_crypt_password
 from kallithea.tests import HG_REPO
 from kallithea.config.environment import load_environment
 rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
 conf = appconfig('config:development.ini', relative_to=rel_path)
 load_environment(conf.global_conf, conf.local_conf)
 setup_cache_regions(conf)
 USER = TEST_USER_ADMIN_LOGIN
 PASS = TEST_USER_ADMIN_PASS
 HOST = 'server.local'
 METHOD = 'pull'
 DEBUG = True
 log = logging.getLogger(__name__)
 class Command(object):
     def __init__(self, cwd):
         self.cwd = cwd
     def execute(self, cmd, *args):
         """Runs command on the system with given ``args``.
         """
         command = cmd + ' ' + ' '.join(args)
         log.debug('Executing %s', command)
         if DEBUG:
             print command
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
         stdout, stderr = p.communicate()
         if DEBUG:
             print stdout, stderr
         return stdout, stderr
 def get_session():
-    engine = engine_from_config(conf, 'sqlalchemy.db1.')
     engine = engine_from_config(conf, 'sqlalchemy.')
     init_model(engine)
     sa = meta.Session
     return sa
 def create_test_user(force=True):
     print 'creating test user'
     sa = get_session()
     user = sa.query(User).filter(User.username == USER).scalar()
     if force and user is not None:
         print 'removing current user'
         for repo in sa.query(Repository).filter(Repository.user == user).all():
             sa.delete(repo)
         sa.delete(user)
         sa.commit()
     if user is None or force:
         print 'creating new one'
         new_usr = User()
         new_usr.username = USER
         new_usr.password = get_crypt_password(PASS)
         new_usr.email = 'mail@example.com'
         new_usr.name = 'test'
         new_usr.lastname = 'lasttestname'
         new_usr.active = True
         new_usr.admin = True
         sa.add(new_usr)
         sa.commit()
     print 'done'
 def create_test_repo(force=True):
     print 'creating test repo'
     from kallithea.model.repo import RepoModel
     sa = get_session()
     user = sa.query(User).filter(User.username == USER).scalar()
     if user is None:
         raise Exception('user not found')
     repo = sa.query(Repository).filter(Repository.repo_name == HG_REPO).scalar()
     if repo is None:
         print 'repo not found creating'
         form_data = {'repo_name': HG_REPO,
                      'repo_type': 'hg',
                      'private':False,
                      'clone_uri': '' }
         rm = RepoModel(sa)
         rm.base_path = '/home/hg'
         rm.create(form_data, user)
     print 'done'
 def set_anonymous_access(enable=True):
     sa = get_session()
     user = sa.query(User).filter(User.username == 'default').one()
     user.active = enable
     sa.add(user)
     sa.commit()
 def get_anonymous_access():
     sa = get_session()
     return sa.query(User).filter(User.username == 'default').one().active
 #==============================================================================
 # TESTS
 #==============================================================================
 def test_clone_with_credentials(no_errors=False, repo=HG_REPO, method=METHOD,
                                 seq=None, backend='hg'):
     cwd = path = os.path.join(Ui.get_by_key('paths', '/').ui_value, repo)
     if seq is None:
         seq = _RandomNameSequence().next()
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % os.path.join(path)
     except OSError:
         raise
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user': USER,
                    'pass': PASS,
                    'host': HOST,
                    'cloned_repo': repo, }
     dest = path + seq
     if method == 'pull':
         stdout, stderr = Command(cwd).execute(backend, method, '--cwd', dest, clone_url)
     else:
         stdout, stderr = Command(cwd).execute(backend, method, clone_url, dest)
         print stdout,'sdasdsadsa'
         if not no_errors:
             if backend == 'hg':
                 assert """adding file changes""" in stdout, 'no messages about cloning'
                 assert """abort""" not in stderr , 'got error from clone'
             elif backend == 'git':
                 assert """Cloning into""" in stdout, 'no messages about cloning'
 if __name__ == '__main__':
     try:
         create_test_user(force=False)
         seq = None
         import time
         try:
             METHOD = sys.argv[3]
         except IndexError:
             pass
         try:
             backend = sys.argv[4]
         except IndexError:
             backend = 'hg'
         if METHOD == 'pull':
             seq = _RandomNameSequence().next()
             test_clone_with_credentials(repo=sys.argv[1], method='clone',
                                         seq=seq, backend=backend)
         s = time.time()
         for i in range(1, int(sys.argv[2]) + 1):
             print 'take', i
             test_clone_with_credentials(repo=sys.argv[1], method=METHOD,
                                         seq=seq, backend=backend)
         print 'time taken %.3f' % (time.time() - s)
     except Exception as e:
         sys.exit('stop on %s' % e)

kallithea/tests/test.ini

➞

Show inline comments

@@ @@ -299,307 +299,307 @@ allow_repo_location_change = True @@
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = True
 ## extra extensions for indexing, space separated and without the leading '.'.
 # index.extensions =
 #    gemfile
 #    lock
 ## extra filenames for indexing, space separated
 # index.filenames =
 #    .dockerignore
 #    .editorconfig
 #    INSTALL
 #    CHANGELOG
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 ## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
 broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
 celery.imports = kallithea.lib.celerylib.tasks
 celery.accept.content = pickle
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 celeryd.concurrency = 2
 celeryd.max.tasks.per.child = 1
 ## If true, tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 #beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.data_dir = %(here)s/../../data/test/cache/data
 #beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.lock_dir = %(here)s/../../data/test/cache/lock
 beaker.cache.regions = short_term,long_term,sql_cache_short
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type = memory
 #beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.expire = 1
 beaker.cache.sql_cache_short.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 ## Name of session cookie. Should be unique for a given host and path, even when running
 ## on different ports. Otherwise, cookie sessions will be shared and messed up.
 beaker.session.key = kallithea
 ## Sessions should always only be accessible by the browser, not directly by JavaScript.
 beaker.session.httponly = true
 ## Session lifetime. 2592000 seconds is 30 days.
 beaker.session.timeout = 2592000
 ## Server secret used with HMAC to ensure integrity of cookies.
 beaker.session.secret = {74e0cd75-b339-478b-b129-07dd221def1f}
 ## Further, encrypt the data with AES.
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 ## Type of storage used for the session, current types are
 ## dbm, file, memcached, database, and memory.
 ## File system storage of session data. (default)
 #beaker.session.type = file
 ## Cookie only, store all session data inside the cookie. Requires secure secrets.
 #beaker.session.type = cookie
 ## Database storage of session data.
 #beaker.session.type = ext:database
 #beaker.session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 #beaker.session.table_name = db_session
 ############################
 ## ERROR HANDLING SYSTEMS ##
 ############################
 ####################
 ### [errormator] ###
 ####################
 ## Errormator is tailored to work with Kallithea, see
 ## http://errormator.com for details how to obtain an account
 ## you must install python package `errormator_client` to make it work
 ## errormator enabled
 errormator = false
 errormator.server_url = https://api.errormator.com
 errormator.api_key = YOUR_API_KEY
 ## TWEAK AMOUNT OF INFO SENT HERE
 ## enables 404 error logging (default False)
 errormator.report_404 = false
 ## time in seconds after request is considered being slow (default 1)
 errormator.slow_request_time = 1
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 errormator.slow_requests = true
 ## enable hooking to application loggers
 #errormator.logging = true
 ## minimum log level for log capture
 #errormator.logging.level = WARNING
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 errormator.logging_on_error = false
 ## list of additional keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 errormator.environ_keys_whitelist =
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 errormator.request_keys_blacklist =
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: errormator_client.client)
 errormator.log_namespace_blacklist =
 ################
 ### [sentry] ###
 ################
 ## sentry is a alternative open source error aggregator
 ## you must install python packages `sentry` and `raven` to enable
 sentry.dsn = YOUR_DNS
 sentry.servers =
 sentry.name =
 sentry.key =
 sentry.public_key =
 sentry.secret_key =
 sentry.project =
 sentry.site =
 sentry.include_paths =
 sentry.exclude_paths =
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 # SQLITE [default]
 #sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 sqlalchemy.db1.url = sqlite:///%(here)s/kallithea_test.sqlite
 #sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 sqlalchemy.url = sqlite:///%(here)s/kallithea_test.sqlite
 # POSTGRESQL
-#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 #sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 # MySQL
-#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 #sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 # see sqlalchemy docs for others
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.echo = false
 sqlalchemy.pool_recycle = 3600
 ################################
 ### ALEMBIC CONFIGURATION   ####
 ################################
 [alembic]
 script_location = kallithea:alembic
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 [handlers]
 keys = console, console_sql
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 [logger_kallithea]
 level = DEBUG
 handlers =
 qualname = kallithea
 propagate = 1
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 ##############
 ## HANDLERS ##
 ##############
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 #level = INFO
 level = DEBUG
 #formatter = generic
 formatter = color_formatter
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 #formatter = generic
 formatter = color_formatter_sql
 ################
 ## FORMATTERS ##
 ################
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter]
 class = kallithea.lib.colored_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 [formatter_color_formatter_sql]
 class = kallithea.lib.colored_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

kallithea/websetup.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.websetup
 ~~~~~~~~~~~~~~~~~~
 Weboperations and setup for kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 11, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 from kallithea.config.environment import load_environment
 from kallithea.lib.db_manage import DbManage
 from kallithea.model.meta import Session
 log = logging.getLogger(__name__)
 def setup_app(command, conf, vars):
     """Place any commands to setup kallithea here"""
-    dbconf = conf['sqlalchemy.db1.url']
     dbconf = conf['sqlalchemy.url']
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=conf['here'],
                         tests=False, cli_args=command.options.__dict__)
     dbmanage.create_tables(override=True)
     opts = dbmanage.config_prompt(None)
     dbmanage.create_settings(opts)
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
     load_environment(conf.global_conf, conf.local_conf)

scripts/dbmigrate-test

➞

Show inline comments

@@ modified file chmod 100755 => 100644 @@
 #!/bin/sh -e
 if [ $# -lt 2 ] || [ $# -gt 3 ]; then
     cat >&2 <<EOD
 usage: $0 CONFIG_FILE FROM_REV [TO_REV]
 Runs a database migration from FROM_REV to TO_REV (default: current
 working directory parent), using the specified CONFIG_FILE (.ini file).
 Test is run using a clean Kallithea install, in a temporary virtual
 environment. FROM_REV and (optional) TO_REV should be Mercurial revision
 identifiers (e.g. changeset hash or a version number tag). The working
 directory is not touched, but the database referenced in the config file
 will be (re)created.
 Only SQLite is available out of the box; for MySQL or PostgreSQL, set
 the EXTRA environment variable to the required package(s), and it'll
 be installed in the virtual environment. (E.g. EXTRA=MySQL-python or
 EXTRA=psycopg2.)
 The temporary directory is not removed, allowing follow-up examination
 of the upgrade results. It is, however, created in /tmp by default,
 which many Linux distributions automatically clean at regular intervals.
 EOD
     exit 1
 fi
 config_file=$(readlink -f "$1")
 from_rev=$2
 to_rev=$3
 source_repo=$(dirname "$(dirname "$(readlink -f "$0")")")
 announce() {
     echo
     echo "$1"
     echo
+}
 quiet_if_ok() (
     local output
     local st
     set +e
     output=$("$@" < /dev/null 2>&1)
     st=$?
     if [ $st -ne 0 ]; then
         echo "$output" >&2
         echo "Command $@ returned exit status $st." >&2
         exit 1
     fi
+)
 HG() {
     "${HG:-hg}" --repository "$source_repo" "$@"
+}
 # If upgrading to "current revision", warn if working directory is dirty.
 if [ ! "$to_rev" ] && [ "$(HG status -mard)" ]; then
     announce "Warning: Uncommitted changes in working directory will be ignored!"
 fi
 from_rev_hash=$(HG id --id --rev "${from_rev:-.}")
 to_rev_hash=$(HG id --id --rev "${to_rev:-.}")
 temp=$(readlink -f "$(mktemp --tmpdir -d 'dbmigrate-test.XXXXXX')")
 cat <<EOD
 Config file:    $config_file
 EOD
-sed -n -e 's/^sqlalchemy\.db1\.url *= */Database URL:   /p' "$config_file"
 sed -n -e 's/^sqlalchemy\.url *= */Database URL:   /p' "$config_file"
 cat <<EOD
 Working dir:    $temp
 Repository:     $source_repo
 Upgrade from:   $from_rev_hash (${from_rev:-current})
 Upgrade to:     $to_rev_hash (${to_rev:-current})
 Extra packages: ${EXTRA:-(none)}
 EOD
 mkdir "$temp/repos" # empty
 # Enable caching for old pip versions (this will cache the pip upgrade)
 # Newer pip versions cache automatically, and don't use this variable.
 if [ ! "$PIP_DOWNLOAD_CACHE" ]; then
     export PIP_DOWNLOAD_CACHE=$HOME/.cache/pip/legacy
 fi
 install_kallithea() {
     local prefix=$1
     local rev=$2
     announce "Installing Kallithea $rev in $prefix..."
     "${VIRTUALENV:-virtualenv}" --quiet "$prefix-env"
     HG archive --rev "$rev" "$prefix"
+    (
         cd "$prefix"
         . "$prefix-env/bin/activate"
         pip install --quiet --upgrade pip setuptools mercurial $EXTRA
         pip install --quiet -e .
+    )
+}
 install_kallithea "$temp/from" "$from_rev_hash"
+(
     cd "$temp/from"
     . "$temp/from-env/bin/activate"
     announce "Initializing database..."
     quiet_if_ok paster setup-db "$config_file" --repos="$temp/repos" --user=doe --email=doe@example.com --password=123456 --no-public-access --force-yes
     alembic -c "$config_file" current -v
+)
 install_kallithea "$temp/to" "$to_rev_hash"
+(
     cd "$temp/to"
     . "$temp/to-env/bin/activate"
     announce "Commencing database upgrade from shown Alembic revision to head..."
     alembic -c "$config_file" current -v
     alembic -c "$config_file" upgrade head
     announce "Upgrade complete, now at the shown Alembic revision:"
     alembic -c "$config_file" current -v
+)

scripts/generate-ini.py

➞

Show inline comments

@@ modified file chmod 100755 => 100644 @@
 #!/usr/bin/env python2
 """
 Based on kallithea/bin/template.ini.mako, generate
   kallithea/config/deployment.ini_tmpl
   development.ini
   kallithea/tests/test.ini
 """
 import re
 makofile = 'kallithea/bin/template.ini.mako'
 # the mako conditionals used in all other ini files and templates
 selected_mako_conditionals = set([
     "database_engine == 'sqlite'",
     "http_server == 'waitress'",
     "error_aggregation_service == 'errormator'",
     "error_aggregation_service == 'sentry'",
 ])
 # the mako variables used in all other ini files and templates
 mako_variable_values = {
     'host': '127.0.0.1',
     'port': '5000',
     'here': '%(here)s',
     'uuid()': '${app_instance_uuid}',
+}
 # files to be generated from the mako template
 ini_files = [
     ('kallithea/config/deployment.ini_tmpl',
         '''
         Kallithea - Example config
         The %(here)s variable will be replaced with the parent directory of this file
         ''',
         {}, # exactly the same settings as template.ini.mako
     ),
     ('kallithea/tests/test.ini',
         '''
         Kallithea - config for tests:
         initial_repo_scan = true
         sqlalchemy and kallithea_test.sqlite
         custom logging
         The %(here)s variable will be replaced with the parent directory of this file
         ''',
+        {
             '[server:main]': {
                 'port': '4999',
             },
             '[app:main]': {
                 'initial_repo_scan': 'true',
                 'app_instance_uuid': 'test',
                 'show_revision_number': 'true',
                 'beaker.cache.sql_cache_short.expire': '1',
                 'beaker.session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
                 'cache_dir': '%(here)s/../../data/test/cache',
                 'index_dir': '%(here)s/../../data/test/index',
                 'archive_cache_dir': '%(here)s/../../data/test/tarballcache',
                 'beaker.cache.data_dir': '%(here)s/../../data/test/cache/data',
                 'beaker.cache.lock_dir': '%(here)s/../../data/test/cache/lock',
-                'sqlalchemy.db1.url': 'sqlite:///%(here)s/kallithea_test.sqlite',
                 'sqlalchemy.url': 'sqlite:///%(here)s/kallithea_test.sqlite',
             },
             '[handler_console]': {
                 'level': 'DEBUG',
                 'formatter': 'color_formatter',
             },
             # The 'handler_console_sql' block is very similar to the one in
             # development.ini, but without the explicit 'level=DEBUG' setting:
             # it causes duplicate sqlalchemy debug logs, one through
             # handler_console_sql and another through another path.
             '[handler_console_sql]': {
                 'formatter': 'color_formatter_sql',
             },
         },
     ),
     ('development.ini',
         '''
         Kallithea - Development config:
         listening on *:5000
         sqlite and kallithea.db
         initial_repo_scan = true
         set debug = true
         verbose and colorful logging
         The %(here)s variable will be replaced with the parent directory of this file
         ''',
+        {
             '[server:main]': {
                 'host': '0.0.0.0',
             },
             '[app:main]': {
                 'initial_repo_scan': 'true',
                 'set debug': 'true',
                 'app_instance_uuid': 'development-not-secret',
                 'beaker.session.secret': 'development-not-secret',
             },
             '[handler_console]': {
                 'level': 'DEBUG',
                 'formatter': 'color_formatter',
             },
             '[handler_console_sql]': {
                 'level': 'DEBUG',
                 'formatter': 'color_formatter_sql',
             },
         },
     ),
+]
 def main():
     # make sure all mako lines starting with '#' (the '##' comments) are marked up as <text>
     print 'reading:', makofile
     mako_org = file(makofile).read()
     mako_no_text_markup = re.sub(r'</?%text>', '', mako_org)
     mako_marked_up = re.sub(r'\n(##.*)', r'\n<%text>\1</%text>', mako_no_text_markup, flags=re.MULTILINE)
     if mako_marked_up != mako_org:
         print 'writing:', makofile
         file(makofile, 'w').write(mako_marked_up)
     # select the right mako conditionals for the other less sophisticated formats
     def sub_conditionals(m):
         """given a %if...%endif match, replace with just the selected
         conditional sections enabled and the rest as comments
         """
         conditional_lines = m.group(1)
         def sub_conditional(m):
             """given a conditional and the corresponding lines, return them raw
             or commented out, based on whether conditional is selected
             """
             criteria, lines = m.groups()
             if criteria not in selected_mako_conditionals:
                 lines = '\n'.join((l if not l or l.startswith('#') else '#' + l) for l in lines.split('\n'))
             return lines
         conditional_lines = re.sub(r'^%(?:el)?if (.*):\n((?:^[^%\n].*\n|\n)*)',
             sub_conditional, conditional_lines, flags=re.MULTILINE)
         return conditional_lines
     mako_no_conditionals = re.sub(r'^(%if .*\n(?:[^%\n].*\n|%elif .*\n|\n)*)%endif\n',
         sub_conditionals, mako_no_text_markup, flags=re.MULTILINE)
     # expand mako variables
     def pyrepl(m):
         return mako_variable_values.get(m.group(1), m.group(0))
     mako_no_variables = re.sub(r'\${([^}]*)}', pyrepl, mako_no_conditionals)
     # remove utf-8 coding header
     base_ini = re.sub(r'^## -\*- coding: utf-8 -\*-\n', '', mako_no_variables)
     # create ini files
     for fn, desc, settings in ini_files:
         print 'updating:', fn
         ini_lines = re.sub(
             '# Kallithea - config file generated with kallithea-config *#\n',
             ''.join('# %-77s#\n' % l.strip() for l in desc.strip().split('\n')),
             base_ini)
         def process_section(m):
             """process a ini section, replacing values as necessary"""
             sectionname, lines = m.groups()
             if sectionname in settings:
                 section_settings = settings[sectionname]
                 def process_line(m):
                     """process a section line and update value if necessary"""
                     setting, value = m.groups()
                     line = m.group(0)
                     if setting in section_settings:
                         line = '%s = %s' % (setting, section_settings[setting])
                         if '$' not in value:
                             line = '#%s = %s\n%s' % (setting, value, line)
                     return line.rstrip()
                 lines = re.sub(r'^([^#\n].*) = ?(.*)', process_line, lines, flags=re.MULTILINE)
             return sectionname + '\n' + lines
         ini_lines = re.sub(r'^(\[.*\])\n((?:(?:[^[\n].*)?\n)*)', process_section, ini_lines, flags=re.MULTILINE)
         file(fn, 'w').write(ini_lines)
 if __name__ == '__main__':
     main()

0 comments (0 inline, 0 general)