kallithea Changeset - 5b551b189459

Changeset - 5b551b189459

Parent rev.

Child rev.

[Not reviewed]

default

0 9 0

Mads Kiilerich - 7 years ago 2019-01-23 00:03:40
mads@kiilerich.com

Grafted from: 3bfbd1f20b0c

utils: refactor make_ui to always read from db, optionally also augmenting with hgrc content

simplehg was the only user of reading hgrc content ... and it would rather just
have everything at once.

The implementation in utils is moved around and re-indented, but without
significant changes.

9 files changed with 29 insertions and 45 deletions:

kallithea/config/app_cfg.py

kallithea/lib/hooks.py

kallithea/lib/middleware/simplegit.py

kallithea/lib/middleware/simplehg.py

kallithea/lib/utils.py

kallithea/model/db.py

kallithea/model/repo.py

kallithea/model/scm.py

kallithea/model/validators.py

0 comments (0 inline, 0 general)

kallithea/config/app_cfg.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Global configuration file for TurboGears2 specific settings in Kallithea.
 This file complements the .ini file.
 """
 import platform
 import os, sys, logging
 import tg
 from tg import hooks
 from tg.configuration import AppConfig
 from tg.support.converters import asbool
 import alembic.config
 from alembic.script.base import ScriptDirectory
 from alembic.migration import MigrationContext
 from sqlalchemy import create_engine
 import mercurial
 from kallithea.lib.middleware.https_fixup import HttpsFixup
 from kallithea.lib.middleware.simplegit import SimpleGit
 from kallithea.lib.middleware.simplehg import SimpleHg
 from kallithea.lib.auth import set_available_permissions
 from kallithea.lib.utils import load_rcextensions, make_ui, set_app_settings, set_vcs_config, \
     set_indexer_config, check_git_version, repo2db_mapper
 from kallithea.lib.utils2 import str2bool
 import kallithea.model.base
 from kallithea.model.scm import ScmModel
 import formencode
 log = logging.getLogger(__name__)
 class KallitheaAppConfig(AppConfig):
     # Note: AppConfig has a misleading name, as it's not the application
     # configuration, but the application configurator. The AppConfig values are
     # used as a template to create the actual configuration, which might
     # overwrite or extend the one provided by the configurator template.
     # To make it clear, AppConfig creates the config and sets into it the same
     # values that AppConfig itself has. Then the values from the config file and
     # gearbox options are loaded and merged into the configuration. Then an
     # after_init_config(conf) method of AppConfig is called for any change that
     # might depend on options provided by configuration files.
     def __init__(self):
         super(KallitheaAppConfig, self).__init__()
         self['package'] = kallithea
         self['prefer_toscawidgets2'] = False
         self['use_toscawidgets'] = False
         self['renderers'] = []
         # Enable json in expose
         self['renderers'].append('json')
         # Configure template rendering
         self['renderers'].append('mako')
         self['default_renderer'] = 'mako'
         self['use_dotted_templatenames'] = False
         # Configure Sessions, store data as JSON to avoid pickle security issues
         self['session.enabled'] = True
         self['session.data_serializer'] = 'json'
         # Configure the base SQLALchemy Setup
         self['use_sqlalchemy'] = True
         self['model'] = kallithea.model.base
         self['DBSession'] = kallithea.model.meta.Session
         # Configure App without an authentication backend.
         self['auth_backend'] = None
         # Use custom error page for these errors. By default, Turbogears2 does not add
         # 400 in this list.
         # Explicitly listing all is considered more robust than appending to defaults,
         # in light of possible future framework changes.
         self['errorpage.status_codes'] = [400, 401, 403, 404]
         # Disable transaction manager -- currently Kallithea takes care of transactions itself
         self['tm.enabled'] = False
 base_config = KallitheaAppConfig()
 # TODO still needed as long as we use pylonslib
 sys.modules['pylons'] = tg
 # DebugBar, a debug toolbar for TurboGears2.
 # (https://github.com/TurboGears/tgext.debugbar)
 # To enable it, install 'tgext.debugbar' and 'kajiki', and run Kallithea with
 # 'debug = true' (not in production!)
 # See the Kallithea documentation for more information.
 try:
     from tgext.debugbar import enable_debugbar
     import kajiki # only to check its existence
 except ImportError:
     pass
 else:
     base_config['renderers'].append('kajiki')
     enable_debugbar(base_config)
 def setup_configuration(app):
     config = app.config
     # Verify that things work when Dulwich passes unicode paths to the file system layer.
     # Note: UTF-8 is preferred, but for example ISO-8859-1 or mbcs should also work under the right cirumstances.
     try:
         u'\xe9'.encode(sys.getfilesystemencoding()) # Test using é (&eacute;)
     except UnicodeEncodeError:
         log.error("Cannot encode Unicode paths to file system encoding %r", sys.getfilesystemencoding())
         for var in ['LC_CTYPE', 'LC_ALL', 'LANG']:
             if var in os.environ:
                 val = os.environ[var]
                 log.error("Note: Environment variable %s is %r - perhaps change it to some other value from 'locale -a', like 'C.UTF-8' or 'en_US.UTF-8'", var, val)
                 break
         else:
             log.error("Note: No locale setting found in environment variables - perhaps set LC_CTYPE to some value from 'locale -a', like 'C.UTF-8' or 'en_US.UTF-8'")
         log.error("Terminating ...")
         sys.exit(1)
     # Mercurial sets encoding at module import time, so we have to monkey patch it
     hgencoding = config.get('hgencoding')
     if hgencoding:
         mercurial.encoding.encoding = hgencoding
     if config.get('ignore_alembic_revision', False):
         log.warn('database alembic revision checking is disabled')
     else:
         dbconf = config['sqlalchemy.url']
         alembic_cfg = alembic.config.Config()
         alembic_cfg.set_main_option('script_location', 'kallithea:alembic')
         alembic_cfg.set_main_option('sqlalchemy.url', dbconf)
         script_dir = ScriptDirectory.from_config(alembic_cfg)
         available_heads = sorted(script_dir.get_heads())
         engine = create_engine(dbconf)
         with engine.connect() as conn:
             context = MigrationContext.configure(conn)
             current_heads = sorted(str(s) for s in context.get_current_heads())
         if current_heads != available_heads:
             log.error('Failed to run Kallithea:\n\n'
                       'The database version does not match the Kallithea version.\n'
                       'Please read the documentation on how to upgrade or downgrade the database.\n'
                       'Current database version id(s): %s\n'
                       'Expected database version id(s): %s\n'
                       'If you are a developer and you know what you are doing, you can add `ignore_alembic_revision = True` '
                       'to your .ini file to skip the check.\n' % (' '.join(current_heads), ' '.join(available_heads)))
             sys.exit(1)
     # store some globals into kallithea
     kallithea.CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
     kallithea.CELERY_EAGER = str2bool(config['app_conf'].get('celery.always.eager'))
     kallithea.CONFIG = config
     load_rcextensions(root_path=config['here'])
     set_available_permissions(config)
-    repos_path = make_ui('db').configitems('paths')[0][1]
     repos_path = make_ui().configitems('paths')[0][1]
     config['base_path'] = repos_path
     set_app_settings(config)
     instance_id = kallithea.CONFIG.get('instance_id', '*')
     if instance_id == '*':
         instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
         kallithea.CONFIG['instance_id'] = instance_id
     # update kallithea.CONFIG with the meanwhile changed 'config'
     kallithea.CONFIG.update(config)
     # configure vcs and indexer libraries (they are supposed to be independent
     # as much as possible and thus avoid importing tg.config or
     # kallithea.CONFIG).
     set_vcs_config(kallithea.CONFIG)
     set_indexer_config(kallithea.CONFIG)
     check_git_version()
 hooks.register('configure_new_app', setup_configuration)
 def setup_application(app):
     config = app.config
     # we want our low level middleware to get to the request ASAP. We don't
     # need any stack middleware in them - especially no StatusCodeRedirect buffering
     app = SimpleHg(app, config)
     app = SimpleGit(app, config)
     # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     if any(asbool(config.get(x)) for x in ['https_fixup', 'force_https', 'use_htsts']):
         app = HttpsFixup(app, config)
     return app
 hooks.register('before_config', setup_application)

kallithea/lib/hooks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.hooks
 ~~~~~~~~~~~~~~~~~~~
 Hooks run by Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import time
 import binascii
 from kallithea.lib.vcs.utils.hgcompat import nullrev, revrange
 from kallithea.lib import helpers as h
 from kallithea.lib.utils import action_logger
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import make_ui, setup_cache_regions
 from kallithea.lib.utils2 import safe_str, safe_unicode, _extract_extras
 from kallithea.model.db import Repository, User, Ui
 def _get_scm_size(alias, root_path):
     if not alias.startswith('.'):
         alias += '.'
     size_scm, size_root = 0, 0
     for path, dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
     size_scm_f = h.format_byte_size(size_scm)
     size_root_f = h.format_byte_size(size_root)
     size_total_f = h.format_byte_size(size_root + size_scm)
     return size_scm_f, size_root_f, size_total_f
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Presents size of repository after push"""
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
     last_cs = repo[len(repo) - 1]
     msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), last_cs.hex()[:12]
+    )
     ui.status(msg)
 def log_pull_action(ui, repo, **kwargs):
     """Logs user last pull action
     Called as Mercurial hook outgoing.pull_logger or from Kallithea before invoking Git.
     """
     ex = _extract_extras()
     user = User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
     return 0
 def log_push_action(ui, repo, **kwargs):
     """
     Register that changes have been pushed - log it *and* invalidate caches.
     Note: It is not only logging, but also the side effect invalidating cahes!
     The function should perhaps be renamed.
     Called as Mercurial hook changegroup.push_logger or from the Git
     post-receive hook calling handle_git_post_receive ... or from scm _handle_push.
     Revisions are passed in different hack-ish ways.
     """
     ex = _extract_extras()
     action_tmpl = ex.action + ':%s'
     revs = []
     if ex.scm == 'hg':
         node = kwargs['node']
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
         stop, start = get_revs(repo, [node + ':'])
         _h = binascii.hexlify
         revs = [_h(repo[r].node()) for r in xrange(start, stop + 1)]
     elif ex.scm == 'git':
         revs = kwargs.get('_git_revs', [])
         if '_git_revs' in kwargs:
             kwargs.pop('_git_revs')
     action = action_tmpl % ','.join(revs)
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
     from kallithea.model.scm import ScmModel
     ScmModel().mark_for_invalidation(ex.repository)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
     if callable(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
     return 0
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook.
     :param repository: dict dump of repository object
     :param created_by: username who created repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PRE_CREATE_USER_HOOK', None)
     if callable(callback):
         allowed, reason = callback(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 def log_create_user(user_dict, created_by, **kwargs):
     """
     Post create user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_USER_HOOK', None)
     if callable(callback):
         return callback(created_by=created_by, **user_dict)
     return 0
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
     Post delete repository Hook.
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def log_delete_user(user_dict, deleted_by, **kwargs):
     """
     Post delete user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
      'inherit_default_permissions'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_USER_HOOK', None)
     if callable(callback):
         return callback(deleted_by=deleted_by, **user_dict)
     return 0
 def _hook_environment(repo_path):
     """
     Create a light-weight environment for stand-alone scripts and return an UI and the
     db repository.
     Git hooks are executed as subprocess of Git while Kallithea is waiting, and
     they thus need enough info to be able to create an app environment and
     connect to the database.
     """
     from paste.deploy import appconfig
     from sqlalchemy import engine_from_config
     from kallithea.config.environment import load_environment
     from kallithea.model.base import init_model
     extras = _extract_extras()
     ini_file_path = extras['config']
     #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
     app_conf = appconfig('config:%s' % ini_file_path)
     conf = load_environment(app_conf.global_conf, app_conf.local_conf)
     setup_cache_regions(conf)
     engine = engine_from_config(conf, 'sqlalchemy.')
     init_model(engine)
     repo_path = safe_unicode(repo_path)
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
     repo = Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database'
                       % (safe_str(repo_path)))
-    baseui = make_ui('db')
     baseui = make_ui()
     return baseui, repo
 def handle_git_pre_receive(repo_path, git_stdin_lines):
     """Called from Git pre-receive hook"""
     return 0
 def handle_git_post_receive(repo_path, git_stdin_lines):
     """Called from Git post-receive hook"""
     baseui, repo = _hook_environment(repo_path)
     # the post push hook should never use the cached instance
     scm_repo = repo.scm_instance_no_cache()
     rev_data = []
     for l in git_stdin_lines:
         old_rev, new_rev, ref = l.strip().split(' ')
         _ref_data = ref.split('/')
         if _ref_data[1] in ['tags', 'heads']:
             rev_data.append({'old_rev': old_rev,
                              'new_rev': new_rev,
                              'ref': ref,
                              'type': _ref_data[1],
                              'name': '/'.join(_ref_data[2:])})
     git_revs = []
     for push_ref in rev_data:
         _type = push_ref['type']
         if _type == 'heads':
             if push_ref['old_rev'] == EmptyChangeset().raw_id:
                 # update the symbolic ref if we push new repo
                 if scm_repo.is_empty():
                     scm_repo._repo.refs.set_symbolic_ref('HEAD',
                                         'refs/heads/%s' % push_ref['name'])
                 # build exclude list without the ref
                 cmd = ['for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout, stderr = scm_repo.run_git_command(cmd)
                 ref = push_ref['ref']
                 heads = [head for head in stdout.splitlines() if head != ref]
                 # now list the git revs while excluding from the list
                 cmd = ['log', push_ref['new_rev'], '--reverse', '--pretty=format:%H']
                 cmd.append('--not')
                 cmd.extend(heads) # empty list is ok
                 stdout, stderr = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
             elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                 # delete branch case
                 git_revs += ['delete_branch=>%s' % push_ref['name']]
             else:
                 cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
                        '--reverse', '--pretty=format:%H']
                 stdout, stderr = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
         elif _type == 'tags':
             git_revs += ['tag=>%s' % push_ref['name']]
     log_push_action(baseui, scm_repo, _git_revs=git_revs)
     return 0

kallithea/lib/middleware/simplegit.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.middleware.simplegit
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 SimpleGit middleware for handling Git protocol requests (push/clone etc.)
 It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import re
 import logging
 import traceback
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
     HTTPNotAcceptable
 from kallithea.model.db import Ui
 from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
     _set_extras
 from kallithea.lib.base import BaseVCSController
 from kallithea.lib.utils import make_ui, is_valid_repo
 log = logging.getLogger(__name__)
 GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
 def is_git(environ):
     path_info = environ['PATH_INFO']
     isgit_path = GIT_PROTO_PAT.match(path_info)
     log.debug('pathinfo: %s detected as Git %s',
         path_info, isgit_path is not None
+    )
     return isgit_path
 class SimpleGit(BaseVCSController):
     _git_stored_op = 'pull' # most recent kind of command
     def _handle_request(self, environ, start_response):
         if not is_git(environ):
             return self.application(environ, start_response)
         ip_addr = self._get_ip_addr(environ)
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
         #======================================================================
         # EXTRACT REPOSITORY NAME FROM ENV
         #======================================================================
         try:
             str_repo_name = self.__get_repository(environ)
             repo_name = safe_unicode(str_repo_name)
             log.debug('Extracted repo name is %s', repo_name)
         except Exception as e:
             log.error('error extracting repo_name: %r', e)
             return HTTPInternalServerError()(environ, start_response)
         # quick check if that dir exists...
         if not is_valid_repo(repo_name, self.basepath, 'git'):
             return HTTPNotFound()(environ, start_response)
         #======================================================================
         # GET ACTION PULL or PUSH
         #======================================================================
         action = self.__get_action(environ)
         #======================================================================
         # CHECK PERMISSIONS
         #======================================================================
         user, response_app = self._authorize(environ, start_response, action, repo_name, ip_addr)
         if response_app is not None:
             return response_app(environ, start_response)
         # extras are injected into Mercurial UI object and later available
         # in hooks executed by Kallithea
         from kallithea import CONFIG
         server_url = get_server_url(environ)
         extras = {
             'ip': ip_addr,
             'username': user.username,
             'action': action,
             'repository': repo_name,
             'scm': 'git',
             'config': CONFIG['__file__'],
             'server_url': server_url,
+        }
         #===================================================================
         # GIT REQUEST HANDLING
         #===================================================================
         repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
         log.debug('Repository path is %s', repo_path)
         fix_PATH()
         log.debug('HOOKS extras is %s', extras)
-        baseui = make_ui('db')
         baseui = make_ui()
         _set_extras(extras or {})
         try:
             self._handle_githooks(repo_name, action, baseui, environ)
             log.info('%s action on Git repo "%s" by "%s" from %s',
                      action, str_repo_name, safe_str(user.username), ip_addr)
             app = self.__make_app(repo_name, repo_path, extras)
             return app(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
     def __make_app(self, repo_name, repo_path, extras):
         """
         Make an wsgi application using dulserver
         :param repo_name: name of the repository
         :param repo_path: full path to the repository
         """
         from kallithea.lib.middleware.pygrack import make_wsgi_app
         app = make_wsgi_app(
             repo_root=safe_str(self.basepath),
             repo_name=safe_unicode(repo_name),
             extras=extras,
+        )
         return app
     def __get_repository(self, environ):
         """
         Gets repository name out of PATH_INFO header
         :param environ: environ where PATH_INFO is stored
         """
         try:
             environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
             repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
         except Exception:
             log.error(traceback.format_exc())
             raise
         return repo_name
     def __get_action(self, environ):
         """
         Maps Git request commands into a pull or push command.
         :param environ:
         """
         service = environ['QUERY_STRING'].split('=')
         if len(service) > 1:
             service_cmd = service[1]
             mapping = {
                 'git-receive-pack': 'push',
                 'git-upload-pack': 'pull',
+            }
             op = mapping[service_cmd]
             self._git_stored_op = op
             # try to fallback to stored variable as we don't know if the last
             # operation is pull/push
         return self._git_stored_op
     def _handle_githooks(self, repo_name, action, baseui, environ):
         """
         Handles pull action, push is handled by post-receive hook
         """
         from kallithea.lib.hooks import log_pull_action
         service = environ['QUERY_STRING'].split('=')
         if len(service) < 2:
             return
         from kallithea.model.db import Repository
         _repo = Repository.get_by_repo_name(repo_name)
         _repo = _repo.scm_instance
         if action == 'pull':
             log_pull_action(ui=baseui, repo=_repo._repo)

kallithea/lib/middleware/simplehg.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.middleware.simplehg
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 SimpleHg middleware for handling Mercurial protocol requests (push/clone etc.).
 It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import logging
 import traceback
 import urllib
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
     HTTPNotAcceptable, HTTPBadRequest
 from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
     _set_extras
 from kallithea.lib.base import BaseVCSController
-from kallithea.lib.utils import make_ui, is_valid_repo, ui_sections
 from kallithea.lib.utils import make_ui, is_valid_repo
 from kallithea.lib.vcs.utils.hgcompat import RepoError, hgweb_mod
 log = logging.getLogger(__name__)
 def is_mercurial(environ):
     """
     Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     path_info = environ['PATH_INFO']
     if http_accept and http_accept.startswith('application/mercurial'):
         ishg_path = True
     else:
         ishg_path = False
     log.debug('pathinfo: %s detected as Mercurial %s',
         path_info, ishg_path
+    )
     return ishg_path
 def get_header_hgarg(environ):
     """Decode the special Mercurial encoding of big requests over multiple headers.
     >>> get_header_hgarg({})
     ''
     >>> get_header_hgarg({'HTTP_X_HGARG_0': ' ', 'HTTP_X_HGARG_1': 'a','HTTP_X_HGARG_2': '','HTTP_X_HGARG_3': 'b+c %20'})
     'ab+c %20'
     """
     chunks = []
     i = 1
     while True:
         v = environ.get('HTTP_X_HGARG_%d' % i)
         if v is None:
             break
         chunks.append(v)
         i += 1
     return ''.join(chunks)
 class SimpleHg(BaseVCSController):
     def _handle_request(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
         ip_addr = self._get_ip_addr(environ)
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
         #======================================================================
         # EXTRACT REPOSITORY NAME FROM ENV
         #======================================================================
         try:
             str_repo_name = self.__get_repository(environ)
             repo_name = safe_unicode(str_repo_name)
             log.debug('Extracted repo name is %s', repo_name)
         except Exception as e:
             log.error('error extracting repo_name: %r', e)
             return HTTPInternalServerError()(environ, start_response)
         # quick check if that dir exists...
         if not is_valid_repo(repo_name, self.basepath, 'hg'):
             return HTTPNotFound()(environ, start_response)
         #======================================================================
         # GET ACTION PULL or PUSH
         #======================================================================
         try:
             action = self.__get_action(environ)
         except HTTPBadRequest as e:
             return e(environ, start_response)
         #======================================================================
         # CHECK PERMISSIONS
         #======================================================================
         user, response_app = self._authorize(environ, start_response, action, repo_name, ip_addr)
         if response_app is not None:
             return response_app(environ, start_response)
         # extras are injected into Mercurial UI object and later available
         # in hooks executed by Kallithea
         from kallithea import CONFIG
         server_url = get_server_url(environ)
         extras = {
             'ip': ip_addr,
             'username': user.username,
             'action': action,
             'repository': repo_name,
             'scm': 'hg',
             'config': CONFIG['__file__'],
             'server_url': server_url,
+        }
         #======================================================================
         # MERCURIAL REQUEST HANDLING
         #======================================================================
         repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
         log.debug('Repository path is %s', repo_path)
         fix_PATH()
         log.debug('HOOKS extras is %s', extras)
         baseui = make_ui('db')
         self._augment_hgrc(repo_path, baseui)
         baseui = make_ui(repo_path=repo_path)
         _set_extras(extras or {})
         try:
             log.info('%s action on Mercurial repo "%s" by "%s" from %s',
                      action, str_repo_name, safe_str(user.username), ip_addr)
             environ['REPO_NAME'] = str_repo_name # used by hgweb_mod.hgweb
             app = self.__make_app(repo_path, baseui, extras)
             return app(environ, start_response)
         except RepoError as e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
     def __make_app(self, repo_name, baseui, extras):
         """
         Make an wsgi application using hgweb, and inject generated baseui
         instance, additionally inject some extras into ui object
         """
         return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
     def __get_repository(self, environ):
         """
         Gets repository name out of PATH_INFO header
         :param environ: environ where PATH_INFO is stored
         """
         try:
             environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
             repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
             if repo_name.endswith('/'):
                 repo_name = repo_name.rstrip('/')
         except Exception:
             log.error(traceback.format_exc())
             raise
         return repo_name
     def __get_action(self, environ):
         """
         Maps Mercurial request commands into 'pull' or 'push'.
         Raises HTTPBadRequest if the request environment doesn't look like a hg client.
         """
         mapping = {
             # 'batch' is not in this list - it is handled explicitly
             'between': 'pull',
             'branches': 'pull',
             'branchmap': 'pull',
             'capabilities': 'pull',
             'changegroup': 'pull',
             'changegroupsubset': 'pull',
             'changesetdata': 'pull',
             'clonebundles': 'pull',
             'debugwireargs': 'pull',
             'filedata': 'pull',
             'getbundle': 'pull',
             'getlfile': 'pull',
             'heads': 'pull',
             'hello': 'pull',
             'known': 'pull',
             'lheads': 'pull',
             'listkeys': 'pull',
             'lookup': 'pull',
             'manifestdata': 'pull',
             'narrow_widen': 'pull',
             'protocaps': 'pull',
             'statlfile': 'pull',
             'stream_out': 'pull',
             'pushkey': 'push',
             'putlfile': 'push',
             'unbundle': 'push',
+            }
         for qry in environ['QUERY_STRING'].split('&'):
             parts = qry.split('=', 1)
             if len(parts) == 2 and parts[0] == 'cmd':
                 cmd = parts[1]
                 if cmd == 'batch':
                     hgarg = get_header_hgarg(environ)
                     if not hgarg.startswith('cmds='):
                         return 'push' # paranoid and safe
                     for cmd_arg in hgarg[5:].split(';'):
                         cmd, _args = urllib.unquote_plus(cmd_arg).split(' ', 1)
                         op = mapping.get(cmd, 'push')
                         if op != 'pull':
                             assert op == 'push'
                             return 'push'
                     return 'pull'
                 return mapping.get(cmd, 'push')
         # Note: the client doesn't get the helpful error message
         raise HTTPBadRequest('Unable to detect pull/push action! Are you using non standard command or client?')
     def _augment_hgrc(self, repo_path, baseui):
         """Augment baseui with config settings from the repo_path repo"""
         hgrc = os.path.join(repo_path, '.hg', 'hgrc')
         repoui = make_ui('file', hgrc)
         for section in ui_sections:
             for k, v in repoui.configitems(section):
                 baseui.setconfig(section, k, v)

kallithea/lib/utils.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils
 ~~~~~~~~~~~~~~~~~~~
 Utilities library for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import re
 import logging
 import datetime
 import traceback
 import beaker
 from tg import request, response
 from tg.i18n import ugettext as _
 from beaker.cache import _cache_decorate
 from kallithea.lib.vcs.utils.hgcompat import ui, config
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.exceptions import VCSError
 from kallithea.lib.exceptions import HgsubversionImportError
 from kallithea.model import meta
 from kallithea.model.db import Repository, User, Ui, \
     UserLog, RepoGroup, Setting, UserGroup
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.lib.utils2 import safe_str, safe_unicode, get_current_authuser
 from kallithea.lib.vcs.utils.fakemod import create_module
 log = logging.getLogger(__name__)
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = request.environ['pylons.routes_dict'].get('repo_name')
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 def get_repo_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('group_name')
     if _group:
         _group = _group.rstrip('/')
     return _group
 def get_user_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('id')
     _group = UserGroup.get(_group)
     if _group:
         return _group.users_group_name
     return None
 def _extract_id_from_repo_name(repo_name):
     if repo_name.startswith('/'):
         repo_name = repo_name.lstrip('/')
     by_id_match = re.match(r'^_(\d{1,})', repo_name)
     if by_id_match:
         return by_id_match.groups()[0]
 def get_repo_by_id(repo_name):
     """
     Extracts repo_name by id from special urls. Example url is _11/repo_name
     :param repo_name:
     :return: repo_name if matched else None
     """
     _repo_id = _extract_id_from_repo_name(repo_name)
     if _repo_id:
         from kallithea.model.db import Repository
         repo = Repository.get(_repo_id)
         if repo:
             # TODO: return repo instead of reponame? or would that be a layering violation?
             return repo.repo_name
     return None
 def action_logger(user, action, repo, ipaddr='', commit=False):
     """
     Action logger for various actions made by users
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional IP address from what the action was made
     """
     # if we don't get explicit IP address try to get one from registered user
     # in tmpl context var
     if not ipaddr:
         ipaddr = getattr(get_current_authuser(), 'ip_addr', '')
     if getattr(user, 'user_id', None):
         user_obj = User.get(user.user_id)
     elif isinstance(user, basestring):
         user_obj = User.get_by_username(user)
     else:
         raise Exception('You have to provide a user object or a username')
     if getattr(repo, 'repo_id', None):
         repo_obj = Repository.get(repo.repo_id)
         repo_name = repo_obj.repo_name
     elif isinstance(repo, basestring):
         repo_name = repo.lstrip('/')
         repo_obj = Repository.get_by_repo_name(repo_name)
     else:
         repo_obj = None
         repo_name = u''
     user_log = UserLog()
     user_log.user_id = user_obj.user_id
     user_log.username = user_obj.username
     user_log.action = safe_unicode(action)
     user_log.repository = repo_obj
     user_log.repository_name = repo_name
     user_log.action_date = datetime.datetime.now()
     user_log.user_ip = ipaddr
     meta.Session().add(user_log)
     log.info('Logging action:%s on %s by user:%s ip:%s',
              action, safe_unicode(repo), user_obj, ipaddr)
     if commit:
         meta.Session().commit()
 def get_filesystem_repos(path):
     """
     Scans given path for repos and return (name,(type,path)) tuple
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
     # remove ending slash for better results
     path = safe_str(path.rstrip(os.sep))
     log.debug('now scanning in %s', path)
     def isdir(*n):
         return os.path.isdir(os.path.join(*n))
     for root, dirs, _files in os.walk(path):
         recurse_dirs = []
         for subdir in dirs:
             # skip removed repos
             if REMOVED_REPO_PAT.match(subdir):
                 continue
             # skip .<something> dirs TODO: rly? then we should prevent creating them ...
             if subdir.startswith('.'):
                 continue
             cur_path = os.path.join(root, subdir)
             if isdir(cur_path, '.git'):
                 log.warning('ignoring non-bare Git repo: %s', cur_path)
                 continue
             if (isdir(cur_path, '.hg') or
                 isdir(cur_path, '.svn') or
                 isdir(cur_path, 'objects') and (isdir(cur_path, 'refs') or
                                                 os.path.isfile(os.path.join(cur_path, 'packed-refs')))):
                 if not os.access(cur_path, os.R_OK) or not os.access(cur_path, os.X_OK):
                     log.warning('ignoring repo path without access: %s', cur_path)
                     continue
                 if not os.access(cur_path, os.W_OK):
                     log.warning('repo path without write access: %s', cur_path)
                 try:
                     scm_info = get_scm(cur_path)
                     assert cur_path.startswith(path)
                     repo_path = cur_path[len(path) + 1:]
                     yield repo_path, scm_info
                     continue # no recursion
                 except VCSError:
                     # We should perhaps ignore such broken repos, but especially
                     # the bare git detection is unreliable so we dive into it
                     pass
             recurse_dirs.append(subdir)
         dirs[:] = recurse_dirs
 def is_valid_repo_uri(repo_type, url, ui):
     """Check if the url seems like a valid remote repo location - raise an Exception if any problems"""
     if repo_type == 'hg':
         from kallithea.lib.vcs.backends.hg.repository import MercurialRepository
         if url.startswith('http') or url.startswith('ssh'):
             # initially check if it's at least the proper URL
             # or does it pass basic auth
             MercurialRepository._check_url(url, ui)
         elif url.startswith('svn+http'):
             try:
                 from hgsubversion.svnrepo import svnremoterepo
             except ImportError:
                 raise HgsubversionImportError(_('Unable to activate hgsubversion support. '
                                                 'The "hgsubversion" library is missing'))
             svnremoterepo(ui, url).svn.uuid
         elif url.startswith('git+http'):
             raise NotImplementedError()
         else:
             raise Exception('URI %s not allowed' % (url,))
     elif repo_type == 'git':
         from kallithea.lib.vcs.backends.git.repository import GitRepository
         if url.startswith('http') or url.startswith('git'):
             # initially check if it's at least the proper URL
             # or does it pass basic auth
             GitRepository._check_url(url)
         elif url.startswith('svn+http'):
             raise NotImplementedError()
         elif url.startswith('hg+http'):
             raise NotImplementedError()
         else:
             raise Exception('URI %s not allowed' % (url))
 def is_valid_repo(repo_name, base_path, scm=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If scm param is given also compare if given scm is the same as expected
     from scm parameter
     :param repo_name:
     :param base_path:
     :param scm:
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     try:
         scm_ = get_scm(full_path)
         if scm:
             return scm_[0] == scm
         return True
     except VCSError:
         return False
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group False otherwise
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         return False
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         get_scm(os.path.dirname(full_path))
         return False
     except VCSError:
         pass
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         return True
     return False
 # propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
-def make_ui(read_from='file', path=None, clear_session=True):
+def make_ui(repo_path=None, clear_session=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
     :param path: path to mercurial config file
     :param read_from: read from 'file' or 'db'
     Create an Mercurial 'ui' object based on database Ui settings, possibly
     augmenting with content from a hgrc file.
     """
     baseui = ui.ui()
     # clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
     if read_from == 'file':
         if not os.path.isfile(path):
             log.debug('hgrc file is not present at %s, skipping...', path)
             return baseui
         log.debug('reading hgrc from %s', path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file: [%s] %s=%s', section, k, v)
                 baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
     elif read_from == 'db':
         sa = meta.Session()
         ret = sa.query(Ui).all()
         hg_ui = ret
         for ui_ in hg_ui:
     for ui_ in sa.query(Ui).all():
             if ui_.ui_active:
                 ui_val = '' if ui_.ui_value is None else safe_str(ui_.ui_value)
-                log.debug('settings ui from db: [%s] %s=%r', ui_.ui_section,
+            log.debug('config from db: [%s] %s=%r', ui_.ui_section,
                           ui_.ui_key, ui_val)
                 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
                                  ui_val)
         if clear_session:
             meta.Session.remove()
         # force set push_ssl requirement to False, Kallithea handles that
         baseui.setconfig('web', 'push_ssl', False)
         baseui.setconfig('web', 'allow_push', '*')
         # prevent interactive questions for ssh password / passphrase
         ssh = baseui.config('ui', 'ssh', default='ssh')
         baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)
         # push / pull hooks
         baseui.setconfig('hooks', 'changegroup.kallithea_log_push_action', 'python:kallithea.lib.hooks.log_push_action')
         baseui.setconfig('hooks', 'outgoing.kallithea_log_pull_action', 'python:kallithea.lib.hooks.log_pull_action')
     if repo_path is not None:
         hgrc_path = os.path.join(repo_path, '.hg', 'hgrc')
         if os.path.isfile(hgrc_path):
             log.debug('reading hgrc from %s', hgrc_path)
             cfg = config.config()
             cfg.read(hgrc_path)
             for section in ui_sections:
                 for k, v in cfg.items(section):
                     log.debug('config from file: [%s] %s=%s', section, k, v)
                     baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
         else:
             log.debug('hgrc file is not present at %s, skipping...', hgrc_path)
     return baseui
 def set_app_settings(config):
     """
     Updates app config with new settings from database
     :param config:
     """
     try:
         hgsettings = Setting.get_app_settings()
         for k, v in hgsettings.items():
             config[k] = v
     finally:
         meta.Session.remove()
 def set_vcs_config(config):
     """
     Patch VCS config with some Kallithea specific stuff
     :param config: kallithea.CONFIG
     """
     from kallithea.lib.vcs import conf
     from kallithea.lib.utils2 import aslist
     conf.settings.BACKENDS = {
         'hg': 'kallithea.lib.vcs.backends.hg.MercurialRepository',
         'git': 'kallithea.lib.vcs.backends.git.GitRepository',
+    }
     conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')
     conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()
     conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
                                                         'utf-8'), sep=',')
 def set_indexer_config(config):
     """
     Update Whoosh index mapping
     :param config: kallithea.CONFIG
     """
     from kallithea.config import conf
     log.debug('adding extra into INDEX_EXTENSIONS')
     conf.INDEX_EXTENSIONS.extend(re.split('\s+', config.get('index.extensions', '')))
     log.debug('adding extra into INDEX_FILENAMES')
     conf.INDEX_FILENAMES.extend(re.split('\s+', config.get('index.filenames', '')))
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
     :param paths: full path to repository
     """
     sa = meta.Session()
     groups = path.split(Repository.url_sep())
     parent = None
     group = None
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel()
     owner = User.get_first_admin()
     for lvl, group_name in enumerate(groups):
         group_name = u'/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                       lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.owner = owner
             sa.add(group)
             rgm._create_default_perms(group)
             sa.flush()
         parent = group
     return group
 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
                    install_git_hooks=False, user=None, overwrite_git_hooks=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hooks: if this is True, also check and install git hook
         for a repo if missing
     :param overwrite_git_hooks: if this is True, overwrite any existing git hooks
         that may be encountered (even if user-deployed)
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.scm import ScmModel
     sa = meta.Session()
     repo_model = RepoModel()
     if user is None:
         user = User.get_first_admin()
     added = []
     # creation defaults
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in Kallithea database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
             new_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
+            )
             sa.commit()
             # we added that repo just now, and make sure it has githook
             # installed, and updated server info
             if new_repo.repo_type == 'git':
                 git_repo = new_repo.scm_instance
                 ScmModel().install_git_hooks(git_repo)
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
             new_repo.update_changeset_cache()
         elif install_git_hooks:
             if db_repo.repo_type == 'git':
                 ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)
     removed = []
     # remove from database those repositories that are not in the filesystem
     unicode_initial_repo_list = set(safe_unicode(name) for name in initial_repo_list)
     for repo in sa.query(Repository).all():
         if repo.repo_name not in unicode_initial_repo_list:
             if remove_obsolete:
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel().delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                 except Exception:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
             removed.append(repo.repo_name)
     return added, removed
 def load_rcextensions(root_path):
     import kallithea
     from kallithea.config import conf
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = kallithea.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...', rcext)
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
         # OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
         if getattr(EXT, 'INDEX_EXTENSIONS', []):
             log.debug('settings custom INDEX_EXTENSIONS')
             conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
         # ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from kallithea.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 #==============================================================================
 # MISC
 #==============================================================================
 def check_git_version():
     """
     Checks what version of git is installed in system, and issues a warning
     if it's too old for Kallithea to work properly.
     """
     from kallithea import BACKENDS
     from kallithea.lib.vcs.backends.git.repository import GitRepository
     from kallithea.lib.vcs.conf import settings
     from distutils.version import StrictVersion
     if 'git' not in BACKENDS:
         return None
     stdout, stderr = GitRepository._run_git_command(['--version'], _bare=True,
                                                     _safe=True)
     m = re.search("\d+.\d+.\d+", stdout)
     if m:
         ver = StrictVersion(m.group(0))
     else:
         ver = StrictVersion('0.0.0')
     req_ver = StrictVersion('1.7.4')
     log.debug('Git executable: "%s" version %s detected: %s',
               settings.GIT_EXECUTABLE_PATH, ver, stdout)
     if stderr:
         log.warning('Error detecting git version: %r', stderr)
     elif ver < req_ver:
         log.warning('Kallithea detected git version %s, which is too old '
                     'for the system to function properly. '
                     'Please upgrade to version %s or later.' % (ver, req_ver))
     return ver
 #===============================================================================
 # CACHE RELATED METHODS
 #===============================================================================
 # set cache regions for beaker so celery can utilise it
 def setup_cache_regions(settings):
     # Create dict with just beaker cache configs with prefix stripped
     cache_settings = {'regions': None}
     prefix = 'beaker.cache.'
     for key in settings:
         if key.startswith(prefix):
             name = key[len(prefix):]
             cache_settings[name] = settings[key]
     # Find all regions, apply defaults, and apply to beaker
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             prefix = region + '.'
             region_settings = {}
             for key in cache_settings:
                 if key.startswith(prefix):
                     name = key[len(prefix):]
                     region_settings[name] = cache_settings[key]
             region_settings.setdefault('expire',
                                        cache_settings.get('expire', '60'))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
             region_settings.setdefault('type',
                                        cache_settings.get('type', 'memory'))
             beaker.cache.cache_regions[region] = region_settings
 def conditional_cache(region, prefix, condition, func):
     """
     Conditional caching function use like::
         def _c(arg):
             #heavy computation function
             return data
         # depending from condition the compute is wrapped in cache or not
         compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)
         return compute(arg)
     :param region: name of cache region
     :param prefix: cache region prefix
     :param condition: condition for cache to be triggered, and return data cached
     :param func: wrapped heavy function to compute
     """
     wrapped = func
     if condition:
         log.debug('conditional_cache: True, wrapping call of '
                   'func: %s into %s region cache' % (region, func))
         wrapped = _cache_decorate((prefix,), None, None, region)(func)
     return wrapped

kallithea/model/db.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.db
 ~~~~~~~~~~~~~~~~~~
 Database Models for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 08, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import time
 import logging
 import datetime
 import traceback
 import hashlib
 import collections
 import functools
 import sqlalchemy
 from sqlalchemy import *
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
 from webob.exc import HTTPNotFound
 from tg.i18n import lazy_ugettext as _
 from kallithea.lib.exceptions import DefaultUserException
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.utils2 import str2bool, safe_str, get_changeset_safe, \
     safe_unicode, remove_prefix, time_to_datetime, aslist, Optional, safe_int, \
     get_clone_url, urlreadable
 from kallithea.lib.compat import json
 from kallithea.lib.caching_query import FromCache
 from kallithea.model.meta import Base, Session
 URL_SEP = '/'
 log = logging.getLogger(__name__)
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 _hash_key = lambda k: hashlib.md5(safe_str(k)).hexdigest()
 class BaseDbModel(object):
     """
     Base Model for all classes
     """
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.iteritems():
                 d[k] = val
         return d
     def get_appstruct(self):
         """return list with keys and values tuples corresponding
         to this model data """
         return [
             (k, getattr(self, k))
             for k in self._get_keys()
+        ]
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
     @classmethod
     def query(cls):
         return Session().query(cls)
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
     @classmethod
     def guess_instance(cls, value, callback=None):
         """Haphazardly attempt to convert `value` to a `cls` instance.
         If `value` is None or already a `cls` instance, return it. If `value`
         is a number (or looks like one if you squint just right), assume it's
         a database primary key and let SQLAlchemy sort things out. Otherwise,
         fall back to resolving it using `callback` (if specified); this could
         e.g. be a function that looks up instances by name (though that won't
         work if the name begins with a digit). Otherwise, raise Exception.
         """
         if value is None:
             return None
         if isinstance(value, cls):
             return value
         if isinstance(value, (int, long)) or safe_str(value).isdigit():
             return cls.get(value)
         if callback is not None:
             return callback(value)
         raise Exception(
             'given object must be int, long or Instance of %s '
             'got %s, no callback provided' % (cls, type(value))
+        )
     @classmethod
     def get_or_404(cls, id_):
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound
         res = cls.query().get(id_)
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session().delete(obj)
     def __repr__(self):
         if hasattr(self, '__unicode__'):
             # python repr needs to return str
             try:
                 return safe_str(self.__unicode__())
             except UnicodeDecodeError:
                 pass
         return '<DB:%s>' % (self.__class__.__name__)
 _table_args_default_dict = {'extend_existing': True,
                             'mysql_engine': 'InnoDB',
                             'mysql_charset': 'utf8',
                             'sqlite_autoincrement': True,
+                           }
 class Setting(Base, BaseDbModel):
     __tablename__ = 'settings'
     __table_args__ = (
         _table_args_default_dict,
+    )
     SETTINGS_TYPES = {
         'str': safe_str,
         'int': safe_int,
         'unicode': safe_unicode,
         'bool': str2bool,
         'list': functools.partial(aslist, sep=',')
+    }
     DEFAULT_UPDATE_URL = ''
     app_settings_id = Column(Integer(), primary_key=True)
     app_settings_name = Column(String(255), nullable=False, unique=True)
     _app_settings_value = Column("app_settings_value", Unicode(4096), nullable=False)
     _app_settings_type = Column("app_settings_type", String(255), nullable=True) # FIXME: not nullable?
     def __init__(self, key='', val='', type='unicode'):
         self.app_settings_name = key
         self.app_settings_value = val
         self.app_settings_type = type
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == unicode
         return val
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         _type = self.app_settings_type
         converter = self.SETTINGS_TYPES.get(_type) or self.SETTINGS_TYPES['unicode']
         return converter(v)
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
     @app_settings_type.setter
     def app_settings_type(self, val):
         if val not in self.SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (self.SETTINGS_TYPES.keys(), val))
         self._app_settings_type = val
     def __unicode__(self):
         return u"<%s('%s:%s[%s]')>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_value, self.app_settings_type
+        )
     @classmethod
     def get_by_name(cls, key):
         return cls.query() \
             .filter(cls.app_settings_name == key).scalar()
     @classmethod
     def get_by_name_or_create(cls, key, val='', type='unicode'):
         res = cls.get_by_name(key)
         if res is None:
             res = cls(key, val, type)
         return res
     @classmethod
     def create_or_update(cls, key, val=Optional(''), type=Optional('unicode')):
         """
         Creates or updates Kallithea setting. If updates are triggered, it will only
         update parameters that are explicitly set. Optional instance will be skipped.
         :param key:
         :param val:
         :param type:
         :return:
         """
         res = cls.get_by_name(key)
         if res is None:
             val = Optional.extract(val)
             type = Optional.extract(type)
             res = cls(key, val, type)
             Session().add(res)
         else:
             res.app_settings_name = key
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
             if not isinstance(type, Optional):
                 # update if set
                 res.app_settings_type = type
         return res
     @classmethod
     def get_app_settings(cls, cache=False):
         ret = cls.query()
         if cache:
             ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
         if ret is None:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings[each.app_settings_name] = \
                 each.app_settings_value
         return settings
     @classmethod
     def get_auth_settings(cls, cache=False):
         ret = cls.query() \
                 .filter(cls.app_settings_name.startswith('auth_')).all()
         fd = {}
         for row in ret:
             fd[row.app_settings_name] = row.app_settings_value
         return fd
     @classmethod
     def get_default_repo_settings(cls, cache=False, strip_prefix=False):
         ret = cls.query() \
                 .filter(cls.app_settings_name.startswith('default_')).all()
         fd = {}
         for row in ret:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             fd.update({key: row.app_settings_value})
         return fd
     @classmethod
     def get_server_info(cls):
         import pkg_resources
         import platform
         import kallithea
         from kallithea.lib.utils import check_git_version
         mods = [(p.project_name, p.version) for p in pkg_resources.working_set]
         info = {
             'modules': sorted(mods, key=lambda k: k[0].lower()),
             'py_version': platform.python_version(),
             'platform': safe_unicode(platform.platform()),
             'kallithea_version': kallithea.__version__,
             'git_version': safe_unicode(check_git_version()),
             'git_path': kallithea.CONFIG.get('git_path')
+        }
         return info
 class Ui(Base, BaseDbModel):
     __tablename__ = 'ui'
     __table_args__ = (
         # FIXME: ui_key as key is wrong and should be removed when the corresponding
         # Ui.get_by_key has been replaced by the composite key
         UniqueConstraint('ui_key'),
         UniqueConstraint('ui_section', 'ui_key'),
         _table_args_default_dict,
+    )
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     ui_id = Column(Integer(), primary_key=True)
     ui_section = Column(String(255), nullable=False)
     ui_key = Column(String(255), nullable=False)
     ui_value = Column(String(255), nullable=True) # FIXME: not nullable?
     ui_active = Column(Boolean(), nullable=False, default=True)
     @classmethod
     def get_by_key(cls, section, key):
         """ Return specified Ui object, or None if not found. """
         return cls.query().filter_by(ui_section=section, ui_key=key).scalar()
     @classmethod
     def get_or_create(cls, section, key):
         """ Return specified Ui object, creating it if necessary. """
         setting = cls.get_by_key(section, key)
         if setting is None:
             setting = cls(ui_section=section, ui_key=key)
             Session().add(setting)
         return setting
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
     @classmethod
     def get_repos_location(cls):
         return cls.get_by_key('paths', '/').ui_value
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_or_create('hooks', key)
         new_ui.ui_active = True
         new_ui.ui_value = val
     def __repr__(self):
         return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section,
                                     self.ui_key, self.ui_value)
 class User(Base, BaseDbModel):
     __tablename__ = 'users'
     __table_args__ = (
         Index('u_username_idx', 'username'),
         Index('u_email_idx', 'email'),
         _table_args_default_dict,
+    )
     DEFAULT_USER = 'default'
     DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
     # The name of the default auth type in extern_type, 'internal' lives in auth_internal.py
     DEFAULT_AUTH_TYPE = 'internal'
     user_id = Column(Integer(), primary_key=True)
     username = Column(String(255), nullable=False, unique=True)
     password = Column(String(255), nullable=False)
     active = Column(Boolean(), nullable=False, default=True)
     admin = Column(Boolean(), nullable=False, default=False)
     name = Column("firstname", Unicode(255), nullable=False)
     lastname = Column(Unicode(255), nullable=False)
     _email = Column("email", String(255), nullable=True, unique=True) # FIXME: not nullable?
     last_login = Column(DateTime(timezone=False), nullable=True)
     extern_type = Column(String(255), nullable=True) # FIXME: not nullable?
     extern_name = Column(String(255), nullable=True) # FIXME: not nullable?
     api_key = Column(String(255), nullable=False)
     inherit_default_permissions = Column(Boolean(), nullable=False, default=True)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _user_data = Column("user_data", LargeBinary(), nullable=True)  # JSON data # FIXME: not nullable?
     user_log = relationship('UserLog')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
     repositories = relationship('Repository')
     repo_groups = relationship('RepoGroup')
     user_groups = relationship('UserGroup')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
     repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all')
     group_member = relationship('UserGroupMember', cascade='all')
     # comments created by this user
     user_comments = relationship('ChangesetComment', cascade='all')
     # extra emails for this user
     user_emails = relationship('UserEmailMap', cascade='all')
     # extra API keys
     user_api_keys = relationship('UserApiKeys', cascade='all')
     @hybrid_property
     def email(self):
         return self._email
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
     @property
     def firstname(self):
         # alias for future
         return self.name
     @property
     def emails(self):
         other = UserEmailMap.query().filter(UserEmailMap.user == self).all()
         return [self.email] + [x.email for x in other]
     @property
     def api_keys(self):
         other = UserApiKeys.query().filter(UserApiKeys.user == self).all()
         return [self.api_key] + [x.api_key for x in other]
     @property
     def ip_addresses(self):
         ret = UserIpMap.query().filter(UserIpMap.user == self).all()
         return [x.ip_addr for x in ret]
     @property
     def full_name(self):
         return '%s %s' % (self.firstname, self.lastname)
     @property
     def full_name_or_username(self):
         """
         Show full name.
         If full name is not set, fall back to username.
         """
         return ('%s %s' % (self.firstname, self.lastname)
                 if (self.firstname and self.lastname) else self.username)
     @property
     def full_name_and_username(self):
         """
         Show full name and username as 'Firstname Lastname (username)'.
         If full name is not set, fall back to username.
         """
         return ('%s %s (%s)' % (self.firstname, self.lastname, self.username)
                 if (self.firstname and self.lastname) else self.username)
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.firstname, self.lastname, self.email)
     @property
     def short_contact(self):
         return '%s %s' % (self.firstname, self.lastname)
     @property
     def is_admin(self):
         return self.admin
     @hybrid_property
     def is_default_user(self):
         return self.username == User.DEFAULT_USER
     @hybrid_property
     def user_data(self):
         if not self._user_data:
             return {}
         try:
             return json.loads(self._user_data)
         except TypeError:
             return {}
     @user_data.setter
     def user_data(self, val):
         try:
             self._user_data = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
                                       self.user_id, self.username)
     @classmethod
     def guess_instance(cls, value):
         return super(User, cls).guess_instance(value, User.get_by_username)
     @classmethod
     def get_or_404(cls, id_, allow_default=True):
         '''
         Overridden version of BaseDbModel.get_or_404, with an extra check on
         the default user.
         '''
         user = super(User, cls).get_or_404(id_)
         if not allow_default and user.is_default_user:
             raise DefaultUserException()
         return user
     @classmethod
     def get_by_username_or_email(cls, username_or_email, case_insensitive=False, cache=False):
         """
         For anything that looks like an email address, look up by the email address (matching
         case insensitively).
         For anything else, try to look up by the user name.
         This assumes no normal username can have '@' symbol.
         """
         if '@' in username_or_email:
             return User.get_by_email(username_or_email, cache=cache)
         else:
             return User.get_by_username(username_or_email, case_insensitive=case_insensitive, cache=cache)
     @classmethod
     def get_by_username(cls, username, case_insensitive=False, cache=False):
         if case_insensitive:
             q = cls.query().filter(func.lower(cls.username) == func.lower(username))
         else:
             q = cls.query().filter(cls.username == username)
         if cache:
             q = q.options(FromCache(
                             "sql_cache_short",
                             "get_user_%s" % _hash_key(username)
+                          )
+            )
         return q.scalar()
     @classmethod
     def get_by_api_key(cls, api_key, cache=False, fallback=True):
         if len(api_key) != 40 or not api_key.isalnum():
             return None
         q = cls.query().filter(cls.api_key == api_key)
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_api_key_%s" % api_key))
         res = q.scalar()
         if fallback and not res:
             # fallback to additional keys
             _res = UserApiKeys.query().filter_by(api_key=api_key, is_expired=False).first()
             if _res:
                 res = _res.user
         return res
     @classmethod
     def get_by_email(cls, email, cache=False):
         q = cls.query().filter(func.lower(cls.email) == func.lower(email))
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_email_key_%s" % email))
         ret = q.scalar()
         if ret is None:
             q = UserEmailMap.query()
             # try fetching in alternate email map
             q = q.filter(func.lower(UserEmailMap.email) == func.lower(email))
             q = q.options(joinedload(UserEmailMap.user))
             if cache:
                 q = q.options(FromCache("sql_cache_short",
                                         "get_email_map_key_%s" % email))
             ret = getattr(q.scalar(), 'user', None)
         return ret
     @classmethod
     def get_from_cs_author(cls, author):
         """
         Tries to get User objects out of commit author string
         :param author:
         """
         from kallithea.lib.helpers import email, author_name
         # Valid email in the attribute passed, see if they're in the system
         _email = email(author)
         if _email:
             user = cls.get_by_email(_email)
             if user is not None:
                 return user
         # Maybe we can match by username?
         _author = author_name(author)
         user = cls.get_by_username(_author, case_insensitive=True)
         if user is not None:
             return user
     def update_lastlogin(self):
         """Update user lastlogin"""
         self.last_login = datetime.datetime.now()
         log.debug('updated user %s lastlogin', self.username)
     @classmethod
     def get_first_admin(cls):
         user = User.query().filter(User.admin == True).first()
         if user is None:
             raise Exception('Missing administrative account!')
         return user
     @classmethod
     def get_default_user(cls, cache=False):
         user = User.get_by_username(User.DEFAULT_USER, cache=cache)
         if user is None:
             raise Exception('Missing default account!')
         return user
     def get_api_data(self, details=False):
         """
         Common function for generating user related data for API
         """
         user = self
         data = dict(
             user_id=user.user_id,
             username=user.username,
             firstname=user.name,
             lastname=user.lastname,
             email=user.email,
             emails=user.emails,
             active=user.active,
             admin=user.admin,
+        )
         if details:
             data.update(dict(
                 extern_type=user.extern_type,
                 extern_name=user.extern_name,
                 api_key=user.api_key,
                 api_keys=user.api_keys,
                 last_login=user.last_login,
                 ip_addresses=user.ip_addresses
                 ))
         return data
     def __json__(self):
         data = dict(
             full_name=self.full_name,
             full_name_or_username=self.full_name_or_username,
             short_contact=self.short_contact,
             full_contact=self.full_contact
+        )
         data.update(self.get_api_data())
         return data
 class UserApiKeys(Base, BaseDbModel):
     __tablename__ = 'user_api_keys'
     __table_args__ = (
         Index('uak_api_key_idx', 'api_key'),
         Index('uak_api_key_expires_idx', 'api_key', 'expires'),
         _table_args_default_dict,
+    )
     __mapper_args__ = {}
     user_api_key_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     api_key = Column(String(255), nullable=False, unique=True)
     description = Column(UnicodeText(), nullable=False)
     expires = Column(Float(53), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     user = relationship('User')
     @hybrid_property
     def is_expired(self):
         return (self.expires != -1) & (time.time() > self.expires)
 class UserEmailMap(Base, BaseDbModel):
     __tablename__ = 'user_email_map'
     __table_args__ = (
         Index('uem_email_idx', 'email'),
         _table_args_default_dict,
+    )
     __mapper_args__ = {}
     email_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     _email = Column("email", String(255), nullable=False, unique=True)
     user = relationship('User')
     @validates('_email')
     def validate_email(self, key, email):
         # check if this email is not main one
         main_email = Session().query(User).filter(User.email == email).scalar()
         if main_email is not None:
             raise AttributeError('email %s is present is user table' % email)
         return email
     @hybrid_property
     def email(self):
         return self._email
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 class UserIpMap(Base, BaseDbModel):
     __tablename__ = 'user_ip_map'
     __table_args__ = (
         UniqueConstraint('user_id', 'ip_addr'),
         _table_args_default_dict,
+    )
     __mapper_args__ = {}
     ip_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     ip_addr = Column(String(255), nullable=False)
     active = Column(Boolean(), nullable=False, default=True)
     user = relationship('User')
     @classmethod
     def _get_ip_range(cls, ip_addr):
         from kallithea.lib import ipaddr
         net = ipaddr.IPNetwork(address=ip_addr)
         return [str(net.network), str(net.broadcast)]
     def __json__(self):
         return dict(
           ip_addr=self.ip_addr,
           ip_range=self._get_ip_range(self.ip_addr)
+        )
     def __unicode__(self):
         return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__,
                                             self.user_id, self.ip_addr)
 class UserLog(Base, BaseDbModel):
     __tablename__ = 'user_logs'
     __table_args__ = (
         _table_args_default_dict,
+    )
     user_log_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     username = Column(String(255), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     repository_name = Column(Unicode(255), nullable=False)
     user_ip = Column(String(255), nullable=True)
     action = Column(UnicodeText(), nullable=False)
     action_date = Column(DateTime(timezone=False), nullable=False)
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
                                       self.repository_name,
                                       self.action)
     @property
     def action_as_day(self):
         return datetime.date(*self.action_date.timetuple()[:3])
     user = relationship('User')
     repository = relationship('Repository', cascade='')
 class UserGroup(Base, BaseDbModel):
     __tablename__ = 'users_groups'
     __table_args__ = (
         _table_args_default_dict,
+    )
     users_group_id = Column(Integer(), primary_key=True)
     users_group_name = Column(Unicode(255), nullable=False, unique=True)
     user_group_description = Column(Unicode(10000), nullable=True) # FIXME: not nullable?
     users_group_active = Column(Boolean(), nullable=False)
     inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, default=True)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _group_data = Column("group_data", LargeBinary(), nullable=True)  # JSON data # FIXME: not nullable?
     members = relationship('UserGroupMember', cascade="all, delete-orphan")
     users_group_to_perm = relationship('UserGroupToPerm', cascade='all')
     users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     user_user_group_to_perm = relationship('UserUserGroupToPerm ', cascade='all')
     user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all')
     owner = relationship('User')
     @hybrid_property
     def group_data(self):
         if not self._group_data:
             return {}
         try:
             return json.loads(self._group_data)
         except TypeError:
             return {}
     @group_data.setter
     def group_data(self, val):
         try:
             self._group_data = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
                                       self.users_group_id,
                                       self.users_group_name)
     @classmethod
     def guess_instance(cls, value):
         return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
     @classmethod
     def get_by_group_name(cls, group_name, cache=False,
                           case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(func.lower(cls.users_group_name) == func.lower(group_name))
         else:
             q = cls.query().filter(cls.users_group_name == group_name)
         if cache:
             q = q.options(FromCache(
                             "sql_cache_short",
                             "get_group_%s" % _hash_key(group_name)
+                          )
+            )
         return q.scalar()
     @classmethod
     def get(cls, user_group_id, cache=False):
         user_group = cls.query()
         if cache:
             user_group = user_group.options(FromCache("sql_cache_short",
                                     "get_users_group_%s" % user_group_id))
         return user_group.get(user_group_id)
     def get_api_data(self, with_members=True):
         user_group = self
         data = dict(
             users_group_id=user_group.users_group_id,
             group_name=user_group.users_group_name,
             group_description=user_group.user_group_description,
             active=user_group.users_group_active,
             owner=user_group.owner.username,
+        )
         if with_members:
             data['members'] = [
                 ugm.user.get_api_data()
                 for ugm in user_group.members
+            ]
         return data
 class UserGroupMember(Base, BaseDbModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = (
         _table_args_default_dict,
+    )
     users_group_member_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     user = relationship('User')
     users_group = relationship('UserGroup')
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 class RepositoryField(Base, BaseDbModel):
     __tablename__ = 'repositories_fields'
     __table_args__ = (
         UniqueConstraint('repository_id', 'field_key'),  # no-multi field
         _table_args_default_dict,
+    )
     PREFIX = 'ex_'  # prefix used in form to not conflict with already existing fields
     repo_field_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     field_key = Column(String(250), nullable=False)
     field_label = Column(String(1024), nullable=False)
     field_value = Column(String(10000), nullable=False)
     field_desc = Column(String(1024), nullable=False)
     field_type = Column(String(255), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     repository = relationship('Repository')
     @property
     def field_key_prefixed(self):
         return 'ex_%s' % self.field_key
     @classmethod
     def un_prefix_key(cls, key):
         if key.startswith(cls.PREFIX):
             return key[len(cls.PREFIX):]
         return key
     @classmethod
     def get_by_key_name(cls, key, repo):
         row = cls.query() \
                 .filter(cls.repository == repo) \
                 .filter(cls.field_key == key).scalar()
         return row
 class Repository(Base, BaseDbModel):
     __tablename__ = 'repositories'
     __table_args__ = (
         Index('r_repo_name_idx', 'repo_name'),
         _table_args_default_dict,
+    )
     DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
     DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}'
     STATE_CREATED = u'repo_state_created'
     STATE_PENDING = u'repo_state_pending'
     STATE_ERROR = u'repo_state_error'
     repo_id = Column(Integer(), primary_key=True)
     repo_name = Column(Unicode(255), nullable=False, unique=True)
     repo_state = Column(String(255), nullable=False)
     clone_uri = Column(String(255), nullable=True) # FIXME: not nullable?
     repo_type = Column(String(255), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     private = Column(Boolean(), nullable=False)
     enable_statistics = Column("statistics", Boolean(), nullable=False, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=False, default=True)
     description = Column(Unicode(10000), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _landing_revision = Column("landing_revision", String(255), nullable=False)
     _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True) # JSON data # FIXME: not nullable?
     fork_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True)
     owner = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
     followers = relationship('UserFollowing',
                              primaryjoin='UserFollowing.follows_repository_id==Repository.repo_id',
                              cascade='all')
     extra_fields = relationship('RepositoryField',
                                 cascade="all, delete-orphan")
     logs = relationship('UserLog')
     comments = relationship('ChangesetComment', cascade="all, delete-orphan")
     pull_requests_org = relationship('PullRequest',
                     primaryjoin='PullRequest.org_repo_id==Repository.repo_id',
                     cascade="all, delete-orphan")
     pull_requests_other = relationship('PullRequest',
                     primaryjoin='PullRequest.other_repo_id==Repository.repo_id',
                     cascade="all, delete-orphan")
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id,
                                    safe_unicode(self.repo_name))
     @hybrid_property
     def landing_rev(self):
         # always should return [rev_type, rev]
         if self._landing_revision:
             _rev_info = self._landing_revision.split(':')
             if len(_rev_info) < 2:
                 _rev_info.insert(0, 'rev')
             return [_rev_info[0], _rev_info[1]]
         return [None, None]
     @landing_rev.setter
     def landing_rev(self, val):
         if ':' not in val:
             raise ValueError('value must be delimited with `:` and consist '
                              'of <rev_type>:<rev>, got %s instead' % val)
         self._landing_revision = val
     @hybrid_property
     def changeset_cache(self):
         try:
             cs_cache = json.loads(self._changeset_cache) # might raise on bad data
             cs_cache['raw_id'] # verify data, raise exception on error
             return cs_cache
         except (TypeError, KeyError, ValueError):
             return EmptyChangeset().__json__()
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
             self._changeset_cache = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
     @classmethod
     def query(cls, sorted=False):
         """Add Repository-specific helpers for common query constructs.
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(Repository, cls).query()
         if sorted:
             q = q.order_by(func.lower(Repository.repo_name))
         return q
     @classmethod
     def url_sep(cls):
         return URL_SEP
     @classmethod
     def normalize_repo_name(cls, repo_name):
         """
         Normalizes os specific repo_name to the format internally stored inside
         database using URL_SEP
         :param cls:
         :param repo_name:
         """
         return cls.url_sep().join(repo_name.split(os.sep))
     @classmethod
     def guess_instance(cls, value):
         return super(Repository, cls).guess_instance(value, Repository.get_by_repo_name)
     @classmethod
     def get_by_repo_name(cls, repo_name, case_insensitive=False):
         """Get the repo, defaulting to database case sensitivity.
         case_insensitive will be slower and should only be specified if necessary."""
         if case_insensitive:
             q = Session().query(cls).filter(func.lower(cls.repo_name) == func.lower(repo_name))
         else:
             q = Session().query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork)) \
                 .options(joinedload(Repository.owner)) \
                 .options(joinedload(Repository.group))
         return q.scalar()
     @classmethod
     def get_by_full_path(cls, repo_full_path):
         base_full_path = os.path.realpath(cls.base_path())
         repo_full_path = os.path.realpath(repo_full_path)
         assert repo_full_path.startswith(base_full_path + os.path.sep)
         repo_name = repo_full_path[len(base_full_path) + 1:]
         repo_name = cls.normalize_repo_name(repo_name)
         return cls.get_by_repo_name(repo_name.strip(URL_SEP))
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
     @classmethod
     def base_path(cls):
         """
         Returns base path where all repos are stored
         :param cls:
         """
         q = Session().query(Ui) \
             .filter(Ui.ui_key == cls.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
     @property
     def forks(self):
         """
         Return forks of this repo
         """
         return Repository.get_repo_forks(self.repo_id)
     @property
     def parent(self):
         """
         Returns fork parent
         """
         return self.fork
     @property
     def just_name(self):
         return self.repo_name.split(Repository.url_sep())[-1]
     @property
     def groups_with_parents(self):
         groups = []
         group = self.group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
     @LazyProperty
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = Session().query(Ui).filter(Ui.ui_key ==
                                               Repository.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(Repository.url_sep())
         return os.path.join(*map(safe_unicode, p))
     @property
     def cache_keys(self):
         """
         Returns associated cache keys for that repo
         """
         return CacheInvalidation.query() \
             .filter(CacheInvalidation.cache_args == self.repo_name) \
             .order_by(CacheInvalidation.cache_key) \
             .all()
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return Repository.url_sep().join(path_prefix + [repo_name])
     @property
     def _ui(self):
         """
         Creates an db based ui object for this repository
         """
         from kallithea.lib.utils import make_ui
-        return make_ui('db', clear_session=False)
         return make_ui(clear_session=False)
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
         :param cls:
         :param repo_name:
         """
         from kallithea.lib.utils import is_valid_repo
         return is_valid_repo(repo_name, cls.base_path())
     def get_api_data(self, with_revision_names=False,
                            with_pullrequests=False):
         """
         Common function for generating repo api data.
         Optionally, also return tags, branches, bookmarks and PRs.
         """
         repo = self
         data = dict(
             repo_id=repo.repo_id,
             repo_name=repo.repo_name,
             repo_type=repo.repo_type,
             clone_uri=repo.clone_uri,
             private=repo.private,
             created_on=repo.created_on,
             description=repo.description,
             landing_rev=repo.landing_rev,
             owner=repo.owner.username,
             fork_of=repo.fork.repo_name if repo.fork else None,
             enable_statistics=repo.enable_statistics,
             enable_downloads=repo.enable_downloads,
             last_changeset=repo.changeset_cache,
+        )
         if with_revision_names:
             scm_repo = repo.scm_instance_no_cache()
             data.update(dict(
                 tags=scm_repo.tags,
                 branches=scm_repo.branches,
                 bookmarks=scm_repo.bookmarks,
             ))
         if with_pullrequests:
             data['pull_requests'] = repo.pull_requests_other
         rc_config = Setting.get_app_settings()
         repository_fields = str2bool(rc_config.get('repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
         return data
     @property
     def last_db_change(self):
         return self.updated_on
     @property
     def clone_uri_hidden(self):
         clone_uri = self.clone_uri
         if clone_uri:
             import urlobject
             url_obj = urlobject.URLObject(self.clone_uri)
             if url_obj.password:
                 clone_uri = url_obj.with_password('*****')
         return clone_uri
     def clone_url(self, **override):
         import kallithea.lib.helpers as h
         qualified_home_url = h.canonical_url('home')
         uri_tmpl = None
         if 'with_id' in override:
             uri_tmpl = self.DEFAULT_CLONE_URI_ID
             del override['with_id']
         if 'uri_tmpl' in override:
             uri_tmpl = override['uri_tmpl']
             del override['uri_tmpl']
         # we didn't override our tmpl from **overrides
         if not uri_tmpl:
             uri_tmpl = self.DEFAULT_CLONE_URI
             try:
                 from tg import tmpl_context as c
                 uri_tmpl = c.clone_uri_tmpl
             except AttributeError:
                 # in any case if we call this outside of request context,
                 # ie, not having tmpl_context set up
                 pass
         return get_clone_url(uri_tmpl=uri_tmpl,
                              qualified_home_url=qualified_home_url,
                              repo_name=self.repo_name,
                              repo_id=self.repo_id, **override)
     def set_state(self, state):
         self.repo_state = state
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
     def get_changeset(self, rev=None):
         return get_changeset_safe(self.scm_instance, rev)
     def get_landing_changeset(self):
         """
         Returns landing changeset, or if that doesn't exist returns the tip
         """
         _rev_type, _rev = self.landing_rev
         cs = self.get_changeset(_rev)
         if isinstance(cs, EmptyChangeset):
             return self.get_changeset()
         return cs
     def update_changeset_cache(self, cs_cache=None):
         """
         Update cache of last changeset for repository, keys should be::
             short_id
             raw_id
             revision
             message
             date
             author
         :param cs_cache:
         """
         from kallithea.lib.vcs.backends.base import BaseChangeset
         if cs_cache is None:
             cs_cache = EmptyChangeset()
             # use no-cache version here
             scm_repo = self.scm_instance_no_cache()
             if scm_repo:
                 cs_cache = scm_repo.get_changeset()
         if isinstance(cs_cache, BaseChangeset):
             cs_cache = cs_cache.__json__()
         if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):
             _default = datetime.datetime.fromtimestamp(0)
             last_change = cs_cache.get('date') or _default
             log.debug('updated repo %s with new cs cache %s',
                       self.repo_name, cs_cache)
             self.updated_on = last_change
             self.changeset_cache = cs_cache
             Session().commit()
         else:
             log.debug('changeset_cache for %s already up to date with %s',
                       self.repo_name, cs_cache['raw_id'])
     @property
     def tip(self):
         return self.get_changeset('tip')
     @property
     def author(self):
         return self.tip.author
     @property
     def last_change(self):
         return self.scm_instance.last_change
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query() \
             .filter(ChangesetComment.repo == self)
         if revisions is not None:
             if not revisions:
                 return {} # don't use sql 'in' on empty set
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = collections.defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
     def statuses(self, revisions):
         """
         Returns statuses for this repository.
         PRs without any votes do _not_ show up as unreviewed.
         :param revisions: list of revisions to get statuses for
         """
         if not revisions:
             return {}
         statuses = ChangesetStatus.query() \
             .filter(ChangesetStatus.repo == self) \
             .filter(ChangesetStatus.version == 0) \
             .filter(ChangesetStatus.revision.in_(revisions))
         grouped = {}
         for stat in statuses.all():
             pr_id = pr_nice_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_nice_id = PullRequest.make_nice_id(pr_id)
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo, pr_nice_id,
                                       stat.author]
         return grouped
     def _repo_size(self):
         from kallithea.lib import helpers as h
         log.debug('calculating repository size...')
         return h.format_byte_size(self.scm_instance.size)
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
     def set_invalidate(self):
         """
         Mark caches of this repo as invalid.
         """
         CacheInvalidation.set_invalidate(self.repo_name)
     _scm_instance = None
     @property
     def scm_instance(self):
         if self._scm_instance is None:
             self._scm_instance = self.scm_instance_cached()
         return self._scm_instance
     def scm_instance_cached(self, valid_cache_keys=None):
         @cache_region('long_term', 'scm_instance_cached')
         def _c(repo_name): # repo_name is just for the cache key
             log.debug('Creating new %s scm_instance and populating cache', repo_name)
             return self.scm_instance_no_cache()
         rn = self.repo_name
         valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
         if not valid:
             log.debug('Cache for %s invalidated, getting new object', rn)
             region_invalidate(_c, None, 'scm_instance_cached', rn)
         else:
             log.debug('Trying to get scm_instance of %s from cache', rn)
         return _c(rn)
     def scm_instance_no_cache(self):
         repo_full_path = safe_str(self.repo_full_path)
         alias = get_scm(repo_full_path)[0]
         log.debug('Creating instance of %s repository from %s',
                   alias, self.repo_full_path)
         backend = get_backend(alias)
         if alias == 'hg':
             repo = backend(repo_full_path, create=False,
                            baseui=self._ui)
         else:
             repo = backend(repo_full_path, create=False)
         return repo
     def __json__(self):
         return dict(
             repo_id=self.repo_id,
             repo_name=self.repo_name,
             landing_rev=self.landing_rev,
+        )
 class RepoGroup(Base, BaseDbModel):
     __tablename__ = 'groups'
     __table_args__ = (
         _table_args_default_dict,
+    )
     __mapper_args__ = {'order_by': 'group_name'} # TODO: Deprecated as of SQLAlchemy 1.1.
     SEP = ' &raquo; '
     group_id = Column(Integer(), primary_key=True)
     group_name = Column(Unicode(255), nullable=False, unique=True) # full path
     parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)
     group_description = Column(Unicode(10000), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     owner = relationship('User')
     @classmethod
     def query(cls, sorted=False):
         """Add RepoGroup-specific helpers for common query constructs.
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(RepoGroup, cls).query()
         if sorted:
             q = q.order_by(func.lower(RepoGroup.group_name))
         return q
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,
                                       self.group_name)
     @classmethod
     def _generate_choice(cls, repo_group):
         """Return tuple with group_id and name as html literal"""
         from webhelpers.html import literal
         if repo_group is None:
             return (-1, u'-- %s --' % _('top level'))
         return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))
     @classmethod
     def groups_choices(cls, groups):
         """Return tuples with group_id and name as html literal."""
         return sorted((cls._generate_choice(g) for g in groups),
                       key=lambda c: c[1].split(cls.SEP))
     @classmethod
     def url_sep(cls):
         return URL_SEP
     @classmethod
     def guess_instance(cls, value):
         return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         group_name = group_name.rstrip('/')
         if case_insensitive:
             gr = cls.query() \
                 .filter(func.lower(cls.group_name) == func.lower(group_name))
         else:
             gr = cls.query() \
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache(
                             "sql_cache_short",
                             "get_group_%s" % _hash_key(group_name)
+                            )
+            )
         return gr.scalar()
     @property
     def parents(self):
         groups = []
         group = self.parent_group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
     @property
     def name(self):
         return self.group_name.split(RepoGroup.url_sep())[-1]
     @property
     def full_path(self):
         return self.group_name
     @property
     def full_path_splitted(self):
         return self.group_name.split(RepoGroup.url_sep())
     @property
     def repositories(self):
         return Repository.query(sorted=True).filter_by(group=self)
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
         return cnt + children_count(self)
     def _recursive_objects(self, include_repos=True):
         all_ = []
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     all_.append(gr)
                     _get_members(gr)
         _get_members(self)
         return [self] + all_
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return RepoGroup.url_sep().join(path_prefix + [group_name])
     def get_api_data(self):
         """
         Common function for generating api data
         """
         group = self
         data = dict(
             group_id=group.group_id,
             group_name=group.group_name,
             group_description=group.group_description,
             parent_group=group.parent_group.group_name if group.parent_group else None,
             repositories=[x.repo_name for x in group.repositories],
             owner=group.owner.username
+        )
         return data
 class Permission(Base, BaseDbModel):
     __tablename__ = 'permissions'
     __table_args__ = (
         Index('p_perm_name_idx', 'permission_name'),
         _table_args_default_dict,
+    )
     PERMS = [
         ('hg.admin', _('Kallithea Administrator')),
         ('repository.none', _('Default user has no access to new repositories')),
         ('repository.read', _('Default user has read access to new repositories')),
         ('repository.write', _('Default user has write access to new repositories')),
         ('repository.admin', _('Default user has admin access to new repositories')),
         ('group.none', _('Default user has no access to new repository groups')),
         ('group.read', _('Default user has read access to new repository groups')),
         ('group.write', _('Default user has write access to new repository groups')),
         ('group.admin', _('Default user has admin access to new repository groups')),
         ('usergroup.none', _('Default user has no access to new user groups')),
         ('usergroup.read', _('Default user has read access to new user groups')),
         ('usergroup.write', _('Default user has write access to new user groups')),
         ('usergroup.admin', _('Default user has admin access to new user groups')),
         ('hg.repogroup.create.false', _('Only admins can create repository groups')),
         ('hg.repogroup.create.true', _('Non-admins can create repository groups')),
         ('hg.usergroup.create.false', _('Only admins can create user groups')),
         ('hg.usergroup.create.true', _('Non-admins can create user groups')),
         ('hg.create.none', _('Only admins can create top level repositories')),
         ('hg.create.repository', _('Non-admins can create top level repositories')),
         ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')),
         ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')),
         ('hg.fork.none', _('Only admins can fork repositories')),
         ('hg.fork.repository', _('Non-admins can fork repositories')),
         ('hg.register.none', _('Registration disabled')),
         ('hg.register.manual_activate', _('User registration with manual account activation')),
         ('hg.register.auto_activate', _('User registration with automatic account activation')),
         ('hg.extern_activate.manual', _('Manual activation of external account')),
         ('hg.extern_activate.auto', _('Automatic activation of external account')),
+    ]
     # definition of system default permissions for DEFAULT user
     DEFAULT_USER_PERMISSIONS = [
         'repository.read',
         'group.read',
         'usergroup.read',
         'hg.create.repository',
         'hg.create.write_on_repogroup.true',
         'hg.fork.repository',
         'hg.register.manual_activate',
         'hg.extern_activate.auto',
+    ]
     # defines which permissions are more important higher the more important
     # Weight defines which permissions are more important.
     # The higher number the more important.
     PERM_WEIGHTS = {
         'repository.none': 0,
         'repository.read': 1,
         'repository.write': 3,
         'repository.admin': 4,
         'group.none': 0,
         'group.read': 1,
         'group.write': 3,
         'group.admin': 4,
         'usergroup.none': 0,
         'usergroup.read': 1,
         'usergroup.write': 3,
         'usergroup.admin': 4,
         'hg.repogroup.create.false': 0,
         'hg.repogroup.create.true': 1,
         'hg.usergroup.create.false': 0,
         'hg.usergroup.create.true': 1,
         'hg.fork.none': 0,
         'hg.fork.repository': 1,
         'hg.create.none': 0,
         'hg.create.repository': 1
+    }
     permission_id = Column(Integer(), primary_key=True)
     permission_name = Column(String(255), nullable=False)
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (
             self.__class__.__name__, self.permission_id, self.permission_name
+        )
     @classmethod
     def guess_instance(cls, value):
         return super(Permission, cls).guess_instance(value, Permission.get_by_key)
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
     @classmethod
     def get_default_perms(cls, default_user_id):
         q = Session().query(UserRepoToPerm, Repository, cls) \
          .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id)) \
          .join((cls, UserRepoToPerm.permission_id == cls.permission_id)) \
          .filter(UserRepoToPerm.user_id == default_user_id)
         return q.all()
     @classmethod
     def get_default_group_perms(cls, default_user_id):
         q = Session().query(UserRepoGroupToPerm, RepoGroup, cls) \
          .join((RepoGroup, UserRepoGroupToPerm.group_id == RepoGroup.group_id)) \
          .join((cls, UserRepoGroupToPerm.permission_id == cls.permission_id)) \
          .filter(UserRepoGroupToPerm.user_id == default_user_id)
         return q.all()
     @classmethod
     def get_default_user_group_perms(cls, default_user_id):
         q = Session().query(UserUserGroupToPerm, UserGroup, cls) \
          .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id)) \
          .join((cls, UserUserGroupToPerm.permission_id == cls.permission_id)) \
          .filter(UserUserGroupToPerm.user_id == default_user_id)
         return q.all()
 class UserRepoToPerm(Base, BaseDbModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'repository_id', 'permission_id'),
         _table_args_default_dict,
+    )
     repo_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user = relationship('User')
     repository = relationship('Repository')
     permission = relationship('Permission')
     @classmethod
     def create(cls, user, repository, permission):
         n = cls()
         n.user = user
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.repository)
 class UserUserGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'user_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'user_group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     user_user_group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     user = relationship('User')
     user_group = relationship('UserGroup')
     permission = relationship('Permission')
     @classmethod
     def create(cls, user, user_group, permission):
         n = cls()
         n.user = user
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.user_group)
 class UserToPerm(Base, BaseDbModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'permission_id'),
         _table_args_default_dict,
+    )
     user_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user = relationship('User')
     permission = relationship('Permission')
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.permission)
 class UserGroupRepoToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (
         UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
     @classmethod
     def create(cls, users_group, repository, permission):
         n = cls()
         n.users_group = users_group
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
     def __unicode__(self):
         return u'<UserGroupRepoToPerm:%s => %s >' % (self.users_group, self.repository)
 class UserGroupUserGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'user_group_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     user_group_user_group_to_perm_id = Column(Integer(), primary_key=True)
     target_user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id')
     user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
     permission = relationship('Permission')
     @classmethod
     def create(cls, target_user_group, user_group, permission):
         n = cls()
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
     def __unicode__(self):
         return u'<UserGroupUserGroup:%s => %s >' % (self.target_user_group, self.user_group)
 class UserGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         _table_args_default_dict,
+    )
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 class UserRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
     @classmethod
     def create(cls, user, repository_group, permission):
         n = cls()
         n.user = user
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class UserGroupRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         _table_args_default_dict,
+    )
     users_group_repo_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
     @classmethod
     def create(cls, user_group, repository_group, permission):
         n = cls()
         n.users_group = user_group
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class Statistics(Base, BaseDbModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          _table_args_default_dict,
+    )
     stat_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True)
     stat_on_revision = Column(Integer(), nullable=False)
     commit_activity = Column(LargeBinary(1000000), nullable=False) # JSON data
     commit_activity_combined = Column(LargeBinary(), nullable=False) # JSON data
     languages = Column(LargeBinary(1000000), nullable=False) # JSON data
     repository = relationship('Repository', single_parent=True)
 class UserFollowing(Base, BaseDbModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id', name='uq_user_followings_user_repo'),
         UniqueConstraint('user_id', 'follows_user_id', name='uq_user_followings_user_user'),
         _table_args_default_dict,
+    )
     user_following_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     follows_repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     follows_user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     follows_from = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by=lambda: func.lower(Repository.repo_name))
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repository_id == repo_id)
 class CacheInvalidation(Base, BaseDbModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (
         Index('key_idx', 'cache_key'),
         _table_args_default_dict,
+    )
     # cache_id, not used
     cache_id = Column(Integer(), primary_key=True)
     # cache_key as created by _get_cache_key
     cache_key = Column(Unicode(255), nullable=False, unique=True)
     # cache_args is a repo_name
     cache_args = Column(Unicode(255), nullable=False)
     # instance sets cache_active True when it is caching, other instances set
     # cache_active to False to indicate that this cache is invalid
     cache_active = Column(Boolean(), nullable=False, default=False)
     def __init__(self, cache_key, repo_name=''):
         self.cache_key = cache_key
         self.cache_args = repo_name
         self.cache_active = False
     def __unicode__(self):
         return u"<%s('%s:%s[%s]')>" % (
             self.__class__.__name__,
             self.cache_id, self.cache_key, self.cache_active)
     def _cache_key_partition(self):
         prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
         return prefix, repo_name, suffix
     def get_prefix(self):
         """
         get prefix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[0]
     def get_suffix(self):
         """
         get suffix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[2]
     @classmethod
     def clear_cache(cls):
         """
         Delete all cache keys from database.
         Should only be run when all instances are down and all entries thus stale.
         """
         cls.query().delete()
         Session().commit()
     @classmethod
     def _get_cache_key(cls, key):
         """
         Wrapper for generating a unique cache key for this instance and "key".
         key must / will start with a repo_name which will be stored in .cache_args .
         """
         import kallithea
         prefix = kallithea.CONFIG.get('instance_id', '')
         return "%s%s" % (prefix, key)
     @classmethod
     def set_invalidate(cls, repo_name):
         """
         Mark all caches of a repo as invalid in the database.
         """
         inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
         log.debug('for repo %s got %s invalidation objects',
                   safe_str(repo_name), inv_objs)
         for inv_obj in inv_objs:
             log.debug('marking %s key for invalidation based on repo_name=%s',
                       inv_obj, safe_str(repo_name))
             Session().delete(inv_obj)
         Session().commit()
     @classmethod
     def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
         """
         Mark this cache key as active and currently cached.
         Return True if the existing cache registration still was valid.
         Return False to indicate that it had been invalidated and caches should be refreshed.
         """
         key = (repo_name + '_' + kind) if kind else repo_name
         cache_key = cls._get_cache_key(key)
         if valid_cache_keys and cache_key in valid_cache_keys:
             return True
         inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
         if inv_obj is None:
             inv_obj = cls(cache_key, repo_name)
             Session().add(inv_obj)
         elif inv_obj.cache_active:
             return True
         inv_obj.cache_active = True
         try:
             Session().commit()
         except sqlalchemy.exc.IntegrityError:
             log.error('commit of CacheInvalidation failed - retrying')
             Session().rollback()
             inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
             if inv_obj is None:
                 log.error('failed to create CacheInvalidation entry')
                 # TODO: fail badly?
             # else: TOCTOU - another thread added the key at the same time; no further action required
         return False
     @classmethod
     def get_valid_cache_keys(cls):
         """
         Return opaque object with information of which caches still are valid
         and can be used without checking for invalidation.
         """
         return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())
 class ChangesetComment(Base, BaseDbModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         Index('cc_pull_request_id_idx', 'pull_request_id'),
         _table_args_default_dict,
+    )
     comment_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     line_no = Column(Unicode(10), nullable=True)
     f_path = Column(Unicode(1000), nullable=True)
     author_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column(UnicodeText(), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     author = relationship('User')
     repo = relationship('Repository')
     # status_change is frequently used directly in templates - make it a lazy
     # join to avoid fetching each related ChangesetStatus on demand.
     # There will only be one ChangesetStatus referencing each comment so the join will not explode.
     status_change = relationship('ChangesetStatus',
                                  cascade="all, delete-orphan", lazy='joined')
     pull_request = relationship('PullRequest')
     def url(self):
         anchor = "comment-%s" % self.comment_id
         import kallithea.lib.helpers as h
         if self.revision:
             return h.url('changeset_home', repo_name=self.repo.repo_name, revision=self.revision, anchor=anchor)
         elif self.pull_request_id is not None:
             return self.pull_request.url(anchor=anchor)
     def __json__(self):
         return dict(
             comment_id=self.comment_id,
             username=self.author.username,
             text=self.text,
+        )
     def deletable(self):
         return self.created_on > datetime.datetime.now() - datetime.timedelta(minutes=5)
 class ChangesetStatus(Base, BaseDbModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         Index('cs_pull_request_id_idx', 'pull_request_id'),
         Index('cs_changeset_comment_id_idx', 'changeset_comment_id'),
         Index('cs_pull_request_id_user_id_version_idx', 'pull_request_id', 'user_id', 'version'),
         Index('cs_repo_id_pull_request_id_idx', 'repo_id', 'pull_request_id'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         _table_args_default_dict,
+    )
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected' # is shown as "Not approved" - TODO: change database content / scheme
     STATUS_UNDER_REVIEW = 'under_review'
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not reviewed")),  # (no icon) and default
         (STATUS_UNDER_REVIEW, _("Under review")),
         (STATUS_REJECTED, _("Not approved")),
         (STATUS_APPROVED, _("Approved")),
+    ]
     STATUSES_DICT = dict(STATUSES)
     changeset_status_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     status = Column(String(128), nullable=False, default=DEFAULT)
     comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)
     modified_at = Column(DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column(Integer(), nullable=False, default=0)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     author = relationship('User')
     repo = relationship('Repository')
     comment = relationship('ChangesetComment')
     pull_request = relationship('PullRequest')
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (
             self.__class__.__name__,
             self.status, self.author
+        )
     @classmethod
     def get_status_lbl(cls, value):
         return cls.STATUSES_DICT.get(value)
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
     def __json__(self):
         return dict(
             status=self.status,
             modified_at=self.modified_at.replace(microsecond=0),
             reviewer=self.author.username,
+            )
 class PullRequest(Base, BaseDbModel):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         Index('pr_org_repo_id_idx', 'org_repo_id'),
         Index('pr_other_repo_id_idx', 'other_repo_id'),
         _table_args_default_dict,
+    )
     # values for .status
     STATUS_NEW = u'new'
     STATUS_CLOSED = u'closed'
     pull_request_id = Column(Integer(), primary_key=True)
     title = Column(Unicode(255), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     status = Column(Unicode(255), nullable=False, default=STATUS_NEW) # only for closedness, not approve/reject/etc
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     _revisions = Column('revisions', UnicodeText(), nullable=False)
     org_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     org_ref = Column(Unicode(255), nullable=False)
     other_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     other_ref = Column(Unicode(255), nullable=False)
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':')
     @revisions.setter
     def revisions(self, val):
         self._revisions = safe_unicode(':'.join(val))
     @property
     def org_ref_parts(self):
         return self.org_ref.split(':')
     @property
     def other_ref_parts(self):
         return self.other_ref.split(':')
     owner = relationship('User')
     reviewers = relationship('PullRequestReviewer',
                              cascade="all, delete-orphan")
     org_repo = relationship('Repository', primaryjoin='PullRequest.org_repo_id==Repository.repo_id')
     other_repo = relationship('Repository', primaryjoin='PullRequest.other_repo_id==Repository.repo_id')
     statuses = relationship('ChangesetStatus', order_by='ChangesetStatus.changeset_status_id')
     comments = relationship('ChangesetComment', order_by='ChangesetComment.comment_id',
                              cascade="all, delete-orphan")
     @classmethod
     def query(cls, reviewer_id=None, include_closed=True, sorted=False):
         """Add PullRequest-specific helpers for common query constructs.
         reviewer_id: only PRs with the specified user added as reviewer.
         include_closed: if False, do not include closed PRs.
         sorted: if True, apply the default ordering (newest first).
         """
         q = super(PullRequest, cls).query()
         if reviewer_id is not None:
             q = q.join(PullRequestReviewer).filter(PullRequestReviewer.user_id == reviewer_id)
         if not include_closed:
             q = q.filter(PullRequest.status != PullRequest.STATUS_CLOSED)
         if sorted:
             q = q.order_by(PullRequest.created_on.desc())
         return q
     def get_reviewer_users(self):
         """Like .reviewers, but actually returning the users"""
         return User.query() \
             .join(PullRequestReviewer) \
             .filter(PullRequestReviewer.pull_request == self) \
             .order_by(PullRequestReviewer.pull_request_reviewers_id) \
             .all()
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
     def user_review_status(self, user_id):
         """Return the user's latest status votes on PR"""
         # note: no filtering on repo - that would be redundant
         status = ChangesetStatus.query() \
             .filter(ChangesetStatus.pull_request == self) \
             .filter(ChangesetStatus.user_id == user_id) \
             .order_by(ChangesetStatus.version) \
             .first()
         return str(status.status) if status else ''
     @classmethod
     def make_nice_id(cls, pull_request_id):
         '''Return pull request id nicely formatted for displaying'''
         return '#%s' % pull_request_id
     def nice_id(self):
         '''Return the id of this pull request, nicely formatted for displaying'''
         return self.make_nice_id(self.pull_request_id)
     def get_api_data(self):
         return self.__json__()
     def __json__(self):
         return dict(
             pull_request_id=self.pull_request_id,
             url=self.url(),
             reviewers=self.reviewers,
             revisions=self.revisions,
             owner=self.owner.username,
             title=self.title,
             description=self.description,
             org_repo_url=self.org_repo.clone_url(),
             org_ref_parts=self.org_ref_parts,
             other_ref_parts=self.other_ref_parts,
             status=self.status,
             comments=self.comments,
             statuses=self.statuses,
+        )
     def url(self, **kwargs):
         canonical = kwargs.pop('canonical', None)
         import kallithea.lib.helpers as h
         b = self.org_ref_parts[1]
         if b != self.other_ref_parts[1]:
             s = '/_/' + b
         else:
             s = '/_/' + self.title
         kwargs['extra'] = urlreadable(s)
         if canonical:
             return h.canonical_url('pullrequest_show', repo_name=self.other_repo.repo_name,
                                    pull_request_id=self.pull_request_id, **kwargs)
         return h.url('pullrequest_show', repo_name=self.other_repo.repo_name,
                      pull_request_id=self.pull_request_id, **kwargs)
 class PullRequestReviewer(Base, BaseDbModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         Index('pull_request_reviewers_user_id_idx', 'user_id'),
         _table_args_default_dict,
+    )
     def __init__(self, user=None, pull_request=None):
         self.user = user
         self.pull_request = pull_request
     pull_request_reviewers_id = Column('pull_requests_reviewers_id', Integer(), primary_key=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     user = relationship('User')
     pull_request = relationship('PullRequest')
     def __json__(self):
         return dict(
             username=self.user.username if self.user else None,
+        )
 class Notification(object):
     __tablename__ = 'notifications'
 class UserNotification(object):
     __tablename__ = 'user_to_notification'
 class Gist(Base, BaseDbModel):
     __tablename__ = 'gists'
     __table_args__ = (
         Index('g_gist_access_id_idx', 'gist_access_id'),
         Index('g_created_on_idx', 'created_on'),
         _table_args_default_dict,
+    )
     GIST_PUBLIC = u'public'
     GIST_PRIVATE = u'private'
     DEFAULT_FILENAME = u'gistfile1.txt'
     gist_id = Column(Integer(), primary_key=True)
     gist_access_id = Column(Unicode(250), nullable=False)
     gist_description = Column(UnicodeText(), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     gist_expires = Column(Float(53), nullable=False)
     gist_type = Column(Unicode(128), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner = relationship('User')
     @hybrid_property
     def is_expired(self):
         return (self.gist_expires != -1) & (time.time() > self.gist_expires)
     def __repr__(self):
         return '<Gist:[%s]%s>' % (self.gist_type, self.gist_access_id)
     @classmethod
     def guess_instance(cls, value):
         return super(Gist, cls).guess_instance(value, Gist.get_by_access_id)
     @classmethod
     def get_or_404(cls, id_):
         res = cls.query().filter(cls.gist_access_id == id_).scalar()
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def get_by_access_id(cls, gist_access_id):
         return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
     def gist_url(self):
         import kallithea
         alias_url = kallithea.CONFIG.get('gist_alias_url')
         if alias_url:
             return alias_url.replace('{gistid}', self.gist_access_id)
         import kallithea.lib.helpers as h
         return h.canonical_url('gist', gist_id=self.gist_access_id)
     @classmethod
     def base_path(cls):
         """
         Returns base path where all gists are stored
         :param cls:
         """
         from kallithea.model.gist import GIST_STORE_LOC
         q = Session().query(Ui) \
             .filter(Ui.ui_key == URL_SEP)
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return os.path.join(q.one().ui_value, GIST_STORE_LOC)
     def get_api_data(self):
         """
         Common function for generating gist related data for API
         """
         gist = self
         data = dict(
             gist_id=gist.gist_id,
             type=gist.gist_type,
             access_id=gist.gist_access_id,
             description=gist.gist_description,
             url=gist.gist_url(),
             expires=gist.gist_expires,
             created_on=gist.created_on,
+        )
         return data
     def __json__(self):
         data = dict(
+        )
         data.update(self.get_api_data())
         return data
     ## SCM functions
     @property
     def scm_instance(self):
         from kallithea.lib.vcs import get_repo
         base_path = self.base_path()
         return get_repo(os.path.join(*map(safe_str,
                                           [base_path, self.gist_access_id])))

kallithea/model/repo.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.repo
 ~~~~~~~~~~~~~~~~~~~~
 Repository model for kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 5, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import shutil
 import logging
 import traceback
 from datetime import datetime
 from sqlalchemy.orm import subqueryload
 import kallithea.lib.utils2
 from kallithea.lib.utils import make_ui, is_valid_repo_uri
 from kallithea.lib.vcs.backends import get_backend
 from kallithea.lib.utils2 import LazyProperty, safe_str, safe_unicode, \
     remove_prefix, obfuscate_url_pw, get_current_authuser
 from kallithea.lib.caching_query import FromCache
 from kallithea.lib.hooks import log_delete_repository
 from kallithea.model.db import Repository, UserRepoToPerm, UserGroupRepoToPerm, \
     UserRepoGroupToPerm, UserGroupRepoGroupToPerm, User, Permission, Session, \
     Statistics, UserGroup, Ui, RepoGroup, RepositoryField
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.exceptions import AttachedForksError
 from kallithea.model.scm import UserGroupList
 log = logging.getLogger(__name__)
 class RepoModel(object):
     URL_SEPARATOR = Repository.url_sep()
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
         default_perm = 'repository.none' if private else default
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
         repo_to_perm.repository = repository
         repo_to_perm.user_id = def_user.user_id
         Session().add(repo_to_perm)
         return repo_to_perm
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = Ui.query().filter(Ui.ui_key == '/').one()
         return q.ui_value
     def get(self, repo_id, cache=False):
         repo = Repository.query() \
             .filter(Repository.repo_id == repo_id)
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_id))
         return repo.scalar()
     def get_repo(self, repository):
         return Repository.guess_instance(repository)
     def get_by_repo_name(self, repo_name, cache=False):
         repo = Repository.query() \
             .filter(Repository.repo_name == repo_name)
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_name))
         return repo.scalar()
     def get_all_user_repos(self, user):
         """
         Gets all repositories that user have at least read access
         :param user:
         """
         from kallithea.lib.auth import AuthUser
         user = User.guess_instance(user)
         repos = AuthUser(dbuser=user).permissions['repositories']
         access_check = lambda r: r[1] in ['repository.read',
                                           'repository.write',
                                           'repository.admin']
         repos = [x[0] for x in filter(access_check, repos.items())]
         return Repository.query().filter(Repository.repo_name.in_(repos))
     @classmethod
     def _render_datatable(cls, tmpl, *args, **kwargs):
         import kallithea
         from tg import tmpl_context as c, request, app_globals
         from tg.i18n import ugettext as _
         _tmpl_lookup = app_globals.mako_lookup
         template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
         tmpl = template.get_def(tmpl)
         kwargs.update(dict(_=_, h=h, c=c, request=request))
         return tmpl.render(*args, **kwargs)
     def get_repos_as_dict(self, repos_list, repo_groups_list=None,
                           admin=False,
                           short_name=False):
         """Return repository list for use by DataTable.
         repos_list: list of repositories - but will be filtered for read permission.
         repo_groups_list: added at top of list without permission check.
         admin: return data for action column.
         """
         _render = self._render_datatable
         from tg import tmpl_context as c
         def repo_lnk(name, rtype, rstate, private, fork_of):
             return _render('repo_name', name, rtype, rstate, private, fork_of,
                            short_name=short_name)
         def last_change(last_change):
             return _render("last_change", last_change)
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'))
         def desc(desc):
             return h.urlify_text(desc, truncate=80, stylize=c.visual.stylify_metalabels)
         def state(repo_state):
             return _render("repo_state", repo_state)
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name)
         def owner_actions(owner_id, username):
             return _render('user_name', owner_id, username)
         repos_data = []
         for gr in repo_groups_list or []:
             repos_data.append(dict(
                 raw_name='\0' + gr.name, # sort before repositories
                 just_name=gr.name,
                 name=_render('group_name_html', group_name=gr.group_name, name=gr.name),
                 desc=gr.group_description))
         for repo in repos_list:
             if not HasRepoPermissionLevel('read')(repo.repo_name, 'get_repos_as_dict check'):
                 continue
             cs_cache = repo.changeset_cache
             row = {
                 "raw_name": repo.repo_name,
                 "just_name": repo.just_name,
                 "name": repo_lnk(repo.repo_name, repo.repo_type,
                                  repo.repo_state, repo.private, repo.fork),
                 "last_change_iso": repo.last_db_change.isoformat(),
                 "last_change": last_change(repo.last_db_change),
                 "last_changeset": last_rev(repo.repo_name, cs_cache),
                 "last_rev_raw": cs_cache.get('revision'),
                 "desc": desc(repo.description),
                 "owner": h.person(repo.owner),
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
+            }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                     "owner": owner_actions(repo.owner_id,
                                            h.person(repo.owner))
                 })
             repos_data.append(row)
         return {
             "sort": "name",
             "dir": "asc",
             "records": repos_data
+        }
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
         :param repo_name:
         """
         repo_info = Repository.get_by_repo_name(repo_name)
         if repo_info is None:
             return None
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
         defaults['repo_group'] = repo_info.group_id
         for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                          (1, 'repo_description'),
                          (1, 'repo_landing_rev'), (0, 'clone_uri'),
                          (1, 'repo_private'), (1, 'repo_enable_statistics')]:
             attr = k
             if strip:
                 attr = remove_prefix(k, 'repo_')
             val = defaults[attr]
             if k == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[k] = val
             if k == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
         # fill owner
         if repo_info.owner:
             defaults.update({'owner': repo_info.owner.username})
         else:
             replacement_user = User.query().filter(User.admin ==
                                                    True).first().username
             defaults.update({'owner': replacement_user})
         # fill repository users
         for p in repo_info.repo_to_perm:
             defaults.update({'u_perm_%s' % p.user.username:
                                  p.permission.permission_name})
         # fill repository groups
         for p in repo_info.users_group_to_perm:
             defaults.update({'g_perm_%s' % p.users_group.users_group_name:
                                  p.permission.permission_name})
         return defaults
     def update(self, repo, **kwargs):
         try:
             cur_repo = Repository.guess_instance(repo)
             org_repo_name = cur_repo.repo_name
             if 'owner' in kwargs:
                 cur_repo.owner = User.get_by_username(kwargs['owner'])
             if 'repo_group' in kwargs:
                 assert kwargs['repo_group'] != u'-1', kwargs # RepoForm should have converted to None
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
                 cur_repo.repo_name = cur_repo.get_new_name(cur_repo.just_name)
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
             for k in ['repo_enable_downloads',
                       'repo_description',
                       'repo_landing_rev',
                       'repo_private',
                       'repo_enable_statistics',
                       ]:
                 if k in kwargs:
                     setattr(cur_repo, remove_prefix(k, 'repo_'), kwargs[k])
             clone_uri = kwargs.get('clone_uri')
             if clone_uri is not None and clone_uri != cur_repo.clone_uri_hidden:
                 # clone_uri is modified - if given a value, check it is valid
                 if clone_uri != '':
                     # will raise exception on error
-                    is_valid_repo_uri(cur_repo.repo_type, clone_uri, make_ui('db', clear_session=False))
                     is_valid_repo_uri(cur_repo.repo_type, clone_uri, make_ui(clear_session=False))
                 cur_repo.clone_uri = clone_uri
             if 'repo_name' in kwargs:
                 repo_name = kwargs['repo_name']
                 if kallithea.lib.utils2.repo_name_slug(repo_name) != repo_name:
                     raise Exception('invalid repo name %s' % repo_name)
                 cur_repo.repo_name = cur_repo.get_new_name(repo_name)
             # if private flag is set, reset default permission to NONE
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user='default', perm=EMPTY_PERM
+                )
                 # handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX),
                                 kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
             if org_repo_name != cur_repo.repo_name:
                 # rename repository
                 self._rename_filesystem_repo(old=org_repo_name, new=cur_repo.repo_name)
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev='rev:tip', fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_downloads=False,
                      copy_group_permissions=False, state=Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state. This should only be
         executed by create() repo, with exception of importing existing repos.
         """
         from kallithea.model.scm import ScmModel
         owner = User.guess_instance(owner)
         fork_of = Repository.guess_instance(fork_of)
         repo_group = RepoGroup.guess_instance(repo_group)
         try:
             repo_name = safe_unicode(repo_name)
             description = safe_unicode(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(self.URL_SEPARATOR)[-1]
             if kallithea.lib.utils2.repo_name_slug(repo_name) != repo_name:
                 raise Exception('invalid repo name %s' % repo_name)
             new_repo = Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.owner = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             if clone_uri:
                 # will raise exception on error
-                is_valid_repo_uri(repo_type, clone_uri, make_ui('db', clear_session=False))
                 is_valid_repo_uri(repo_type, clone_uri, make_ui(clear_session=False))
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_downloads = enable_downloads
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
             Session().add(new_repo)
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = UserRepoToPerm.query() \
                     .filter(UserRepoToPerm.repository == repo).all()
                 group_perms = UserGroupRepoToPerm.query() \
                     .filter(UserGroupRepoToPerm.repository == repo).all()
                 for perm in user_perms:
                     UserRepoToPerm.create(perm.user, new_repo, perm.permission)
                 for perm in group_perms:
                     UserGroupRepoToPerm.create(perm.users_group, new_repo,
                                                perm.permission)
             elif repo_group and copy_group_permissions:
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == repo_group).all()
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == repo_group).all()
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserRepoToPerm.create(perm.user, new_repo, perm_obj)
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace('group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
             else:
                 self._create_default_perms(new_repo, private)
             # now automatically start following this repository as owner
             ScmModel().toggle_following_repo(new_repo.repo_id, owner.user_id)
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             Session().flush()
             return new_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo(form_data, cur_user)
     def _update_permissions(self, repo, perms_new=None, perms_updates=None,
                             check_perms=True):
         if not perms_new:
             perms_new = []
         if not perms_updates:
             perms_updates = []
         # update permissions
         for member, perm, member_type in perms_updates:
             if member_type == 'user':
                 # this updates existing one
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
+                )
             else:
                 # check if we have permissions to alter this usergroup's access
                 if not check_perms or HasUserGroupPermissionLevel('read')(member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
+                    )
             # set new permissions
         for member, perm, member_type in perms_new:
             if member_type == 'user':
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
+                )
             else:
                 # check if we have permissions to alter this usergroup's access
                 if not check_perms or HasUserGroupPermissionLevel('read')(member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
+                    )
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo_fork(form_data, cur_user)
     def delete(self, repo, forks=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
         repo = Repository.guess_instance(repo)
         if repo is not None:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
             old_repo_dict = repo.get_dict()
             try:
                 Session().delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 log_delete_repository(old_repo_dict,
                                       deleted_by=cur_user)
             except Exception:
                 log.error(traceback.format_exc())
                 raise
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = User.guess_instance(user)
         repo = Repository.guess_instance(repo)
         permission = Permission.guess_instance(perm)
         # check if we have that permission already
         obj = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
             Session().add(obj)
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         return obj
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
         user = User.guess_instance(user)
         repo = Repository.guess_instance(repo)
         obj = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj is not None:
             Session().delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = Repository.guess_instance(repo)
         group_name = UserGroup.guess_instance(group_name)
         permission = Permission.guess_instance(perm)
         # check if we have that permission already
         obj = UserGroupRepoToPerm.query() \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
             Session().add(obj)
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         return obj
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = Repository.guess_instance(repo)
         group_name = UserGroup.guess_instance(group_name)
         obj = UserGroupRepoToPerm.query() \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj is not None:
             Session().delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
         :param repo_name:
         """
         repo = Repository.guess_instance(repo_name)
         try:
             obj = Statistics.query() \
                 .filter(Statistics.repository == repo).scalar()
             if obj is not None:
                 Session().delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None):
         """
         Makes repository on filesystem. Operation is group aware, meaning that it will create
         a repository within a group, and alter the paths accordingly to the group location.
         Note: clone_uri is low level and not validated - it might be a file system path used for validated cloning
         """
         from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
         from kallithea.model.scm import ScmModel
         if '/' in repo_name:
             raise ValueError('repo_name must not contain groups got `%s`' % repo_name)
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
         log.info('creating repo %s in %s from url: `%s`',
             repo_name, safe_unicode(repo_path),
             obfuscate_url_pw(clone_uri))
         backend = get_backend(repo_type)
         if repo_type == 'hg':
-            baseui = make_ui('db', clear_session=False)
             baseui = make_ui(clear_session=False)
             # patch and reset hooks section of UI config to not run any
             # hooks on creating remote repo
             for k, v in baseui.configitems('hooks'):
                 baseui.setconfig('hooks', k, None)
             repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui)
         elif repo_type == 'git':
             repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)
             # add kallithea hook into this repo
             ScmModel().install_git_hooks(repo=repo)
         else:
             raise Exception('Not supported repo_type %s expected hg/git' % repo_type)
         log.debug('Created repo %s with %s backend',
                   safe_unicode(repo_name), safe_unicode(repo_type))
         return repo
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
         old_path = safe_str(os.path.join(self.repos_path, old))
         new_path = safe_str(os.path.join(self.repos_path, new))
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
+            )
         shutil.move(old_path, new_path)
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is actually done by
         renaming dir to a 'rm__*' prefix which Kallithea will skip.
         It can be undeleted later by reverting the rename.
         :param repo: repo object
         """
         rm_path = safe_str(os.path.join(self.repos_path, repo.repo_name))
         log.info("Removing %s", rm_path)
         _now = datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo.group:
             args = repo.group.full_path_splitted + [_d]
             _d = os.path.join(*args)
         if os.path.exists(rm_path):
             shutil.move(rm_path, safe_str(os.path.join(self.repos_path, _d)))
         else:
             log.error("Can't find repo to delete in %r", rm_path)

kallithea/model/scm.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.scm
 ~~~~~~~~~~~~~~~~~~~
 Scm model for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 9, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import sys
 import posixpath
 import re
 import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 from sqlalchemy import func
 from tg.i18n import ugettext as _
 import kallithea
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea import BACKENDS
 from kallithea.lib import helpers as h
 from kallithea.lib.utils2 import safe_str, safe_unicode, get_server_url, \
     _set_extras
 from kallithea.lib.auth import HasRepoPermissionLevel, HasRepoGroupPermissionLevel, \
     HasUserGroupPermissionLevel, HasPermissionAny, HasPermissionAny
 from kallithea.lib.utils import get_filesystem_repos, make_ui, \
     action_logger
 from kallithea.model.db import Repository, Session, Ui, CacheInvalidation, \
     UserFollowing, UserLog, User, RepoGroup, PullRequest
 from kallithea.lib.hooks import log_push_action
 from kallithea.lib.exceptions import NonRelativePathError, IMCCommitError
 log = logging.getLogger(__name__)
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
     def __len__(self):
         return len(self.obj_list)
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(
                     name, self.__class__.__name__, **self.extra_kwargs):
                 continue
             yield db_obj
 class RepoList(_PermCheckIterator):
     def __init__(self, db_repo_list, perm_level, extra_kwargs=None):
         super(RepoList, self).__init__(obj_list=db_repo_list,
                     obj_attr='repo_name', perm_set=[perm_level],
                     perm_checker=HasRepoPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class RepoGroupList(_PermCheckIterator):
     def __init__(self, db_repo_group_list, perm_level, extra_kwargs=None):
         super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
                     obj_attr='group_name', perm_set=[perm_level],
                     perm_checker=HasRepoGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class UserGroupList(_PermCheckIterator):
     def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):
         super(UserGroupList, self).__init__(obj_list=db_user_group_list,
                     obj_attr='users_group_name', perm_set=[perm_level],
                     perm_checker=HasUserGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class ScmModel(object):
     """
     Generic Scm Model
     """
     def __get_repo(self, instance):
         cls = Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int) or safe_str(instance).isdigit():
             return cls.get(instance)
         elif isinstance(instance, basestring):
             return cls.get_by_repo_name(instance)
         elif instance is not None:
             raise Exception('given object must be int, basestr or Instance'
                             ' of %s got %s' % (type(cls), type(instance)))
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = Ui.query().filter(Ui.ui_key == '/').one()
         return q.ui_value
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
         :param repos_path: path to directory containing repositories
         """
         if repos_path is None:
             repos_path = self.repos_path
         log.info('scanning for repositories in %s', repos_path)
-        baseui = make_ui('db')
         baseui = make_ui()
         repos = {}
         for name, path in get_filesystem_repos(repos_path):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = Repository.normalize_repo_name(name)
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos[name] = klass(safe_str(path[1]), baseui=baseui)
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos[name] = klass(path[1])
             except OSError:
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
     def get_repos(self, repos):
         """Return the repos the user has access to"""
         return RepoList(repos, perm_level='read')
     def get_repo_groups(self, groups=None):
         """Return the repo groups the user has access to
         If no groups are specified, use top level groups.
         """
         if groups is None:
             groups = RepoGroup.query() \
                 .filter(RepoGroup.parent_group_id == None).all()
         return RepoGroupList(groups, perm_level='read')
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
         :param repo_name: the repo for which caches should be marked invalid
         """
         CacheInvalidation.set_invalidate(repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         if repo is not None:
             repo.update_changeset_cache()
     def toggle_following_repo(self, follow_repo_id, user_id):
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_repository_id == follow_repo_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 Session().delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repository_id = follow_repo_id
             Session().add(f)
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
     def toggle_following_user(self, follow_user_id, user_id):
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_user_id == follow_user_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 Session().delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             Session().add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = Repository.query() \
             .filter(Repository.repo_name == repo_name).scalar()
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_repository == r) \
             .filter(UserFollowing.user_id == user_id).scalar()
         return f is not None
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_user == u) \
             .filter(UserFollowing.user_id == user_id).scalar()
         return f is not None
     def get_followers(self, repo):
         repo = Repository.guess_instance(repo)
         return UserFollowing.query() \
                 .filter(UserFollowing.follows_repository == repo).count()
     def get_forks(self, repo):
         repo = Repository.guess_instance(repo)
         return Repository.query() \
                 .filter(Repository.fork == repo).count()
     def get_pull_requests(self, repo):
         repo = Repository.guess_instance(repo)
         return PullRequest.query() \
                 .filter(PullRequest.other_repo == repo) \
                 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError("Cannot set repository as fork of repository with other type")
         repo.fork = fork
         return repo
     def _handle_rc_scm_extras(self, username, repo_name, repo_alias,
                               action=None):
         from kallithea import CONFIG
         from kallithea.lib.base import _get_ip_addr
         try:
             from tg import request
             environ = request.environ
         except TypeError:
             # we might use this outside of request context, let's fake the
             # environ data
             from webob import Request
             environ = Request.blank('').environ
         extras = {
             'ip': _get_ip_addr(environ),
             'username': username,
             'action': action or 'push_local',
             'repository': repo_name,
             'scm': repo_alias,
             'config': CONFIG['__file__'],
             'server_url': get_server_url(environ),
+        }
         _set_extras(extras)
     def _handle_push(self, repo, username, action, repo_name, revisions):
         """
         Triggers push action hooks
         :param repo: SCM repo
         :param username: username who pushes
         :param action: push/push_local/push_remote
         :param repo_name: name of repo
         :param revisions: list of revisions that we pushed
         """
         self._handle_rc_scm_extras(username, repo_name, repo_alias=repo.alias, action=action)
         _scm_repo = repo._repo
         # trigger push hook
         if repo.alias == 'hg':
             log_push_action(_scm_repo.ui, _scm_repo, node=revisions[0])
         elif repo.alias == 'git':
             log_push_action(None, _scm_repo, _git_revs=revisions)
     def _get_IMC_module(self, scm_type):
         """
         Returns InMemoryCommit class based on scm_type
         :param scm_type:
         """
         if scm_type == 'hg':
             from kallithea.lib.vcs.backends.hg import MercurialInMemoryChangeset
             return MercurialInMemoryChangeset
         if scm_type == 'git':
             from kallithea.lib.vcs.backends.git import GitInMemoryChangeset
             return GitInMemoryChangeset
         raise Exception('Invalid scm_type, must be one of hg,git got %s'
                         % (scm_type,))
     def pull_changes(self, repo, username, clone_uri=None):
         """
         Pull from "clone URL" or fork origin.
         """
         dbrepo = self.__get_repo(repo)
         if clone_uri is None:
             clone_uri = dbrepo.clone_uri or dbrepo.fork and dbrepo.fork.repo_full_path
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
         repo = dbrepo.scm_instance
         repo_name = dbrepo.repo_name
         try:
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
                 # git doesn't really have something like post-fetch action
                 # we fake that now. #TODO: extract fetched revisions somehow
                 # here
                 self._handle_push(repo,
                                   username=username,
                                   action='push_remote',
                                   repo_name=repo_name,
                                   revisions=[])
             else:
                 self._handle_rc_scm_extras(username, dbrepo.repo_name,
                                            repo.alias, action='push_remote')
                 repo.pull(clone_uri)
             self.mark_for_invalidation(repo_name)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def commit_change(self, repo, repo_name, cs, user, author, message,
                       content, f_path):
         """
         Commit a change to a single file
         :param repo: a db_repo.scm_instance
         """
         user = User.guess_instance(user)
         IMC = self._get_IMC_module(repo.alias)
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         imc = IMC(repo)
         imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
         try:
             tip = imc.commit(message=message, author=author,
                              parents=[cs], branch=cs.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo_name)
         self._handle_push(repo,
                           username=user.username,
                           action='push_local',
                           repo_name=repo_name,
                           revisions=[tip.raw_id])
         return tip
     def _sanitize_path(self, f_path):
         if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = posixpath.normpath(f_path)
         return f_path
     def get_nodes(self, repo_name, revision, root_path='/', flat=True):
         """
         Recursively walk root dir and return a set of all paths found.
         :param repo_name: name of repository
         :param revision: revision for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self.__get_repo(repo_name)
             changeset = _repo.scm_instance.get_changeset(revision)
             root_path = root_path.lstrip('/')
             for topnode, dirs, files in changeset.walk(root_path):
                 for f in files:
                     _files.append(f.path if flat else {"name": f.path,
                                                        "type": "file"})
                 for d in dirs:
                     _dirs.append(d.path if flat else {"name": d.path,
                                                       "type": "dir"})
         except RepositoryError:
             log.debug(traceback.format_exc())
             raise
         return _dirs, _files
     def create_nodes(self, user, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo.
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
         :returns: new committed changeset
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         processed_nodes = []
         for f_path in nodes:
             content = nodes[f_path]['content']
             f_path = self._sanitize_path(f_path)
             f_path = safe_str(f_path)
             # decoding here will force that we have proper encoded values
             # in any other case this will throw exceptions and deny commit
             if isinstance(content, (basestring,)):
                 content = safe_str(content)
             elif isinstance(content, (file, cStringIO.OutputType,)):
                 content = content.read()
             else:
                 raise Exception('Content is of unrecognized type %s' % (
                     type(content)
                 ))
             processed_nodes.append((f_path, content))
         message = safe_unicode(message)
         committer = user.full_contact
         author = safe_unicode(author) if author else committer
         IMC = self._get_IMC_module(scm_instance.alias)
         imc = IMC(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         return tip
     def update_nodes(self, user, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo. Again.
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         message = safe_unicode(message)
         committer = user.full_contact
         author = safe_unicode(author) if author else committer
         imc_class = self._get_IMC_module(scm_instance.alias)
         imc = imc_class(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
             filenode = FileNode(old_filename, content=content)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames, needs vcs lib changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content))
                 else:
                     imc.change(filenode)
         # commit changes
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
     def delete_nodes(self, user, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes specified nodes from repo.
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
         :returns: new committed changeset after deletion
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatibility it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
         message = safe_unicode(message)
         committer = user.full_contact
         author = safe_unicode(author) if author else committer
         IMC = self._get_IMC_module(scm_instance.alias)
         imc = IMC(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         return tip
     def get_unread_journal(self):
         return UserLog.query().count()
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
         :param repo:
         """
         hist_l = []
         choices = []
         repo = self.__get_repo(repo)
         hist_l.append(['rev:tip', _('latest tip')])
         choices.append('rev:tip')
         if repo is None:
             return choices, hist_l
         repo = repo.scm_instance
         branches_group = ([(u'branch:%s' % k, k) for k, v in
                            repo.branches.iteritems()], _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
         if repo.alias == 'hg':
             bookmarks_group = ([(u'book:%s' % k, k) for k, v in
                                 repo.bookmarks.iteritems()], _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
         tags_group = ([(u'tag:%s' % k, k) for k, v in
                        repo.tags.iteritems()], _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
         return choices, hist_l
     def _get_git_hook_interpreter(self):
         """Return a suitable interpreter for Git hooks.
         Return a suitable string to be written in the POSIX #! shebang line for
         Git hook scripts so they invoke Kallithea code with the right Python
         interpreter and in the right environment.
         """
         # Note: sys.executable might not point at a usable Python interpreter. For
         # example, when using uwsgi, it will point at the uwsgi program itself.
         # FIXME This may not work on Windows and may need a shell wrapper script.
         return (kallithea.CONFIG.get('git_hook_interpreter')
                 or sys.executable
                 or '/usr/bin/env python2')
     def install_git_hooks(self, repo, force_create=False):
         """
         Creates a kallithea hook inside a git repository
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
         loc = os.path.join(repo.path, 'hooks')
         if not repo.bare:
             loc = os.path.join(repo.path, '.git', 'hooks')
         if not os.path.isdir(loc):
             os.makedirs(loc)
         tmpl_post = "#!%s\n" % self._get_git_hook_interpreter()
         tmpl_post += pkg_resources.resource_string(
             'kallithea', os.path.join('config', 'post_receive_tmpl.py')
+        )
         tmpl_pre = "#!%s\n" % self._get_git_hook_interpreter()
         tmpl_pre += pkg_resources.resource_string(
             'kallithea', os.path.join('config', 'pre_receive_tmpl.py')
+        )
         for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
             _hook_file = os.path.join(loc, '%s-receive' % h_type)
             has_hook = False
             log.debug('Installing git hook in repo %s', repo)
             if os.path.exists(_hook_file):
                 # let's take a look at this hook, maybe it's kallithea ?
                 log.debug('hook exists, checking if it is from kallithea')
                 with open(_hook_file, 'rb') as f:
                     data = f.read()
                     matches = re.compile(r'(?:%s)\s*=\s*(.*)'
                                          % 'KALLITHEA_HOOK_VER').search(data)
                     if matches:
                         try:
                             ver = matches.groups()[0]
                             log.debug('got %s it is kallithea', ver)
                             has_hook = True
                         except Exception:
                             log.error(traceback.format_exc())
             else:
                 # there is no hook in this dir, so we want to create one
                 has_hook = True
             if has_hook or force_create:
                 log.debug('writing %s hook file !', h_type)
                 try:
                     with open(_hook_file, 'wb') as f:
                         tmpl = tmpl.replace('_TMPL_', kallithea.__version__)
                         f.write(tmpl)
                     os.chmod(_hook_file, 0755)
                 except IOError as e:
                     log.error('error writing %s: %s', _hook_file, e)
             else:
                 log.debug('skipping writing hook file')
 def AvailableRepoGroupChoices(top_perms, repo_group_perm_level, extras=()):
     """Return group_id,string tuples with choices for all the repo groups where
     the user has the necessary permissions.
     Top level is -1.
     """
     groups = RepoGroup.query().all()
     if HasPermissionAny('hg.admin')('available repo groups'):
         groups.append(None)
     else:
         groups = list(RepoGroupList(groups, perm_level=repo_group_perm_level))
         if top_perms and HasPermissionAny(*top_perms)('available repo groups'):
             groups.append(None)
         for extra in extras:
             if not any(rg == extra for rg in groups):
                 groups.append(extra)
     return RepoGroup.groups_choices(groups=groups)

kallithea/model/validators.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Set of generic validators
 """
 import os
 import re
 import formencode
 import logging
 from collections import defaultdict
 from tg.i18n import ugettext as _
 from sqlalchemy import func
 from webhelpers.pylonslib.secure_form import authentication_token
 import sqlalchemy
 from formencode.validators import (
     UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set,
     NotEmpty, IPAddress, CIDR, String, FancyValidator
+)
 from kallithea.lib.compat import OrderedSet
 from kallithea.lib import ipaddr
 from kallithea.lib.utils import is_valid_repo_uri
 from kallithea.lib.utils2 import str2bool, aslist, repo_name_slug
 from kallithea.model.db import RepoGroup, Repository, UserGroup, User
 from kallithea.lib.exceptions import LdapImportError
 from kallithea.config.routing import ADMIN_PREFIX
 from kallithea.lib.auth import HasRepoGroupPermissionLevel, HasPermissionAny
 # silence warnings and pylint
 UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set, \
     NotEmpty, IPAddress, CIDR, String, FancyValidator
 log = logging.getLogger(__name__)
 def UniqueListFromString():
     class _UniqueListFromString(formencode.FancyValidator):
         """
         Split value on ',' and make unique while preserving order
         """
         messages = dict(
             empty=_('Value cannot be an empty list'),
             missing_value=_('Value cannot be an empty list'),
+        )
         def _to_python(self, value, state):
             value = aslist(value, ',')
             seen = set()
             return [c for c in value if not (c in seen or seen.add(c))]
         def empty_value(self, value):
             return []
     return _UniqueListFromString
 def ValidUsername(edit=False, old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'username_exists': _('Username "%(username)s" already exists'),
             'system_invalid_username':
                 _('Username "%(username)s" cannot be used'),
             'invalid_username':
                 _('Username may only contain alphanumeric characters '
                   'underscores, periods or dashes and must begin with an '
                   'alphanumeric character or underscore')
+        }
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 msg = self.message('system_invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state)
             # check if user is unique
             old_un = None
             if edit:
                 old_un = User.get(old_data.get('user_id')).username
             if old_un != value or not edit:
                 if User.get_by_username(value, case_insensitive=True):
                     msg = self.message('username_exists', state, username=value)
                     raise formencode.Invalid(msg, value, state)
             if re.match(r'^[a-zA-Z0-9\_]{1}[a-zA-Z0-9\-\_\.]*$', value) is None:
                 msg = self.message('invalid_username', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 def ValidRegex(msg=None):
     class _validator(formencode.validators.Regex):
         messages = dict(invalid=msg or _('The input is not valid'))
     return _validator
 def ValidRepoUser():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_username': _('Username %(username)s is not valid')
+        }
         def validate_python(self, value, state):
             try:
                 User.query().filter(User.active == True) \
                     .filter(User.username == value).one()
             except sqlalchemy.exc.InvalidRequestError: # NoResultFound/MultipleResultsFound
                 msg = self.message('invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(username=msg)
+                )
     return _validator
 def ValidUserGroup(edit=False, old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_group': _('Invalid user group name'),
             'group_exist': _('User group "%(usergroup)s" already exists'),
             'invalid_usergroup_name':
                 _('user group name may only contain alphanumeric '
                   'characters underscores, periods or dashes and must begin '
                   'with alphanumeric character')
+        }
         def validate_python(self, value, state):
             if value in ['default']:
                 msg = self.message('invalid_group', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
+                )
             # check if group is unique
             old_ugname = None
             if edit:
                 old_id = old_data.get('users_group_id')
                 old_ugname = UserGroup.get(old_id).users_group_name
             if old_ugname != value or not edit:
                 is_existing_group = UserGroup.get_by_group_name(value,
                                                         case_insensitive=True)
                 if is_existing_group:
                     msg = self.message('group_exist', state, usergroup=value)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(users_group_name=msg)
+                    )
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 msg = self.message('invalid_usergroup_name', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
+                )
     return _validator
 def ValidRepoGroup(edit=False, old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'parent_group_id': _('Cannot assign this group as parent'),
             'group_exists': _('Group "%(group_name)s" already exists'),
             'repo_exists':
                 _('Repository with name "%(group_name)s" already exists')
+        }
         def validate_python(self, value, state):
             # TODO WRITE VALIDATIONS
             group_name = value.get('group_name')
             parent_group_id = value.get('parent_group_id')
             # slugify repo group just in case :)
             slug = repo_name_slug(group_name)
             # check for parent of self
             parent_of_self = lambda: (
                 old_data['group_id'] == parent_group_id
                 if parent_group_id else False
+            )
             if edit and parent_of_self():
                 msg = self.message('parent_group_id', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(parent_group_id=msg)
+                )
             old_gname = None
             if edit:
                 old_gname = RepoGroup.get(old_data.get('group_id')).group_name
             if old_gname != group_name or not edit:
                 # check group
                 gr = RepoGroup.query() \
                       .filter(func.lower(RepoGroup.group_name) == func.lower(slug)) \
                       .filter(RepoGroup.parent_group_id == parent_group_id) \
                       .scalar()
                 if gr is not None:
                     msg = self.message('group_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
+                    )
                 # check for same repo
                 repo = Repository.query() \
                       .filter(func.lower(Repository.repo_name) == func.lower(slug)) \
                       .scalar()
                 if repo is not None:
                     msg = self.message('repo_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
+                    )
     return _validator
 def ValidPassword():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password':
                 _('Invalid characters (non-ascii) in password')
+        }
         def validate_python(self, value, state):
             try:
                 (value or '').decode('ascii')
             except UnicodeError:
                 msg = self.message('invalid_password', state)
                 raise formencode.Invalid(msg, value, state,)
     return _validator
 def ValidOldPassword(username):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password': _('Invalid old password')
+        }
         def validate_python(self, value, state):
             from kallithea.lib import auth_modules
             if auth_modules.authenticate(username, value, '') is None:
                 msg = self.message('invalid_password', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(current_password=msg)
+                )
     return _validator
 def ValidPasswordsMatch(password_field, password_confirmation_field):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'password_mismatch': _('Passwords do not match'),
+        }
         def validate_python(self, value, state):
             if value.get(password_field) != value[password_confirmation_field]:
                 msg = self.message('password_mismatch', state)
                 raise formencode.Invalid(msg, value, state,
                      error_dict={password_field:msg, password_confirmation_field: msg}
+                )
     return _validator
 def ValidAuth():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_auth': _(u'Invalid username or password'),
+        }
         def validate_python(self, value, state):
             from kallithea.lib import auth_modules
             password = value['password']
             username = value['username']
             # authenticate returns unused dict but has called
             # plugin._authenticate which has create_or_update'ed the username user in db
             if auth_modules.authenticate(username, password) is None:
                 user = User.get_by_username_or_email(username)
                 if user and not user.active:
                     log.warning('user %s is disabled', username)
                     msg = self.message('invalid_auth', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=' ', password=msg)
+                    )
                 else:
                     log.warning('user %s failed to authenticate', username)
                     msg = self.message('invalid_auth', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=' ', password=msg)
+                    )
     return _validator
 def ValidAuthToken():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_token': _('Token mismatch')
+        }
         def validate_python(self, value, state):
             if value != authentication_token():
                 msg = self.message('invalid_token', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 def ValidRepoName(edit=False, old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_repo_name':
                 _('Repository name %(repo)s is not allowed'),
             'repository_exists':
                 _('Repository named %(repo)s already exists'),
             'repository_in_group_exists': _('Repository "%(repo)s" already '
                                             'exists in group "%(group)s"'),
             'same_group_exists': _('Repository group with name "%(repo)s" '
                                    'already exists')
+        }
         def _to_python(self, value, state):
             repo_name = repo_name_slug(value.get('repo_name', ''))
             repo_group = value.get('repo_group')
             if repo_group:
                 gr = RepoGroup.get(repo_group)
                 group_path = gr.full_path
                 group_name = gr.group_name
                 # value needs to be aware of group name in order to check
                 # db key This is an actual just the name to store in the
                 # database
                 repo_name_full = group_path + RepoGroup.url_sep() + repo_name
             else:
                 group_name = group_path = ''
                 repo_name_full = repo_name
             value['repo_name'] = repo_name
             value['repo_name_full'] = repo_name_full
             value['group_path'] = group_path
             value['group_name'] = group_name
             return value
         def validate_python(self, value, state):
             repo_name = value.get('repo_name')
             repo_name_full = value.get('repo_name_full')
             group_path = value.get('group_path')
             group_name = value.get('group_name')
             if repo_name in [ADMIN_PREFIX, '']:
                 msg = self.message('invalid_repo_name', state, repo=repo_name)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_name=msg)
+                )
             rename = old_data.get('repo_name') != repo_name_full
             create = not edit
             if rename or create:
                 repo = Repository.get_by_repo_name(repo_name_full, case_insensitive=True)
                 repo_group = RepoGroup.get_by_group_name(repo_name_full, case_insensitive=True)
                 if group_path != '':
                     if repo is not None:
                         msg = self.message('repository_in_group_exists', state,
                                 repo=repo.repo_name, group=group_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
+                        )
                 elif repo_group is not None:
                         msg = self.message('same_group_exists', state,
                                 repo=repo_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
+                        )
                 elif repo is not None:
                         msg = self.message('repository_exists', state,
                                 repo=repo.repo_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
+                        )
             return value
     return _validator
 def ValidForkName(*args, **kwargs):
     return ValidRepoName(*args, **kwargs)
 def SlugifyName():
     class _validator(formencode.validators.FancyValidator):
         def _to_python(self, value, state):
             return repo_name_slug(value)
         def validate_python(self, value, state):
             pass
     return _validator
 def ValidCloneUri():
     from kallithea.lib.utils import make_ui
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'clone_uri': _('Invalid repository URL'),
             'invalid_clone_uri': _('Invalid repository URL. It must be a '
                                    'valid http, https, ssh, svn+http or svn+https URL'),
+        }
         def validate_python(self, value, state):
             repo_type = value.get('repo_type')
             url = value.get('clone_uri')
             if url and url != value.get('clone_uri_hidden'):
                 try:
-                    is_valid_repo_uri(repo_type, url, make_ui('db', clear_session=False))
                     is_valid_repo_uri(repo_type, url, make_ui(clear_session=False))
                 except Exception:
                     log.exception('URL validation failed')
                     msg = self.message('clone_uri', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(clone_uri=msg)
+                    )
     return _validator
 def ValidForkType(old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_fork_type': _('Fork has to be the same type as parent')
+        }
         def validate_python(self, value, state):
             if old_data['repo_type'] != value:
                 msg = self.message('invalid_fork_type', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_type=msg)
+                )
     return _validator
 def CanWriteGroup(old_data=None):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _("You don't have permissions "
                                    "to create repository in this group"),
             'permission_denied_root': _("no permission to create repository "
                                         "in root location")
+        }
         def _to_python(self, value, state):
             # root location
             if value == -1:
                 return None
             return value
         def validate_python(self, value, state):
             gr = RepoGroup.get(value)
             gr_name = gr.group_name if gr is not None else None # None means ROOT location
             # create repositories with write permission on group is set to true
             create_on_write = HasPermissionAny('hg.create.write_on_repogroup.true')()
             group_admin = HasRepoGroupPermissionLevel('admin')(gr_name,
                                             'can write into group validator')
             group_write = HasRepoGroupPermissionLevel('write')(gr_name,
                                             'can write into group validator')
             forbidden = not (group_admin or (group_write and create_on_write))
             can_create_repos = HasPermissionAny('hg.admin', 'hg.create.repository')
             gid = (old_data['repo_group'].get('group_id')
                    if (old_data and 'repo_group' in old_data) else None)
             value_changed = gid != value
             new = not old_data
             # do check if we changed the value, there's a case that someone got
             # revoked write permissions to a repository, he still created, we
             # don't need to check permission if he didn't change the value of
             # groups in form box
             if value_changed or new:
                 # parent group need to be existing
                 if gr and forbidden:
                     msg = self.message('permission_denied', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
+                    )
                 ## check if we can write to root location !
                 elif gr is None and not can_create_repos():
                     msg = self.message('permission_denied_root', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
+                    )
     return _validator
 def CanCreateGroup(can_create_in_root=False):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _("You don't have permissions "
                                    "to create a group in this location")
+        }
         def to_python(self, value, state):
             # root location
             if value == -1:
                 return None
             return value
         def validate_python(self, value, state):
             gr = RepoGroup.get(value)
             gr_name = gr.group_name if gr is not None else None # None means ROOT location
             if can_create_in_root and gr is None:
                 # we can create in root, we're fine no validations required
                 return
             forbidden_in_root = gr is None and not can_create_in_root
             forbidden = not HasRepoGroupPermissionLevel('admin')(gr_name, 'can create group validator')
             if forbidden_in_root or forbidden:
                 msg = self.message('permission_denied', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(parent_group_id=msg)
+                )
     return _validator
 def ValidPerms(type_='repo'):
     if type_ == 'repo_group':
         EMPTY_PERM = 'group.none'
     elif type_ == 'repo':
         EMPTY_PERM = 'repository.none'
     elif type_ == 'user_group':
         EMPTY_PERM = 'usergroup.none'
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'perm_new_member_name':
                 _('This username or user group name is not valid')
+        }
         def to_python(self, value, state):
             perms_update = OrderedSet()
             perms_new = OrderedSet()
             # build a list of permission to update and new permission to create
             # CLEAN OUT ORG VALUE FROM NEW MEMBERS, and group them using
             new_perms_group = defaultdict(dict)
             for k, v in value.copy().iteritems():
                 if k.startswith('perm_new_member'):
                     del value[k]
                     _type, part = k.split('perm_new_member_')
                     args = part.split('_')
                     if len(args) == 1:
                         new_perms_group[args[0]]['perm'] = v
                     elif len(args) == 2:
                         _key, pos = args
                         new_perms_group[pos][_key] = v
             # fill new permissions in order of how they were added
             for k in sorted(map(int, new_perms_group.keys())):
                 perm_dict = new_perms_group[str(k)]
                 new_member = perm_dict.get('name')
                 new_perm = perm_dict.get('perm')
                 new_type = perm_dict.get('type')
                 if new_member and new_perm and new_type:
                     perms_new.add((new_member, new_perm, new_type))
             for k, v in value.iteritems():
                 if k.startswith('u_perm_') or k.startswith('g_perm_'):
                     member = k[7:]
                     t = {'u': 'user',
                          'g': 'users_group'
                     }[k[0]]
                     if member == User.DEFAULT_USER:
                         if str2bool(value.get('repo_private')):
                             # set none for default when updating to
                             # private repo protects against form manipulation
                             v = EMPTY_PERM
                     perms_update.add((member, v, t))
             value['perms_updates'] = list(perms_update)
             value['perms_new'] = list(perms_new)
             # update permissions
             for k, v, t in perms_new:
                 try:
                     if t is 'user':
                         self.user_db = User.query() \
                             .filter(User.active == True) \
                             .filter(User.username == k).one()
                     if t is 'users_group':
                         self.user_db = UserGroup.query() \
                             .filter(UserGroup.users_group_active == True) \
                             .filter(UserGroup.users_group_name == k).one()
                 except Exception:
                     log.exception('Updated permission failed')
                     msg = self.message('perm_new_member_type', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(perm_new_member_name=msg)
+                    )
             return value
     return _validator
 def ValidSettings():
     class _validator(formencode.validators.FancyValidator):
         def _to_python(self, value, state):
             # settings  form for users that are not admin
             # can't edit certain parameters, it's extra backup if they mangle
             # with forms
             forbidden_params = [
                 'user', 'repo_type',
                 'repo_enable_downloads', 'repo_enable_statistics'
+            ]
             for param in forbidden_params:
                 if param in value:
                     del value[param]
             return value
         def validate_python(self, value, state):
             pass
     return _validator
 def ValidPath():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_path': _('This is not a valid path')
+        }
         def validate_python(self, value, state):
             if not os.path.isdir(value):
                 msg = self.message('invalid_path', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(paths_root_path=msg)
+                )
     return _validator
 def UniqSystemEmail(old_data=None):
     old_data = old_data or {}
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'email_taken': _('This email address is already in use')
+        }
         def _to_python(self, value, state):
             return value.lower()
         def validate_python(self, value, state):
             if (old_data.get('email') or '').lower() != value:
                 user = User.get_by_email(value)
                 if user is not None:
                     msg = self.message('email_taken', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(email=msg)
+                    )
     return _validator
 def ValidSystemEmail():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'non_existing_email': _('Email address "%(email)s" not found')
+        }
         def _to_python(self, value, state):
             return value.lower()
         def validate_python(self, value, state):
             user = User.get_by_email(value)
             if user is None:
                 msg = self.message('non_existing_email', state, email=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(email=msg)
+                )
     return _validator
 def LdapLibValidator():
     class _validator(formencode.validators.FancyValidator):
         messages = {
+        }
         def validate_python(self, value, state):
             try:
                 import ldap
                 ldap  # pyflakes silence !
             except ImportError:
                 raise LdapImportError()
     return _validator
 def AttrLoginValidator():
     class _validator(formencode.validators.UnicodeString):
         messages = {
             'invalid_cn':
                   _('The LDAP Login attribute of the CN must be specified - '
                     'this is the name of the attribute that is equivalent '
                     'to "username"')
+        }
         messages['empty'] = messages['invalid_cn']
     return _validator
 def ValidIp():
     class _validator(CIDR):
         messages = dict(
             badFormat=_('Please enter a valid IPv4 or IPv6 address'),
             illegalBits=_('The network size (bits) must be within the range'
                 ' of 0-32 (not %(bits)r)')
+        )
         def to_python(self, value, state):
             v = super(_validator, self).to_python(value, state)
             v = v.strip()
             net = ipaddr.IPNetwork(address=v)
             if isinstance(net, ipaddr.IPv4Network):
                 # if IPv4 doesn't end with a mask, add /32
                 if '/' not in value:
                     v += '/32'
             if isinstance(net, ipaddr.IPv6Network):
                 # if IPv6 doesn't end with a mask, add /128
                 if '/' not in value:
                     v += '/128'
             return v
         def validate_python(self, value, state):
             try:
                 addr = value.strip()
                 # this raises an ValueError if address is not IPv4 or IPv6
                 ipaddr.IPNetwork(address=addr)
             except ValueError:
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator
 def FieldKey():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             badFormat=_('Key name can only consist of letters, '
                         'underscore, dash or numbers')
+        )
         def validate_python(self, value, state):
             if not re.match('[a-zA-Z0-9_-]+$', value):
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator
 def BasePath():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             badPath=_('Filename cannot be inside a directory')
+        )
         def _to_python(self, value, state):
             return value
         def validate_python(self, value, state):
             if value != os.path.basename(value):
                 raise formencode.Invalid(self.message('badPath', state),
                                          value, state)
     return _validator
 def ValidAuthPlugins():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             import_duplicate=_('Plugins %(loaded)s and %(next_to_load)s both export the same name')
+        )
         def _to_python(self, value, state):
             # filter empty values
             return filter(lambda s: s not in [None, ''], value)
         def validate_python(self, value, state):
             from kallithea.lib import auth_modules
             module_list = value
             unique_names = {}
             try:
                 for module in module_list:
                     plugin = auth_modules.loadplugin(module)
                     plugin_name = plugin.name
                     if plugin_name in unique_names:
                         msg = self.message('import_duplicate', state,
                                 loaded=unique_names[plugin_name],
                                 next_to_load=plugin_name)
                         raise formencode.Invalid(msg, value, state)
                     unique_names[plugin_name] = plugin
             except (ImportError, AttributeError, TypeError) as e:
                 raise formencode.Invalid(str(e), value, state)
     return _validator

0 comments (0 inline, 0 general)