Changeset - 82ed7ad0dc48
[Not reviewed]
Merge default
0 5 0
Mads Kiilerich - 10 years ago 2016-01-05 16:30:05
madski@unity3d.com
Merge stable
5 files changed with 33 insertions and 19 deletions:
0 comments (0 inline, 0 general)
kallithea/__init__.py
Show inline comments
 
@@ -48,48 +48,54 @@ EXTENSIONS = {}
 
# own name.
 
#
 
# NOTE: If you want compatibility with a database that was originally created
 
#  for use with the RhodeCode software product, change BRAND to "rhodecode",
 
#  either by editing here or by creating a new file:
 
#  echo "BRAND = 'rhodecode'" > kallithea/brand.py
 

	
 
BRAND = "kallithea"
 
try:
 
    from kallithea.brand import BRAND
 
except ImportError:
 
    pass
 

	
 
# Prefix for the ui and settings table names
 
DB_PREFIX = (BRAND + "_") if BRAND != "kallithea" else ""
 

	
 
# Users.extern_type and .extern_name value for local users
 
EXTERN_TYPE_INTERNAL = BRAND if BRAND != 'kallithea' else 'internal'
 

	
 
# db_migrate_version.repository_id value, same as kallithea/lib/dbmigrate/migrate.cfg
 
DB_MIGRATIONS = BRAND + "_db_migrations"
 

	
 
try:
 
    from kallithea.lib import get_current_revision
 
    _rev = get_current_revision(quiet=True)
 
    if _rev and len(VERSION) > 3:
 
        VERSION += (_rev[0],)
 
except ImportError:
 
    pass
 

	
 
__version__ = ('.'.join((str(each) for each in VERSION[:3])))
 
__dbversion__ = 31  # defines current db version for migrations
 
__platform__ = platform.system()
 
__license__ = 'GPLv3'
 
__py_version__ = sys.version_info
 
__author__ = "Various Authors"
 
__url__ = 'https://kallithea-scm.org/'
 

	
 
is_windows = __platform__ in ['Windows']
 
is_unix = not is_windows
 

	
 
if len(VERSION) > 3:
 
    __version__ += '.'+VERSION[3]
 

	
 
    if len(VERSION) > 4:
 
        __version__ += VERSION[4]
 
    else:
 
        __version__ += '0'
 

	
 
# Hack for making the celery dependency kombu==1.5.1 compatible with Python
 
# 2.7.11 which has https://hg.python.org/releases/2.7.11/rev/24bdc4940e81
 
import uuid
 
if not hasattr(uuid, '_uuid_generate_random'):
 
    uuid._uuid_generate_random = None
kallithea/lib/middleware/simplegit.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.lib.middleware.simplegit
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
SimpleGit middleware for handling git protocol request (push/clone etc.)
 
It's implemented with basic auth function
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 28, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 

	
 
"""
 

	
 

	
 
import os
 
import re
 
import logging
 
import traceback
 

	
 
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
 
    HTTPNotAcceptable
 
from kallithea.model.db import User, Ui
 

	
 
from kallithea.lib.utils2 import safe_str, fix_PATH, get_server_url, \
 
from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
 
    _set_extras
 
from kallithea.lib.base import BaseVCSController, WSGIResultCloseCallback
 
from kallithea.lib.utils import make_ui, is_valid_repo
 
from kallithea.lib.exceptions import HTTPLockedRC
 
from kallithea.lib.hooks import pre_pull
 
from kallithea.lib import auth_modules
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
 

	
 

	
 
def is_git(environ):
 
    path_info = environ['PATH_INFO']
 
    isgit_path = GIT_PROTO_PAT.match(path_info)
 
    log.debug('pathinfo: %s detected as Git %s',
 
        path_info, isgit_path is not None
 
    )
 
    return isgit_path
 

	
 

	
 
class SimpleGit(BaseVCSController):
 

	
 
    def _handle_request(self, environ, start_response):
 
        if not is_git(environ):
 
            return self.application(environ, start_response)
 
        if not self._check_ssl(environ):
 
            return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
 

	
 
        ip_addr = self._get_ip_addr(environ)
 
        username = None
 
        self._git_first_op = False
 
        # skip passing error to error controller
 
        environ['pylons.status_code_redirect'] = True
 

	
 
        #======================================================================
 
        # EXTRACT REPOSITORY NAME FROM ENV
 
        #======================================================================
 
        try:
 
            repo_name = self.__get_repository(environ)
 
            str_repo_name = self.__get_repository(environ)
 
            repo_name = safe_unicode(str_repo_name)
 
            log.debug('Extracted repo name is %s', repo_name)
 
        except Exception:
 
        except Exception as e:
 
            log.error('error extracting repo_name: %r', e)
 
            return HTTPInternalServerError()(environ, start_response)
 

	
 
        # quick check if that dir exists...
 
        if not is_valid_repo(repo_name, self.basepath, 'git'):
 
            return HTTPNotFound()(environ, start_response)
 

	
 
        #======================================================================
 
        # GET ACTION PULL or PUSH
 
        #======================================================================
 
        action = self.__get_action(environ)
 

	
 
        #======================================================================
 
        # CHECK ANONYMOUS PERMISSION
 
        #======================================================================
 
        if action in ['pull', 'push']:
 
            anonymous_user = self.__get_user('default')
 
            username = anonymous_user.username
 
            if anonymous_user.active:
 
                # ONLY check permissions if the user is activated
 
                anonymous_perm = self._check_permission(action, anonymous_user,
 
                                                        repo_name, ip_addr)
 
            else:
 
                anonymous_perm = False
 

	
 
            if not anonymous_user.active or not anonymous_perm:
 
                if not anonymous_user.active:
 
                    log.debug('Anonymous access is disabled, running '
 
                              'authentication')
 

	
 
                if not anonymous_perm:
 
                    log.debug('Not enough credentials to access this '
 
                              'repository as anonymous user')
 

	
 
                username = None
 
                #==============================================================
 
                # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
 
                # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
 
                #==============================================================
 

	
 
                # try to auth based on environ, container auth methods
 
                log.debug('Running PRE-AUTH for container based authentication')
 
                pre_auth = auth_modules.authenticate('', '', environ)
 
                if pre_auth is not None and pre_auth.get('username'):
 
                    username = pre_auth['username']
 
                log.debug('PRE-AUTH got %s as username', username)
 

	
 
                # If not authenticated by the container, running basic auth
 
                if not username:
 
                    self.authenticate.realm = \
 
                        safe_str(self.config['realm'])
 
                    result = self.authenticate(environ)
 
                    if isinstance(result, str):
 
                        AUTH_TYPE.update(environ, 'basic')
 
                        REMOTE_USER.update(environ, result)
 
                        username = result
 
                    else:
 
                        return result.wsgi_application(environ, start_response)
 

	
 
                #==============================================================
 
                # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
 
                #==============================================================
 
                try:
 
                    user = self.__get_user(username)
 
                    if user is None or not user.active:
 
                        return HTTPForbidden()(environ, start_response)
 
                    username = user.username
 
                except Exception:
 
                    log.error(traceback.format_exc())
 
                    return HTTPInternalServerError()(environ, start_response)
 

	
 
                #check permissions for this repository
 
                perm = self._check_permission(action, user, repo_name, ip_addr)
 
                if not perm:
 
                    return HTTPForbidden()(environ, start_response)
 

	
 
        # extras are injected into UI object and later available
 
        # in hooks executed by kallithea
 
        from kallithea import CONFIG
 
        server_url = get_server_url(environ)
 
        extras = {
 
            'ip': ip_addr,
 
            'username': username,
 
            'action': action,
 
            'repository': repo_name,
 
            'scm': 'git',
 
            'config': CONFIG['__file__'],
 
            'server_url': server_url,
 
            'make_lock': None,
 
            'locked_by': [None, None]
 
        }
 

	
 
        #===================================================================
 
        # GIT REQUEST HANDLING
 
        #===================================================================
 
        str_repo_name = safe_str(repo_name)
 
        repo_path = os.path.join(safe_str(self.basepath),str_repo_name)
 
        log.debug('Repository path is %s', repo_path)
 

	
 
        # CHECK LOCKING only if it's not ANONYMOUS USER
 
        if username != User.DEFAULT_USER:
 
            log.debug('Checking locking on repository')
 
            (make_lock,
 
             locked,
 
             locked_by) = self._check_locking_state(
 
                            environ=environ, action=action,
 
                            repo=repo_name, user_id=user.user_id
 
                       )
 
            # store the make_lock for later evaluation in hooks
 
            extras.update({'make_lock': make_lock,
 
                           'locked_by': locked_by})
 

	
 
        fix_PATH()
 
        log.debug('HOOKS extras is %s', extras)
 
        baseui = make_ui('db')
 
        self.__inject_extras(repo_path, baseui, extras)
 

	
 
        try:
 
            self._handle_githooks(repo_name, action, baseui, environ)
 
            log.info('%s action on Git repo "%s" by "%s" from %s',
 
                     action, str_repo_name, safe_str(username), ip_addr)
 
            app = self.__make_app(repo_name, repo_path, extras)
 
            result = app(environ, start_response)
 
            if action == 'push':
 
                result = WSGIResultCloseCallback(result,
 
                    lambda: self._invalidate_cache(repo_name))
 
            return result
 
        except HTTPLockedRC as e:
 
            _code = CONFIG.get('lock_ret_code')
 
            log.debug('Repository LOCKED ret code %s!', _code)
 
            return e(environ, start_response)
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            return HTTPInternalServerError()(environ, start_response)
 

	
 
    def __make_app(self, repo_name, repo_path, extras):
 
        """
 
        Make an wsgi application using dulserver
 

	
 
        :param repo_name: name of the repository
 
        :param repo_path: full path to the repository
 
        """
 

	
 
        from kallithea.lib.middleware.pygrack import make_wsgi_app
kallithea/lib/middleware/simplehg.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.lib.middleware.simplehg
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
SimpleHg middleware for handling mercurial protocol request
 
(push/clone etc.). It's implemented with basic auth function
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 28, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 

	
 
"""
 

	
 

	
 
import os
 
import logging
 
import traceback
 

	
 
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
 
    HTTPNotAcceptable
 
from kallithea.model.db import User
 

	
 
from kallithea.lib.utils2 import safe_str, fix_PATH, get_server_url, \
 
from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
 
    _set_extras
 
from kallithea.lib.base import BaseVCSController, WSGIResultCloseCallback
 
from kallithea.lib.utils import make_ui, is_valid_repo, ui_sections
 
from kallithea.lib.vcs.utils.hgcompat import RepoError, hgweb_mod
 
from kallithea.lib.exceptions import HTTPLockedRC
 
from kallithea.lib import auth_modules
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def is_mercurial(environ):
 
    """
 
    Returns True if request's target is mercurial server - header
 
    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
 
    """
 
    http_accept = environ.get('HTTP_ACCEPT')
 
    path_info = environ['PATH_INFO']
 
    if http_accept and http_accept.startswith('application/mercurial'):
 
        ishg_path = True
 
    else:
 
        ishg_path = False
 

	
 
    log.debug('pathinfo: %s detected as Mercurial %s',
 
        path_info, ishg_path
 
    )
 
    return ishg_path
 

	
 

	
 
class SimpleHg(BaseVCSController):
 

	
 
    def _handle_request(self, environ, start_response):
 
        if not is_mercurial(environ):
 
            return self.application(environ, start_response)
 
        if not self._check_ssl(environ):
 
            return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
 

	
 
        ip_addr = self._get_ip_addr(environ)
 
        username = None
 
        # skip passing error to error controller
 
        environ['pylons.status_code_redirect'] = True
 

	
 
        #======================================================================
 
        # EXTRACT REPOSITORY NAME FROM ENV
 
        #======================================================================
 
        try:
 
            repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
 
            str_repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
 
            assert isinstance(str_repo_name, str)
 
            repo_name = safe_unicode(str_repo_name)
 
            assert safe_str(repo_name) == str_repo_name, (str_repo_name, repo_name)
 
            log.debug('Extracted repo name is %s', repo_name)
 
        except Exception:
 
        except Exception as e:
 
            log.error('error extracting repo_name: %r', e)
 
            return HTTPInternalServerError()(environ, start_response)
 

	
 
        # quick check if that dir exists...
 
        if not is_valid_repo(repo_name, self.basepath, 'hg'):
 
            return HTTPNotFound()(environ, start_response)
 

	
 
        #======================================================================
 
        # GET ACTION PULL or PUSH
 
        #======================================================================
 
        action = self.__get_action(environ)
 

	
 
        #======================================================================
 
        # CHECK ANONYMOUS PERMISSION
 
        #======================================================================
 
        if action in ['pull', 'push']:
 
            anonymous_user = self.__get_user('default')
 
            username = anonymous_user.username
 
            if anonymous_user.active:
 
                # ONLY check permissions if the user is activated
 
                anonymous_perm = self._check_permission(action, anonymous_user,
 
                                                        repo_name, ip_addr)
 
            else:
 
                anonymous_perm = False
 

	
 
            if not anonymous_user.active or not anonymous_perm:
 
                if not anonymous_user.active:
 
                    log.debug('Anonymous access is disabled, running '
 
                              'authentication')
 

	
 
                if not anonymous_perm:
 
                    log.debug('Not enough credentials to access this '
 
                              'repository as anonymous user')
 

	
 
                username = None
 
                #==============================================================
 
                # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
 
                # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
 
                #==============================================================
 

	
 
                # try to auth based on environ, container auth methods
 
                log.debug('Running PRE-AUTH for container based authentication')
 
                pre_auth = auth_modules.authenticate('', '', environ)
 
                if pre_auth is not None and pre_auth.get('username'):
 
                    username = pre_auth['username']
 
                log.debug('PRE-AUTH got %s as username', username)
 

	
 
                # If not authenticated by the container, running basic auth
 
                if not username:
 
                    self.authenticate.realm = \
 
                        safe_str(self.config['realm'])
 
                    result = self.authenticate(environ)
 
                    if isinstance(result, str):
 
                        AUTH_TYPE.update(environ, 'basic')
 
                        REMOTE_USER.update(environ, result)
 
                        username = result
 
                    else:
 
                        return result.wsgi_application(environ, start_response)
 

	
 
                #==============================================================
 
                # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
 
                #==============================================================
 
                try:
 
                    user = self.__get_user(username)
 
                    if user is None or not user.active:
 
                        return HTTPForbidden()(environ, start_response)
 
                    username = user.username
 
                except Exception:
 
                    log.error(traceback.format_exc())
 
                    return HTTPInternalServerError()(environ, start_response)
 

	
 
                #check permissions for this repository
 
                perm = self._check_permission(action, user, repo_name, ip_addr)
 
                if not perm:
 
                    return HTTPForbidden()(environ, start_response)
 

	
 
        # extras are injected into mercurial UI object and later available
 
        # in hg hooks executed by kallithea
 
        from kallithea import CONFIG
 
        server_url = get_server_url(environ)
 
        extras = {
 
            'ip': ip_addr,
 
            'username': username,
 
            'action': action,
 
            'repository': repo_name,
 
            'scm': 'hg',
 
            'config': CONFIG['__file__'],
 
            'server_url': server_url,
 
            'make_lock': None,
 
            'locked_by': [None, None]
 
        }
 
        #======================================================================
 
        # MERCURIAL REQUEST HANDLING
 
        #======================================================================
 
        str_repo_name = safe_str(repo_name)
 
        repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
 
        log.debug('Repository path is %s', repo_path)
 

	
 
        # CHECK LOCKING only if it's not ANONYMOUS USER
 
        if username != User.DEFAULT_USER:
 
            log.debug('Checking locking on repository')
 
            (make_lock,
 
             locked,
 
             locked_by) = self._check_locking_state(
 
                            environ=environ, action=action,
 
                            repo=repo_name, user_id=user.user_id
 
                       )
 
            # store the make_lock for later evaluation in hooks
 
            extras.update({'make_lock': make_lock,
 
                           'locked_by': locked_by})
 

	
 
        fix_PATH()
 
        log.debug('HOOKS extras is %s', extras)
 
        baseui = make_ui('db')
 
        self.__inject_extras(repo_path, baseui, extras)
 

	
 
        try:
 
            log.info('%s action on Mercurial repo "%s" by "%s" from %s',
 
                     action, str_repo_name, safe_str(username), ip_addr)
 
            app = self.__make_app(repo_path, baseui, extras)
 
            result = app(environ, start_response)
 
            if action == 'push':
 
                result = WSGIResultCloseCallback(result,
 
                    lambda: self._invalidate_cache(repo_name))
 
            return result
 
        except RepoError as e:
 
            if str(e).find('not found') != -1:
 
                return HTTPNotFound()(environ, start_response)
 
        except HTTPLockedRC as e:
 
            _code = CONFIG.get('lock_ret_code')
 
            log.debug('Repository LOCKED ret code %s!', _code)
 
            return e(environ, start_response)
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            return HTTPInternalServerError()(environ, start_response)
 

	
 
    def __make_app(self, repo_name, baseui, extras):
 
        """
 
        Make an wsgi application using hgweb, and inject generated baseui
 
        instance, additionally inject some extras into ui object
 
        """
 
        return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
 

	
kallithea/model/db.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.model.db
 
~~~~~~~~~~~~~~~~~~
 

	
 
Database Models for Kallithea
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 08, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import os
 
import time
 
import logging
 
import datetime
 
import traceback
 
import hashlib
 
import collections
 
import functools
 

	
 
import sqlalchemy
 
from sqlalchemy import *
 
from sqlalchemy.ext.hybrid import hybrid_property
 
from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 
from beaker.cache import cache_region, region_invalidate
 
from webob.exc import HTTPNotFound
 

	
 
from pylons.i18n.translation import lazy_ugettext as _
 

	
 
from kallithea import DB_PREFIX
 
from kallithea.lib.exceptions import DefaultUserException
 
from kallithea.lib.vcs import get_backend
 
from kallithea.lib.vcs.utils.helpers import get_scm
 
from kallithea.lib.vcs.utils.lazy import LazyProperty
 
from kallithea.lib.vcs.backends.base import EmptyChangeset
 

	
 
from kallithea.lib.utils2 import str2bool, safe_str, get_changeset_safe, \
 
    safe_unicode, remove_prefix, time_to_datetime, aslist, Optional, safe_int, \
 
    get_clone_url, urlreadable
 
from kallithea.lib.compat import json
 
from kallithea.lib.caching_query import FromCache
 

	
 
from kallithea.model.meta import Base, Session
 

	
 
URL_SEP = '/'
 
log = logging.getLogger(__name__)
 

	
 
#==============================================================================
 
# BASE CLASSES
 
#==============================================================================
 

	
 
_hash_key = lambda k: hashlib.md5(safe_str(k)).hexdigest()
 

	
 

	
 
class BaseModel(object):
 
    """
 
    Base Model for all classes
 
    """
 

	
 
    @classmethod
 
    def _get_keys(cls):
 
        """return column names for this model """
 
        return class_mapper(cls).c.keys()
 

	
 
    def get_dict(self):
 
        """
 
        return dict with keys and values corresponding
 
        to this model data """
 

	
 
@@ -1412,97 +1413,97 @@ class Repository(Base, BaseModel):
 
                pr_repo = stat.pull_request.other_repo.repo_name
 
            grouped[stat.revision] = [str(stat.status), stat.status_lbl,
 
                                      pr_id, pr_repo, pr_nice_id]
 
        return grouped
 

	
 
    def _repo_size(self):
 
        from kallithea.lib import helpers as h
 
        log.debug('calculating repository size...')
 
        return h.format_byte_size(self.scm_instance.size)
 

	
 
    #==========================================================================
 
    # SCM CACHE INSTANCE
 
    #==========================================================================
 

	
 
    def set_invalidate(self):
 
        """
 
        Mark caches of this repo as invalid.
 
        """
 
        CacheInvalidation.set_invalidate(self.repo_name)
 

	
 
    def scm_instance_no_cache(self):
 
        return self.__get_instance()
 

	
 
    @property
 
    def scm_instance(self):
 
        import kallithea
 
        full_cache = str2bool(kallithea.CONFIG.get('vcs_full_cache'))
 
        if full_cache:
 
            return self.scm_instance_cached()
 
        return self.__get_instance()
 

	
 
    def scm_instance_cached(self, valid_cache_keys=None):
 
        @cache_region('long_term')
 
        def _c(repo_name):
 
            return self.__get_instance()
 
        rn = self.repo_name
 

	
 
        valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
 
        if not valid:
 
            log.debug('Cache for %s invalidated, getting new object', rn)
 
            region_invalidate(_c, None, rn)
 
        else:
 
            log.debug('Getting scm_instance of %s from cache', rn)
 
        return _c(rn)
 

	
 
    def __get_instance(self):
 
        repo_full_path = self.repo_full_path
 

	
 
        alias = get_scm(repo_full_path)[0]
 
        alias = get_scm(safe_str(repo_full_path))[0]
 
        log.debug('Creating instance of %s repository from %s',
 
                  alias, repo_full_path)
 
        backend = get_backend(alias)
 

	
 
        if alias == 'hg':
 
            repo = backend(safe_str(repo_full_path), create=False,
 
                           baseui=self._ui)
 
        else:
 
            repo = backend(repo_full_path, create=False)
 

	
 
        return repo
 

	
 
    def __json__(self):
 
        return dict(landing_rev = self.landing_rev)
 

	
 
class RepoGroup(Base, BaseModel):
 
    __tablename__ = 'groups'
 
    __table_args__ = (
 
        UniqueConstraint('group_name', 'group_parent_id'),
 
        CheckConstraint('group_id != group_parent_id'),
 
        {'extend_existing': True, 'mysql_engine': 'InnoDB',
 
         'mysql_charset': 'utf8', 'sqlite_autoincrement': True},
 
    )
 
    __mapper_args__ = {'order_by': 'group_name'}
 

	
 
    SEP = ' &raquo; '
 

	
 
    group_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    group_name = Column(String(255, convert_unicode=False), nullable=False, unique=True, default=None)
 
    group_parent_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
 
    group_description = Column(String(10000, convert_unicode=False), nullable=True, unique=None, default=None)
 
    enable_locking = Column(Boolean(), nullable=False, unique=None, default=False)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 

	
 
    repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
 
    users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
 
    parent_group = relationship('RepoGroup', remote_side=group_id)
 
    user = relationship('User')
 

	
 
    def __init__(self, group_name='', parent_group=None):
 
        self.group_name = group_name
 
        self.parent_group = parent_group
 

	
 
    def __unicode__(self):
 
        return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,
 
                                      self.group_name)
 

	
 
@@ -2087,109 +2088,112 @@ class CacheInvalidation(Base, BaseModel)
 
    def clear_cache(cls):
 
        """
 
        Delete all cache keys from database.
 
        Should only be run when all instances are down and all entries thus stale.
 
        """
 
        cls.query().delete()
 
        Session().commit()
 

	
 
    @classmethod
 
    def _get_cache_key(cls, key):
 
        """
 
        Wrapper for generating a unique cache key for this instance and "key".
 
        key must / will start with a repo_name which will be stored in .cache_args .
 
        """
 
        import kallithea
 
        prefix = kallithea.CONFIG.get('instance_id', '')
 
        return "%s%s" % (prefix, key)
 

	
 
    @classmethod
 
    def set_invalidate(cls, repo_name):
 
        """
 
        Mark all caches of a repo as invalid in the database.
 
        """
 
        inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
 
        log.debug('for repo %s got %s invalidation objects',
 
                  safe_str(repo_name), inv_objs)
 

	
 
        for inv_obj in inv_objs:
 
            log.debug('marking %s key for invalidation based on repo_name=%s',
 
                      inv_obj, safe_str(repo_name))
 
            Session().delete(inv_obj)
 
        Session().commit()
 

	
 
    @classmethod
 
    def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
 
        """
 
        Mark this cache key as active and currently cached.
 
        Return True if the existing cache registration still was valid.
 
        Return False to indicate that it had been invalidated and caches should be refreshed.
 
        """
 

	
 
        key = (repo_name + '_' + kind) if kind else repo_name
 
        cache_key = cls._get_cache_key(key)
 

	
 
        if valid_cache_keys and cache_key in valid_cache_keys:
 
            return True
 

	
 
        inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
 
        if not inv_obj:
 
        if inv_obj is None:
 
            inv_obj = CacheInvalidation(cache_key, repo_name)
 
        if inv_obj.cache_active:
 
        elif inv_obj.cache_active:
 
            return True
 
        inv_obj.cache_active = True
 
        Session().add(inv_obj)
 
        try:
 
            Session().commit()
 
        except exc.IntegrityError:
 
        except sqlalchemy.exc.IntegrityError:
 
            log.error('commit of CacheInvalidation failed - retrying')
 
            Session().rollback()
 
            inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
 
            if not inv_obj:
 
                raise
 
            # TOCTOU - another thread added the key at the same time; no further action required
 
            if inv_obj is None:
 
                log.error('failed to create CacheInvalidation entry')
 
                # TODO: fail badly?
 
            # else: TOCTOU - another thread added the key at the same time; no further action required
 
        return False
 

	
 
    @classmethod
 
    def get_valid_cache_keys(cls):
 
        """
 
        Return opaque object with information of which caches still are valid
 
        and can be used without checking for invalidation.
 
        """
 
        return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())
 

	
 

	
 
class ChangesetComment(Base, BaseModel):
 
    __tablename__ = 'changeset_comments'
 
    __table_args__ = (
 
        Index('cc_revision_idx', 'revision'),
 
        Index('cc_pull_request_id_idx', 'pull_request_id'),
 
        {'extend_existing': True, 'mysql_engine': 'InnoDB',
 
         'mysql_charset': 'utf8', 'sqlite_autoincrement': True},
 
    )
 
    comment_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
 
    revision = Column(String(40))
 
    pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'))
 
    line_no = Column(Unicode(10))
 
    f_path = Column(Unicode(1000))
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
 
    text = Column(UnicodeText(25000), nullable=False)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
    modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 

	
 
    author = relationship('User')
 
    repo = relationship('Repository')
 
    # status_change is frequently used directly in templates - make it a lazy
 
    # join to avoid fetching each related ChangesetStatus on demand.
 
    # There will only be one ChangesetStatus referencing each comment so the join will not explode.
 
    status_change = relationship('ChangesetStatus',
 
                                 cascade="all, delete-orphan", lazy='joined')
 
    pull_request = relationship('PullRequest')
 

	
 
    @classmethod
 
    def get_users(cls, revision=None, pull_request_id=None):
 
        """
 
        Returns user associated with this ChangesetComment. ie those
 
        who actually commented
 

	
 
        :param cls:
 
        :param revision:
 
        """
kallithea/model/repo.py
Show inline comments
 
@@ -697,75 +697,75 @@ class RepoModel(BaseModel):
 
            _paths = [repo_store_location]
 
        else:
 
            _paths = [self.repos_path, new_parent_path, repo_name]
 
            # we need to make it str for mercurial
 
        repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
 

	
 
        # check if this path is not a repository
 
        if is_valid_repo(repo_path, self.repos_path):
 
            raise Exception('This path %s is a valid repository' % repo_path)
 

	
 
        # check if this path is a group
 
        if is_valid_repo_group(repo_path, self.repos_path):
 
            raise Exception('This path %s is a valid group' % repo_path)
 

	
 
        log.info('creating repo %s in %s from url: `%s`',
 
            repo_name, safe_unicode(repo_path),
 
            obfuscate_url_pw(clone_uri))
 

	
 
        backend = get_backend(repo_type)
 

	
 
        if repo_type == 'hg':
 
            baseui = make_ui('db', clear_session=False)
 
            # patch and reset hooks section of UI config to not run any
 
            # hooks on creating remote repo
 
            for k, v in baseui.configitems('hooks'):
 
                baseui.setconfig('hooks', k, None)
 

	
 
            repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui)
 
        elif repo_type == 'git':
 
            repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)
 
            # add kallithea hook into this repo
 
            ScmModel().install_git_hooks(repo=repo)
 
        else:
 
            raise Exception('Not supported repo_type %s expected hg/git' % repo_type)
 

	
 
        log.debug('Created repo %s with %s backend',
 
                  safe_unicode(repo_name), safe_unicode(repo_type))
 
        return repo
 

	
 
    def _rename_filesystem_repo(self, old, new):
 
        """
 
        renames repository on filesystem
 

	
 
        :param old: old name
 
        :param new: new name
 
        """
 
        log.info('renaming repo from %s to %s', old, new)
 

	
 
        old_path = os.path.join(self.repos_path, old)
 
        new_path = os.path.join(self.repos_path, new)
 
        old_path = safe_str(os.path.join(self.repos_path, old))
 
        new_path = safe_str(os.path.join(self.repos_path, new))
 
        if os.path.isdir(new_path):
 
            raise Exception(
 
                'Was trying to rename to already existing dir %s' % new_path
 
            )
 
        shutil.move(old_path, new_path)
 

	
 
    def _delete_filesystem_repo(self, repo):
 
        """
 
        removes repo from filesystem, the removal is actually done by
 
        renaming dir to a 'rm__*' prefix which Kallithea will skip.
 
        It can be undeleted later by reverting the rename.
 

	
 
        :param repo: repo object
 
        """
 
        rm_path = os.path.join(self.repos_path, repo.repo_name)
 
        rm_path = safe_str(os.path.join(self.repos_path, repo.repo_name))
 
        log.info("Removing %s", rm_path)
 

	
 
        _now = datetime.now()
 
        _ms = str(_now.microsecond).rjust(6, '0')
 
        _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
 
                             repo.just_name)
 
        if repo.group:
 
            args = repo.group.full_path_splitted + [_d]
 
            _d = os.path.join(*args)
 
        shutil.move(rm_path, os.path.join(self.repos_path, _d))
 
        shutil.move(rm_path, safe_str(os.path.join(self.repos_path, _d)))
0 comments (0 inline, 0 general)