Changeset - 7fdefd3c5bd5
[Not reviewed]
default
0 4 0
Mads Kiilerich - 6 years ago 2020-01-03 01:55:06
mads@kiilerich.com
Grafted from: 79e4b8640b6d
cache: drop setup_cache_regions - tg will already have done that and coerced the types correctly

The configuration and type fixing will be invoked from make_base_app, and we
will thus not have to do it:

File "kallithea/config/middleware.py", line 31, in make_app_without_logging
return make_base_app(global_conf, full_stack=full_stack, **app_conf)
File ".../python3.7/site-packages/tg/configuration/app_config.py", line 176, in make_base_app
wrap_app)
File ".../python3.7/site-packages/tg/configurator/application.py", line 112, in _make_app
app = TGApp(conf)
File ".../python3.7/site-packages/tg/wsgiapp.py", line 49, in __init__
app_wrapper = wrapper(self.wrapped_dispatch, self.config)
File ".../python3.7/site-packages/tg/appwrappers/caching.py", line 36, in __init__
self.options = parse_cache_config_options(config)
File ".../python3.7/site-packages/beaker/util.py", line 430, in parse_cache_config_options

This will fix a py3 problem where setup_cache_regions was run *after* beaker
had coerced types, thus introducing string types in the config where beaker
expected the integers it had put there.
4 files changed with 2 insertions and 39 deletions:
0 comments (0 inline, 0 general)
kallithea/bin/kallithea_cli_base.py
Show inline comments
 
@@ -30,53 +30,52 @@ import kallithea
 
# entry_points. The script will be using the right virtualenv (if any), and for
 
# Unix, it will contain #! pointing at the right python executable. The script
 
# also makes sure sys.argv[0] points back at the script path, and that is what
 
# can be used to invoke 'kallithea-cli' later.
 
kallithea_cli_path = sys.argv[0]
 

	
 

	
 
def read_config(ini_file_name, strip_section_prefix):
 
    """Read ini_file_name content, and for all sections like '[X:Y]' where X is
 
    strip_section_prefix, replace the section name with '[Y]'."""
 

	
 
    def repl(m):
 
        if m.group(1) == strip_section_prefix:
 
            return '[%s]' % m.group(2)
 
        return m.group(0)
 

	
 
    with open(ini_file_name) as f:
 
        return re.sub(r'^\[([^:]+):(.*)]', repl, f.read().decode(), flags=re.MULTILINE)
 

	
 

	
 
# This placeholder is the main entry point for the kallithea-cli command
 
@click.group(context_settings=dict(help_option_names=['-h', '--help']))
 
def cli():
 
    """Various commands to manage a Kallithea instance."""
 

	
 
def register_command(config_file=False, config_file_initialize_app=False, hidden=False):
 
    """Register a kallithea-cli subcommand.
 

	
 
    If one of the config_file flags are true, a config file must be specified
 
    with -c and it is read and logging is configured. The configuration is
 
    available in the kallithea.CONFIG dict.
 

	
 
    If config_file_initialize_app is true, Kallithea, TurboGears global state
 
    (including tg.config), and database access will also be fully initialized.
 
    """
 
    cli_command = cli.command(hidden=hidden)
 
    if config_file or config_file_initialize_app:
 
        def annotator(annotated):
 
            @click.option('--config_file', '-c', help="Path to .ini file with app configuration.",
 
                type=click.Path(dir_okay=False, exists=True, readable=True), required=True)
 
            @functools.wraps(annotated) # reuse meta data from the wrapped function so click can see other options
 
            def runtime_wrapper(config_file, *args, **kwargs):
 
                path_to_ini_file = os.path.realpath(config_file)
 
                kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file)
 
                config_string = read_config(path_to_ini_file, strip_section_prefix=annotated.__name__)
 
                logging.config.fileConfig(io.StringIO(config_string))
 
                if config_file_initialize_app:
 
                    kallithea.config.middleware.make_app_without_logging(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf)
 
                    kallithea.lib.utils.setup_cache_regions(kallithea.CONFIG)
 
                return annotated(*args, **kwargs)
 
            return cli_command(runtime_wrapper)
 
        return annotator
 
    return cli_command
kallithea/lib/hooks.py
Show inline comments
 
@@ -258,105 +258,103 @@ def log_delete_repository(repository_dic
 

	
 
def log_delete_user(user_dict, deleted_by, **kwargs):
 
    """
 
    Post delete user Hook.
 

	
 
    :param user_dict: dict dump of user object
 

	
 
    available keys for user_dict:
 

	
 
     'username',
 
     'full_name_or_username',
 
     'full_contact',
 
     'user_id',
 
     'name',
 
     'firstname',
 
     'short_contact',
 
     'admin',
 
     'lastname',
 
     'ip_addresses',
 
     'ldap_dn',
 
     'email',
 
     'api_key',
 
     'last_login',
 
     'full_name',
 
     'active',
 
     'password',
 
     'emails',
 

	
 
    """
 
    from kallithea import EXTENSIONS
 
    callback = getattr(EXTENSIONS, 'DELETE_USER_HOOK', None)
 
    if callable(callback):
 
        return callback(deleted_by=deleted_by, **user_dict)
 

	
 
    return 0
 

	
 

	
 
def _hook_environment(repo_path):
 
    """
 
    Create a light-weight environment for stand-alone scripts and return an UI and the
 
    db repository.
 

	
 
    Git hooks are executed as subprocess of Git while Kallithea is waiting, and
 
    they thus need enough info to be able to create an app environment and
 
    connect to the database.
 
    """
 
    import paste.deploy
 
    import kallithea.config.middleware
 
    import kallithea.lib.utils
 

	
 
    extras = get_hook_environment()
 

	
 
    path_to_ini_file = extras['config']
 
    kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file)
 
    #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
 
    kallithea.config.middleware.make_app_without_logging(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf)
 
    kallithea.lib.utils.setup_cache_regions(kallithea.CONFIG)
 

	
 
    repo_path = safe_unicode(repo_path)
 
    # fix if it's not a bare repo
 
    if repo_path.endswith(os.sep + '.git'):
 
        repo_path = repo_path[:-5]
 

	
 
    repo = Repository.get_by_full_path(repo_path)
 
    if not repo:
 
        raise OSError('Repository %s not found in database'
 
                      % (safe_str(repo_path)))
 

	
 
    baseui = make_ui()
 
    return baseui, repo
 

	
 

	
 
def handle_git_pre_receive(repo_path, git_stdin_lines):
 
    """Called from Git pre-receive hook"""
 
    # Currently unused. TODO: remove?
 
    return 0
 

	
 

	
 
def handle_git_post_receive(repo_path, git_stdin_lines):
 
    """Called from Git post-receive hook"""
 
    baseui, repo = _hook_environment(repo_path)
 

	
 
    # the post push hook should never use the cached instance
 
    scm_repo = repo.scm_instance_no_cache()
 

	
 
    rev_data = []
 
    for l in git_stdin_lines:
 
        old_rev, new_rev, ref = l.strip().split(' ')
 
        _ref_data = ref.split('/')
 
        if _ref_data[1] in ['tags', 'heads']:
 
            rev_data.append({'old_rev': old_rev,
 
                             'new_rev': new_rev,
 
                             'ref': ref,
 
                             'type': _ref_data[1],
 
                             'name': '/'.join(_ref_data[2:])})
 

	
 
    git_revs = []
 
    for push_ref in rev_data:
 
        _type = push_ref['type']
 
        if _type == 'heads':
 
            if push_ref['old_rev'] == EmptyChangeset().raw_id:
 
                # update the symbolic ref if we push new repo
 
                if scm_repo.is_empty():
 
                    scm_repo._repo.refs.set_symbolic_ref(
 
                        b'HEAD',
kallithea/lib/utils.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.lib.utils
 
~~~~~~~~~~~~~~~~~~~
 

	
 
Utilities library for Kallithea
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 18, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import datetime
 
import logging
 
import os
 
import re
 
import sys
 
import traceback
 
from distutils.version import StrictVersion
 

	
 
import beaker
 
import beaker.cache
 
import mercurial.config
 
import mercurial.ui
 
from beaker.cache import _cache_decorate
 
from tg.i18n import ugettext as _
 

	
 
import kallithea.config.conf
 
from kallithea.lib.exceptions import HgsubversionImportError
 
from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes, safe_str, safe_unicode
 
from kallithea.lib.vcs.backends.git.repository import GitRepository
 
from kallithea.lib.vcs.backends.hg.repository import MercurialRepository
 
from kallithea.lib.vcs.conf import settings
 
from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 
from kallithea.lib.vcs.utils.fakemod import create_module
 
from kallithea.lib.vcs.utils.helpers import get_scm
 
from kallithea.model import meta
 
from kallithea.model.db import RepoGroup, Repository, Setting, Ui, User, UserGroup, UserLog
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 
REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 

	
 

	
 
#==============================================================================
 
# PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 
#==============================================================================
 
def get_repo_slug(request):
 
    _repo = request.environ['pylons.routes_dict'].get('repo_name')
 
    if _repo:
 
        _repo = _repo.rstrip('/')
 
    return _repo
 

	
 

	
 
def get_repo_group_slug(request):
 
    _group = request.environ['pylons.routes_dict'].get('group_name')
 
    if _group:
 
        _group = _group.rstrip('/')
 
    return _group
 

	
 

	
 
def get_user_group_slug(request):
 
    _group = request.environ['pylons.routes_dict'].get('id')
 
    _group = UserGroup.get(_group)
 
    if _group:
 
        return _group.users_group_name
 
    return None
 

	
 

	
 
def _get_permanent_id(s):
 
    """Helper for decoding stable URLs with repo ID. For a string like '_123'
 
    return 123.
 
@@ -577,103 +576,73 @@ def check_git_version():
 
    if 'git' not in kallithea.BACKENDS:
 
        return None
 

	
 
    if not settings.GIT_EXECUTABLE_PATH:
 
        log.warning('No git executable configured - check "git_path" in the ini file.')
 
        return None
 

	
 
    try:
 
        stdout, stderr = GitRepository._run_git_command(['--version'])
 
    except RepositoryError as e:
 
        # message will already have been logged as error
 
        log.warning('No working git executable found - check "git_path" in the ini file.')
 
        return None
 

	
 
    if stderr:
 
        log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, stderr)
 

	
 
    if not stdout:
 
        log.warning('No working git executable found - check "git_path" in the ini file.')
 
        return None
 

	
 
    output = stdout.strip()
 
    m = re.search(r"\d+.\d+.\d+", output)
 
    if m:
 
        ver = StrictVersion(m.group(0))
 
        log.debug('Git executable: "%s", version %s (parsed from: "%s")',
 
                  settings.GIT_EXECUTABLE_PATH, ver, output)
 
        if ver < git_req_ver:
 
            log.error('Kallithea detected %s version %s, which is too old '
 
                      'for the system to function properly. '
 
                      'Please upgrade to version %s or later. '
 
                      'If you strictly need Mercurial repositories, you can '
 
                      'clear the "git_path" setting in the ini file.',
 
                      settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)
 
            log.error("Terminating ...")
 
            sys.exit(1)
 
    else:
 
        ver = StrictVersion('0.0.0')
 
        log.warning('Error finding version number in "%s --version" stdout:\n%s',
 
                    settings.GIT_EXECUTABLE_PATH, output)
 

	
 
    return ver
 

	
 

	
 
#===============================================================================
 
# CACHE RELATED METHODS
 
#===============================================================================
 

	
 
# set cache regions for beaker so celery can utilise it
 
def setup_cache_regions(settings):
 
    # Create dict with just beaker cache configs with prefix stripped
 
    cache_settings = {'regions': None}
 
    prefix = 'beaker.cache.'
 
    for key in settings:
 
        if key.startswith(prefix):
 
            name = key[len(prefix):]
 
            cache_settings[name] = settings[key]
 
    # Find all regions, apply defaults, and apply to beaker
 
    if cache_settings['regions']:
 
        for region in cache_settings['regions'].split(','):
 
            region = region.strip()
 
            prefix = region + '.'
 
            region_settings = {}
 
            for key in cache_settings:
 
                if key.startswith(prefix):
 
                    name = key[len(prefix):]
 
                    region_settings[name] = cache_settings[key]
 
            region_settings.setdefault('expire',
 
                                       cache_settings.get('expire', '60'))
 
            region_settings.setdefault('lock_dir',
 
                                       cache_settings.get('lock_dir'))
 
            region_settings.setdefault('data_dir',
 
                                       cache_settings.get('data_dir'))
 
            region_settings.setdefault('type',
 
                                       cache_settings.get('type', 'memory'))
 
            beaker.cache.cache_regions[region] = region_settings
 

	
 

	
 
def conditional_cache(region, prefix, condition, func):
 
    """
 

	
 
    Conditional caching function use like::
 
        def _c(arg):
 
            #heavy computation function
 
            return data
 

	
 
        # depending from condition the compute is wrapped in cache or not
 
        compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)
 
        return compute(arg)
 

	
 
    :param region: name of cache region
 
    :param prefix: cache region prefix
 
    :param condition: condition for cache to be triggered, and return data cached
 
    :param func: wrapped heavy function to compute
 

	
 
    """
 
    wrapped = func
 
    if condition:
 
        log.debug('conditional_cache: True, wrapping call of '
 
                  'func: %s into %s region cache' % (region, func))
 
        wrapped = _cache_decorate((prefix,), None, None, region)(func)
 
        wrapped = beaker.cache._cache_decorate((prefix,), None, None, region)(func)
 

	
 
    return wrapped
kallithea/tests/scripts/manual_test_concurrency.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.tests.scripts.manual_test_concurrency
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
Test suite for making push/pull operations
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Dec 30, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 

	
 
"""
 

	
 
from __future__ import print_function
 

	
 
import logging
 
import os
 
import shutil
 
import sys
 
import tempfile
 
from os.path import dirname
 
from subprocess import PIPE, Popen
 

	
 
from paste.deploy import appconfig
 
from sqlalchemy import engine_from_config
 

	
 
from kallithea.config.environment import load_environment
 
from kallithea.lib.auth import get_crypt_password
 
from kallithea.lib.utils import setup_cache_regions
 
from kallithea.model import meta
 
from kallithea.model.base import init_model
 
from kallithea.model.db import Repository, Ui, User
 
from kallithea.tests.base import HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 

	
 

	
 
rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
 
conf = appconfig('config:development.ini', relative_to=rel_path)
 
load_environment(conf.global_conf, conf.local_conf)
 

	
 
setup_cache_regions(conf)
 

	
 
USER = TEST_USER_ADMIN_LOGIN
 
PASS = TEST_USER_ADMIN_PASS
 
HOST = 'server.local'
 
METHOD = 'pull'
 
DEBUG = True
 
log = logging.getLogger(__name__)
 

	
 

	
 
class Command(object):
 

	
 
    def __init__(self, cwd):
 
        self.cwd = cwd
 

	
 
    def execute(self, cmd, *args):
 
        """Runs command on the system with given ``args``.
 
        """
 

	
 
        command = cmd + ' ' + ' '.join(args)
 
        log.debug('Executing %s', command)
 
        if DEBUG:
 
            print(command)
 
        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
 
        stdout, stderr = p.communicate()
 
        if DEBUG:
 
            print(stdout, stderr)
 
        return stdout, stderr
 

	
 

	
 
def get_session():
 
    engine = engine_from_config(conf, 'sqlalchemy.')
 
    init_model(engine)
 
    sa = meta.Session
 
    return sa
 

	
 

	
 
def create_test_user(force=True):
 
    print('creating test user')
 
    sa = get_session()
 

	
 
    user = sa.query(User).filter(User.username == USER).scalar()
 

	
 
    if force and user is not None:
 
        print('removing current user')
 
        for repo in sa.query(Repository).filter(Repository.user == user).all():
 
            sa.delete(repo)
 
        sa.delete(user)
 
        sa.commit()
 

	
0 comments (0 inline, 0 general)