from kallithea.lib.utils import BasePasterCommand, Command, load_rcextensions

from kallithea.lib.utils import ask_ok

from kallithea.lib.utils import BasePasterCommand

from kallithea.lib.utils import BasePasterCommand, ask_ok, REMOVED_REPO_PAT

kallithea.lib.utils

~~~~~~~~~~~~~~~~~~~

Utilities library for Kallithea

import re

import datetime

import traceback

import beaker

import tarfile

import shutil

import decorator

import warnings

from os.path import abspath

from os.path import dirname

from webhelpers.text import collapse, remove_formatting, strip_tags

from beaker.cache import _cache_decorate

from kallithea.lib.vcs.utils.hgcompat import ui, config

from kallithea.lib.vcs.utils.helpers import get_scm

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.model import meta

from kallithea.model.db import Repository, User, Ui, \

    UserLog, RepoGroup, Setting, UserGroup

from kallithea.model.meta import Session

from kallithea.model.repo_group import RepoGroupModel

from kallithea.lib.utils2 import safe_str, safe_unicode, get_current_authuser

from kallithea.lib.vcs.utils.fakemod import create_module

log = logging.getLogger(__name__)

REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')

def recursive_replace(str_, replace=' '):

    """

    Recursive replace of given sign to just one instance

    :param str_: given string

    :param replace: char to find and replace multiple instances

    Examples::

    >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')

    'Mighty-Mighty-Bo-sstones'

    """

    if str_.find(replace * 2) == -1:

        return str_

    else:

        str_ = str_.replace(replace * 2, replace)

        return recursive_replace(str_, replace)

def repo_name_slug(value):

    """

    Return slug of name of repository

    This function is called on each creation/modification

    of repository to prevent bad names in repo

    """

    slug = remove_formatting(value)

    slug = strip_tags(slug)

    for c in """`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:

        slug = slug.replace(c, '-')

    slug = recursive_replace(slug, '-')

    slug = collapse(slug, '-')

    return slug

#==============================================================================

# PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS

#==============================================================================

def get_repo_slug(request):

    _repo = request.environ['pylons.routes_dict'].get('repo_name')

    if _repo:

        _repo = _repo.rstrip('/')

    return _repo

def get_repo_group_slug(request):

    _group = request.environ['pylons.routes_dict'].get('group_name')

    if _group:

        _group = _group.rstrip('/')

    return _group

def get_user_group_slug(request):

    _group = request.environ['pylons.routes_dict'].get('id')

    _group = UserGroup.get(_group)

    if _group:

        return _group.users_group_name

    return None

def _extract_id_from_repo_name(repo_name):

    if repo_name.startswith('/'):

        repo_name = repo_name.lstrip('/')

    by_id_match = re.match(r'^_(\d{1,})', repo_name)

    if by_id_match:

        return by_id_match.groups()[0]

def get_repo_by_id(repo_name):

    """

    Extracts repo_name by id from special urls. Example url is _11/repo_name

    :param repo_name:

    :return: repo_name if matched else None

    """

    _repo_id = _extract_id_from_repo_name(repo_name)

    if _repo_id:

        from kallithea.model.db import Repository

        repo = Repository.get(_repo_id)

        if repo:

            # TODO: return repo instead of reponame? or would that be a layering violation?

            return repo.repo_name

    return None

def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):

    """

    Action logger for various actions made by users

    :param user: user that made this action, can be a unique username string or

        object containing user_id attribute

    :param action: action to log, should be on of predefined unique actions for

        easy translations

    :param repo: string name of repository or object containing repo_id,

        that action was made on

    :param ipaddr: optional IP address from what the action was made

    :param sa: optional sqlalchemy session

    """

    if not sa:

        sa = meta.Session()

    # if we don't get explicit IP address try to get one from registered user

    # in tmpl context var

    if not ipaddr:

        ipaddr = getattr(get_current_authuser(), 'ip_addr', '')

    if getattr(user, 'user_id', None):

        user_obj = User.get(user.user_id)

    elif isinstance(user, basestring):

        user_obj = User.get_by_username(user)

    else:

        raise Exception('You have to provide a user object or a username')

    if getattr(repo, 'repo_id', None):

        repo_obj = Repository.get(repo.repo_id)

        repo_name = repo_obj.repo_name

    elif isinstance(repo, basestring):

        repo_name = repo.lstrip('/')

        repo_obj = Repository.get_by_repo_name(repo_name)

    else:

        repo_obj = None

        repo_name = u''

    user_log = UserLog()

    user_log.user_id = user_obj.user_id

    user_log.username = user_obj.username

    user_log.action = safe_unicode(action)

    user_log.repository = repo_obj

    user_log.repository_name = repo_name

    user_log.action_date = datetime.datetime.now()

    user_log.user_ip = ipaddr

    sa.add(user_log)

    log.info('Logging action:%s on %s by user:%s ip:%s',

             action, safe_unicode(repo), user_obj, ipaddr)

    if commit:

        sa.commit()

def get_filesystem_repos(path):

    """

    Scans given path for repos and return (name,(type,path)) tuple

    :param path: path to scan for repositories

    :param recursive: recursive search and return names with subdirs in front

    """

    # remove ending slash for better results

    path = safe_str(path.rstrip(os.sep))

    log.debug('now scanning in %s', path)

    def isdir(*n):

        return os.path.isdir(os.path.join(*n))

    for root, dirs, _files in os.walk(path):

        recurse_dirs = []

        for subdir in dirs:

            # skip removed repos

            if REMOVED_REPO_PAT.match(subdir):

                continue

            #skip .<something> dirs TODO: rly? then we should prevent creating them ...

            if subdir.startswith('.'):

                continue

            cur_path = os.path.join(root, subdir)

            if (isdir(cur_path, '.hg') or

                isdir(cur_path, '.git') or

                isdir(cur_path, '.svn') or

                isdir(cur_path, 'objects') and (isdir(cur_path, 'refs') or isfile(cur_path, 'packed-refs'))):

                if not os.access(cur_path, os.R_OK) or not os.access(cur_path, os.X_OK):

                    log.warning('ignoring repo path without access: %s', cur_path)

                    continue

                if not os.access(cur_path, os.W_OK):

                    log.warning('repo path without write access: %s', cur_path)

                try:

                    scm_info = get_scm(cur_path)

                    assert cur_path.startswith(path)

                    repo_path = cur_path[len(path) + 1:]

                    yield repo_path, scm_info

                    continue # no recursion

                except VCSError:

                    # We should perhaps ignore such broken repos, but especially

                    # the bare git detection is unreliable so we dive into it

                    pass

            recurse_dirs.append(subdir)

        dirs[:] = recurse_dirs

def is_valid_repo(repo_name, base_path, scm=None):

    """

    Returns True if given path is a valid repository False otherwise.

    If scm param is given also compare if given scm is the same as expected

    from scm parameter

    :param repo_name:

    :param base_path:

    :param scm:

    :return True: if given path is a valid repository

    """

    full_path = os.path.join(safe_str(base_path), safe_str(repo_name))

    try:

        scm_ = get_scm(full_path)

        if scm:

            return scm_[0] == scm

        return True

    except VCSError:

        return False

def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):

    """

    Returns True if given path is a repository group False otherwise

    :param repo_name:

    :param base_path:

    """

    full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))

    # check if it's not a repo

    if is_valid_repo(repo_group_name, base_path):

        return False

    try:

        # we need to check bare git repos at higher level

        # since we might match branches/hooks/info/objects or possible

        # other things inside bare git repo

        get_scm(os.path.dirname(full_path))

        return False

    except VCSError:

        pass

    # check if it's a valid path

    if skip_path_check or os.path.isdir(full_path):

        return True

    return False

#propagated from mercurial documentation

ui_sections = ['alias', 'auth',

                'decode/encode', 'defaults',

                'diff', 'email',

                'extensions', 'format',

                'merge-patterns', 'merge-tools',

                'hooks', 'http_proxy',

                'smtp', 'patch',

                'paths', 'profiling',

                'server', 'trusted',

                'ui', 'web', ]

def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True):

    """

    A function that will read python rc files or database

    and make an mercurial ui object from read options

    :param path: path to mercurial config file

    :param checkpaths: check the path

    :param read_from: read from 'file' or 'db'

    """

    baseui = ui.ui()

    # clean the baseui object

    baseui._ocfg = config.config()

    baseui._ucfg = config.config()

    baseui._tcfg = config.config()

    if read_from == 'file':

        if not os.path.isfile(path):

            log.debug('hgrc file is not present at %s, skipping...', path)

            return False

        log.debug('reading hgrc from %s', path)

        cfg = config.config()

        cfg.read(path)

        for section in ui_sections:

            for k, v in cfg.items(section):

                log.debug('settings ui from file: [%s] %s=%s', section, k, v)

                baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))

    elif read_from == 'db':

        sa = meta.Session()

        ret = sa.query(Ui).all()

        hg_ui = ret

        for ui_ in hg_ui:

            if ui_.ui_active:

                ui_val = safe_str(ui_.ui_value)

                log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section,

                          ui_.ui_key, ui_val)

                baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),

                                 ui_val)

            if ui_.ui_key == 'push_ssl':

                # force set push_ssl requirement to False, kallithea

                # handles that

                baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),

                                 False)

        if clear_session:

            meta.Session.remove()

        # prevent interactive questions for ssh password / passphrase

        ssh = baseui.config('ui', 'ssh', default='ssh')

        baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)

    return baseui

def set_app_settings(config):

    """

    Updates pylons config with new settings from database

    :param config:

    """

    hgsettings = Setting.get_app_settings()

    for k, v in hgsettings.items():

        config[k] = v

def set_vcs_config(config):

    """

    Patch VCS config with some Kallithea specific stuff

    :param config: kallithea.CONFIG

    """

    from kallithea.lib.vcs import conf

    from kallithea.lib.utils2 import aslist

    conf.settings.BACKENDS = {

        'hg': 'kallithea.lib.vcs.backends.hg.MercurialRepository',

        'git': 'kallithea.lib.vcs.backends.git.GitRepository',

    }

    conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')

    conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()

    conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',

                                                        'utf8'), sep=',')

def set_indexer_config(config):

    """

    Update Whoosh index mapping

    :param config: kallithea.CONFIG

    """

    from kallithea.config import conf

    log.debug('adding extra into INDEX_EXTENSIONS')

    conf.INDEX_EXTENSIONS.extend(re.split('\s+', config.get('index.extensions', '')))

    log.debug('adding extra into INDEX_FILENAMES')

    conf.INDEX_FILENAMES.extend(re.split('\s+', config.get('index.filenames', '')))

def map_groups(path):

    """

    Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = RepoGroupModel(sa)

    owner = User.get_first_admin()

    for lvl, group_name in enumerate(groups):

        group_name = u'/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s',

                      lvl, group_name)

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            group.user = owner

            sa.add(group)

            perm_obj = rgm._create_default_perms(group)

            sa.add(perm_obj)

            sa.flush()

        parent = group

    return group

def repo2db_mapper(initial_repo_list, remove_obsolete=False,

                   install_git_hooks=False, user=None, overwrite_git_hooks=False):

    """

    maps all repos given in initial_repo_list, non existing repositories

    are created, if remove_obsolete is True it also check for db entries

    that are not in initial_repo_list and removes them.

    :param initial_repo_list: list of repositories found by scanning methods

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hooks: if this is True, also check and install git hook

        for a repo if missing

    :param overwrite_git_hooks: if this is True, overwrite any existing git hooks

        that may be encountered (even if user-deployed)

    """

    from kallithea.model.repo import RepoModel

    from kallithea.model.scm import ScmModel

    sa = meta.Session()

    repo_model = RepoModel()

    if user is None:

        user = User.get_first_admin()

    added = []

    ##creation defaults

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_locking = defs.get('repo_enable_locking')

    enable_downloads = defs.get('repo_enable_downloads')

    private = defs.get('repo_private')

    for name, repo in initial_repo_list.items():

        group = map_groups(name)

        unicode_name = safe_unicode(name)

        db_repo = repo_model.get_by_repo_name(unicode_name)

        # found repo that is on filesystem not in Kallithea database

        if not db_repo:

            log.info('repository %s not found, creating now', name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = repo_model._create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repo_group=getattr(group, 'group_id', None),

                owner=user,

                enable_locking=enable_locking,

                enable_downloads=enable_downloads,

                enable_statistics=enable_statistics,

                private=private,

                state=Repository.STATE_CREATED

            )

            sa.commit()

            # we added that repo just now, and make sure it has githook

            # installed, and updated server info

            if new_repo.repo_type == 'git':

                git_repo = new_repo.scm_instance

                ScmModel().install_git_hooks(git_repo)

                # update repository server-info

                log.debug('Running update server info')

                git_repo._update_server_info()

            new_repo.update_changeset_cache()

        elif install_git_hooks:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)

    removed = []

    # remove from database those repositories that are not in the filesystem

    unicode_initial_repo_list = set(safe_unicode(name) for name in initial_repo_list)

    for repo in sa.query(Repository).all():

        if repo.repo_name not in unicode_initial_repo_list:

            if remove_obsolete:

                log.debug("Removing non-existing repository found in db `%s`",

                          repo.repo_name)

                try:

                    RepoModel(sa).delete(repo, forks='detach', fs_remove=False)

                    sa.commit()

                except Exception:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

            removed.append(repo.repo_name)

    return added, removed

# set cache regions for beaker so celery can utilise it

def add_cache(settings):

    cache_settings = {'regions': None}

    for key in settings.keys():

        for prefix in ['beaker.cache.', 'cache.']:

            if key.startswith(prefix):

                name = key.split(prefix)[1].strip()

                cache_settings[name] = settings[key].strip()

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            region_settings = {}

            for key, value in cache_settings.items():

                if key.startswith(region):

                    region_settings[key.split('.')[1]] = value

            region_settings['expire'] = int(region_settings.get('expire',

                                                                60))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            if 'type' not in region_settings:

                region_settings['type'] = cache_settings.get('type',

                                                             'memory')

            beaker.cache.cache_regions[region] = region_settings

def load_rcextensions(root_path):

    import kallithea

    from kallithea.config import conf

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = kallithea.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...', rcext)

        # Additional mappings that are not present in the pygments lexers

        conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []):

            log.debug('settings custom INDEX_EXTENSIONS')

            conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])

        #ADDITIONAL MAPPINGS

        log.debug('adding extra into INDEX_EXTENSIONS')

        conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))

        # auto check if the module is not missing any data, set to default if is

        # this will help autoupdate new feature of rcext module

        #from kallithea.config import rcextensions

        #for k in dir(rcextensions):

        #    if not k.startswith('_') and not hasattr(EXT, k):

        #        setattr(EXT, k, getattr(rcextensions, k))

def get_custom_lexer(extension):

    """

    returns a custom lexer if it's defined in rcextensions module, or None

    if there's no custom lexer defined

    """

    import kallithea

    from pygments import lexers

    #check if we didn't define this extension as other lexer

    if kallithea.EXTENSIONS and extension in kallithea.EXTENSIONS.EXTRA_LEXERS:

        _lexer_name = kallithea.EXTENSIONS.EXTRA_LEXERS[extension]

        return lexers.get_lexer_by_name(_lexer_name)

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    :param config: test config

    :param full_index:

    """

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    from kallithea.lib.pidlock import DaemonLock, LockHeld

    repo_location = repo_location

    index_location = os.path.join(config['app_conf']['index_dir'])

    if not os.path.exists(index_location):

        os.makedirs(index_location)

    try:

        l = DaemonLock(file_=os.path.join(dirname(index_location), 'make_index.lock'))

        WhooshIndexingDaemon(index_location=index_location,

                             repo_location=repo_location) \

            .run(full_index=full_index)

        l.release()

    except LockHeld:

        pass

def create_test_env(repos_test_path, config):

    """

    Makes a fresh database and

    install test repository into tmp dir

    """

    from kallithea.lib.db_manage import DbManage

    from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH

    # PART ONE create db

    dbconf = config['sqlalchemy.db1.url']

    log.debug('making test db %s', dbconf)

    # create test dir if it doesn't exist

    if not os.path.isdir(repos_test_path):

        log.debug('Creating testdir %s', repos_test_path)

        os.makedirs(repos_test_path)

    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],

                        tests=True)

    dbmanage.create_tables(override=True)

    # for tests dynamically set new root paths based on generated content

    dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))

    dbmanage.create_default_user()

    dbmanage.admin_prompt()

    dbmanage.create_permissions()

    dbmanage.populate_default_permissions()

    Session().commit()

    # PART TWO make test repo

    log.debug('making test vcs repositories')

    idx_path = config['app_conf']['index_dir']

    data_path = config['app_conf']['cache_dir']

    #clean index and data

    if idx_path and os.path.exists(idx_path):

        log.debug('remove %s', idx_path)

        shutil.rmtree(idx_path)

    if data_path and os.path.exists(data_path):

        log.debug('remove %s', data_path)

        shutil.rmtree(data_path)

    #CREATE DEFAULT TEST REPOS

    cur_dir = dirname(dirname(abspath(__file__)))

    tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))

    tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))

    tar.close()

    cur_dir = dirname(dirname(abspath(__file__)))

    tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))

    tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))

    tar.close()

    #LOAD VCS test stuff

    from kallithea.tests.vcs import setup_package

    setup_package()

#==============================================================================

# PASTER COMMANDS

#==============================================================================

def check_git_version():

    """

    Checks what version of git is installed in system, and issues a warning

    if it's too old for Kallithea to work properly.

    """

    from kallithea import BACKENDS

    from kallithea.lib.vcs.backends.git.repository import GitRepository

    from kallithea.lib.vcs.conf import settings

    from distutils.version import StrictVersion

    if 'git' not in BACKENDS:

        return None

    stdout, stderr = GitRepository._run_git_command(['--version'], _bare=True,

                                                    _safe=True)

    m = re.search("\d+.\d+.\d+", stdout)

    if m:

        ver = StrictVersion(m.group(0))

    else:

        ver = StrictVersion('0.0.0')

    req_ver = StrictVersion('1.7.4')

    log.debug('Git executable: "%s" version %s detected: %s',

              settings.GIT_EXECUTABLE_PATH, ver, stdout)

    if stderr:

        log.warning('Error detecting git version: %r', stderr)

    elif ver < req_ver:

        log.warning('Kallithea detected git version %s, which is too old '

                    'for the system to function properly. '

                    'Please upgrade to version %s or later.' % (ver, req_ver))

    return ver

@decorator.decorator

def jsonify(func, *args, **kwargs):

    """Action decorator that formats output for JSON

    Given a function that will return content, this decorator will turn

    the result into JSON, with a content-type of 'application/json' and

    output it.

    """

    from pylons.decorators.util import get_pylons

    from kallithea.lib.compat import json

    pylons = get_pylons(args)

    pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'

    data = func(*args, **kwargs)

    if isinstance(data, (list, tuple)):

        msg = "JSON responses with Array envelopes are susceptible to " \

              "cross-site data leak attacks, see " \

              "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"

        warnings.warn(msg, Warning, 2)

        log.warning(msg)