kallithea.bin.base

~~~~~~~~~~~~~~~~~~

Base utils for shell scripts

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: May 09, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import pprint

import random

import sys

import urllib2

from kallithea.lib import ext_json

from kallithea.lib.utils2 import ascii_bytes

CONFIG_NAME = '.config/kallithea'

FORMAT_PRETTY = 'pretty'

FORMAT_JSON = 'json'

def api_call(apikey, apihost, method=None, **kw):

    """

    Api_call wrapper for Kallithea.

    :param apikey:

    :param apihost:

    :param format: formatting, pretty means prints and pprint of json

     json returns unparsed json

    :param method:

    :returns: json response from server

    """

    def _build_data(random_id):

        """

        Builds API data with given random ID

        :param random_id:

        """

        return {

            "id": random_id,

            "api_key": apikey,

            "method": method,

            "args": kw

        }

    if not method:

        raise Exception('please specify method name !')

    apihost = apihost.rstrip('/')

    id_ = random.randrange(1, 9999)

    req = urllib2.Request('%s/_admin/api' % apihost,

                      data=ascii_bytes(ext_json.dumps(_build_data(id_))),

                      headers={'content-type': 'text/plain'})

    ret = urllib2.urlopen(req)

    raw_json = ret.read()

    json_data = ext_json.loads(raw_json)

    id_ret = json_data['id']

    if id_ret == id_:

        return json_data

    else:

        _formatted_json = pprint.pformat(json_data)

        raise Exception('something went wrong. '

                        'ID mismatch got %s, expected %s | %s' % (

                                            id_ret, id_, _formatted_json))

class RcConf(object):

    """

    Kallithea config for API

    conf = RcConf()

    conf['key']

    """

    def __init__(self, config_location=None, autoload=True, autocreate=False,

                 config=None):

        HOME = os.getenv('HOME', os.getenv('USERPROFILE')) or ''

        HOME_CONF = os.path.abspath(os.path.join(HOME, CONFIG_NAME))

        self._conf_name = HOME_CONF if not config_location else config_location

        self._conf = {}

        if autocreate:

            self.make_config(config)

        if autoload:

            self._conf = self.load_config()

    def __getitem__(self, key):

        return self._conf[key]

        if self._conf:

            return True

        return False

    def __eq__(self, other):

        return self._conf.__eq__(other)

    def __repr__(self):

        return 'RcConf<%s>' % self._conf.__repr__()

    def make_config(self, config):

        """

        Saves given config as a JSON dump in the _conf_name location

        :param config:

        """

        update = False

        if os.path.exists(self._conf_name):

            update = True

        with open(self._conf_name, 'wb') as f:

            ext_json.dump(config, f, indent=4)

            f.write('\n')

        if update:

            sys.stdout.write('Updated config in %s\n' % self._conf_name)

        else:

            sys.stdout.write('Created new config in %s\n' % self._conf_name)

    def update_config(self, new_config):

        """

        Reads the JSON config updates it's values with new_config and

        saves it back as JSON dump

        :param new_config:

        """

        config = {}

        try:

            with open(self._conf_name, 'rb') as conf:

                config = ext_json.load(conf)

        except IOError as e:

            sys.stderr.write(str(e) + '\n')

        config.update(new_config)

        self.make_config(config)

    def load_config(self):

        """

        Loads config from file and returns loaded JSON object

        """

        try:

            with open(self._conf_name, 'rb') as conf:

                return ext_json.load(conf)

        except IOError as e:

            #sys.stderr.write(str(e) + '\n')

            pass

        if user.is_default_user:

            raise _redirect_to_login(_('You need to be a registered user to '

                                       'perform this action'))

        else:

            return func(*fargs, **fkwargs)

class _PermsDecorator(object):

    """Base class for controller decorators with multiple permissions"""

    def __init__(self, *required_perms):

        self.required_perms = required_perms # usually very short - a list is thus fine

    def __call__(self, func):

        return decorator(self.__wrapper, func)

    def __wrapper(self, func, *fargs, **fkwargs):

        cls = fargs[0]

        user = request.authuser

        log.debug('checking %s permissions %s for %s %s',

          self.__class__.__name__, self.required_perms, cls, user)

        if self.check_permissions(user):

            log.debug('Permission granted for %s %s', cls, user)

            return func(*fargs, **fkwargs)

        else:

            log.info('Permission denied for %s %s', cls, user)

            if user.is_default_user:

                raise _redirect_to_login(_('You need to be signed in to view this page'))

            else:

                raise HTTPForbidden()

    def check_permissions(self, user):

        raise NotImplementedError()

class HasPermissionAnyDecorator(_PermsDecorator):

    """

    Checks the user has any of the given global permissions.

    """

    def check_permissions(self, user):

        global_permissions = user.permissions['global'] # usually very short

        return any(p in global_permissions for p in self.required_perms)

class _PermDecorator(_PermsDecorator):

    """Base class for controller decorators with a single permission"""

    def __init__(self, required_perm):

        _PermsDecorator.__init__(self, [required_perm])

        self.required_perm = required_perm

class HasRepoPermissionLevelDecorator(_PermDecorator):

    """

    Checks the user has at least the specified permission level for the requested repository.

    """

    def check_permissions(self, user):

        repo_name = get_repo_slug(request)

        return user.has_repository_permission_level(repo_name, self.required_perm)

class HasRepoGroupPermissionLevelDecorator(_PermDecorator):

    """

    Checks the user has any of given permissions for the requested repository group.

    """

    def check_permissions(self, user):

        repo_group_name = get_repo_group_slug(request)

        return user.has_repository_group_permission_level(repo_group_name, self.required_perm)

class HasUserGroupPermissionLevelDecorator(_PermDecorator):

    """

    Checks for access permission for any of given predicates for specific

    user group. In order to fulfill the request any of predicates must be meet

    """

    def check_permissions(self, user):

        user_group_name = get_user_group_slug(request)

        return user.has_user_group_permission_level(user_group_name, self.required_perm)

#==============================================================================

# CHECK FUNCTIONS

#==============================================================================

class _PermsFunction(object):

    """Base function for other check functions with multiple permissions"""

    def __init__(self, *required_perms):

        self.required_perms = required_perms # usually very short - a list is thus fine

        """ Defend against accidentally forgetting to call the object

            and instead evaluating it directly in a boolean context,

            which could have security implications.

        """

        raise AssertionError(self.__class__.__name__ + ' is not a bool and must be called!')

    def __call__(self, *a, **b):

        raise NotImplementedError()

class HasPermissionAny(_PermsFunction):

    def __call__(self, purpose=None):

        global_permissions = request.authuser.permissions['global'] # usually very short

        ok = any(p in global_permissions for p in self.required_perms)

        log.debug('Check %s for global %s (%s): %s',

            request.authuser.username, self.required_perms, purpose, ok)

        return ok

class _PermFunction(_PermsFunction):

    """Base function for other check functions with a single permission"""

    def __init__(self, required_perm):

        _PermsFunction.__init__(self, [required_perm])

        self.required_perm = required_perm

class HasRepoPermissionLevel(_PermFunction):

    def __call__(self, repo_name, purpose=None):

        return request.authuser.has_repository_permission_level(repo_name, self.required_perm, purpose)

class HasRepoGroupPermissionLevel(_PermFunction):

    def __call__(self, group_name, purpose=None):

        return request.authuser.has_repository_group_permission_level(group_name, self.required_perm, purpose)

class HasUserGroupPermissionLevel(_PermFunction):

    def __call__(self, user_group_name, purpose=None):

        return request.authuser.has_user_group_permission_level(user_group_name, self.required_perm, purpose)

#==============================================================================

# SPECIAL VERSION TO HANDLE MIDDLEWARE AUTH

#==============================================================================

class HasPermissionAnyMiddleware(object):

    def __init__(self, *perms):

        self.required_perms = set(perms)

    def __call__(self, authuser, repo_name, purpose=None):

        # repo_name MUST be unicode, since we handle keys in ok

        # dict by unicode

        repo_name = safe_unicode(repo_name)

        try:

            ok = authuser.permissions['repositories'][repo_name] in self.required_perms

        except KeyError:

            ok = False

        log.debug('Middleware check %s for %s for repo %s (%s): %s', authuser.username, self.required_perms, repo_name, purpose, ok)

        return ok

def check_ip_access(source_ip, allowed_ips=None):

    """

    Checks if source_ip is a subnet of any of allowed_ips.

    :param source_ip:

    :param allowed_ips: list of allowed ips together with mask

    """

    source_ip = source_ip.split('%', 1)[0]

    log.debug('checking if ip:%s is subnet of %s', source_ip, allowed_ips)

    if isinstance(allowed_ips, (tuple, list, set)):

        for ip in allowed_ips:

            if ipaddr.IPAddress(source_ip) in ipaddr.IPNetwork(ip):

                log.debug('IP %s is network %s',

                          ipaddr.IPAddress(source_ip), ipaddr.IPNetwork(ip))

                return True

    return False

# The code in this module is entirely lifted from the Lamson project

# (http://lamsonproject.org/).  Its copyright is:

# Copyright (c) 2008, Zed A. Shaw

# All rights reserved.

# It is provided under this license:

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are met:

# * Redistributions of source code must retain the above copyright notice, this

#   list of conditions and the following disclaimer.

# * Redistributions in binary form must reproduce the above copyright notice,

#   this list of conditions and the following disclaimer in the documentation

#   and/or other materials provided with the distribution.

# * Neither the name of the Zed A. Shaw nor the names of its contributors may

#   be used to endorse or promote products derived from this software without

#   specific prior written permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS

# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE

# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,

# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,

# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

import mimetypes

import os

import string

from email import encoders

from email.charset import Charset

from email.mime.base import MIMEBase

from email.utils import parseaddr

ADDRESS_HEADERS_WHITELIST = ['From', 'To', 'Delivered-To', 'Cc']

DEFAULT_ENCODING = "utf-8"

VALUE_IS_EMAIL_ADDRESS = lambda v: '@' in v

def normalize_header(header):

    return string.capwords(header.lower(), '-')

class EncodingError(Exception):

    """Thrown when there is an encoding error."""

    pass

class MailBase(object):

    """MailBase is used as the basis of lamson.mail and contains the basics of

    encoding an email.  You actually can do all your email processing with this

    class, but it's more raw.

    """

    def __init__(self, items=()):

        self.headers = dict(items)

        self.parts = []

        self.body = None

        self.content_encoding = {'Content-Type': (None, {}),

                                 'Content-Disposition': (None, {}),

                                 'Content-Transfer-Encoding': (None, {})}

    def __getitem__(self, key):

        return self.headers.get(normalize_header(key), None)

    def __len__(self):

        return len(self.headers)

    def __iter__(self):

        return iter(self.headers)

    def __contains__(self, key):

        return normalize_header(key) in self.headers

    def __setitem__(self, key, value):

        self.headers[normalize_header(key)] = value

    def __delitem__(self, key):

        del self.headers[normalize_header(key)]

        return self.body is not None or len(self.headers) > 0 or len(self.parts) > 0

    def keys(self):

        """Returns the sorted keys."""

        return sorted(self.headers.keys())

    def attach_file(self, filename, data, ctype, disposition):

        """

        A file attachment is a raw attachment with a disposition that

        indicates the file name.

        """

        assert filename, "You can't attach a file without a filename."

        ctype = ctype.lower()

        part = MailBase()

        part.body = data

        part.content_encoding['Content-Type'] = (ctype, {'name': filename})

        part.content_encoding['Content-Disposition'] = (disposition,

                                                        {'filename': filename})

        self.parts.append(part)

    def attach_text(self, data, ctype):

        """

        This attaches a simpler text encoded part, which doesn't have a

        filename.

        """

        ctype = ctype.lower()

        part = MailBase()

        part.body = data

        part.content_encoding['Content-Type'] = (ctype, {})

        self.parts.append(part)

    def walk(self):

        for p in self.parts:

            yield p

            for x in p.walk():

                yield x

class MailResponse(object):

    """

    You are given MailResponse objects from the lamson.view methods, and

    whenever you want to generate an email to send to someone.  It has the

    same basic functionality as MailRequest, but it is designed to be written

    to, rather than read from (although you can do both).

    You can easily set a Body or Html during creation or after by passing it

    as __init__ parameters, or by setting those attributes.

    You can initially set the From, To, and Subject, but they are headers so

    use the dict notation to change them: msg['From'] = 'joe@example.com'.

    The message is not fully crafted until right when you convert it with

    MailResponse.to_message.  This lets you change it and work with it, then

    send it out when it's ready.

    """

    def __init__(self, To=None, From=None, Subject=None, Body=None, Html=None,

                 separator="; "):

        self.Body = Body

        self.Html = Html

        self.base = MailBase([('To', To), ('From', From), ('Subject', Subject)])

        self.multipart = self.Body and self.Html

        self.attachments = []

        self.separator = separator

    def __contains__(self, key):

        return self.base.__contains__(key)

    def __getitem__(self, key):

        return self.base.__getitem__(key)

    def __setitem__(self, key, val):

        return self.base.__setitem__(key, val)

    def __delitem__(self, name):

        del self.base[name]

    def attach(self, filename=None, content_type=None, data=None,

               disposition=None):

        """

        Simplifies attaching files from disk or data as files.  To attach

        simple text simple give data and a content_type.  To attach a file,

        give the data/content_type/filename/disposition combination.

        For convenience, if you don't give data and only a filename, then it

        will read that file's contents when you call to_message() later.  If

        you give data and filename then it will assume you've filled data

        with what the file's contents are and filename is just the name to

        use.

        """

        assert filename or data, ("You must give a filename or some data to "

                                  "attach.")

        assert data or os.path.exists(filename), ("File doesn't exist, and no "