# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.annotate

~~~~~~~~~~~~~~~~~~~~~~

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Dec 4, 2011

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import StringIO

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.lib.vcs.nodes import FileNode

from pygments.formatters import HtmlFormatter

from pygments import highlight

def annotate_highlight(filenode, annotate_from_changeset_func=None,

        order=None, headers=None, **options):

    """

    Returns html portion containing annotated table with 3 columns: line

    numbers, changeset information and pygmentized line of code.

    :param filenode: FileNode object

        # repositories

        for perm in default_repo_perms:

            r_k = perm.UserRepoToPerm.repository.repo_name

            p = 'repository.admin'

            permissions[RK][r_k] = p

        # repository groups

        for perm in default_repo_groups_perms:

            rg_k = perm.UserRepoGroupToPerm.group.group_name

            p = 'group.admin'

            permissions[GK][rg_k] = p

        # user groups

        for perm in default_user_group_perms:

            u_k = perm.UserUserGroupToPerm.user_group.users_group_name

            p = 'usergroup.admin'

            permissions[UK][u_k] = p

        return permissions

    #==================================================================

    # SET DEFAULTS GLOBAL, REPOS, REPOSITORY GROUPS

    #==================================================================

    uid = user_id

    default_global_perms = UserToPerm.query()\

        .filter(UserToPerm.user_id == default_user_id)\

        .options(joinedload(UserToPerm.permission))

    for perm in default_global_perms:

        permissions[GLOBAL].add(perm.permission.permission_name)

    # defaults for repositories, taken from default user

    for perm in default_repo_perms:

        r_k = perm.UserRepoToPerm.repository.repo_name

        if perm.Repository.private and not (perm.Repository.user_id == uid):

            # disable defaults for private repos,

            p = 'repository.none'

        elif perm.Repository.user_id == uid:

            # set admin if owner

            p = 'repository.admin'

        else:

            p = perm.Permission.permission_name

        permissions[RK][r_k] = p

    # defaults for repository groups taken from default user permission

    # on given group

    for perm in default_repo_groups_perms:

        if not is_user_loaded:

            # if we cannot authenticate user try anonymous

            if self.anonymous_user.active:

                user_model.fill_data(self, user_id=self.anonymous_user.user_id)

                # then we set this user is logged in

                self.is_authenticated = True

            else:

                self.user_id = None

                self.username = None

                self.is_authenticated = False

        if not self.username:

            self.username = 'None'

        log.debug('Auth User is now %s' % self)

    def get_perms(self, user, explicit=True, algo='higherwin', cache=False):

        """

        Fills user permission attribute with permissions taken from database

        works for permissions given for repositories, and for permissions that

        are granted to groups

        :param user: instance of User object from database

        :param explicit: In case there are permissions both for user and a group

            explicitly override permissions from group, if it's False it will

            make decision based on the algo

        :param algo: algorithm to decide what permission should be choose if

            it's multiple defined, eg user in two different groups. It also

            decides if explicit flag is turned off how to specify the permission

            for case when user is in a group + have defined separate permission

        """

        user_id = user.user_id

        user_is_admin = user.is_admin

        user_inherit_default_permissions = user.inherit_default_permissions

        log.debug('Getting PERMISSION tree')

        compute = conditional_cache('short_term', 'cache_desc',

                                    condition=cache, func=_cached_perms_data)

        return compute(user_id, user_is_admin,

                       user_inherit_default_permissions, explicit, algo)

    def get_api_keys(self):

        api_keys = [self.api_key]

        for api_key in UserApiKeys.query()\

                .filter(UserApiKeys.user_id == self.user_id)\

                .filter(or_(UserApiKeys.expires == -1,

                            UserApiKeys.expires >= time.time())).all():

            api_keys.append(api_key.api_key)

        return formencode_obj(*self.args, **self.kwargs)

class KallitheaAuthPluginBase(object):

    auth_func_attrs = {

        "username": "unique username",

        "firstname": "first name",

        "lastname": "last name",

        "email": "email address",

        "groups": '["list", "of", "groups"]',

        "extern_name": "name in external source of record",

        "extern_type": "type of external source of record",

        "admin": 'True|False defines if user should be Kallithea super admin',

        "active": 'True|False defines active state of user internally for Kallithea',

        "active_from_extern": "True|False\None, active state from the external auth, "

                              "None means use definition from Kallithea extern_type active value"

    }

    @property

    def validators(self):

        """

        Exposes Kallithea validators modules

        """

        # this is a hack to overcome issues with pylons threadlocals and

        class LazyCaller(object):

            def __init__(self, name):

                self.validator_name = name

            def __call__(self, *args, **kwargs):

                from kallithea.model import validators as v

                obj = getattr(v, self.validator_name)

                #log.debug('Initializing lazy formencode object: %s' % obj)

                return LazyFormencode(obj, *args, **kwargs)

        class ProxyGet(object):

            def __getattribute__(self, name):

                return LazyCaller(name)

        return ProxyGet()

    @hybrid_property

    def name(self):

        """

        Returns the name of this authentication plugin.

        :returns: string

        """

    def accepts(self, user, accepts_empty=True):

        """

        Checks if this authentication module should accept a request for

        the current user.

        :param user: user object fetched using plugin's get_user() method.

        :param accepts_empty: if True accepts don't allow the user to be empty

        :returns: boolean

        """

        plugin_name = self.name

        if not user and not accepts_empty:

            log.debug('User is empty not allowed to authenticate')

            return False

        if user and user.extern_type and user.extern_type != plugin_name:

            log.debug('User %s should authenticate using %s this is %s, skipping'

                      % (user, user.extern_type, plugin_name))

            return False

        return True

    def get_user(self, username=None, **kwargs):

        """

        Helper method for user fetching in plugins, by default it's using

        eg. container auth plugin to fetch user by environ params

        :param username: username if given to fetch from database

        :param kwargs: extra arguments needed for user fetching.

        """

        user = None

        log.debug('Trying to fetch user `%s` from Kallithea database'

                  % (username))

        if username:

            user = User.get_by_username(username)

            if not user:

                log.debug('Fallback to fetch user in case insensitive mode')

                user = User.get_by_username(username, case_insensitive=True)

        else:

            log.debug('provided username:`%s` is empty skipping...' % username)

        return user

    def settings(self):

        """

        Return a list of the form:

        [

            {

                "name": "OPTION_NAME",

                "type": "[bool|password|string|int|select]",

        for k in self.auth_func_attrs:

            if k not in ret:

                raise Exception('Missing %s attribute from returned data' % k)

        return ret

class KallitheaExternalAuthPlugin(KallitheaAuthPluginBase):

    def use_fake_password(self):

        """

        Return a boolean that indicates whether or not we should set the user's

        password to a random value when it is authenticated by this plugin.

        If your plugin provides authentication, then you will generally want this.

        :returns: boolean

        """

        raise NotImplementedError("Not implemented in base class")

    def _authenticate(self, userobj, username, passwd, settings, **kwargs):

        auth = super(KallitheaExternalAuthPlugin, self)._authenticate(

            userobj, username, passwd, settings, **kwargs)

        if auth:

            # maybe plugin will clean the username ?

            # we should use the return value

            username = auth['username']

            # this can prevent from creating users in Kallithea when using

            # external authentication, but if it's inactive user we shouldn't

            # create that user anyway

            if auth['active_from_extern'] is False:

                log.warning("User %s authenticated against %s, but is inactive"

                            % (username, self.__module__))

                return None

            if self.use_fake_password():

                # Randomize the PW because we don't need it, but don't want

                # them blank either

                passwd = PasswordGenerator().gen_password(length=8)

            log.debug('Updating or creating user info from %s plugin'

                      % self.name)

            user = UserModel().create_or_update(

                username=username,

                password=passwd,

                email=auth["email"],

                firstname=auth["firstname"],

                lastname=auth["lastname"],

                active=auth["active"],

                admin=auth["admin"],

                extern_name=auth["extern_name"],

    """

    Imports and returns the authentication plugin in the module named by plugin

    (e.g., plugin='kallithea.lib.auth_modules.auth_internal'). Returns the

    KallitheaAuthPluginBase subclass on success, raises exceptions on failure.

    raises:

        AttributeError -- no KallitheaAuthPlugin class in the module

        TypeError -- if the KallitheaAuthPlugin is not a subclass of ours KallitheaAuthPluginBase

        ImportError -- if we couldn't import the plugin at all

    """

    log.debug("Importing %s" % plugin)

    if not plugin.startswith(u'kallithea.lib.auth_modules.auth_'):

        parts = plugin.split(u'.lib.auth_modules.auth_', 1)

        if len(parts) == 2:

            _module, pn = parts

            if pn == EXTERN_TYPE_INTERNAL:

                pn = "internal"

            plugin = u'kallithea.lib.auth_modules.auth_' + pn

    PLUGIN_CLASS_NAME = "KallitheaAuthPlugin"

    try:

        module = importlib.import_module(plugin)

    except (ImportError, TypeError):

        log.error(traceback.format_exc())

        # TODO: make this more error prone, if by some accident we screw up

        raise

    log.debug("Loaded auth plugin from %s (module:%s, file:%s)"

              % (plugin, module.__name__, module.__file__))

    pluginclass = getattr(module, PLUGIN_CLASS_NAME)

    if not issubclass(pluginclass, KallitheaAuthPluginBase):

        raise TypeError("Authentication class %s.KallitheaAuthPlugin is not "

                        "a subclass of %s" % (plugin, KallitheaAuthPluginBase))

    return pluginclass

def loadplugin(plugin):

    """

    Loads and returns an instantiated authentication plugin.

        see: importplugin

    """

    plugin = importplugin(plugin)()

    if plugin.plugin_settings.im_func != KallitheaAuthPluginBase.plugin_settings.im_func:

        raise TypeError("Authentication class %s.KallitheaAuthPluginBase "

                        "forbidden." % plugin)

    return plugin

def authenticate(username, password, environ=None):

    """

    Authentication function used for access control,

    It tries to authenticate based on enabled authentication modules.

    :param username: username can be empty for container auth

    :param password: password can be empty for container auth

    :param environ: environ headers passed for container auth

    :returns: None if auth failed, plugin_user dict if auth is correct

    """

    auth_plugins = Setting.get_auth_plugins()

    log.debug('Authentication against %s plugins' % (auth_plugins,))

    for module in auth_plugins:

        try:

            plugin = loadplugin(module)

        except (ImportError, AttributeError, TypeError), e:

            raise ImportError('Failed to load authentication module %s : %s'

                              % (module, str(e)))

        log.debug('Trying authentication using ** %s **' % (module,))

            conf_key = "auth_%s_%s" % (plugin_name, v["name"])

            setting = Setting.get_by_name(conf_key)

            plugin_settings[v["name"]] = setting.app_settings_value if setting else None

        log.debug('Plugin settings \n%s' % formatted_json(plugin_settings))

        if not str2bool(plugin_settings["enabled"]):

            log.info("Authentication plugin %s is disabled, skipping for %s"

                     % (module, username))

            continue

        # use plugin's method of user extraction.

        user = plugin.get_user(username, environ=environ,

                               settings=plugin_settings)

        log.debug('Plugin %s extracted user is `%s`' % (module, user))

        if not plugin.accepts(user):

            log.debug('Plugin %s does not accept user `%s` for authentication'

                      % (module, user))

            continue

        else:

            log.debug('Plugin %s accepted user `%s` for authentication'

                      % (module, user))

        log.info('Authenticating user using %s plugin' % plugin.__module__)

        # _authenticate is a wrapper for .auth() method of plugin.

        # it also maps users to Database and maps the attributes returned

        # from .auth() to Kallithea database. If this function returns data

        # then auth is correct.

        plugin_user = plugin._authenticate(user, username, password,

                                           plugin_settings,

                                           environ=environ or {})

        log.debug('PLUGIN USER DATA: %s' % plugin_user)

        if plugin_user:

            log.debug('Plugin returned proper authentication data')

            return plugin_user

        # we failed to Auth because .auth() method didn't return proper the user

        if username:

            log.warning("User `%s` failed to authenticate against %s"

                        % (username, plugin.__module__))

    return None

        if not environ:

            log.debug('got empty environ: %s' % environ)

        settings = settings or {}

        if settings.get('header'):

            header = settings.get('header')

            username = environ.get(header)

            log.debug('extracted %s:%s' % (header, username))

        # fallback mode

        if not username and settings.get('fallback_header'):

            header = settings.get('fallback_header')

            username = environ.get(header)

            log.debug('extracted %s:%s' % (header, username))

        if username and str2bool(settings.get('clean_username')):

            log.debug('Received username %s from container' % username)

            username = self._clean_username(username)

            log.debug('New cleanup user is: %s' % username)

        return username

    def get_user(self, username=None, **kwargs):

        """

        Helper method for user fetching in plugins, by default it's using

        eg. container auth plugin to fetch user by environ params

        :param username: username if given to fetch

        :param kwargs: extra arguments needed for user fetching.

        """

        environ = kwargs.get('environ') or {}

        settings = kwargs.get('settings') or {}

        username = self._get_username(environ, settings)

        # we got the username, so use default method now

        return super(KallitheaAuthPlugin, self).get_user(username)

    def auth(self, userobj, username, password, settings, **kwargs):

        """

        from REMOTE_USER if this plugin is enabled, if that fails

        it tries to get username from HTTP_X_FORWARDED_USER if fallback header

        is set. clean_username extracts the username from this data if it's

        having @ in it.

        Return None on failure. On success, return a dictionary of the form:

            see: KallitheaAuthPluginBase.auth_func_attrs

        :param userobj:

        :param username:

        :param password:

        :param settings:

        :param kwargs:

        """

        environ = kwargs.get('environ')

        if not environ:

            log.debug('Empty environ data skipping...')

            return None

        if not userobj:

            userobj = self.get_user('', environ=environ, settings=settings)

        # we don't care passed username/password for container auth plugins.

        # only way to log in is using environ

        username = None

        if userobj:

            username = getattr(userobj, 'username')

        if not username:

            username = self._get_username(environ, settings)

        # if cannot fetch username, it's a no-go for this plugin to proceed

        if not username:

            return None

        # old attrs fetched from Kallithea database

        admin = getattr(userobj, 'admin', False)

        active = getattr(userobj, 'active', True)

        email = getattr(userobj, 'email', '')

        firstname = getattr(userobj, 'firstname', '')

        lastname = getattr(userobj, 'lastname', '')

        extern_type = getattr(userobj, 'extern_type', '')

        user_attrs = {

            'username': username,

            'firstname': safe_unicode(firstname or username),

            'lastname': safe_unicode(lastname or ''),

            'groups': [],

            'email': email or '',

            'admin': admin or False,

            'active': active,

            'active_from_extern': True,

            'extern_name': username,

from kallithea.lib.compat import formatted_json, hybrid_property

from kallithea.model.db import User

log = logging.getLogger(__name__)

class KallitheaAuthPlugin(auth_modules.KallitheaAuthPluginBase):

    def __init__(self):

        pass

    @hybrid_property

    def name(self):

        return EXTERN_TYPE_INTERNAL

    def settings(self):

        return []

    def user_activation_state(self):

        def_user_perms = User.get_default_user().AuthUser.permissions['global']

        return 'hg.register.auto_activate' in def_user_perms

    def accepts(self, user, accepts_empty=True):

        """

        Custom accepts for this auth that doesn't accept empty users. We

        """

        return super(KallitheaAuthPlugin, self).accepts(user,

                                                        accepts_empty=False)

    def auth(self, userobj, username, password, settings, **kwargs):

        if not userobj:

            log.debug('userobj was:%s skipping' % (userobj, ))

            return None

        if userobj.extern_type != self.name:

            log.warning("userobj:%s extern_type mismatch got:`%s` expected:`%s`"

                     % (userobj, userobj.extern_type, self.name))

            return None

        user_attrs = {

            "username": userobj.username,

            "firstname": userobj.firstname,

            "lastname": userobj.lastname,

            "groups": [],

            "email": userobj.email,

            "admin": userobj.admin,

            "active": userobj.active,

            "active_from_extern": userobj.active,

            "extern_name": userobj.user_id,

            'extern_type': userobj.extern_type,

                return username

        return self.build_authentication()

    __call__ = authenticate

class BaseVCSController(object):

    def __init__(self, application, config):

        self.application = application

        self.config = config

        # base path of repo locations

        self.basepath = self.config['base_path']

        #authenticate this VCS request using authfunc

        self.authenticate = BasicAuth('', auth_modules.authenticate,

                                      config.get('auth_ret_code'))

        self.ip_addr = '0.0.0.0'

    def _handle_request(self, environ, start_response):

        raise NotImplementedError()

    def _get_by_id(self, repo_name):

        """

        Gets a special pattern _<ID> from clone url and tries to replace it

        :param repo_name:

        """

        data = repo_name.split('/')

        if len(data) >= 2:

            from kallithea.lib.utils import get_repo_by_id

            by_id_match = get_repo_by_id(repo_name)

            if by_id_match:

                data[1] = by_id_match

        return '/'.join(data)

    def _invalidate_cache(self, repo_name):

        """

        :param repo_name: full repo name, also a cache key

        """

        ScmModel().mark_for_invalidation(repo_name)

    def _check_permission(self, action, user, repo_name, ip_addr=None):

        """

        Checks permissions using action (push/pull) user and repository

        name

        :param action: push or pull action

        :param user: user instance

        :param repo_name: repository name

        """

        # check IP

        inherit = user.inherit_default_permissions

        ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,

                                               inherit_from_default=inherit)

        if ip_allowed:

            log.info('Access for IP:%s allowed' % (ip_addr,))

        else:

            return False

        if action == 'push':

    def set_value(self, value):

        """Set the value in the cache for this query."""

        cache, cache_key = _get_cache_parameters(self)

        cache.put(cache_key, value)

def query_callable(manager, query_cls=CachingQuery):

    def query(*arg, **kw):

        return query_cls(manager, *arg, **kw)

    return query

def get_cache_region(name, region):

    if region not in beaker.cache.cache_regions:

        raise BeakerException('Cache region `%s` not configured '

            'Check if proper cache settings are in the .ini files' % region)

    kw = beaker.cache.cache_regions[region]

    return beaker.cache.Cache._get_cache(name, kw)

def _get_cache_parameters(query):

    """For a query with cache_region and cache_namespace configured,

    on this query's current criterion and parameter values.

    """

    if not hasattr(query, '_cache_parameters'):

        raise ValueError("This Query does not have caching "

                         "parameters configured.")

    region, namespace, cache_key = query._cache_parameters

    namespace = _namespace_from_query(namespace, query)

    if cache_key is None:

        # cache key - the value arguments from this query's parameters.

        args = [safe_str(x) for x in _params_from_query(query)]

        args.extend(filter(lambda k: k not in ['None', None, u'None'],

                           [str(query._limit), str(query._offset)]))

        cache_key = " ".join(args)

    if cache_key is None:

        raise Exception('Cache key cannot be None')

    # get cache

    #cache = query.cache_manager.get_cache_region(namespace, region)

                                                 }],

                                        "schema": ["commits"],

                                        }

            #gather all data by day

            if k in commits_by_day_aggregate:

                commits_by_day_aggregate[k] += 1

            else:

                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),

                               key=itemgetter(0))

        if not co_day_auth_aggr:

            co_day_auth_aggr[akc(repo.contact)] = {

                "label": akc(repo.contact),

                "data": [0, 1],

                "schema": ["commits"],

            }

        stats = cur_stats if cur_stats else Statistics()

        stats.commit_activity = json.dumps(co_day_auth_aggr)

        stats.commit_activity_combined = json.dumps(overview_data)

        leftovers = len(repo.revisions[last_rev:])

        log.debug('revisions to parse %s' % leftovers)

        if last_rev == 0 or leftovers < parse_limit:

            log.debug('getting code trending stats')

            stats.languages = json.dumps(__get_codes_stats(repo_name))

        try:

            stats.repository = dbrepo

            stats.stat_on_revision = last_cs.revision if last_cs else 0

            DBS.add(stats)

            DBS.commit()

        except:

            log.error(traceback.format_exc())

            DBS.rollback()

            lock.release()

            return False

        # final release

        lock.release()

        # execute another task if celery is enabled

        if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:

            recurse_limit -= 1

        pools = map(tuple, args) * kwds.get('repeat', 1)

        result = [[]]

        for pool in pools:

            result = [x + [y] for x in result for y in pool]

        for prod in result:

            yield tuple(prod)

#==============================================================================

# BytesIO

#==============================================================================

try:

    from io import BytesIO

except ImportError:

    from cStringIO import StringIO as BytesIO

#==============================================================================

# bytes

#==============================================================================

if __py_version__ >= (2, 6):

    _bytes = bytes

else:

    _bytes = str

if __py_version__ >= (2, 6):

    _bytearray = bytearray

else:

    import array

    # no idea if this is correct but all integration tests are passing

    # i think we never use bytearray anyway

    _bytearray = array

#==============================================================================

# deque

#==============================================================================

if __py_version__ >= (2, 6):

    from collections import deque

else:

    #need to implement our own deque with maxlen

    class deque(object):

        def __init__(self, iterable=(), maxlen= -1):

            if not hasattr(self, 'data'):

                self.left = self.right = 0

            if skip_existing and Setting.get_by_name(k) is not None:

                log.debug('Skipping option %s' % k)

                continue

            setting = Setting(k, v, t)

            self.sa.add(setting)

    def fixup_groups(self):

        def_usr = User.get_default_user()

        for g in RepoGroup.query().all():

            g.group_name = g.get_new_name(g.name)

            self.sa.add(g)

            # get default perm

            default = UserRepoGroupToPerm.query()\

                .filter(UserRepoGroupToPerm.group == g)\

                .filter(UserRepoGroupToPerm.user == def_usr)\