log = logging.getLogger(__name__)

def _journal_filter(user_log, search_term):

    """

    Filters sqlalchemy user_log based on search_term with whoosh Query language

    http://packages.python.org/Whoosh/querylang.html

    :param user_log:

    :param search_term:

    """

    qry = None

    if search_term:

        qp = QueryParser('repository', schema=JOURNAL_SCHEMA)

        qp.add_plugin(DateParserPlugin())

        qry = qp.parse(unicode(search_term))

    def wildcard_handler(col, wc_term):

        if wc_term.startswith('*') and not wc_term.endswith('*'):

            #postfix == endswith

            wc_term = remove_prefix(wc_term, prefix='*')

            return func.lower(col).endswith(wc_term)

        elif wc_term.startswith('*') and wc_term.endswith('*'):

            #wildcard == ilike

            wc_term = remove_prefix(wc_term, prefix='*')

            wc_term = remove_suffix(wc_term, suffix='*')

            return func.lower(col).contains(wc_term)

    def get_filterion(field, val, term):

        if field == 'repository':

            field = getattr(UserLog, 'repository_name')

        elif field == 'ip':

            field = getattr(UserLog, 'user_ip')

        elif field == 'date':

            field = getattr(UserLog, 'action_date')

        elif field == 'username':

            field = getattr(UserLog, 'username')

        else:

            field = getattr(UserLog, field)

        #sql filtering

        if isinstance(term, query.Wildcard):

            return wildcard_handler(field, val)

        elif isinstance(term, query.Prefix):

            return func.lower(field).startswith(func.lower(val))

        elif isinstance(term, query.DateRange):

            return and_(field >= val[0], field <= val[1])

        return func.lower(field) == func.lower(val)

    if isinstance(qry, (query.And, query.Term, query.Prefix, query.Wildcard,

                        query.DateRange)):

            # since the plugin settings will not be in the POST data.

            c.enabled_plugins = [ p for p in c.enabled_plugins if p in new_enabled_plugins ]

        # Next, parse everything including plugin settings.

        _form = AuthSettingsForm(c.enabled_plugins)()

        try:

            form_result = _form.to_python(dict(request.POST))

            for k, v in form_result.items():

                if k == 'auth_plugins':

                    # we want to store it comma separated inside our settings

                    v = ','.join(v)

                setting = Setting.create_or_update(k, v)

                Session().add(setting)

            Session().commit()

            h.flash(_('Auth settings updated successfully'),

                       category='success')

        except formencode.Invalid as errors:

            log.error(traceback.format_exc())

            e = errors.error_dict or {}

            return self.__render(

                defaults=errors.value,

                errors=e,

            )

        return redirect(url('gists'))

    @LoginRequired()

    def show(self, gist_id, revision='tip', format='html', f_path=None):

        """GET /admin/gists/gist_id: Show a specific item"""

        # url('gist', gist_id=ID)

        c.gist = Gist.get_or_404(gist_id)

        #check if this gist is not expired

        if c.gist.gist_expires != -1:

            if time.time() > c.gist.gist_expires:

                raise HTTPNotFound()

        try:

            c.file_changeset, c.files = GistModel().get_gist_files(gist_id,

                                                            revision=revision)

        except VCSError:

            log.error(traceback.format_exc())

            raise HTTPNotFound()

        if format == 'raw':

            content = '\n\n'.join([f.content for f in c.files if (f_path is None or f.path == f_path)])

            response.content_type = 'text/plain'

            return content

        return render('admin/gists/show.html')

    @LoginRequired()

    @NotAnonymous()

    def edit(self, gist_id, format='html'):

        """GET /admin/gists/gist_id/edit: Form to edit an existing item"""

        # url('edit_gist', gist_id=ID)

        c.gist = Gist.get_or_404(gist_id)

        #check if this gist is not expired

        if c.gist.gist_expires != -1:

            if time.time() > c.gist.gist_expires:

                raise HTTPNotFound()

        try:

            c.file_changeset, c.files = GistModel().get_gist_files(gist_id)

        except VCSError:

            log.error(traceback.format_exc())

            raise HTTPNotFound()

        self.__load_defaults(extra_values=('0', _('Unmodified')))

        rendered = render('admin/gists/edit.html')

        if request.POST:

            rpost = request.POST

    @LoginRequired()

    @NotAnonymous()

    @jsonify

    def check_revision(self, gist_id):

        c.gist = Gist.get_or_404(gist_id)

        last_rev = c.gist.scm_instance.get_changeset()

        success = True

        revision = request.POST.get('revision')

        ##TODO: maybe move this to model ?

        if revision != last_rev.raw_id:

            # our gist has newer version than we

            success = False

        return {'success': success}

        from kallithea.lib.verlib import NormalizedVersion

        from kallithea import __version__

        defaults = Setting.get_app_settings()

        defaults.update(self._get_hg_ui_settings())

        _update_url = defaults.get('update_url', '')

        _update_url = "" # FIXME: disabled

        _err = lambda s: '<div style="color:#ff8888; padding:4px 0px">%s</div>' % (s)

        try:

            import kallithea

            ver = kallithea.__version__

            opener = urllib2.build_opener()

            opener.addheaders = [('User-agent', 'Kallithea-SCM/%s' % ver)]

            response = opener.open(_update_url)

            response_data = response.read()

            data = json.loads(response_data)

        except urllib2.URLError as e:

            log.error(traceback.format_exc())

            return _err('Failed to contact upgrade server: %r' % e)

        except ValueError as e:

            log.error(traceback.format_exc())

            return _err('Bad data sent from update server')

    def _handle_request(self, environ, start_response):

        start = time.time()

        ip_addr = self.ip_addr = self._get_ip_addr(environ)

        self._req_id = None

        if 'CONTENT_LENGTH' not in environ:

            log.debug("No Content-Length")

            return jsonrpc_error(retid=self._req_id,

                                 message="No Content-Length in request")

        else:

            length = environ['CONTENT_LENGTH'] or 0

            length = int(environ['CONTENT_LENGTH'])

        if length == 0:

            log.debug("Content-Length is 0")

            return jsonrpc_error(retid=self._req_id,

                                 message="Content-Length is 0")

        raw_body = environ['wsgi.input'].read(length)

        try:

            json_body = json.loads(raw_body)

        except ValueError as e:

            # catch JSON errors Here

                                 % (e, raw_body))

        # check AUTH based on API key

        try:

            self._req_api_key = json_body['api_key']

            self._req_id = json_body['id']

            self._req_method = json_body['method']

            self._request_params = json_body['args']

            if not isinstance(self._request_params, dict):

                self._request_params = {}

            log.debug(

            )

        except KeyError as e:

            return jsonrpc_error(retid=self._req_id,

                                 message='Incorrect JSON query missing %s' % e)

        # check if we can find this session using api_key

        try:

            u = User.get_by_api_key(self._req_api_key)

            if u is None:

                return jsonrpc_error(retid=self._req_id,

                                     message='Invalid API key')

            auth_u = AuthUser(dbuser=u)

            if not AuthUser.check_ip_allowed(auth_u, ip_addr):

                return jsonrpc_error(retid=self._req_id,

                        message='request from IP:%s not allowed' % (ip_addr,))

            else:

        except Exception as e:

            return jsonrpc_error(retid=self._req_id,

                                 message='Invalid API key')

        self._error = None

        try:

            self._func = self._find_method()

        except AttributeError as e:

            return jsonrpc_error(retid=self._req_id,

                                 message=str(e))

    def _dispatch_call(self):

        """

        Implement dispatch interface specified by WSGIController

        """

        raw_response = ''

        try:

            raw_response = self._inspect_call(self._func)

            if isinstance(raw_response, HTTPError):

                self._error = str(raw_response)

        except JSONRPCError as e:

            self._error = safe_str(e)

        except Exception as e:

            json_exc = JSONRPCError('Internal server error')

            self._error = safe_str(json_exc)

        if self._error is not None:

            raw_response = None

        response = dict(id=self._req_id, result=raw_response, error=self._error)

        try:

            return json.dumps(response)

        except TypeError as e:

            return json.dumps(

                dict(

                    id=self._req_id,

                    result=None,

                    error="Error encoding response"

                )

            )

    def _find_method(self):

        """

        Return method named by `self._req_method` in controller if able

        """

        if self._req_method.startswith('_'):

            raise AttributeError("Method not allowed")

        try:

            func = getattr(self, self._req_method, None)

        except UnicodeEncodeError:

            raise AttributeError("Problem decoding unicode in requested "

                                 "method name.")

        if isinstance(func, types.MethodType):

            return func

        else:

            branch_name not in c.db_repo_scm_instance.closed_branches and

            not revision):

            return redirect(url('changelog_file_home', repo_name=c.repo_name,

                                    revision=branch_name, f_path=f_path or ''))

        if revision == 'tip':

            revision = None

        c.changelog_for_path = f_path

        try:

            if f_path:

                # get the history for the file !

                tip_cs = c.db_repo_scm_instance.get_changeset()

                try:

                    collection = tip_cs.get_file_history(f_path)

                except (NodeDoesNotExistError, ChangesetError):

                    #this node is not present at tip !

                    try:

                        cs = self.__get_cs(revision, repo_name)

                        collection = cs.get_file_history(f_path)

                    except RepositoryError as e:

                        h.flash(safe_str(e), category='warning')

                        redirect(h.url('changelog_home', repo_name=repo_name))

        c.cs_comments = other_repo.get_comments(raw_ids)

        c.statuses = other_repo.statuses(raw_ids)

        revs = [ctx.revision for ctx in reversed(c.cs_ranges)]

        c.jsdata = json.dumps(graph_data(c.cs_repo.scm_instance, revs))

        if partial:

            return render('compare/compare_cs.html')

        if merge and c.ancestor:

            # case we want a simple diff without incoming changesets,

            # previewing what will be merged.

            # Make the diff on the other repo (which is known to have other_rev)

            rev1 = c.ancestor

            org_repo = other_repo

        else: # comparing tips, not necessarily linearly related

            if merge:

                log.error('Unable to find ancestor revision')

            if org_repo != other_repo:

                # TODO: we could do this by using hg unionrepo

                log.error('cannot compare across repos %s and %s', org_repo, other_repo)

                h.flash(_('Cannot compare repositories without using common ancestor'), category='error')

                raise HTTPBadRequest

            rev1 = c.a_rev

        diff_limit = self.cut_off_limit if not c.fulldiff else None

        txtdiff = org_repo.scm_instance.get_diff(rev1=rev1, rev2=c.cs_rev,

                                      ignore_whitespace=ignore_whitespace,

                                      context=line_context)

        diff_processor = diffs.DiffProcessor(txtdiff or '', format='gitdiff',

                                             diff_limit=diff_limit)

        _parsed = diff_processor.prepare()

        c.limited_diff = False

        if isinstance(_parsed, LimitedDiffContainer):

            c.limited_diff = True

    This behavior can be altered by changing the parameters to the

    ErrorDocuments middleware in your config/middleware.py file.

    """

    def __before__(self):

        # disable all base actions since we don't need them here

        pass

    def document(self):

        resp = request.environ.get('pylons.original_response')

        c.site_name = config.get('title')

        e = request.environ

        c.serv_p = r'%(protocol)s://%(host)s/' % {

            'protocol': e.get('wsgi.url_scheme'),

            'host': e.get('HTTP_HOST'), }

        if resp:

            c.error_message = cgi.escape(request.GET.get('code',

                                                         str(resp.status)))

            c.error_explanation = self.get_error_explanation(resp.status_int)

        else:

            c.error_message = _('No response')

            c.error_explanation = _('Unknown error')

        rev_name = cs.raw_id[:12]

        archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),

                                    safe_str(rev_name), ext)

        archive_path = None

        cached_archive_path = None

        archive_cache_dir = CONFIG.get('archive_cache_dir')

        if archive_cache_dir and not subrepos: # TOOD: subrepo caching?

            if not os.path.isdir(archive_cache_dir):

                os.makedirs(archive_cache_dir)

            cached_archive_path = os.path.join(archive_cache_dir, archive_name)

            if os.path.isfile(cached_archive_path):

                archive_path = cached_archive_path

            else:

        if archive_path is None:

            # generate new archive

            fd, archive_path = tempfile.mkstemp()

            with os.fdopen(fd, 'wb') as stream:

                cs.fill_archive(stream=stream, kind=fileformat, subrepos=subrepos)

                # stream (and thus fd) has been closed by cs.fill_archive

            if cached_archive_path is not None:

                # we generated the archive - move it to cache

                shutil.move(archive_path, cached_archive_path)

                archive_path = cached_archive_path

        def get_chunked_archive(archive_path):

            stream = open(archive_path, 'rb')

            while True:

                data = stream.read(16 * 1024)

                if not data:

                    break

                yield data

            stream.close()

            if archive_path != cached_archive_path:

                os.remove(archive_path)

        action_logger(user=c.authuser,

                      action='user_downloaded_archive:%s' % (archive_name),

                      repo=repo_name, ipaddr=self.ip_addr, commit=True)

        response.content_disposition = str('attachment; filename=%s' % (archive_name))

        response.content_type = str(content_type)

        return get_chunked_archive(archive_path)

    @LoginRequired()

    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',

    def __before__(self):

        super(LoginController, self).__before__()

    def _validate_came_from(self, came_from):

        """Return True if came_from is valid and can and should be used"""

        if not came_from:

            return False

        parsed = urlparse.urlparse(came_from)

        server_parsed = urlparse.urlparse(url.current())

        allowed_schemes = ['http', 'https']

        if parsed.scheme and parsed.scheme not in allowed_schemes:

            return False

        if server_parsed.netloc != parsed.netloc:

            log.error('Suspicious NETLOC detected %s for url %s server url '

                      'is: %s' % (parsed.netloc, parsed, server_parsed))

            return False

        return True

    def _redirect_to_origin(self, origin):

        '''redirect to the original page, preserving any get arguments given'''

        request.GET.pop('came_from', None)

        raise HTTPFound(location=url(origin, **request.GET))

        raw_ids = [x.raw_id for x in c.cs_ranges]

        c.cs_comments = c.cs_repo.get_comments(raw_ids)

        c.statuses = c.cs_repo.statuses(raw_ids)

        ignore_whitespace = request.GET.get('ignorews') == '1'

        line_context = request.GET.get('context', 3)

        c.ignorews_url = _ignorews_url

        c.context_url = _context_url

        c.fulldiff = request.GET.get('fulldiff')

        diff_limit = self.cut_off_limit if not c.fulldiff else None

        # we swap org/other ref since we run a simple diff on one repo

        txtdiff = org_scm_instance.get_diff(rev1=safe_str(c.a_rev), rev2=safe_str(c.cs_rev),

                                      ignore_whitespace=ignore_whitespace,

                                      context=line_context)

        diff_processor = diffs.DiffProcessor(txtdiff or '', format='gitdiff',

                                             diff_limit=diff_limit)

        _parsed = diff_processor.prepare()

        c.limited_diff = False

        if isinstance(_parsed, LimitedDiffContainer):

            c.limited_diff = True

        index_name = {

            'content': IDX_NAME,

            'commit': CHGSET_IDX_NAME,

            'path': IDX_NAME

        }.get(c.cur_type, IDX_NAME)

        schema_defn = {

            'content': SCHEMA,

            'commit': CHGSETS_SCHEMA,

            'path': SCHEMA

        }.get(c.cur_type, SCHEMA)

        if c.cur_query:

            cur_query = c.cur_query.lower()

            log.debug(cur_query)

        if c.cur_query:

            p = safe_int(request.GET.get('page', 1), 1)

            highlight_items = set()

            try:

                idx = open_dir(config['app_conf']['index_dir'],

                               indexname=index_name)

                searcher = idx.searcher()

                    # extract words for highlight

                    if isinstance(query, Phrase):

                        highlight_items.update(query.words)

                    elif isinstance(query, Prefix):

                        highlight_items.add(query.text)

                    else:

                        for i in query.all_terms():

                            if i[0] in ['content', 'message']:

                                highlight_items.add(i[1])

                    matcher = query.matcher(searcher)

                    results = searcher.search(query)

                    res_ln = len(results)

                    c.runtime = '%s results (%.3f seconds)' % (

                        res_ln, results.runtime

                    )

                    def url_generator(**kw):

                        q = urllib.quote(safe_str(c.cur_query))

                        return update_params("?q=%s&type=%s" \

                        % (q, safe_str(c.cur_type)), **kw)

                    repo_location = RepoModel().repos_path

                    c.formated_results = Page(

            try:

                # gets the landing revision! or tip if fails

                cs = db_repo.get_landing_changeset()

                if isinstance(cs, EmptyChangeset):

                    raise EmptyRepositoryError()

                renderer = MarkupRenderer()

                for f in README_FILES:

                    try:

                        readme = cs.get_node(f)

                        if not isinstance(readme, FileNode):

                            continue

                        readme_file = f

                                  readme_file)

                        readme_data = renderer.render(readme.content,

                                                      filename=f)

                        break

                    except NodeDoesNotExistError:

                        continue

            except ChangesetError:

                log.error(traceback.format_exc())

                pass

            except EmptyRepositoryError:

                pass

    return permissions

def allowed_api_access(controller_name, whitelist=None, api_key=None):

    """

    Check if given controller_name is in whitelist API access

    """

    if not whitelist:

        from kallithea import CONFIG

        whitelist = aslist(CONFIG.get('api_access_controllers_whitelist'),

                           sep=',')

    api_access_valid = controller_name in whitelist

    if api_access_valid:

    else:

        msg = 'controller: %s is *NOT* in API whitelist' % (controller_name)

        if api_key:

            #if we use API key and don't have access it's a warning

            log.warning(msg)

        else:

            log.debug(msg)

    return api_access_valid

class AuthUser(object):

    """

        # not enabled.

        self.user_id = None

        self.username = None

        self.api_key = None

        self.name = ''

        self.lastname = ''

        self.email = ''

        self.admin = False

        self.inherit_default_permissions = False

        # Look up database user, if necessary.

        if user_id is not None:

            dbuser = user_model.get(user_id)

        else:

            # Note: dbuser is allowed to be None.

            log.debug('Auth User lookup by database user %s', dbuser)

        is_user_loaded = self._fill_data(dbuser)

        # If user cannot be found, try falling back to anonymous.

        if not is_user_loaded:

            is_user_loaded =  self._fill_data(self.anonymous_user)

        # The anonymous user is always "logged in".

        if self.user_id == self.anonymous_user.user_id:

            self.is_authenticated = True

        if not self.username:

            self.username = 'None'

    def _fill_data(self, dbuser):

        """

        Copies database fields from a `db.User` to this `AuthUser`. Does

        not copy `api_keys` and `permissions` attributes.

        Checks that `dbuser` is `active` (and not None) before copying;

        returns True on success.

        """

        if dbuser is not None and dbuser.active:

            log.debug('filling %s data', dbuser)

            for k, v in dbuser.get_dict().iteritems():

        return [x[0] for x in self.permissions['user_groups'].iteritems()

                if x[1] == 'usergroup.admin']

    @staticmethod

    def check_ip_allowed(user, ip_addr):

        """

        Check if the given IP address (a `str`) is allowed for the given

        user (an `AuthUser` or `db.User`).

        """

        allowed_ips = AuthUser.get_allowed_ips(user.user_id, cache=True,

            inherit_from_default=user.inherit_default_permissions)

        if check_ip_access(source_ip=ip_addr, allowed_ips=allowed_ips):

            return True

        else:

            log.info('Access for IP:%s forbidden, '

                     'not in %s' % (ip_addr, allowed_ips))

            return False

    def __repr__(self):

        return "<AuthUser('id:%s[%s] auth:%s')>"\

            % (self.user_id, self.username, self.is_authenticated)

    def set_authenticated(self, authenticated=True):

        if self.user_id != self.anonymous_user.user_id:

        meta.Session.remove()

#==============================================================================

# CHECK DECORATORS

#==============================================================================

def redirect_to_login(message=None):

    from kallithea.lib import helpers as h

    p = url.current()

    if message:

        h.flash(h.literal(message), category='warning')

    return redirect(url('login_home', came_from=p, **request.GET))

class LoginRequired(object):

    """

    Must be logged in to execute this function else

    redirect to login page

    :param api_access: if enabled this checks only for valid auth token

        and grants access based on valid token

    """

    def __init__(self, api_access=False):

        self.api_access = api_access

    def __call__(self, func):

        return decorator(self.__wrapper, func)

    def __wrapper(self, func, *fargs, **fkwargs):

        controller = fargs[0]

        user = controller.authuser

        loc = "%s:%s" % (controller.__class__.__name__, func.__name__)

        if not AuthUser.check_ip_allowed(user, controller.ip_addr):

            return redirect_to_login(_('IP %s not allowed') % controller.ip_addr)

        # check if we used an API key and it's a valid one

        api_key = request.GET.get('api_key')

        if api_key is not None:

            # explicit controller is enabled or API is in our whitelist

            if self.api_access or allowed_api_access(loc, api_key=api_key):

                if api_key in user.api_keys: