Changeset - cc7eedb5323c
[Not reviewed]
beta
0 6 0
Marcin Kuzminski - 13 years ago 2012-12-06 23:55:12
marcin@python-works.com
final implementation of #210 journal filtering.
6 files changed with 105 insertions and 42 deletions:
0 comments (0 inline, 0 general)
rhodecode/controllers/admin/admin.py
Show inline comments
 
@@ -21,48 +21,50 @@
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import logging
 

	
 
from pylons import request, tmpl_context as c, url
 
from sqlalchemy.orm import joinedload
 
from webhelpers.paginate import Page
 
from whoosh.qparser.default import QueryParser
 
from whoosh import query
 
from sqlalchemy.sql.expression import or_
 
from sqlalchemy.sql.expression import or_, and_
 

	
 
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.model.db import UserLog, User
 
from rhodecode.lib.utils2 import safe_int, remove_prefix, remove_suffix
 
from rhodecode.lib.indexers import JOURNAL_SCHEMA
 
from whoosh.qparser.dateparse import DateParserPlugin
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def _filter(user_log, search_term):
 
def _journal_filter(user_log, search_term):
 
    """
 
    Filters sqlalchemy user_log based on search_term with whoosh Query language
 
    http://packages.python.org/Whoosh/querylang.html
 

	
 
    :param user_log:
 
    :param search_term:
 
    """
 
    log.debug('Initial search term: %r' % search_term)
 
    qry = None
 
    if search_term:
 
        qp = QueryParser('repository', schema=JOURNAL_SCHEMA)
 
        qp.add_plugin(DateParserPlugin())
 
        qry = qp.parse(unicode(search_term))
 
        log.debug('Filtering using parsed query %r' % qry)
 

	
 
    def wildcard_handler(col, wc_term):
 
        if wc_term.startswith('*') and not wc_term.endswith('*'):
 
            #postfix == endswith
 
            wc_term = remove_prefix(wc_term, prefix='*')
 
            return getattr(col, 'endswith')(wc_term)
 
        elif wc_term.startswith('*') and wc_term.endswith('*'):
 
            #wildcard == ilike
 
            wc_term = remove_prefix(wc_term, prefix='*')
 
            wc_term = remove_suffix(wc_term, suffix='*')
 
@@ -78,60 +80,65 @@ def _filter(user_log, search_term):
 
            field = getattr(UserLog, 'action_date')
 
        elif field == 'username':
 
            field = getattr(UserLog, 'username')
 
        else:
 
            field = getattr(UserLog, field)
 
        log.debug('filter field: %s val=>%s' % (field, val))
 

	
 
        #sql filtering
 
        if isinstance(term, query.Wildcard):
 
            return wildcard_handler(field, val)
 
        elif isinstance(term, query.Prefix):
 
            return field.startswith(val)
 
        elif isinstance(term, query.DateRange):
 
            return and_(field >= val[0], field <= val[1])
 
        return field == val
 

	
 
    if isinstance(qry, (query.And, query.Term, query.Prefix, query.Wildcard)):
 
    if isinstance(qry, (query.And, query.Term, query.Prefix, query.Wildcard,
 
                        query.DateRange)):
 
        if not isinstance(qry, query.And):
 
            qry = [qry]
 
        for term in qry:
 
            field = term.fieldname
 
            val = term.text
 
            val = (term.text if not isinstance(term, query.DateRange)
 
                   else [term.startdate, term.enddate])
 
            user_log = user_log.filter(get_filterion(field, val, term))
 
    elif isinstance(qry, query.Or):
 
        filters = []
 
        for term in qry:
 
            field = term.fieldname
 
            val = term.text
 
            val = (term.text if not isinstance(term, query.DateRange)
 
                   else [term.startdate, term.enddate])
 
            filters.append(get_filterion(field, val, term))
 
        user_log = user_log.filter(or_(*filters))
 

	
 
    return user_log
 

	
 

	
 
class AdminController(BaseController):
 

	
 
    @LoginRequired()
 
    def __before__(self):
 
        super(AdminController, self).__before__()
 

	
 
    @HasPermissionAllDecorator('hg.admin')
 
    def index(self):
 
        users_log = UserLog.query()\
 
                .options(joinedload(UserLog.user))\
 
                .options(joinedload(UserLog.repository))
 

	
 
        #FILTERING
 
        c.search_term = request.GET.get('filter')
 
        try:
 
            users_log = _filter(users_log, c.search_term)
 
            users_log = _journal_filter(users_log, c.search_term)
 
        except:
 
            # we want this to crash for now
 
            raise
 

	
 
        users_log = users_log.order_by(UserLog.action_date.desc())
 

	
 
        p = safe_int(request.params.get('page', 1), 1)
 

	
 
        def url_generator(**kw):
 
            return url.current(filter=c.search_term, **kw)
 

	
 
        c.users_log = Page(users_log, page=p, items_per_page=10, url=url_generator)
rhodecode/controllers/journal.py
Show inline comments
 
@@ -33,50 +33,56 @@ from webhelpers.feedgenerator import Ato
 
from webob.exc import HTTPBadRequest
 
from pylons import request, tmpl_context as c, response, url
 
from pylons.i18n.translation import _
 

	
 
import rhodecode.lib.helpers as h
 
from rhodecode.lib.auth import LoginRequired, NotAnonymous
 
from rhodecode.lib.base import BaseController, render
 
from rhodecode.model.db import UserLog, UserFollowing, Repository, User
 
from rhodecode.model.meta import Session
 
from sqlalchemy.sql.expression import func
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.lib.utils2 import safe_int
 
from rhodecode.controllers.admin.admin import _journal_filter
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class JournalController(BaseController):
 

	
 
    def __before__(self):
 
        super(JournalController, self).__before__()
 
        self.language = 'en-us'
 
        self.ttl = "5"
 
        self.feed_nr = 20
 

	
 
    @LoginRequired()
 
    @NotAnonymous()
 
    def index(self):
 
        # Return a rendered template
 
        p = safe_int(request.params.get('page', 1), 1)
 
        c.user = User.get(self.rhodecode_user.user_id)
 
        c.following = self.sa.query(UserFollowing)\
 
            .filter(UserFollowing.user_id == self.rhodecode_user.user_id)\
 
            .options(joinedload(UserFollowing.follows_repository))\
 
            .all()
 

	
 
        #FILTERING
 
        c.search_term = request.GET.get('filter')
 
        journal = self._get_journal_data(c.following)
 

	
 
        c.journal_pager = Page(journal, page=p, items_per_page=20)
 
        def url_generator(**kw):
 
            return url.current(filter=c.search_term, **kw)
 

	
 
        c.journal_pager = Page(journal, page=p, items_per_page=20, url=url_generator)
 
        c.journal_day_aggreagate = self._get_daily_aggregate(c.journal_pager)
 

	
 
        c.journal_data = render('journal/journal_data.html')
 
        if request.environ.get('HTTP_X_PARTIAL_XHR'):
 
            return c.journal_data
 
        return render('journal/journal.html')
 

	
 
    @LoginRequired()
 
    @NotAnonymous()
 
    def index_my_repos(self):
 
        c.user = User.get(self.rhodecode_user.user_id)
 
        if request.environ.get('HTTP_X_PARTIAL_XHR'):
 
@@ -132,26 +138,32 @@ class JournalController(BaseController):
 
        filtering_criterion = None
 

	
 
        if repo_ids and user_ids:
 
            filtering_criterion = or_(UserLog.repository_id.in_(repo_ids),
 
                        UserLog.user_id.in_(user_ids))
 
        if repo_ids and not user_ids:
 
            filtering_criterion = UserLog.repository_id.in_(repo_ids)
 
        if not repo_ids and user_ids:
 
            filtering_criterion = UserLog.user_id.in_(user_ids)
 
        if filtering_criterion is not None:
 
            journal = self.sa.query(UserLog)\
 
                .options(joinedload(UserLog.user))\
 
                .options(joinedload(UserLog.repository))\
 
                .filter(filtering_criterion)\
 
                .options(joinedload(UserLog.repository))
 
            #filter
 
            try:
 
                journal = _journal_filter(journal, c.search_term)
 
            except:
 
                # we want this to crash for now
 
                raise
 
            journal = journal.filter(filtering_criterion)\
 
                .order_by(UserLog.action_date.desc())
 
        else:
 
            journal = []
 

	
 
        return journal
 

	
 
    @LoginRequired()
 
    @NotAnonymous()
 
    def toggle_following(self):
 
        cur_token = request.POST.get('auth_token')
 
        token = h.get_token()
 
        if cur_token == token:
rhodecode/lib/helpers.py
Show inline comments
 
@@ -2,24 +2,25 @@
 

	
 
Consists of functions to typically be used within templates, but also
 
available to Controllers. This module is available to both as 'h'.
 
"""
 
import random
 
import hashlib
 
import StringIO
 
import urllib
 
import math
 
import logging
 
import re
 
import urlparse
 
import textwrap
 

	
 
from datetime import datetime
 
from pygments.formatters.html import HtmlFormatter
 
from pygments import highlight as code_highlight
 
from pylons import url, request, config
 
from pylons.i18n.translation import _, ungettext
 
from hashlib import md5
 

	
 
from webhelpers.html import literal, HTML, escape
 
from webhelpers.html.tools import *
 
from webhelpers.html.builder import make_tag
 
from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
 
@@ -1126,12 +1127,32 @@ def rst_w_mentions(source):
 

	
 

	
 
def changeset_status(repo, revision):
 
    return ChangesetStatusModel().get_status(repo, revision)
 

	
 

	
 
def changeset_status_lbl(changeset_status):
 
    return dict(ChangesetStatus.STATUSES).get(changeset_status)
 

	
 

	
 
def get_permission_name(key):
 
    return dict(Permission.PERMS).get(key)
 

	
 

	
 
def journal_filter_help():
 
    return _(textwrap.dedent('''
 
        Example filter terms:
 
            repository:vcs
 
            username:marcin
 
            action:*push*
 
            ip:127.0.0.1
 
            date:20120101
 
            date:[20120101100000 TO 20120102]
 

	
 
        Generate wildcards using '*' character:
 
            "repositroy:vcs*" - search everything starting with 'vcs'
 
            "repository:*vcs*" - search for repository containing 'vcs'
 

	
 
        Optional AND / OR operators in queries
 
            "repository:vcs OR repository:test"
 
            "username:test AND repository:test*"
 
    '''))
rhodecode/templates/admin/admin.html
Show inline comments
 
## -*- coding: utf-8 -*-
 
<%inherit file="/base/base.html"/>
 

	
 
<%def name="title()">
 
    ${_('Admin journal')} - ${c.rhodecode_name}
 
</%def>
 

	
 
<%def name="breadcrumbs_links()">
 
    <form id="filter_form">
 
    <input class="q_filter_box ${'' if c.search_term else 'initial'}" id="q_filter" size="15" type="text" name="filter" value="${c.search_term or _('quick filter...')}"/>
 
    <span class="tooltip" title="${h.tooltip(_('''
 
        Example search query:
 
            "repository:vcs"
 
            "username:marcin"
 
        
 
        You can use wildcards using '*'
 
            "repositroy:vcs*" - search everything starting with 'vcs'
 
            "repository:*vcs*" - search for repository containing 'vcs'
 
        Use AND / OR operators in queries
 
            "repository:vcs OR repository:test"
 
            "username:test AND repository:test*"
 
        List of valid search filters:
 
            repository:
 
            username:
 
            action:
 
            ip:
 
            date:
 
    '''))}">?</span>
 
    <input class="q_filter_box ${'' if c.search_term else 'initial'}" id="j_filter" size="15" type="text" name="filter" value="${c.search_term or _('journal filter...')}"/>
 
    <span class="tooltip" title="${h.tooltip(h.journal_filter_help())}">?</span>
 
    <input type='submit' value="${_('filter')}" class="ui-btn" style="padding:0px 2px 0px 2px;margin:0px"/>
 
    ${_('Admin journal')} - ${ungettext('%s entry', '%s entries', c.users_log.item_count) % (c.users_log.item_count)}
 
    </form>
 
    ${h.end_form()}
 
</%def>
 

	
 
<%def name="page_nav()">
 
	${self.menu('admin')}
 
</%def>
 
<%def name="main()">
 
<div class="box">
 
	<!-- box / title -->
 
	<div class="title">
 
	    ${self.breadcrumbs()}
 
	</div>
 
	<!-- end box / title -->
 
	<div class="table">
 
	    <div id="user_log">
 
	        ${c.log_data}
 
	    </div>
 
	</div>
 
</div>
 

	
 
<script>
 
YUE.on('q_filter','click',function(){
 
	var qfilter = YUD.get('q_filter');
 
	if(YUD.hasClass(qfilter, 'initial')){
 
		qfilter.value = '';
 
YUE.on('j_filter','click',function(){
 
	var jfilter = YUD.get('j_filter');
 
	if(YUD.hasClass(jfilter, 'initial')){
 
		jfilter.value = '';
 
	}
 
});
 
var fix_q_filter_width = function(len){
 
    YUD.setStyle(YUD.get('q_filter'),'width',Math.max(80, len*6.50)+'px');
 
var fix_j_filter_width = function(len){
 
    YUD.setStyle(YUD.get('j_filter'),'width',Math.max(80, len*6.50)+'px');
 
}
 
YUE.on('q_filter','keyup',function(){
 
	fix_q_filter_width(YUD.get('q_filter').value.length);
 
YUE.on('j_filter','keyup',function(){
 
	fix_j_filter_width(YUD.get('j_filter').value.length);
 
});
 
YUE.on('filter_form','submit',function(e){
 
	YUE.preventDefault(e)
 
    var val = YUD.get('q_filter').value;
 
    var val = YUD.get('j_filter').value;
 
	window.location = "${url.current(filter='__FILTER__')}".replace('__FILTER__',val);
 
});
 
fix_q_filter_width(YUD.get('q_filter').value.length);
 
fix_j_filter_width(YUD.get('j_filter').value.length);
 
</script>
 
</%def>
 

	
rhodecode/templates/journal/journal.html
Show inline comments
 
## -*- coding: utf-8 -*-
 
<%inherit file="/base/base.html"/>
 
<%def name="title()">
 
    ${_('Journal')} - ${c.rhodecode_name}
 
</%def>
 
<%def name="breadcrumbs()">
 
	${c.rhodecode_name}
 
    <h5>
 
    <form id="filter_form">
 
    <input class="q_filter_box ${'' if c.search_term else 'initial'}" id="j_filter" size="15" type="text" name="filter" value="${c.search_term or _('quick filter...')}"/>
 
    <span class="tooltip" title="${h.tooltip(h.journal_filter_help())}">?</span>
 
    <input type='submit' value="${_('filter')}" class="ui-btn" style="padding:0px 2px 0px 2px;margin:0px"/>
 
    ${_('journal')} - ${ungettext('%s entry', '%s entries', c.journal_pager.item_count) % (c.journal_pager.item_count)}
 
    </form>
 
    ${h.end_form()}
 
    </h5>
 
</%def>
 
<%def name="page_nav()">
 
	${self.menu('home')}
 
</%def>
 
<%def name="head_extra()">
 
<link href="${h.url('journal_atom', api_key=c.rhodecode_user.api_key)}" rel="alternate" title="${_('ATOM journal feed')}" type="application/atom+xml" />
 
<link href="${h.url('journal_rss', api_key=c.rhodecode_user.api_key)}" rel="alternate" title="${_('RSS journal feed')}" type="application/rss+xml" />
 
</%def>
 
<%def name="main()">
 

	
 
    <div class="box box-left">
 
	    <!-- box / title -->
 
	    <div class="title">
 
	        <h5>${_('Journal')}</h5>
 
         ${self.breadcrumbs()}
 
             <ul class="links">
 
               <li>
 
                 <span><a id="refresh" href="${h.url('journal')}"><img class="icon" title="${_('Refresh')}" alt="${_('Refresh')}" src="${h.url('/images/icons/arrow_refresh.png')}"/></a></span>
 
               </li>
 
               <li>
 
                 <span><a href="${h.url('journal_rss', api_key=c.rhodecode_user.api_key)}"><img class="icon" title="${_('RSS feed')}" alt="${_('RSS feed')}" src="${h.url('/images/icons/rss_16.png')}"/></a></span>
 
               </li>
 
               <li>
 
                 <span><a href="${h.url('journal_atom', api_key=c.rhodecode_user.api_key)}"><img class="icon" title="${_('ATOM feed')}" alt="${_('ATOM feed')}" src="${h.url('/images/icons/atom.png')}"/></a></span>
 
               </li>
 
             </ul>
 
	    </div>
 
@@ -97,24 +105,44 @@
 
                %endfor
 
            </tbody>
 
            </table>
 
          %else:
 
              <div style="padding:5px 0px 10px 0px;">
 
              ${_('You are not following any users or repositories')}
 
              </div>
 
          %endif
 
        </div>
 
    </div>
 

	
 
    <script type="text/javascript">
 
    
 
    YUE.on('j_filter','click',function(){
 
        var jfilter = YUD.get('j_filter');
 
        if(YUD.hasClass(jfilter, 'initial')){
 
            jfilter.value = '';
 
        }
 
    });
 
    var fix_j_filter_width = function(len){
 
        YUD.setStyle(YUD.get('j_filter'),'width',Math.max(80, len*6.50)+'px');
 
    }
 
    YUE.on('j_filter','keyup',function(){
 
        fix_j_filter_width(YUD.get('j_filter').value.length);
 
    });
 
    YUE.on('filter_form','submit',function(e){
 
        YUE.preventDefault(e)
 
        var val = YUD.get('j_filter').value;
 
        window.location = "${url.current(filter='__FILTER__')}".replace('__FILTER__',val);
 
    });
 
    fix_j_filter_width(YUD.get('j_filter').value.length);    
 
    
 
    var show_my = function(e){
 
        YUD.setStyle('watched','display','none');
 
        YUD.setStyle('my','display','');
 

	
 
        var url = "${h.url('admin_settings_my_repos')}";
 
        ypjax(url, 'my', function(){
 
            tooltip_activate();
 
            quick_repo_menu();
 
            var nodes = YUQ('#my tr td a.repo_name');
 
            var func = function(node){
 
                return node.parentNode.parentNode.parentNode;
 
            }
 
@@ -144,25 +172,25 @@
 
    var tabs = {
 
        'watched': show_watched,
 
        'my': show_my,
 
    }
 
    var url = location.href.split('#');
 
    if (url[1]) {
 
        //We have a hash
 
        var tabHash = url[1];
 
        tabs[tabHash]();
 
    }
 

	
 
    YUE.on('refresh','click',function(e){
 
        ypjax(e.currentTarget.href,"journal",function(){
 
        ypjax("${h.url.current(filter=c.search_term)}","journal",function(){
 
        	show_more_event();
 
        	tooltip_activate();
 
        	show_changeset_tooltip();
 
        	});
 
        YUE.preventDefault(e);
 
    });
 

	
 

	
 
    // main table sorting
 
    var myColumnDefs = [
 
        {key:"menu",label:"",sortable:false,className:"quick_repo_menu hidden"},
 
        {key:"name",label:"${_('Name')}",sortable:true,
rhodecode/tests/functional/test_admin.py
Show inline comments
 
@@ -91,13 +91,25 @@ class TestAdminController(TestController
 
        response.mustcontain('1101 entries')
 

	
 
    def test_filter_journal_filter_prefix_on_user_or_other_user(self):
 
        self.log_user()
 
        response = self.app.get(url(controller='admin/admin', action='index',
 
                                    filter='username:demo OR username:volcan'))
 
        response.mustcontain('1095 entries')  # 1087 + 8
 

	
 
    def test_filter_journal_filter_wildcard_on_action(self):
 
        self.log_user()
 
        response = self.app.get(url(controller='admin/admin', action='index',
 
                                    filter='action:*pull_request*'))
 
        response.mustcontain('187 entries')
 
\ No newline at end of file
 
        response.mustcontain('187 entries')
 

	
 
    def test_filter_journal_filter_on_date(self):
 
        self.log_user()
 
        response = self.app.get(url(controller='admin/admin', action='index',
 
                                    filter='date:20121010'))
 
        response.mustcontain('47 entries')
 

	
 
    def test_filter_journal_filter_on_date_2(self):
 
        self.log_user()
 
        response = self.app.get(url(controller='admin/admin', action='index',
 
                                    filter='date:20121020'))
 
        response.mustcontain('17 entries')
 
\ No newline at end of file
0 comments (0 inline, 0 general)