kallithea Changeset - b6c902d88472

Changeset - b6c902d88472

Parent rev.

Child rev.

[Not reviewed]

beta

0 7 0

Marcin Kuzminski - 14 years ago 2012-02-16 04:41:58
marcin@python-works.com

bumbed whoosh to 2.3.X series
- fixed some html issues on search results

7 files changed with 62 insertions and 46 deletions:

requires.txt

rhodecode/__init__.py

rhodecode/controllers/search.py

rhodecode/lib/indexers/__init__.py

rhodecode/lib/indexers/daemon.py

rhodecode/public/css/pygments.css

rhodecode/templates/search/search_content.html

0 comments (0 inline, 0 general)

requires.txt

➞

Show inline comments

@@ @@ -5,7 +5,7 @@ formencode==1.2.4 @@
 SQLAlchemy==0.7.4
 Mako==0.5.0
 pygments>=1.4
-whoosh<1.8
+whoosh<2.4
 celery>=2.2.5,<2.3
 babel
 python-dateutil>=1.5.0,<2.0.0

rhodecode/__init__.py

➞

Show inline comments

@@ @@ -44,7 +44,7 @@ requirements = [ @@
     "SQLAlchemy==0.7.4",
     "Mako==0.5.0",
     "pygments>=1.4",
-    "whoosh<1.8",
+    "whoosh<2.4",
     "celery>=2.2.5,<2.3",
     "babel",
     "python-dateutil>=1.5.0,<2.0.0",

rhodecode/controllers/search.py

➞

Show inline comments

@@ @@ -26,7 +26,7 @@ import logging @@
 import traceback
 from pylons.i18n.translation import _
-from pylons import request, config, session, tmpl_context as c
 from pylons import request, config, tmpl_context as c
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
@@ @@ -76,7 +76,7 @@ class SearchController(BaseController): @@
                     cur_query = u'repository:%s %s' % (c.repo_name, cur_query)
                 try:
                     query = qp.parse(unicode(cur_query))
+                    # extract words for highlight
                     if isinstance(query, Phrase):
                         highlight_items.update(query.words)
                     elif isinstance(query, Prefix):
@@ @@ -92,18 +92,22 @@ class SearchController(BaseController): @@
                     log.debug(highlight_items)
                     results = searcher.search(query)
                     res_ln = len(results)
                     c.runtime = '%s results (%.3f seconds)' \
                         % (res_ln, results.runtime)
                     c.runtime = '%s results (%.3f seconds)' % (
                         res_ln, results.runtime
+                    )
                     def url_generator(**kw):
                         return update_params("?q=%s&type=%s" \
                                            % (c.cur_query, c.cur_search), **kw)
                     c.formated_results = Page(
                                 ResultWrapper(search_type, searcher, matcher,
                                               highlight_items),
                                 page=p, item_count=res_ln,
                                 items_per_page=10, url=url_generator)
                         ResultWrapper(search_type, searcher, matcher,
                                       highlight_items),
                         page=p,
                         item_count=res_ln,
                         items_per_page=10,
                         url=url_generator
+                    )
                 except QueryParserError:
                     c.runtime = _('Invalid search query. Try quoting it.')
@@ @@ -117,5 +121,6 @@ class SearchController(BaseController): @@
                 log.error(traceback.format_exc())
                 c.runtime = _('An error occurred during this search operation')
         # Return a rendered template
         return render('/search/search.html')

rhodecode/lib/indexers/__init__.py

➞

Show inline comments

@@ @@ -37,38 +37,39 @@ from whoosh.analysis import RegexTokeniz @@
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
-from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
+from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter
 from webhelpers.html.builder import escape
 from sqlalchemy import engine_from_config
 from vcs.utils.lazy import LazyProperty
 from rhodecode.model import init_model
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.config.environment import load_environment
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP
+from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, LazyProperty
 from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
+# EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys()
 #CUSTOM ANALYZER wordsplit + lowercase filter
+# CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=TEXT(stored=True),
                 content=FieldType(format=Characters(ANALYZER),
                              scorable=True, stored=True),
                 modtime=STORED(), extension=TEXT(stored=True))
 SCHEMA = Schema(
     owner=TEXT(),
     repository=TEXT(stored=True),
     path=TEXT(stored=True),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     extension=TEXT(stored=True)
+)
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
-FRAGMENTER = SimpleFragmenter(200)
+FRAGMENTER = ContextFragmenter(200)
 class MakeIndex(BasePasterCommand):
@@ @@ -136,7 +137,7 @@ class ResultWrapper(object): @@
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
-        self.fragment_size = 200 / 2
         self.fragment_size = 200
     @LazyProperty
     def doc_ids(self):
@@ @@ -172,10 +173,10 @@ class ResultWrapper(object): @@
         """
         i, j = key.start, key.stop
-        slice = []
+        slices = []
         for docid in self.doc_ids[i:j]:
             slice.append(self.get_full_content(docid))
         return slice
             slices.append(self.get_full_content(docid))
         return slices
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
@@ @@ -183,9 +184,9 @@ class ResultWrapper(object): @@
                              + len(res['repository']):].lstrip('/')
         content_short = self.get_short_content(res, docid[1])
         res.update({'content_short':content_short,
                     'content_short_hl':self.highlight(content_short),
                     'f_path':f_path})
         res.update({'content_short': content_short,
                     'content_short_hl': self.highlight(content_short),
                     'f_path': f_path})
         return res
@@ @@ -217,10 +218,12 @@ class ResultWrapper(object): @@
     def highlight(self, content, top=5):
         if self.search_type != 'content':
             return ''
         hl = highlight(escape(content),
                  self.highlight_items,
                  analyzer=ANALYZER,
                  fragmenter=FRAGMENTER,
                  formatter=FORMATTER,
                  top=top)
         hl = highlight(
             text=escape(content),
             terms=self.highlight_items,
             analyzer=ANALYZER,
             fragmenter=FRAGMENTER,
             formatter=FORMATTER,
             top=top
+        )
         return hl

rhodecode/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -49,7 +49,6 @@ from vcs.exceptions import ChangesetErro @@
 from whoosh.index import create_in, open_dir
 log = logging.getLogger('whooshIndexer')
 # create logger
 log.setLevel(logging.DEBUG)
@@ @@ -68,12 +67,13 @@ ch.setFormatter(formatter) @@
 # add ch to logger
 log.addHandler(ch)
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic jobs
     """
-    def __init__(self, indexname='HG_INDEX', index_location=None,
+    def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, sa=None, repo_list=None):
         self.indexname = indexname
@@ @@ -95,7 +95,6 @@ class WhooshIndexingDaemon(object): @@
             self.repo_paths = filtered_repo_paths
         self.initial = False
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
@@ @@ -155,7 +154,6 @@ class WhooshIndexingDaemon(object): @@
                         modtime=self.get_node_mtime(node),
                         extension=node.extension)
     def build_index(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
@@ @@ -177,7 +175,6 @@ class WhooshIndexingDaemon(object): @@
         writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')

rhodecode/public/css/pygments.css

➞

Show inline comments

@@ @@ -65,10 +65,20 @@ div.codeblock .code-body table td { @@
 div.code-body {
 	background-color: #FFFFFF;
+}
 div.code-body pre .match{
 div.codeblock .code-header .search-path {
 	padding: 0px 0px 0px 10px;
+}
 div.search-code-body {
     background-color: #FFFFFF;
     padding: 5px 0px 5px 10px;
+}
 div.search-code-body pre .match{
 	background-color: #FAFFA6;
+}
 div.code-body pre .break{
+div.search-code-body pre .break{
 	background-color: #DDE7EF;
 	width: 100%;
 	color: #747474;

rhodecode/templates/search/search_content.html

➞

Show inline comments

@@ @@ -5,10 +5,11 @@ @@
     <div class="table">
         <div id="body${cnt}" class="codeblock">
             <div class="code-header">
                 <div class="revision">${h.link_to(h.literal('%s &raquo; %s' % (sr['repository'],sr['f_path'])),
                 h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}</div>
                 <div class="search-path">${h.link_to(h.literal('%s &raquo; %s' % (sr['repository'],sr['f_path'])),
                 h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}
                 </div>
             </div>
             <div class="code-body">
+            <div class="search-code-body">
                 <pre>${h.literal(sr['content_short_hl'])}</pre>
             </div>
         </div>

0 comments (0 inline, 0 general)