kallithea Changeset - 3072935bdeed

Changeset - 3072935bdeed

Parent rev.

Child rev.

[Not reviewed]

default

1 2 0

Marcin Kuzminski - 15 years ago 2010-10-09 00:22:19
marcin@python-works.com

rewrote whoosh indexing to run internal repository.walk() instead of filesystem.
Disabled default hg update hook (not needed since whoosh is not dependent on file system files to index)

3 files changed with 26 insertions and 202 deletions:

rhodecode/lib/db_manage.py

rhodecode/lib/indexers/daemon.py

rhodecode/lib/indexers/multiprocessing_indexer.py

176

0 comments (0 inline, 0 general)

rhodecode/lib/db_manage.py

➞

Show inline comments

@@ @@ -112,12 +112,13 @@ class DbManage(object): @@
             sys.exit()
         hooks1 = RhodeCodeUi()
         hooks1.ui_section = 'hooks'
         hooks1.ui_key = 'changegroup.update'
         hooks1.ui_value = 'hg update >&2'
         hooks1.ui_active = False
         hooks2 = RhodeCodeUi()
         hooks2.ui_section = 'hooks'
         hooks2.ui_key = 'changegroup.repo_size'
         hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'

rhodecode/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -36,12 +36,15 @@ from rhodecode.lib.pidlock import LockHe @@
 from rhodecode.model.hg_model import HgModel
 from rhodecode.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
 from rhodecode.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
 from time import mktime
 from vcs.backends import hg
 import logging
 log = logging.getLogger('whooshIndexer')
 # create logger
 log.setLevel(logging.DEBUG)
 log.propagate = False
@@ @@ -59,72 +62,68 @@ ch.setFormatter(formatter) @@
 log.addHandler(ch)
 def scan_paths(root_location):
     return HgModel.repo_scan('/', root_location, None, True)
 class WhooshIndexingDaemon(object):
     """Deamon for atomic jobs"""
     """
     Deamon for atomic jobs
     """
     def __init__(self, indexname='HG_INDEX', repo_location=None):
         self.indexname = indexname
         self.repo_location = repo_location
         self.initial = False
         if not os.path.isdir(IDX_LOCATION):
             os.mkdir(IDX_LOCATION)
             log.info('Cannot run incremental index since it does not'
                      ' yet exist running full build')
             self.initial = True
     def get_paths(self, root_dir):
         """recursive walk in root dir and return a set of all path in that dir
         excluding files in .hg dir"""
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         repo = hg.MercurialRepository(root_dir)
         index_paths_ = set()
         for path, dirs, files in os.walk(root_dir):
             if path.find('.hg') == -1:
         for topnode, dirs, files in repo.walk('/', 'tip'):
                 for f in files:
                     index_paths_.add(jn(path, f))
                 index_paths_.add(jn(root_dir, f.path))
             for dir in dirs:
                 for f in files:
                     index_paths_.add(jn(root_dir, f.path))
         return index_paths_
     def add_doc(self, writer, path, repo):
         """Adding doc to writer"""
         n_path = path[len(repo.path) + 1:]
         node = repo.get_changeset().get_node(n_path)
         ext = unicode(path.split('/')[-1].split('.')[-1].lower())
         #we just index the content of choosen files
         if ext in INDEX_EXTENSIONS:
         #we just index the content of chosen files
         if node.extension in INDEX_EXTENSIONS:
             log.debug('    >> %s [WITH CONTENT]' % path)
             fobj = open(path, 'rb')
             content = fobj.read()
             fobj.close()
             u_content = safe_unicode(content)
             u_content = node.content
         else:
             log.debug('    >> %s' % path)
             #just index file name without it's content
             u_content = u''
         try:
             os.stat(path)
             writer.add_document(owner=unicode(repo.contact),
                             repository=safe_unicode(repo.name),
                             path=safe_unicode(path),
                             content=u_content,
                             modtime=os.path.getmtime(path),
                             extension=ext)
         except OSError, e:
             import errno
             if e.errno == errno.ENOENT:
                 log.debug('path %s does not exist or is a broken symlink' % path)
             else:
                 raise e
                         modtime=mktime(node.last_changeset.date.timetuple()),
                         extension=node.extension)
     def build_index(self):
         if os.path.exists(IDX_LOCATION):
-            log.debug('removing previos index')
+            log.debug('removing previous index')
             rmtree(IDX_LOCATION)
         if not os.path.exists(IDX_LOCATION):
             os.mkdir(IDX_LOCATION)
         idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)

rhodecode/lib/indexers/multiprocessing_indexer.py

➞

Show inline comments

deleted file

0 comments (0 inline, 0 general)