kallithea Changeset - 9b67cebe6609

Changeset - 9b67cebe6609

Parent rev.

Child rev.

[Not reviewed]

default

0 1 0

Marcin Kuzminski - 15 years ago 2010-08-18 19:41:08
marcin@python-works.com

some fixes to whoosh indexer daemon

1 file changed with 16 insertions and 16 deletions:

pylons_app/lib/indexers/daemon.py

0 comments (0 inline, 0 general)

pylons_app/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -22,41 +22,42 @@ Created on Jan 26, 2010 @@
 @author: marcink
 A deamon will read from task table and run tasks
 """
 import sys
 import os
 from pidlock import LockHeld, DaemonLock
 import traceback
 from os.path import dirname as dn
 from os.path import join as jn
 #to get the pylons_app import
 sys.path.append(dn(dn(dn(dn(os.path.realpath(__file__))))))
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 from pidlock import LockHeld, DaemonLock
 import traceback
 from pylons_app.config.environment import load_environment
 from pylons_app.model.hg_model import HgModel
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
 from pylons_app.lib.indexers import ANALYZER, EXCLUDE_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
 from pylons_app.lib.indexers import ANALYZER, EXCLUDE_EXTENSIONS, IDX_LOCATION, \
 SCHEMA, IDX_NAME
 import logging
 log = logging.getLogger(__name__)
 location = '/home/marcink/python_workspace_dirty/*'
 import logging.config
 logging.config.fileConfig(jn(project_path, 'development.ini'))
 log = logging.getLogger('whooshIndexer')
 def scan_paths(root_location):
     return HgModel.repo_scan('/', root_location, None, True)
 class WhooshIndexingDaemon(object):
     """Deamon for atomic jobs"""
     def __init__(self, indexname='HG_INDEX'):
+    def __init__(self, indexname='HG_INDEX', repo_location=None):
         self.indexname = indexname
+        self.repo_location = repo_location
     def get_paths(self, root_dir):
         """recursive walk in root dir and return a set of all path in that dir
         excluding files in .hg dir"""
         index_paths_ = set()
         for path, dirs, files in os.walk(root_dir):
@@ @@ -94,13 +95,13 @@ class WhooshIndexingDaemon(object): @@
         if not os.path.exists(IDX_LOCATION):
             os.mkdir(IDX_LOCATION)
         idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
-        for cnt, repo in enumerate(scan_paths(location).values()):
+        for cnt, repo in enumerate(scan_paths(self.repo_location).values()):
             log.debug('building index @ %s' % repo.path)
             for idx_path in self.get_paths(repo.path):
                 log.debug('    >> %s' % idx_path)
                 self.add_doc(writer, idx_path, repo)
         writer.commit(merge=True)
@@ @@ -146,13 +147,13 @@ class WhooshIndexingDaemon(object): @@
                     to_index.add(indexed_path)
                     #writer.commit()
         # Loop over the files in the filesystem
         # Assume we have a function that gathers the filenames of the
         # documents to be indexed
-        for repo in scan_paths(location).values():
+        for repo in scan_paths(self.repo_location).values():
             for path in self.get_paths(repo.path):
                 if path in to_index or path not in indexed_paths:
                     # This is either a file that's changed, or a new file
                     # that wasn't indexed before. So index it!
                     self.add_doc(writer, path, repo)
                     log.debug('reindexing %s' % path)
@@ @@ -166,16 +167,15 @@ class WhooshIndexingDaemon(object): @@
         if full_index:
             self.build_index()
         else:
             self.update_index()
 if __name__ == "__main__":
     repo_location = '/home/marcink/python_workspace_dirty/*'
     #config = load_environment()
     #print config
     try:
         l = DaemonLock()
         WhooshIndexingDaemon().run(full_index=True)
+        WhooshIndexingDaemon(repo_location=repo_location).run(full_index=True)
         l.release()
     except LockHeld:
         sys.exit(1)

0 comments (0 inline, 0 general)