kallithea Changeset - 341beaa9edba

Changeset - 341beaa9edba

Parent rev.

Child rev.

[Not reviewed]

beta

0 10 0

Marcin Kuzminski - 15 years ago 2010-11-13 02:29:46
marcin@python-works.com

Implemented whoosh index building as paster command.
docs update
fixed manifest.in for missing yui file.
Fixed setup to beta
added base for paster upgrade-db command

10 files changed with 57 insertions and 35 deletions:

MANIFEST.in

development.ini

docs/changelog.rst

docs/setup.rst

production.ini

rhodecode/config/deployment.ini_tmpl

rhodecode/lib/indexers/__init__.py

rhodecode/lib/indexers/daemon.py

setup.cfg

setup.py

0 comments (0 inline, 0 general)

MANIFEST.in

➞

Show inline comments

@@ @@ -7,7 +7,7 @@ recursive-include rhodecode/i18n/ * @@
 recursive-include rhodecode/public/css *
 recursive-include rhodecode/public/images *
 #js
-include rhodecode/public/js/yui2.js
+include rhodecode/public/js/yui2a.js
 include rhodecode/public/js/excanvas.min.js
 include rhodecode/public/js/yui.flot.js
 include rhodecode/public/js/graph.js

development.ini

➞

Show inline comments

@@ @@ -43,6 +43,7 @@ full_stack = true @@
 static_files = true
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 ####################################
 ###         BEAKER CACHE        ####

docs/changelog.rst

➞

Show inline comments

@@ @@ -17,6 +17,7 @@ Changelog @@
 - introduced new enhanced changelog for merges that shows more accurate results
 - gui optimizations, fixed application width to 1024px
 - numerous small bugfixes
 - whoosh index moved to paster command
 .0.2 (**2010-11-12**)
 ----------------------

docs/setup.rst

➞

Show inline comments

@@ @@ -41,21 +41,40 @@ Setting up the application @@
   remember to update these if needed.
 Setting up Whoosh
 -----------------
 Setting up Whoosh full text search
 ----------------------------------
 Index for whoosh can be build starting from version 1.1 using paster command
 passing repo locations to index, as well as Your config file that stores
 whoosh index files locations. There is possible to pass `-f` to the options
 to enable full index rebuild. Without that indexing will run always in in
 incremental mode.
 ::
  paster make-index --repo-location=<location for repos> production.ini
 for full index rebuild You can use
 ::
  paster make-index -f --repo-location=<location for repos> production.ini
 - For full text search You can either put crontab entry for
 This command can be run even from crontab in order to do periodical
 index builds and keep Your index always up to date. An example entry might
 look like this
 ::
- python /var/www/rhodecode/<rhodecode_installation_path>/lib/indexers/daemon.py incremental <put_here_path_to_repos>
+ /path/to/python/bin/paster --repo-location=<location for repos> /path/to/rhodecode/production.ini
 When using incremental mode whoosh will check last modification date of each file
 and add it to reindex if newer file is available. Also indexing daemon checks
 for removed files and removes them from index. Sometime You might want to rebuild
 index from scrach, in admin pannel You can check `build from scratch` flag
 and in standalone daemon You can pass `full` instead on incremental to build
 remove previos index and build new one.
 When using incremental(default) mode whoosh will check last modification date
 of each file and add it to reindex if newer file is available. Also indexing
 daemon checks for removed files and removes them from index.
 Sometime You might want to rebuild index from scratch. You can do that using
 the `-f` flag passed to paster command or, in admin panel You can check
 `build from scratch` flag.
 Nginx virtual host example
 --------------------------

production.ini

➞

Show inline comments

@@ @@ -43,6 +43,7 @@ full_stack = true @@
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 ####################################
 ###         BEAKER CACHE        ####

rhodecode/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -43,6 +43,7 @@ full_stack = true @@
 static_files = true
 lang=en
 cache_dir = %(here)s/data
 index_dir = %(here)s/data/index
 app_instance_uuid = ${app_instance_uuid}
 ####################################

rhodecode/lib/indexers/__init__.py

➞

Show inline comments

@@ @@ -59,14 +59,14 @@ class MakeIndex(command.Command): @@
     usage = "CONFIG_FILE"
     summary = "Creates index for full text search given configuration file"
     group_name = "Whoosh indexing"
     group_name = "RhodeCode"
     takes_config_file = -1
     parser = command.Command.standard_parser(verbose=True)
 #    parser.add_option('--repo-location',
 #                      action='store',
 #                      dest='repo_location',
 #                      help="Specifies repositories location to index",
 #                      )
     parser.add_option('--repo-location',
                       action='store',
                       dest='repo_location',
                       help="Specifies repositories location to index REQUIRED",
+                      )
     parser.add_option('-f',
                       action='store_true',
                       dest='full_index',
@@ @@ -75,27 +75,23 @@ class MakeIndex(command.Command): @@
                       default=False)
     def command(self):
         config_name = self.args[0]
         p = config_name.split('/')
         if len(p) == 1:
             root = '.'
         else:
             root = '/'.join(p[:-1])
         print root
         root = '.' if len(p) == 1 else '/'.join(p[:-1])
         config = ConfigParser.ConfigParser({'here':root})
         config.read(config_name)
-        print dict(config.items('app:main'))['index_dir']
         index_location = dict(config.items('app:main'))['index_dir']
-        #return
+        repo_location = self.options.repo_location
-        #=======================================================================
         #======================================================================
         # WHOOSH DAEMON
-        #=======================================================================
         #======================================================================
         from rhodecode.lib.pidlock import LockHeld, DaemonLock
         from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
         try:
             l = DaemonLock()
             WhooshIndexingDaemon(index_location=index_location)\
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:

rhodecode/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -78,9 +78,7 @@ class WhooshIndexingDaemon(object): @@
         if not repo_location:
             raise Exception('You have to provide repositories location')
         self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True)
         self.repo_paths = HgModel().repo_scan(self.repo_location, None, True)
         self.initial = False
         if not os.path.isdir(self.index_location):
             os.mkdir(self.index_location)
@@ @@ -89,8 +87,7 @@ class WhooshIndexingDaemon(object): @@
             self.initial = True
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         """recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
@@ @@ -115,7 +112,8 @@ class WhooshIndexingDaemon(object): @@
         return mktime(node.last_changeset.date.timetuple())
     def add_doc(self, writer, path, repo):
         """Adding doc to writer"""
         """Adding doc to writer this function itself fetches data from
         the instance of vcs backend"""
         node = self.get_node(repo, path)
         #we just index the content of chosen files

setup.cfg

➞

Show inline comments

 [egg_info]
-tag_build = rc4
+tag_build = beta
 tag_svn_revision = true
 [easy_install]

setup.py

➞

Show inline comments

@@ @@ -89,5 +89,10 @@ setup( @@
     [paste.app_install]
     main = pylons.util:PylonsInstaller
     [paste.global_paster_command]
     make-index = rhodecode.lib.indexers:MakeIndex
     upgrade-db = rhodecode.lib.utils:UpgradeDb
     """,
+)

0 comments (0 inline, 0 general)