Changeset - 932c84e8fa92
[Not reviewed]
beta
0 1 0
Marcin Kuzminski - 12 years ago 2013-05-30 22:37:08
marcin@python-works.com
fixed #851 and #563 make-index crashes on non-ascii files
1 file changed with 11 insertions and 2 deletions:
0 comments (0 inline, 0 general)
rhodecode/lib/indexers/daemon.py
Show inline comments
 
@@ -119,51 +119,60 @@ class WhooshIndexingDaemon(object):
 

	
 
    def _get_index_changeset(self, repo):
 
        index_rev = self._get_index_revision(repo)
 
        cs = repo.get_changeset(index_rev)
 
        return cs
 

	
 
    def get_paths(self, repo):
 
        """
 
        recursive walk in root dir and return a set of all path in that dir
 
        based on repository walk function
 
        """
 
        index_paths_ = set()
 
        try:
 
            cs = self._get_index_changeset(repo)
 
            for _topnode, _dirs, files in cs.walk('/'):
 
                for f in files:
 
                    index_paths_.add(jn(safe_str(repo.path), safe_str(f.path)))
 

	
 
        except RepositoryError:
 
            log.debug(traceback.format_exc())
 
            pass
 
        return index_paths_
 

	
 
    def get_node(self, repo, path):
 
        n_path = path[len(repo.path) + 1:]
 
        """
 
        gets a filenode based on given full path.It operates on string for
 
        hg git compatability.
 

	
 
        :param repo: scm repo instance
 
        :param path: full path including root location
 
        :return: FileNode
 
        """
 
        root_path = safe_str(repo.path)+'/'
 
        parts = safe_str(path).partition(root_path)
 
        cs = self._get_index_changeset(repo)
 
        node = cs.get_node(n_path)
 
        node = cs.get_node(parts[-1])
 
        return node
 

	
 
    def get_node_mtime(self, node):
 
        return mktime(node.last_changeset.date.timetuple())
 

	
 
    def add_doc(self, writer, path, repo, repo_name):
 
        """
 
        Adding doc to writer this function itself fetches data from
 
        the instance of vcs backend
 
        """
 

	
 
        node = self.get_node(repo, path)
 
        indexed = indexed_w_content = 0
 
        # we just index the content of chosen files, and skip binary files
 
        if node.extension in INDEX_EXTENSIONS and not node.is_binary:
 
            u_content = node.content
 
            if not isinstance(u_content, unicode):
 
                log.warning('  >> %s Could not get this content as unicode '
 
                            'replacing with empty content' % path)
 
                u_content = u''
 
            else:
 
                log.debug('    >> %s [WITH CONTENT]' % path)
 
                indexed_w_content += 1
 

	
0 comments (0 inline, 0 general)