kallithea Changeset - b7563ad4e7ee

Changeset - b7563ad4e7ee

Parent rev.

Child rev.

[Not reviewed]

beta

0 12 0

Marcin Kuzminski - 14 years ago 2011-06-30 01:25:37
marcin@python-works.com

Unicode fixes, added safe_str method for global str() operations +better test sandboxing

12 files changed with 93 insertions and 61 deletions:

rhodecode/config/environment.py

rhodecode/controllers/files.py

rhodecode/lib/__init__.py

rhodecode/lib/celerylib/tasks.py

rhodecode/lib/helpers.py

rhodecode/lib/middleware/simplegit.py

rhodecode/lib/middleware/simplehg.py

rhodecode/lib/utils.py

rhodecode/model/db.py

rhodecode/model/repo.py

rhodecode/model/scm.py

rhodecode/tests/functional/test_search.py

0 comments (0 inline, 0 general)

rhodecode/config/environment.py

➞

Show inline comments

@@ @@ -61,7 +61,7 @@ def load_environment(global_conf, app_co @@
         from rhodecode.lib.utils import create_test_env, create_test_index
         from rhodecode.tests import  TESTS_TMP_PATH
         create_test_env(TESTS_TMP_PATH, config)
         create_test_index(TESTS_TMP_PATH, True)
+        create_test_index(TESTS_TMP_PATH, config, True)
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine

rhodecode/controllers/files.py

➞

Show inline comments

@@ @@ -38,7 +38,7 @@ from vcs.exceptions import RepositoryErr @@
 from vcs.nodes import FileNode, NodeKind
 from vcs.utils import diffs as differ
 from rhodecode.lib import convert_line_endings, detect_mode
+from rhodecode.lib import convert_line_endings, detect_mode, safe_str
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
@@ @@ -153,7 +153,7 @@ class FilesController(BaseRepoController @@
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
         response.content_disposition = 'attachment; filename=%s' % \
-            f_path.split(os.sep)[-1].encode('utf8', 'replace')
+            safe_str(f_path.split(os.sep)[-1])
         response.content_type = file_node.mimetype
         return file_node.content
@@ @@ -198,7 +198,7 @@ class FilesController(BaseRepoController @@
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
-                        f_path.split(os.sep)[-1].encode('utf8', 'replace')
+                        safe_str(f_path.split(os.sep)[-1])
         response.content_disposition = dispo
         response.content_type = mimetype

rhodecode/lib/__init__.py

➞

Show inline comments

@@ @@ -160,7 +160,7 @@ def generate_api_key(username, salt=None @@
 def safe_unicode(_str, from_encoding='utf8'):
     """
     safe unicode function. In case of UnicodeDecode error we try to return
-    unicode with errors replace
+    unicode with errors replaceed
     :param _str: string to decode
     :rtype: unicode
@@ @@ -178,6 +178,28 @@ def safe_unicode(_str, from_encoding='ut @@
     return u_str
 def safe_str(_unicode, to_encoding='utf8'):
     """
     safe str function. In case of UnicodeEncode error we try to return
     str with errors replaceed
     :param _unicode: unicode to encode
     :rtype: str
     :returns: str object
     """
     if isinstance(_unicode, str):
         return _unicode
     try:
         safe_str = str(_unicode)
     except UnicodeEncodeError:
         safe_str = _unicode.encode(to_encoding, 'replace')
     return safe_str
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for

rhodecode/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -37,7 +37,7 @@ from string import lower @@
 from pylons import config
 from pylons.i18n.translation import _
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP
+from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \
     __get_lockkey, LockHeld, DaemonLock
 from rhodecode.lib.helpers import person
@@ @@ -66,7 +66,6 @@ __all__ = ['whoosh_index', 'get_commits_ @@
 CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
@@ @@ -112,8 +111,7 @@ def get_commits_stats(repo_name, ts_min_ @@
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repos_path = get_repos_path()
         p = os.path.join(repos_path, repo_name)
         repo = get_repo(p)
         repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
         repo_size = len(repo.revisions)
         #return if repo have no revisions
         if repo_size < 1:
@@ @@ -358,8 +356,7 @@ def create_repo_fork(form_data, cur_user @@
 def __get_codes_stats(repo_name):
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
     tip = repo.get_changeset()
     code_stats = {}

rhodecode/lib/helpers.py

➞

Show inline comments

@@ @@ -36,7 +36,7 @@ from webhelpers.html.tags import _set_in @@
 from vcs.utils.annotate import annotate_highlight
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib import str2bool, safe_unicode
+from rhodecode.lib import str2bool, safe_unicode, safe_str
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
@@ @@ -502,7 +502,7 @@ def gravatar_url(email_address, size=30) @@
     if isinstance(email_address, unicode):
         #hashlib crashes on unicode items
-        email_address = email_address.encode('utf8', 'replace')
+        email_address = safe_str(email_address)
     # construct the url
     gravatar_url = baseurl + hashlib.md5(email_address.lower()).hexdigest() + "?"
     gravatar_url += urllib.urlencode({'d':default, 's':str(size)})

rhodecode/lib/middleware/simplegit.py

➞

Show inline comments

@@ @@ -69,6 +69,7 @@ from dulwich.web import HTTPGitApplicati @@
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import invalidate_cache, check_repo_fast
 from rhodecode.model.user import UserModel
@@ @@ -147,8 +148,8 @@ class SimpleGit(object): @@
                 #==============================================================
                 if not REMOTE_USER(environ):
                     self.authenticate.realm = self.config['rhodecode_realm'].\
                         encode('utf8', 'replace')
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')

rhodecode/lib/middleware/simplehg.py

➞

Show inline comments

@@ @@ -35,6 +35,7 @@ from mercurial.hgweb.request import wsgi @@
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
@@ @@ -112,8 +113,8 @@ class SimpleHg(object): @@
                 #==============================================================
                 if not REMOTE_USER(environ):
                     self.authenticate.realm = self.config['rhodecode_realm'].\
                         encode('utf8', 'replace')
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -458,18 +458,22 @@ def get_current_revision(): @@
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
     :param repo_location:
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
     :param config: test config
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     import shutil
     repo_location = repo_location
     index_location = os.path.join(repo_location, 'index')
     if os.path.exists(index_location):
         shutil.rmtree(index_location)
     index_location = os.path.join(config['app_conf']['index_dir'], 'index')
     if not os.path.exists(index_location):
         os.makedirs(index_location)
     try:
         l = DaemonLock(file=jn(dn(index_location), 'make_index.lock'))

rhodecode/model/db.py

➞

Show inline comments

@@ @@ -42,7 +42,7 @@ from vcs.exceptions import RepositoryErr @@
 from vcs.utils.lazy import LazyProperty
 from vcs.nodes import FileNode
 from rhodecode.lib import str2bool, json
+from rhodecode.lib import str2bool, json, safe_str
 from rhodecode.model.meta import Base, Session
 from rhodecode.model.caching_query import FromCache
@@ @@ -479,7 +479,11 @@ class Repository(Base, BaseModel): @@
             Session.add(inv)
             Session.commit()
         return _c(self.repo_name)
         # TODO: remove this trick when beaker 1.6 is released
         # and have fixed this issue
         rn = safe_str(self.repo_name)
         return _c(rn)
     def __get_instance(self):
@@ @@ -497,7 +501,8 @@ class Repository(Base, BaseModel): @@
             return
         if alias == 'hg':
             repo = backend(repo_full_path, create=False,
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             #skip hidden web repository
             if repo._get_hidden():

rhodecode/model/repo.py

➞

Show inline comments

@@ @@ -33,6 +33,8 @@ from sqlalchemy.orm import joinedload, m @@
 from vcs.utils.lazy import LazyProperty
 from vcs.backends import get_backend
 from rhodecode.lib import safe_str
 from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
@@ @@ -171,13 +173,12 @@ class RepoModel(BaseModel): @@
         try:
             if fork:
                 #force str since hg doesn't go with unicode
                 repo_name = str(form_data['fork_name'])
                 org_name = str(form_data['repo_name'])
                 org_full_name = org_name#str(form_data['fork_name_full'])
                 repo_name = form_data['fork_name']
                 org_name = form_data['repo_name']
                 org_full_name = org_name
             else:
-                org_name = repo_name = str(form_data['repo_name'])
                 org_name = repo_name = form_data['repo_name']
                 repo_name_full = form_data['repo_name_full']
             new_repo = Repository()
@@ @@ -302,21 +303,23 @@ class RepoModel(BaseModel): @@
         """
         from rhodecode.lib.utils import check_repo
         if new_parent_id:
             paths = Group.get(new_parent_id).full_path.split(Group.url_sep())
             new_parent_path = os.sep.join(paths)
         else:
             new_parent_path = ''
         repo_path = os.path.join(self.repos_path, new_parent_path, repo_name)
         repo_path = os.path.join(*map(lambda x:safe_str(x),
                                 [self.repos_path, new_parent_path, repo_name]))
-        if check_repo(repo_name, self.repos_path):
+        if check_repo(repo_path, self.repos_path):
             log.info('creating repo %s in %s @ %s', repo_name, repo_path,
                     clone_uri)
+                     clone_uri)
             backend = get_backend(alias)
             backend(repo_path, create=True, src_url=clone_uri)
     def __rename_repo(self, old, new):
         """
         renames repository on filesystem

rhodecode/model/scm.py

➞

Show inline comments

@@ @@ -27,12 +27,7 @@ import time @@
 import traceback
 import logging
 from mercurial import ui
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import make_transient
 from beaker.cache import cache_region, region_invalidate
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
@@ @@ -42,15 +37,14 @@ from vcs.nodes import FileNode @@
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
     action_logger
 from rhodecode.model import BaseModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
     UserFollowing, UserLog
 from rhodecode.model.caching_query import FromCache
 log = logging.getLogger(__name__)
@@ @@ -182,7 +176,10 @@ class ScmModel(BaseModel): @@
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1], baseui=baseui)
                         # for mercurial we need to have an str path
                         repos_list[name] = klass(safe_str(path[1]),
                                                  baseui=baseui)
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1])
@@ @@ -364,10 +361,10 @@ class ScmModel(BaseModel): @@
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = content.encode('utf8')
         message = message.encode('utf8')
         path = f_path.encode('utf8')
         author = author.encode('utf8')
         content = safe_str(content)
         message = safe_str(message)
         path = safe_str(f_path)
         author = safe_str(author)
         m = IMC(repo)
         m.change(FileNode(path, content))
         tip = m.commit(message=message,

rhodecode/tests/functional/test_search.py

➞

Show inline comments

@@ @@ -7,8 +7,9 @@ class TestSearchController(TestControlle @@
     def test_index(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'))
         print response.body
         assert 'class="small" id="q" name="q" type="text"' in response.body, 'Search box content error'
         self.assertTrue('class="small" id="q" name="q" type="text"' in
                         response.body)
         # Test response...
     def test_empty_search(self):
@@ @@ -16,20 +17,21 @@ class TestSearchController(TestControlle @@
             raise SkipTest('skipped due to existing index')
         else:
             self.log_user()
             response = self.app.get(url(controller='search', action='index'), {'q':HG_REPO})
             assert 'There is no index to search in. Please run whoosh indexer' in response.body, 'No error message about empty index'
             response = self.app.get(url(controller='search', action='index'),
                                     {'q':HG_REPO})
             self.assertTrue('There is no index to search in. '
                             'Please run whoosh indexer' in response.body)
     def test_normal_search(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'), {'q':'def repo'})
         print response.body
         assert '10 results' in response.body, 'no message about proper search results'
         assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user'
         response = self.app.get(url(controller='search', action='index'),
                                 {'q':'def repo'})
         self.assertTrue('10 results' in response.body)
         self.assertTrue('Permission denied' not in response.body)
     def test_repo_search(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'), {'q':'repository:%s def test' % HG_REPO})
         print response.body
         assert '4 results' in response.body, 'no message about proper search results'
         assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user'
         response = self.app.get(url(controller='search', action='index'),
                                 {'q':'repository:%s def test' % HG_REPO})
         self.assertTrue('4 results' in response.body)
         self.assertTrue('Permission denied' not in response.body)

0 comments (0 inline, 0 general)