kallithea Changeset - b7563ad4e7ee

Changeset - b7563ad4e7ee

Parent rev.

Child rev.

[Not reviewed]

beta

0 12 0

Marcin Kuzminski - 14 years ago 2011-06-30 01:25:37
marcin@python-works.com

Unicode fixes, added safe_str method for global str() operations +better test sandboxing

12 files changed with 92 insertions and 60 deletions:

rhodecode/config/environment.py

rhodecode/controllers/files.py

rhodecode/lib/__init__.py

rhodecode/lib/celerylib/tasks.py

rhodecode/lib/helpers.py

rhodecode/lib/middleware/simplegit.py

rhodecode/lib/middleware/simplehg.py

rhodecode/lib/utils.py

rhodecode/model/db.py

rhodecode/model/repo.py

rhodecode/model/scm.py

rhodecode/tests/functional/test_search.py

0 comments (0 inline, 0 general)

rhodecode/config/environment.py

➞

Show inline comments

@@ @@ -58,13 +58,13 @@ def load_environment(global_conf, app_co @@
     config['pylons.strict_tmpl_context'] = True
     test = os.path.split(config['__file__'])[-1] == 'test.ini'
     if test:
         from rhodecode.lib.utils import create_test_env, create_test_index
         from rhodecode.tests import  TESTS_TMP_PATH
         create_test_env(TESTS_TMP_PATH, config)
         create_test_index(TESTS_TMP_PATH, True)
+        create_test_index(TESTS_TMP_PATH, config, True)
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
     sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(sa_engine_db1)

rhodecode/controllers/files.py

➞

Show inline comments

@@ @@ -35,13 +35,13 @@ from pylons.controllers.util import redi @@
 from vcs.backends import ARCHIVE_SPECS
 from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
     EmptyRepositoryError, ImproperArchiveTypeError, VCSError
 from vcs.nodes import FileNode, NodeKind
 from vcs.utils import diffs as differ
 from rhodecode.lib import convert_line_endings, detect_mode
+from rhodecode.lib import convert_line_endings, detect_mode, safe_str
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 import rhodecode.lib.helpers as h
 from rhodecode.model.repo import RepoModel
@@ @@ -150,13 +150,13 @@ class FilesController(BaseRepoController @@
                                    'repository.admin')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
         response.content_disposition = 'attachment; filename=%s' % \
-            f_path.split(os.sep)[-1].encode('utf8', 'replace')
+            safe_str(f_path.split(os.sep)[-1])
         response.content_type = file_node.mimetype
         return file_node.content
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
@@ @@ -195,13 +195,13 @@ class FilesController(BaseRepoController @@
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
-                        f_path.split(os.sep)[-1].encode('utf8', 'replace')
+                        safe_str(f_path.split(os.sep)[-1])
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',

rhodecode/lib/__init__.py

➞

Show inline comments

@@ @@ -157,13 +157,13 @@ def generate_api_key(username, salt=None @@
     return hashlib.sha1(username + salt).hexdigest()
 def safe_unicode(_str, from_encoding='utf8'):
     """
     safe unicode function. In case of UnicodeDecode error we try to return
-    unicode with errors replace
+    unicode with errors replaceed
     :param _str: string to decode
     :rtype: unicode
     :returns: unicode object
     """
@@ @@ -175,12 +175,34 @@ def safe_unicode(_str, from_encoding='ut @@
     except UnicodeDecodeError:
         u_str = unicode(_str, from_encoding, 'replace')
     return u_str
 def safe_str(_unicode, to_encoding='utf8'):
     """
     safe str function. In case of UnicodeEncode error we try to return
     str with errors replaceed
     :param _unicode: unicode to encode
     :rtype: str
     :returns: str object
     """
     if isinstance(_unicode, str):
         return _unicode
     try:
         safe_str = str(_unicode)
     except UnicodeEncodeError:
         safe_str = _unicode.encode(to_encoding, 'replace')
     return safe_str
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for
     file based sqlite databases. This prevents errors on sqlite. This only
     applies to sqlalchemy versions < 0.7.0

rhodecode/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -34,13 +34,13 @@ from time import mktime @@
 from operator import itemgetter
 from string import lower
 from pylons import config
 from pylons.i18n.translation import _
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP
+from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \
     __get_lockkey, LockHeld, DaemonLock
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import add_cache
 from rhodecode.lib.odict import OrderedDict
@@ @@ -63,13 +63,12 @@ except ImportError: @@
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session()
     return sa
@@ @@ -109,14 +108,13 @@ def get_commits_stats(repo_name, ts_min_ @@
         #for js data compatibilty cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repos_path = get_repos_path()
         p = os.path.join(repos_path, repo_name)
         repo = get_repo(p)
         repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
         repo_size = len(repo.revisions)
         #return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
@@ @@ -355,14 +353,13 @@ def create_repo_fork(form_data, cur_user @@
     backend = get_backend(alias)
     backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 def __get_codes_stats(repo_name):
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
     tip = repo.get_changeset()
     code_stats = {}
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)

rhodecode/lib/helpers.py

➞

Show inline comments

@@ @@ -33,13 +33,13 @@ from webhelpers.date import time_ago_in_ @@
 from webhelpers.paginate import Page
 from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
     convert_boolean_attrs, NotGiven
 from vcs.utils.annotate import annotate_highlight
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib import str2bool, safe_unicode
+from rhodecode.lib import str2bool, safe_unicode, safe_str
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     _set_input_attrs(attrs, type, name, value)
@@ @@ -499,13 +499,13 @@ def gravatar_url(email_address, size=30) @@
     baseurl_nossl = "http://www.gravatar.com/avatar/"
     baseurl_ssl = "https://secure.gravatar.com/avatar/"
     baseurl = baseurl_ssl if ssl_enabled else baseurl_nossl
     if isinstance(email_address, unicode):
         #hashlib crashes on unicode items
-        email_address = email_address.encode('utf8', 'replace')
+        email_address = safe_str(email_address)
     # construct the url
     gravatar_url = baseurl + hashlib.md5(email_address.lower()).hexdigest() + "?"
     gravatar_url += urllib.urlencode({'d':default, 's':str(size)})
     return gravatar_url

rhodecode/lib/middleware/simplegit.py

➞

Show inline comments

@@ @@ -66,12 +66,13 @@ dulserver.DEFAULT_HANDLERS = { @@
 from dulwich.repo import Repo
 from dulwich.web import HTTPGitApplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import invalidate_cache, check_repo_fast
 from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
@@ @@ -144,14 +145,14 @@ class SimpleGit(object): @@
                 #==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 #==============================================================
                 if not REMOTE_USER(environ):
                     self.authenticate.realm = self.config['rhodecode_realm'].\
                         encode('utf8', 'replace')
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                     else:
                         return result.wsgi_application(environ, start_response)

rhodecode/lib/middleware/simplehg.py

➞

Show inline comments

@@ @@ -32,12 +32,13 @@ from mercurial.error import RepoError @@
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
 from rhodecode.model.user import UserModel
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
@@ @@ -109,14 +110,14 @@ class SimpleHg(object): @@
                 #==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 #==============================================================
                 if not REMOTE_USER(environ):
                     self.authenticate.realm = self.config['rhodecode_realm'].\
                         encode('utf8', 'replace')
                     self.authenticate.realm = \
                         safe_str(self.config['rhodecode_realm'])
                     result = self.authenticate(environ)
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                     else:
                         return result.wsgi_application(environ, start_response)

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -455,24 +455,28 @@ def get_current_revision(): @@
         return None
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
     :param repo_location:
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
     :param config: test config
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     import shutil
     repo_location = repo_location
     index_location = os.path.join(repo_location, 'index')
     if os.path.exists(index_location):
         shutil.rmtree(index_location)
     index_location = os.path.join(config['app_conf']['index_dir'], 'index')
     if not os.path.exists(index_location):
         os.makedirs(index_location)
     try:
         l = DaemonLock(file=jn(dn(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)

rhodecode/model/db.py

➞

Show inline comments

@@ @@ -39,13 +39,13 @@ from beaker.cache import cache_region, r @@
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
 from vcs.nodes import FileNode
 from rhodecode.lib import str2bool, json
+from rhodecode.lib import str2bool, json, safe_str
 from rhodecode.model.meta import Base, Session
 from rhodecode.model.caching_query import FromCache
 log = logging.getLogger(__name__)
 #==============================================================================
@@ @@ -476,13 +476,17 @@ class Repository(Base, BaseModel): @@
             region_invalidate(_c, None, self.repo_name)
             #update our cache
             inv.cache_key.cache_active = True
             Session.add(inv)
             Session.commit()
         return _c(self.repo_name)
         # TODO: remove this trick when beaker 1.6 is released
         # and have fixed this issue
         rn = safe_str(self.repo_name)
         return _c(rn)
     def __get_instance(self):
         repo_full_path = self.repo_full_path
         try:
@@ @@ -494,13 +498,14 @@ class Repository(Base, BaseModel): @@
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
         if alias == 'hg':
             repo = backend(repo_full_path, create=False,
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             #skip hidden web repository
             if repo._get_hidden():
                 return
         else:
             repo = backend(repo_full_path, create=False)

rhodecode/model/repo.py

➞

Show inline comments

@@ @@ -30,12 +30,14 @@ from datetime import datetime @@
 from sqlalchemy.orm import joinedload, make_transient
 from vcs.utils.lazy import LazyProperty
 from vcs.backends import get_backend
 from rhodecode.lib import safe_str
 from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
     Statistics, UsersGroup, UsersGroupRepoToPerm, RhodeCodeUi, Group
 from rhodecode.model.user import UserModel
@@ @@ -168,19 +170,18 @@ class RepoModel(BaseModel): @@
             raise
     def create(self, form_data, cur_user, just_db=False, fork=False):
         try:
             if fork:
                 #force str since hg doesn't go with unicode
                 repo_name = str(form_data['fork_name'])
                 org_name = str(form_data['repo_name'])
                 org_full_name = org_name#str(form_data['fork_name_full'])
                 repo_name = form_data['fork_name']
                 org_name = form_data['repo_name']
                 org_full_name = org_name
             else:
-                org_name = repo_name = str(form_data['repo_name'])
                 org_name = repo_name = form_data['repo_name']
                 repo_name_full = form_data['repo_name_full']
             new_repo = Repository()
             new_repo.enable_statistics = False
             for k, v in form_data.items():
                 if k == 'repo_name':
@@ @@ -299,27 +300,29 @@ class RepoModel(BaseModel): @@
         :param alias:
         :param parent_id:
         :param clone_uri:
         """
         from rhodecode.lib.utils import check_repo
         if new_parent_id:
             paths = Group.get(new_parent_id).full_path.split(Group.url_sep())
             new_parent_path = os.sep.join(paths)
         else:
             new_parent_path = ''
         repo_path = os.path.join(self.repos_path, new_parent_path, repo_name)
         repo_path = os.path.join(*map(lambda x:safe_str(x),
                                 [self.repos_path, new_parent_path, repo_name]))
-        if check_repo(repo_name, self.repos_path):
+        if check_repo(repo_path, self.repos_path):
             log.info('creating repo %s in %s @ %s', repo_name, repo_path,
                     clone_uri)
             backend = get_backend(alias)
             backend(repo_path, create=True, src_url=clone_uri)
     def __rename_repo(self, old, new):
         """
         renames repository on filesystem
         :param old: old name
         :param new: new name

rhodecode/model/scm.py

➞

Show inline comments

@@ @@ -24,36 +24,30 @@ @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import os
 import time
 import traceback
 import logging
 from mercurial import ui
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import make_transient
 from beaker.cache import cache_region, region_invalidate
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
 from vcs.nodes import FileNode
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
 from rhodecode.lib import safe_str
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
     action_logger
 from rhodecode.model import BaseModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
     UserFollowing, UserLog
 from rhodecode.model.caching_query import FromCache
 log = logging.getLogger(__name__)
 class UserTemp(object):
     def __init__(self, user_id):
@@ @@ -179,13 +173,16 @@ class ScmModel(BaseModel): @@
                                     'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1], baseui=baseui)
                         # for mercurial we need to have an str path
                         repos_list[name] = klass(safe_str(path[1]),
                                                  baseui=baseui)
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1])
             except OSError:
                 continue
@@ @@ -361,16 +358,16 @@ class ScmModel(BaseModel): @@
             from vcs.backends.hg import MercurialInMemoryChangeset as IMC
         elif repo.alias == 'git':
             from vcs.backends.git import GitInMemoryChangeset as IMC
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = content.encode('utf8')
         message = message.encode('utf8')
         path = f_path.encode('utf8')
         author = author.encode('utf8')
         content = safe_str(content)
         message = safe_str(message)
         path = safe_str(f_path)
         author = safe_str(author)
         m = IMC(repo)
         m.change(FileNode(path, content))
         tip = m.commit(message=message,
                  author=author,
                  parents=[cs], branch=cs.branch)

rhodecode/tests/functional/test_search.py

➞

Show inline comments

@@ @@ -4,32 +4,34 @@ from nose.plugins.skip import SkipTest @@
 class TestSearchController(TestController):
     def test_index(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'))
         print response.body
         assert 'class="small" id="q" name="q" type="text"' in response.body, 'Search box content error'
         self.assertTrue('class="small" id="q" name="q" type="text"' in
                         response.body)
         # Test response...
     def test_empty_search(self):
         if os.path.isdir(self.index_location):
             raise SkipTest('skipped due to existing index')
         else:
             self.log_user()
             response = self.app.get(url(controller='search', action='index'), {'q':HG_REPO})
             assert 'There is no index to search in. Please run whoosh indexer' in response.body, 'No error message about empty index'
             response = self.app.get(url(controller='search', action='index'),
                                     {'q':HG_REPO})
             self.assertTrue('There is no index to search in. '
                             'Please run whoosh indexer' in response.body)
     def test_normal_search(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'), {'q':'def repo'})
         print response.body
         assert '10 results' in response.body, 'no message about proper search results'
         assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user'
         response = self.app.get(url(controller='search', action='index'),
                                 {'q':'def repo'})
         self.assertTrue('10 results' in response.body)
         self.assertTrue('Permission denied' not in response.body)
     def test_repo_search(self):
         self.log_user()
         response = self.app.get(url(controller='search', action='index'), {'q':'repository:%s def test' % HG_REPO})
         print response.body
         assert '4 results' in response.body, 'no message about proper search results'
         assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user'
         response = self.app.get(url(controller='search', action='index'),
                                 {'q':'repository:%s def test' % HG_REPO})
         self.assertTrue('4 results' in response.body)
         self.assertTrue('Permission denied' not in response.body)

0 comments (0 inline, 0 general)