kallithea Changeset - 6e76b4892d72

Changeset - 6e76b4892d72

Parent rev.

Child rev.

[Not reviewed]

beta

0 5 0

Marcin Kuzminski - 13 years ago 2012-11-15 00:57:52
marcin@python-works.com

Implemented #647, option to pass list of default encoding used to encode to/decode from unicode

5 files changed with 53 insertions and 16 deletions:

development.ini

production.ini

rhodecode/config/deployment.ini_tmpl

rhodecode/lib/utils2.py

rhodecode/lib/vcs/utils/__init__.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -73,12 +73,14 @@ use_gravatar = true @@
 ## {netloc}       network location from RhodeCode server
 #alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 #alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 container_auth_enabled = false
 proxypass_auth_enabled = false
 ## default encoding used to convert from and to unicode
 ## can be also a comma seperated list of encoding in case of mixed encodings
 default_encoding = utf8
 ## overwrite schema of clone url
 ## available vars:
 ## scheme - http/https
 ## user - current user

production.ini

➞

Show inline comments

@@ @@ -73,12 +73,14 @@ use_gravatar = true @@
 ## {netloc}       network location from RhodeCode server
 #alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 #alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 container_auth_enabled = false
 proxypass_auth_enabled = false
 ## default encoding used to convert from and to unicode
 ## can be also a comma seperated list of encoding in case of mixed encodings
 default_encoding = utf8
 ## overwrite schema of clone url
 ## available vars:
 ## scheme - http/https
 ## user - current user

rhodecode/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -73,12 +73,14 @@ use_gravatar = true @@
 ## {netloc}       network location from RhodeCode server
 #alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 #alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 container_auth_enabled = false
 proxypass_auth_enabled = false
 ## default encoding used to convert from and to unicode
 ## can be also a comma seperated list of encoding in case of mixed encodings
 default_encoding = utf8
 ## overwrite schema of clone url
 ## available vars:
 ## scheme - http/https
 ## user - current user

rhodecode/lib/utils2.py

➞

Show inline comments

@@ @@ -63,12 +63,13 @@ def __get_lem(): @@
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
     return dict(d)
 def str2bool(_str):
     """
     returs True/False value from given string, it tries to translate the
     string into boolean
     :param _str: string value to translate into boolean
@@ @@ -80,12 +81,33 @@ def str2bool(_str): @@
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 def aslist(obj, sep=None, strip=True):
     """
     Returns given string separated by sep as list
     :param obj:
     :param sep:
     :param strip:
     """
     if isinstance(obj, (basestring)):
         lst = obj.split(sep)
         if strip:
             lst = [v.strip() for v in lst]
         return lst
     elif isinstance(obj, (list, tuple)):
         return obj
     elif obj is None:
         return []
     else:
         return [obj]
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" accordingly to given mode
     Available modes are::
 - Unix
@@ @@ -179,33 +201,38 @@ def safe_unicode(str_, from_encoding=Non @@
     """
     if isinstance(str_, unicode):
         return str_
     if not from_encoding:
         import rhodecode
         DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
         from_encoding = DEFAULT_ENCODING
         DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
                                                         'utf8'), sep=',')
         from_encoding = DEFAULT_ENCODINGS
     if not isinstance(from_encoding, (list, tuple)):
         from_encoding = [from_encoding]
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
     for enc in from_encoding:
     try:
-        return unicode(str_, from_encoding)
+            return unicode(str_, enc)
     except UnicodeDecodeError:
         pass
     try:
         import chardet
         encoding = chardet.detect(str_)['encoding']
         if encoding is None:
             raise Exception()
         return str_.decode(encoding)
     except (ImportError, UnicodeDecodeError, Exception):
         return unicode(str_, from_encoding, 'replace')
+        return unicode(str_, from_encoding[0], 'replace')
 def safe_str(unicode_, to_encoding=None):
     """
     safe str function. Does few trick to turn unicode_ into string
@@ @@ -223,29 +250,34 @@ def safe_str(unicode_, to_encoding=None) @@
     if isinstance(unicode_, str):
         return unicode_
     if not to_encoding:
         import rhodecode
         DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
         to_encoding = DEFAULT_ENCODING
         DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
                                                         'utf8'), sep=',')
         to_encoding = DEFAULT_ENCODINGS
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
     for enc in to_encoding:
     try:
-        return unicode_.encode(to_encoding)
+            return unicode_.encode(enc)
     except UnicodeEncodeError:
         pass
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         if encoding is None:
             raise UnicodeEncodeError()
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding, 'replace')
+        return unicode_.encode(to_encoding[0], 'replace')
     return safe_str
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """

rhodecode/lib/vcs/utils/__init__.py

➞

Show inline comments

@@ @@ -35,18 +35,18 @@ def safe_unicode(str_, from_encoding=Non @@
     by chardet library if it fails fallback to unicode with errors replaced
     :param str_: string to decode
     :rtype: unicode
     :returns: unicode object
     """
     from rhodecode.lib.utils2 import safe_unicode
     return safe_unicode(str_, from_encoding)
     if isinstance(str_, unicode):
         return str_
     if not from_encoding:
         import rhodecode
         DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding', 'utf8')
         from_encoding = DEFAULT_ENCODING
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
     try:
@@ @@ -72,19 +72,18 @@ def safe_str(unicode_, to_encoding=None) @@
     by chardet library if it fails fallback to string with errors replaced
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
     from rhodecode.lib.utils2 import safe_str
     return safe_str(unicode_, to_encoding)
     if isinstance(unicode_, str):
         return unicode_
     if not to_encoding:
         import rhodecode
         DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding', 'utf8')
         to_encoding = DEFAULT_ENCODING
     try:
         return unicode_.encode(to_encoding)
     except UnicodeEncodeError:
         pass
     try:

0 comments (0 inline, 0 general)