kallithea Changeset - 58b6e4cd6fe9

Changeset - 58b6e4cd6fe9

Parent rev.

Child rev.

[Not reviewed]

default

0 15 0

Mads Kiilerich - 6 years ago 2019-12-28 13:38:22
mads@kiilerich.com

Grafted from: d6c192808c3d

lib: clean up ext_json and how it is used - avoid monkey patching

Note that py3 json.dumps will return ASCII (with all unicode escaped) encoded
as str. But we generally want JSON as bytes (which json.loads also can read),
so also wrap the result with ascii_bytes in many places.

15 files changed with 94 insertions and 95 deletions:

kallithea/bin/base.py

kallithea/bin/kallithea_api.py

kallithea/bin/kallithea_gist.py

kallithea/bin/ldap_sync.py

kallithea/controllers/api/__init__.py

kallithea/controllers/summary.py

kallithea/lib/auth_modules/auth_crowd.py

kallithea/lib/base.py

kallithea/lib/celerylib/tasks.py

kallithea/lib/compat.py

kallithea/lib/ext_json.py

kallithea/lib/utils2.py

kallithea/model/db.py

kallithea/model/gist.py

kallithea/tests/api/api_base.py

0 comments (0 inline, 0 general)

kallithea/bin/base.py

➞

Show inline comments

@@ @@ -22,25 +22,26 @@ Original author and date, and relevant c @@
 :created_on: May 09, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import pprint
 import random
 import sys
 import urllib2
 from kallithea.lib.compat import json
 from kallithea.lib import ext_json
 from kallithea.lib.utils2 import ascii_bytes
 CONFIG_NAME = '.config/kallithea'
 FORMAT_PRETTY = 'pretty'
 FORMAT_JSON = 'json'
 def api_call(apikey, apihost, method=None, **kw):
     """
     Api_call wrapper for Kallithea.
     :param apikey:
@@ @@ -59,29 +60,29 @@ def api_call(apikey, apihost, method=Non @@
         return {
             "id": random_id,
             "api_key": apikey,
             "method": method,
             "args": kw
+        }
     if not method:
         raise Exception('please specify method name !')
     apihost = apihost.rstrip('/')
     id_ = random.randrange(1, 9999)
     req = urllib2.Request('%s/_admin/api' % apihost,
-                      data=json.dumps(_build_data(id_)),
+                      data=ascii_bytes(ext_json.dumps(_build_data(id_))),
                       headers={'content-type': 'text/plain'})
     ret = urllib2.urlopen(req)
     raw_json = ret.read()
-    json_data = json.loads(raw_json)
+    json_data = ext_json.loads(raw_json)
     id_ret = json_data['id']
     if id_ret == id_:
         return json_data
     else:
         _formatted_json = pprint.pformat(json_data)
         raise Exception('something went wrong. '
                         'ID mismatch got %s, expected %s | %s' % (
                                             id_ret, id_, _formatted_json))
 class RcConf(object):
@@ @@ -119,47 +120,47 @@ class RcConf(object): @@
         return 'RcConf<%s>' % self._conf.__repr__()
     def make_config(self, config):
         """
         Saves given config as a JSON dump in the _conf_name location
         :param config:
         """
         update = False
         if os.path.exists(self._conf_name):
             update = True
         with open(self._conf_name, 'wb') as f:
-            json.dump(config, f, indent=4)
+            ext_json.dump(config, f, indent=4)
             f.write('\n')
         if update:
             sys.stdout.write('Updated config in %s\n' % self._conf_name)
         else:
             sys.stdout.write('Created new config in %s\n' % self._conf_name)
     def update_config(self, new_config):
         """
         Reads the JSON config updates it's values with new_config and
         saves it back as JSON dump
         :param new_config:
         """
         config = {}
         try:
             with open(self._conf_name, 'rb') as conf:
-                config = json.load(conf)
+                config = ext_json.load(conf)
         except IOError as e:
             sys.stderr.write(str(e) + '\n')
         config.update(new_config)
         self.make_config(config)
     def load_config(self):
         """
         Loads config from file and returns loaded JSON object
         """
         try:
             with open(self._conf_name, 'rb') as conf:
-                return json.load(conf)
+                return ext_json.load(conf)
         except IOError as e:
             #sys.stderr.write(str(e) + '\n')
             pass

kallithea/bin/kallithea_api.py

➞

Show inline comments

@@ @@ -19,27 +19,28 @@ Api CLI client for Kallithea @@
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 3, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import argparse
 import json
 import sys
-from kallithea.bin.base import FORMAT_JSON, FORMAT_PRETTY, RcConf, api_call, json
 from kallithea.bin.base import FORMAT_JSON, FORMAT_PRETTY, RcConf, api_call
 def argparser(argv):
     usage = (
       "kallithea-api [-h] [--format=FORMAT] [--apikey=APIKEY] [--apihost=APIHOST] "
       "[--config=CONFIG] [--save-config] "
       "METHOD <key:val> <key2:val> ...\n"
       "Create config file: kallithea-api --apikey=<key> --apihost=http://kallithea.example.com --save-config"
+    )
     parser = argparse.ArgumentParser(description='Kallithea API cli',
                                      usage=usage)

kallithea/bin/kallithea_gist.py

➞

Show inline comments

@@ @@ -20,29 +20,30 @@ Gist CLI client for Kallithea @@
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 9, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import argparse
 import fileinput
 import json
 import os
 import stat
 import sys
-from kallithea.bin.base import FORMAT_JSON, FORMAT_PRETTY, RcConf, api_call, json
 from kallithea.bin.base import FORMAT_JSON, FORMAT_PRETTY, RcConf, api_call
 def argparser(argv):
     usage = (
       "kallithea-gist [-h] [--format=FORMAT] [--apikey=APIKEY] [--apihost=APIHOST] "
       "[--config=CONFIG] [--save-config] [GIST OPTIONS] "
       "[filename or stdin use - for terminal stdin ]\n"
       "Create config file: kallithea-gist --apikey=<key> --apihost=http://kallithea.example.com --save-config"
+    )
     parser = argparse.ArgumentParser(description='Kallithea Gist cli',
                                      usage=usage)

kallithea/bin/ldap_sync.py

➞

Show inline comments

@@ @@ -24,25 +24,26 @@ Original author and date, and relevant c @@
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import urllib2
 import uuid
 from ConfigParser import ConfigParser
 import ldap
 from kallithea.lib.compat import json
 from kallithea.lib import ext_json
 from kallithea.lib.utils2 import ascii_bytes
 config = ConfigParser()
 config.read('ldap_sync.conf')
 class InvalidResponseIDError(Exception):
     """ Request and response don't have the same UUID. """
 class ResponseError(Exception):
     """ Response has an error, something went wrong with request execution. """
@@ @@ -71,30 +72,30 @@ class API(object): @@
             "args": args
+        }
     def post(self, method, args):
         """Send a generic API post to Kallithea.
         This will generate the UUID for validation check after the
         response is returned. Handle errors and get the result back.
         """
         uid = str(uuid.uuid1())
         data = self.get_api_data(uid, method, args)
-        data = json.dumps(data)
+        data = ascii_bytes(ext_json.dumps(data))
         headers = {'content-type': 'text/plain'}
         req = urllib2.Request(self.url, data, headers)
         response = urllib2.urlopen(req)
-        response = json.load(response)
+        response = ext_json.load(response)
         if uid != response["id"]:
             raise InvalidResponseIDError("UUID does not match.")
         if response["error"] is not None:
             raise ResponseError(response["error"])
         return response["result"]
     def create_group(self, name, active=True):
         """Create the Kallithea user group."""
         args = {

kallithea/controllers/api/__init__.py

➞

Show inline comments

@@ @@ -26,29 +26,29 @@ Original author and date, and relevant c @@
 """
 import inspect
 import itertools
 import logging
 import time
 import traceback
 import types
 from tg import Response, TGController, request, response
 from webob.exc import HTTPError, HTTPException
 from kallithea.lib import ext_json
 from kallithea.lib.auth import AuthUser
 from kallithea.lib.base import _get_ip_addr as _get_ip
 from kallithea.lib.base import get_path_info
 from kallithea.lib.compat import json
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.utils2 import ascii_bytes, safe_str
 from kallithea.model.db import User
 log = logging.getLogger('JSONRPC')
 class JSONRPCError(BaseException):
     def __init__(self, message):
         self.message = message
         super(JSONRPCError, self).__init__()
@@ @@ -112,25 +112,25 @@ class JSONRPCController(TGController): @@
         else:
             length = environ['CONTENT_LENGTH'] or 0
             length = int(environ['CONTENT_LENGTH'])
             log.debug('Content-Length: %s', length)
         if length == 0:
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message="Content-Length is 0")
         raw_body = environ['wsgi.input'].read(length)
         try:
-            json_body = json.loads(raw_body)
+            json_body = ext_json.loads(raw_body)
         except ValueError as e:
             # catch JSON errors Here
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message="JSON parse error ERR:%s RAW:%r"
                                                 % (e, raw_body))
         # check AUTH based on API key
         try:
             self._req_api_key = json_body['api_key']
             self._req_id = json_body['id']
             self._req_method = json_body['method']
             self._request_params = json_body['args']
@@ @@ -229,34 +229,34 @@ class JSONRPCController(TGController): @@
             self._error = unicode(e)
         except Exception as e:
             log.error('Encountered unhandled exception: %s',
                       traceback.format_exc(),)
             json_exc = JSONRPCError('Internal server error')
             self._error = unicode(json_exc)
         if self._error is not None:
             raw_response = None
         response = dict(id=self._req_id, result=raw_response, error=self._error)
         try:
-            return json.dumps(response)
+            return ascii_bytes(ext_json.dumps(response))
         except TypeError as e:
             log.error('API FAILED. Error encoding response for %s %s: %s\n%s', action, rpc_args, e, traceback.format_exc())
-            return json.dumps(
+            return ascii_bytes(ext_json.dumps(
                 dict(
                     id=self._req_id,
                     result=None,
                     error="Error encoding response"
+                    error="Error encoding response",
+                )
+            )
+            ))
     def _find_method(self):
         """
         Return method named by `self._req_method` in controller if able
         """
         log.debug('Trying to find JSON-RPC method: %s', self._req_method)
         if self._req_method.startswith('_'):
             raise AttributeError("Method not allowed")
         try:
             func = getattr(self, self._req_method, None)
         except UnicodeEncodeError:

kallithea/controllers/summary.py

➞

Show inline comments

@@ @@ -31,28 +31,28 @@ import logging @@
 import traceback
 from datetime import date, timedelta
 from time import mktime
 from beaker.cache import cache_region
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest
 import kallithea.lib.helpers as h
 from kallithea.config.conf import ALL_EXTS, ALL_READMES, LANGUAGES_EXTENSIONS_MAP
 from kallithea.lib import ext_json
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.celerylib.tasks import get_commits_stats
 from kallithea.lib.compat import json
 from kallithea.lib.markup_renderer import MarkupRenderer
 from kallithea.lib.page import Page
 from kallithea.lib.utils2 import safe_int, safe_unicode
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.model.db import Statistics
 log = logging.getLogger(__name__)
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
@@ @@ -130,25 +130,25 @@ class SummaryController(BaseRepoControll @@
             c.show_stats = True
         else:
             c.show_stats = False
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
-            lang_stats_d = json.loads(stats.languages)
+            lang_stats_d = ext_json.loads(stats.languages)
             lang_stats = [(x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})
                           for x, y in lang_stats_d.items()]
             lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))
             c.trending_languages = lang_stats[:10]
         else:
             c.no_data = True
             c.trending_languages = []
         c.enable_downloads = c.db_repo.enable_downloads
@@ @@ -182,27 +182,27 @@ class SummaryController(BaseRepoControll @@
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
         stats = Statistics.query() \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = json.loads(stats.commit_activity)
             c.overview_data = json.loads(stats.commit_activity_combined)
             lang_stats_d = ext_json.loads(stats.languages)
             c.commit_data = ext_json.loads(stats.commit_activity)
             c.overview_data = ext_json.loads(stats.commit_activity_combined)
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
             c.trending_languages = (
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
+            )
             last_rev = stats.stat_on_revision + 1
             c.repo_last_rev = c.db_repo_scm_instance.count() \
                 if c.db_repo_scm_instance.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:

kallithea/lib/auth_modules/auth_crowd.py

➞

Show inline comments

@@ @@ -21,26 +21,27 @@ This file was forked by the Kallithea pr @@
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Created on Nov 17, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import base64
 import logging
 import urllib2
 from kallithea.lib import auth_modules
 from kallithea.lib.compat import formatted_json, hybrid_property, json
 from kallithea.lib import auth_modules, ext_json
 from kallithea.lib.compat import formatted_json, hybrid_property
 from kallithea.lib.utils2 import ascii_bytes
 log = logging.getLogger(__name__)
 class CrowdServer(object):
     def __init__(self, *args, **kwargs):
         """
         Create a new CrowdServer object that points to IP/Address 'host',
         on the given port, and using the given method (https/http). user and
         passwd can be set here or with set_credentials. If unspecified,
         "version" defaults to "latest".
@@ @@ -94,43 +95,43 @@ class CrowdServer(object): @@
             req.get_method = lambda: method
         global msg
         msg = ""
         try:
             rdoc = self.opener.open(req)
             msg = "".join(rdoc.readlines())
             if not msg and empty_response_ok:
                 rval = {}
                 rval["status"] = True
                 rval["error"] = "Response body was empty"
             elif not noformat:
-                rval = json.loads(msg)
+                rval = ext_json.loads(msg)
                 rval["status"] = True
             else:
                 rval = "".join(rdoc.readlines())
         except Exception as e:
             if not noformat:
                 rval = {"status": False,
                         "body": body,
                         "error": str(e) + "\n" + msg}
             else:
                 rval = None
         return rval
     def user_auth(self, username, password):
         """Authenticate a user against crowd. Returns brief information about
         the user."""
         url = ("%s/rest/usermanagement/%s/authentication?username=%s"
                % (self._uri, self._version, urllib2.quote(username)))
-        body = json.dumps({"value": password})
+        body = ascii_bytes(ext_json.dumps({"value": password}))
         return self._request(url, body)
     def user_groups(self, username):
         """Retrieve a list of groups to which this user belongs."""
         url = ("%s/rest/usermanagement/%s/user/group/nested?username=%s"
                % (self._uri, self._version, urllib2.quote(username)))
         return self._request(url)
 class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):
     def __init__(self):
         self._protocol_values = ["http", "https"]

kallithea/lib/base.py

➞

Show inline comments

@@ @@ -36,30 +36,29 @@ import warnings @@
 import decorator
 import paste.auth.basic
 import paste.httpexceptions
 import paste.httpheaders
 import webob.exc
 from tg import TGController, config, render_template, request, response, session
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from kallithea import BACKENDS, __version__
 from kallithea.config.routing import url
 from kallithea.lib import auth_modules
+from kallithea.lib import auth_modules, ext_json
 from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from kallithea.lib.compat import json
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import get_repo_slug, is_valid_repo
 from kallithea.lib.utils2 import AttributeDict, safe_int, safe_str, safe_unicode, set_hook_environment, str2bool
+from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_str, safe_unicode, set_hook_environment, str2bool
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError
 from kallithea.model import meta
 from kallithea.model.db import PullRequest, Repository, Setting, User
 from kallithea.model.scm import ScmModel
 log = logging.getLogger(__name__)
 def render(template_path):
     return render_template({'url': url}, 'mako', template_path)
@@ @@ -621,25 +620,25 @@ def jsonify(func, *args, **kwargs): @@
     if isinstance(data, (list, tuple)):
         # A JSON list response is syntactically valid JavaScript and can be
         # loaded and executed as JavaScript by a malicious third-party site
         # using <script>, which can lead to cross-site data leaks.
         # JSON responses should therefore be scalars or objects (i.e. Python
         # dicts), because a JSON object is a syntax error if intepreted as JS.
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "https://web.archive.org/web/20120519231904/http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
-    return json.dumps(data)
+    return ascii_bytes(ext_json.dumps(data))
 @decorator.decorator
 def IfSshEnabled(func, *args, **kwargs):
     """Decorator for functions that can only be called if SSH access is enabled.
     If SSH access is disabled in the configuration file, HTTPNotFound is raised.
     """
     if not c.ssh_enabled:
         from kallithea.lib import helpers as h
         h.flash(_("SSH access is disabled."), category='warning')
         raise webob.exc.HTTPNotFound()
     return func(*args, **kwargs)

kallithea/lib/celerylib/tasks.py

➞

Show inline comments

@@ @@ -28,31 +28,30 @@ Original author and date, and relevant c @@
 import email.utils
 import logging
 import os
 import traceback
 from collections import OrderedDict
 from operator import itemgetter
 from time import mktime
 from tg import config
 from kallithea import CELERY_ON
 from kallithea.lib import celerylib
 from kallithea.lib.compat import json
 from kallithea.lib import celerylib, ext_json
 from kallithea.lib.helpers import person
 from kallithea.lib.hooks import log_create_repository
 from kallithea.lib.rcmail.smtp_mailer import SmtpMailer
 from kallithea.lib.utils import action_logger
 from kallithea.lib.utils2 import str2bool
+from kallithea.lib.utils2 import ascii_bytes, str2bool
 from kallithea.lib.vcs.utils import author_email
 from kallithea.model.db import RepoGroup, Repository, Statistics, User
 __all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
 log = logging.getLogger(__name__)
 @celerylib.task
 @celerylib.locked_task
@@ @@ -109,27 +108,27 @@ def get_commits_stats(repo_name, ts_min_ @@
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             # pass silently without any work if we're not on first revision or
             # current state of parsing revision(from db marker) is the
             # last revision
             lock.release()
             return True
         if cur_stats:
-            commits_by_day_aggregate = OrderedDict(json.loads(
+            commits_by_day_aggregate = OrderedDict(ext_json.loads(
                                         cur_stats.commit_activity_combined))
-            co_day_auth_aggr = json.loads(cur_stats.commit_activity)
+            co_day_auth_aggr = ext_json.loads(cur_stats.commit_activity)
         log.debug('starting parsing %s', parse_limit)
         last_rev = last_rev + 1 if last_rev and last_rev >= 0 else 0
         log.debug('Getting revisions from %s to %s',
              last_rev, last_rev + parse_limit
+        )
         for cs in repo[last_rev:last_rev + parse_limit]:
             log.debug('parsing %s', cs)
             last_cs = cs  # remember last parsed changeset
             tt = cs.date.timetuple()
             k = mktime(tt[:3] + (0, 0, 0, 0, 0, 0))
@@ @@ -184,34 +183,34 @@ def get_commits_stats(repo_name, ts_min_ @@
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
+            }
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
         stats.commit_activity = ascii_bytes(ext_json.dumps(co_day_auth_aggr))
         stats.commit_activity_combined = ascii_bytes(ext_json.dumps(overview_data))
         log.debug('last revision %s', last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s', leftovers)
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
-            stats.languages = json.dumps(__get_codes_stats(repo_name))
+            stats.languages = ascii_bytes(ext_json.dumps(__get_codes_stats(repo_name)))
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             DBS.add(stats)
             DBS.commit()
         except:
             log.error(traceback.format_exc())
             DBS.rollback()
             lock.release()
             return False

kallithea/lib/compat.py

➞

Show inline comments

@@ @@ -33,33 +33,28 @@ import os @@
 #==============================================================================
 # Hybrid property/method
 #==============================================================================
 from sqlalchemy.ext.hybrid import hybrid_property
 #==============================================================================
 # OrderedSet
 #==============================================================================
 from sqlalchemy.util import OrderedSet
 #==============================================================================
 # json
 #==============================================================================
-from kallithea.lib.ext_json import json
+from kallithea.lib import ext_json
 # alias for formatted json
 formatted_json = functools.partial(json.dumps, indent=4, sort_keys=True)
 formatted_json = functools.partial(ext_json.dumps, indent=4, sort_keys=True)
 #==============================================================================
 # kill
 #==============================================================================
 if os.name == 'nt': # Windows
     import ctypes
     def kill(pid, sig):
         """kill function for Win32"""
         kernel32 = ctypes.windll.kernel32
         handle = kernel32.OpenProcess(1, 0, pid)

kallithea/lib/ext_json.py

➞

Show inline comments

 """
-Extended JSON encoder for json
+Extended JSON encoder with support for more data types
-json.org does not specify how date time can be represented - monkeypatch it to do something.
+json.org does not specify how date time can be represented - just encode it somehow and ignore decoding ...
 """
 import datetime
 import decimal
 import functools
-import json  # is re-exported after monkey patching
 import json
-__all__ = ['json']
+__all__ = ['dumps', 'dump', 'load', 'loads']
 def _is_tz_aware(value):
     """
     Determines if a given datetime.time is timezone aware.
     The logic is described in Python's docs:
     http://docs.python.org/library/datetime.html#datetime.tzinfo
     """
     return (value.tzinfo is not None
             and value.tzinfo.utcoffset(value) is not None)
@@ @@ -61,19 +61,21 @@ def _obj_dump(obj): @@
             return obj.__json__()
         else:
             return obj.__json__
     else:
         raise NotImplementedError
 class ExtendedEncoder(json.JSONEncoder):
     def default(self, obj):
         try:
             return _obj_dump(obj)
         except NotImplementedError:
             pass
+            pass  # quiet skipping of unsupported types!
         raise TypeError("%r is not JSON serializable" % (obj,))
 # monkey-patch and export JSON encoder to use custom encoding method
 json.dumps = functools.partial(json.dumps, cls=ExtendedEncoder)
 json.dump = functools.partial(json.dump, cls=ExtendedEncoder)
 dumps = functools.partial(json.dumps, cls=ExtendedEncoder)
 dump = functools.partial(json.dump, cls=ExtendedEncoder)
 # No special support for loading these types back!!!
 load = json.load
 loads = json.loads

kallithea/lib/utils2.py

➞

Show inline comments

@@ @@ -22,36 +22,36 @@ models, controllers, etc. to prevent im @@
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 5, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import binascii
 import datetime
 import json
 import os
 import pwd
 import re
 import time
 import urllib
 import urlobject
 from tg.i18n import ugettext as _
 from tg.i18n import ungettext
 from webhelpers2.text import collapse, remove_formatting, strip_tags
 from kallithea.lib.compat import json
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str, safe_unicode  # re-export
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 def str2bool(_str):
     """
     returns True/False value from given string, it tries to translate the
     string into boolean
     :param _str: string value to translate into boolean
     :rtype: boolean
     :returns: boolean from given string

kallithea/model/db.py

➞

Show inline comments

@@ @@ -36,29 +36,29 @@ import time @@
 import traceback
 import ipaddr
 import sqlalchemy
 from beaker.cache import cache_region, region_invalidate
 from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, String, Unicode, UnicodeText, UniqueConstraint
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
 from tg.i18n import lazy_ugettext as _
 from webob.exc import HTTPNotFound
 import kallithea
 from kallithea.lib import ext_json
 from kallithea.lib.caching_query import FromCache
 from kallithea.lib.compat import json
 from kallithea.lib.exceptions import DefaultUserException
 from kallithea.lib.utils2 import (
     Optional, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, safe_unicode, str2bool, urlreadable)
+    Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, safe_unicode, str2bool, urlreadable)
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.model.meta import Base, Session
 URL_SEP = '/'
 log = logging.getLogger(__name__)
 #==============================================================================
 # BASE CLASSES
@@ @@ -512,32 +512,32 @@ class User(Base, BaseDbModel): @@
         return self.admin
     @hybrid_property
     def is_default_user(self):
         return self.username == User.DEFAULT_USER
     @hybrid_property
     def user_data(self):
         if not self._user_data:
             return {}
         try:
-            return json.loads(self._user_data)
+            return ext_json.loads(self._user_data)
         except TypeError:
             return {}
     @user_data.setter
     def user_data(self, val):
         try:
-            self._user_data = json.dumps(val)
+            self._user_data = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
     def __repr__(self):
         return "<%s %s: %r')>" % (self.__class__.__name__, self.user_id, self.username)
     @classmethod
     def guess_instance(cls, value):
         return super(User, cls).guess_instance(value, User.get_by_username)
     @classmethod
     def get_or_404(cls, id_, allow_default=True):
@@ @@ -824,32 +824,32 @@ class UserGroup(Base, BaseDbModel): @@
     users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     user_user_group_to_perm = relationship('UserUserGroupToPerm ', cascade='all')
     user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all')
     owner = relationship('User')
     @hybrid_property
     def group_data(self):
         if not self._group_data:
             return {}
         try:
-            return json.loads(self._group_data)
+            return ext_json.loads(self._group_data)
         except TypeError:
             return {}
     @group_data.setter
     def group_data(self, val):
         try:
-            self._group_data = json.dumps(val)
+            self._group_data = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
     def __repr__(self):
         return "<%s %s: %r')>" % (self.__class__.__name__,
                                   self.users_group_id,
                                   self.users_group_name)
     @classmethod
     def guess_instance(cls, value):
         return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
@@ @@ -1023,34 +1023,34 @@ class Repository(Base, BaseDbModel): @@
         return [None, None]
     @landing_rev.setter
     def landing_rev(self, val):
         if ':' not in val:
             raise ValueError('value must be delimited with `:` and consist '
                              'of <rev_type>:<rev>, got %s instead' % val)
         self._landing_revision = val
     @hybrid_property
     def changeset_cache(self):
         try:
-            cs_cache = json.loads(self._changeset_cache) # might raise on bad data
+            cs_cache = ext_json.loads(self._changeset_cache) # might raise on bad data
             cs_cache['raw_id'] # verify data, raise exception on error
             return cs_cache
         except (TypeError, KeyError, ValueError):
             return EmptyChangeset().__json__()
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
-            self._changeset_cache = json.dumps(val)
+            self._changeset_cache = ascii_bytes(ext_json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
     @classmethod
     def query(cls, sorted=False):
         """Add Repository-specific helpers for common query constructs.
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(Repository, cls).query()
         if sorted:

kallithea/model/gist.py

➞

Show inline comments

@@ @@ -23,26 +23,26 @@ Original author and date, and relevant c @@
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import random
 import shutil
 import time
 import traceback
 from kallithea.lib.compat import json
 from kallithea.lib.utils2 import AttributeDict, safe_int, safe_unicode, time_to_datetime
 from kallithea.lib import ext_json
 from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_unicode, time_to_datetime
 from kallithea.model.db import Gist, Session, User
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import ScmModel
 log = logging.getLogger(__name__)
 GIST_STORE_LOC = '.rc_gist_store'
 GIST_METADATA_FILE = '.rc_gist_metadata'
 def make_gist_access_id():
@@ @@ -73,25 +73,25 @@ class GistModel(object): @@
         or gist identification
         """
         metadata = {
             'metadata_version': '1',
             'gist_db_id': gist_id,
             'gist_access_id': gist_access_id,
             'gist_owner_id': user_id,
             'gist_type': gist_type,
             'gist_expires': gist_expires,
             'gist_updated': time.time(),
+        }
         with open(os.path.join(repo.path, '.hg', GIST_METADATA_FILE), 'wb') as f:
-            f.write(json.dumps(metadata))
+            f.write(ascii_bytes(ext_json.dumps(metadata)))
     def get_gist(self, gist):
         return Gist.guess_instance(gist)
     def get_gist_files(self, gist_access_id, revision=None):
         """
         Get files for given gist
         :param gist_access_id:
         """
         repo = Gist.get_by_access_id(gist_access_id)
         cs = repo.scm_instance.get_changeset(revision)

kallithea/tests/api/api_base.py

➞

Show inline comments

@@ @@ -14,62 +14,62 @@ @@
 """
 Tests for the JSON-RPC web api.
 """
 import os
 import random
 import re
 import mock
 import pytest
 from kallithea.lib import ext_json
 from kallithea.lib.auth import AuthUser
-from kallithea.lib.compat import json
+from kallithea.lib.utils2 import ascii_bytes
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.db import ChangesetStatus, PullRequest, RepoGroup, Repository, Setting, Ui, User
 from kallithea.model.gist import GistModel
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.scm import ScmModel
 from kallithea.model.user import UserModel
 from kallithea.model.user_group import UserGroupModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 API_URL = '/_admin/api'
 TEST_USER_GROUP = u'test_user_group'
 TEST_REPO_GROUP = u'test_repo_group'
 fixture = Fixture()
 def _build_data(apikey, method, **kw):
     """
     Builds API data with given random ID
     :param random_id:
     For convenience, the json is returned as str
     """
     random_id = random.randrange(1, 9999)
-    return random_id, json.dumps({
+    return random_id, ext_json.dumps({
         "id": random_id,
         "api_key": apikey,
         "method": method,
         "args": kw
     })
-jsonify = lambda obj: json.loads(json.dumps(obj))
+jsonify = lambda obj: ext_json.loads(ext_json.dumps(obj))
 def crash(*args, **kwargs):
     raise Exception('Total Crash !')
 def api_call(test_obj, params):
     response = test_obj.app.post(API_URL, content_type='application/json',
                                  params=params)
     return response
@@ @@ -118,34 +118,34 @@ class _BaseTestApi(object): @@
     def teardown_method(self, method):
         fixture.destroy_user_group(TEST_USER_GROUP)
         fixture.destroy_gists()
         fixture.destroy_repo_group(TEST_REPO_GROUP)
     def _compare_ok(self, id_, expected, given):
         expected = jsonify({
             'id': id_,
             'error': None,
             'result': expected
         })
-        given = json.loads(given)
+        given = ext_json.loads(given)
         assert expected == given, (expected, given)
     def _compare_error(self, id_, expected, given):
         expected = jsonify({
             'id': id_,
             'error': expected,
             'result': None
         })
-        given = json.loads(given)
+        given = ext_json.loads(given)
         assert expected == given, (expected, given)
     def test_Optional_object(self):
         from kallithea.controllers.api.api import Optional
         option1 = Optional(None)
         assert '<Optional:%s>' % None == repr(option1)
         assert option1() is None
         assert 1 == Optional.extract(Optional(1))
         assert 'trololo' == Optional.extract('trololo')
@@ @@ -2313,34 +2313,33 @@ class _BaseTestApi(object): @@
                                   userid=base.TEST_USER_ADMIN_LOGIN)
         response = api_call(self, params)
         expected = 'userid is not the same as your user'
         self._compare_error(id_, expected, given=response.body)
     def test_api_create_gist(self):
         id_, params = _build_data(self.apikey_regular, 'create_gist',
                                   lifetime=10,
                                   description='foobar-gist',
                                   gist_type='public',
                                   files={'foobar': {'content': 'foo'}})
         response = api_call(self, params)
         response_json = response.json
         expected = {
             'gist': {
                 'access_id': response_json['result']['gist']['access_id'],
                 'created_on': response_json['result']['gist']['created_on'],
                 'access_id': response.json['result']['gist']['access_id'],
                 'created_on': response.json['result']['gist']['created_on'],
                 'description': 'foobar-gist',
                 'expires': response_json['result']['gist']['expires'],
                 'gist_id': response_json['result']['gist']['gist_id'],
                 'expires': response.json['result']['gist']['expires'],
                 'gist_id': response.json['result']['gist']['gist_id'],
                 'type': 'public',
-                'url': response_json['result']['gist']['url']
+                'url': response.json['result']['gist']['url']
             },
             'msg': 'created new gist'
+        }
         self._compare_ok(id_, expected, given=response.body)
     @mock.patch.object(GistModel, 'create', crash)
     def test_api_create_gist_exception_occurred(self):
         id_, params = _build_data(self.apikey_regular, 'create_gist',
                                   files={})
         response = api_call(self, params)
         expected = 'failed to create gist'
         self._compare_error(id_, expected, given=response.body)
@@ @@ -2388,76 +2387,76 @@ class _BaseTestApi(object): @@
         self._compare_ok(id_, expected, given=response.body)
     def test_api_get_server_info(self):
         id_, params = _build_data(self.apikey, 'get_server_info')
         response = api_call(self, params)
         expected = Setting.get_server_info()
         self._compare_ok(id_, expected, given=response.body)
     def test_api_get_changesets(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start=0, end=2)
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert len(result) == 3
         assert 'message' in result[0]
         assert 'added' not in result[0]
     def test_api_get_changesets_with_max_revisions(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start_date="2011-02-24T00:00:00", max_revisions=10)
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert len(result) == 10
         assert 'message' in result[0]
         assert 'added' not in result[0]
     def test_api_get_changesets_with_branch(self):
         if self.REPO == 'vcs_test_hg':
             branch = 'stable'
         else:
             pytest.skip("skipping due to missing branches in git test repo")
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, branch_name=branch, start_date="2011-02-24T00:00:00")
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert len(result) == 5
         assert 'message' in result[0]
         assert 'added' not in result[0]
     def test_api_get_changesets_with_file_list(self):
         id_, params = _build_data(self.apikey, 'get_changesets',
                                   repoid=self.REPO, start_date="2010-04-07T23:30:30", end_date="2010-04-08T00:31:14", with_file_list=True)
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert len(result) == 3
         assert 'message' in result[0]
         assert 'added' in result[0]
     def test_api_get_changeset(self):
         review = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         id_, params = _build_data(self.apikey, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION)
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert result["raw_id"] == self.TEST_REVISION
         assert "reviews" not in result
     def test_api_get_changeset_with_reviews(self):
         reviewobjs = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         id_, params = _build_data(self.apikey, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION,
                                   with_reviews=True)
         response = api_call(self, params)
-        result = json.loads(response.body)["result"]
+        result = ext_json.loads(response.body)["result"]
         assert result["raw_id"] == self.TEST_REVISION
         assert "reviews" in result
         assert len(result["reviews"]) == 1
         review = result["reviews"][0]
         expected = {
             'status': 'approved',
             'modified_at': reviewobjs[0].modified_at.replace(microsecond=0).isoformat(),
             'reviewer': 'test_admin',
+        }
         assert review == expected
     def test_api_get_changeset_that_does_not_exist(self):
@@ @@ -2475,30 +2474,30 @@ class _BaseTestApi(object): @@
         review = fixture.review_changeset(self.REPO, self.TEST_REVISION, "approved")
         RepoModel().revoke_user_permission(repo=self.REPO, user=self.TEST_USER_LOGIN)
         RepoModel().revoke_user_permission(repo=self.REPO, user="default")
         id_, params = _build_data(self.apikey_regular, 'get_changeset',
                                   repoid=self.REPO, raw_id=self.TEST_REVISION)
         response = api_call(self, params)
         expected = u'Access denied to repo %s' % self.REPO
         self._compare_error(id_, expected, given=response.body)
     def test_api_get_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, u'get test')
         random_id = random.randrange(1, 9999)
-        params = json.dumps({
+        params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": 'get_pullrequest',
             "args": {"pullrequest_id": pull_request_id},
         })
+        }))
         response = api_call(self, params)
         pullrequest = PullRequest().get(pull_request_id)
         expected = {
             "status": "new",
             "pull_request_id": pull_request_id,
             "description": "No description",
             "url": "/%s/pull-request/%s/_/%s" % (self.REPO, pull_request_id, "stable"),
             "reviewers": [{"username": "test_regular"}],
             "org_repo_url": "http://localhost:80/%s" % self.REPO,
             "org_ref_parts": ["branch", "stable", self.TEST_PR_SRC],
             "other_ref_parts": ["branch", "default", self.TEST_PR_DST],
             "comments": [{"username": base.TEST_USER_ADMIN_LOGIN, "text": "",
@@ @@ -2506,63 +2505,63 @@ class _BaseTestApi(object): @@
             "owner": base.TEST_USER_ADMIN_LOGIN,
             "statuses": [{"status": "under_review", "reviewer": base.TEST_USER_ADMIN_LOGIN, "modified_at": "2000-01-01T00:00:00"} for i in range(0, len(self.TEST_PR_REVISIONS))],
             "title": "get test",
             "revisions": self.TEST_PR_REVISIONS,
+        }
         self._compare_ok(random_id, expected,
                          given=re.sub(br"\d\d\d\d\-\d\d\-\d\dT\d\d\:\d\d\:\d\d",
                                       b"2000-01-01T00:00:00", response.body))
     def test_api_close_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, u'close test')
         random_id = random.randrange(1, 9999)
-        params = json.dumps({
+        params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "close_pr": True},
         })
+        }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = PullRequest().get(pull_request_id)
         assert pullrequest.comments[-1].text == ''
         assert pullrequest.status == PullRequest.STATUS_CLOSED
         assert pullrequest.is_closed() == True
     def test_api_status_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, u"status test")
         random_id = random.randrange(1, 9999)
-        params = json.dumps({
+        params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": User.get_by_username(base.TEST_USER_REGULAR2_LOGIN).api_key,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "status": ChangesetStatus.STATUS_APPROVED},
         })
+        }))
         response = api_call(self, params)
         pullrequest = PullRequest().get(pull_request_id)
         self._compare_error(random_id, "No permission to change pull request status. User needs to be admin, owner or reviewer.", given=response.body)
         assert ChangesetStatus.STATUS_UNDER_REVIEW == ChangesetStatusModel().calculate_pull_request_result(pullrequest)[2]
-        params = json.dumps({
+        params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": User.get_by_username(base.TEST_USER_REGULAR_LOGIN).api_key,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "status": ChangesetStatus.STATUS_APPROVED},
         })
+        }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = PullRequest().get(pull_request_id)
         assert ChangesetStatus.STATUS_APPROVED == ChangesetStatusModel().calculate_pull_request_result(pullrequest)[2]
     def test_api_comment_pullrequest(self):
         pull_request_id = fixture.create_pullrequest(self, self.REPO, self.TEST_PR_SRC, self.TEST_PR_DST, u"comment test")
         random_id = random.randrange(1, 9999)
-        params = json.dumps({
+        params = ascii_bytes(ext_json.dumps({
             "id": random_id,
             "api_key": self.apikey,
             "method": "comment_pullrequest",
             "args": {"pull_request_id": pull_request_id, "comment_msg": "Looks good to me"},
         })
+        }))
         response = api_call(self, params)
         self._compare_ok(random_id, True, given=response.body)
         pullrequest = PullRequest().get(pull_request_id)
         assert pullrequest.comments[-1].text == u'Looks good to me'

0 comments (0 inline, 0 general)