kallithea Changeset - f4d128af1a01

Changeset - f4d128af1a01

Parent rev.

Child rev.

[Not reviewed]

default

0 6 0

Mads Kiilerich - 9 years ago 2016-09-12 17:41:19
madski@unity3d.com

compat: drop unnecessary wrappers for old Python versions

Made unnecessary by other cleanup or upgrade to Python 2.6+.

6 files changed with 31 insertions and 529 deletions:

kallithea/controllers/api/__init__.py

kallithea/controllers/summary.py

kallithea/lib/auth_modules/__init__.py

kallithea/lib/compat.py

204

kallithea/lib/vcs/subprocessio.py

kallithea/lib/vcs/utils/compat.py

307

0 comments (0 inline, 0 general)

kallithea/controllers/api/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.api
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 JSON RPC controller
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 20, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import inspect
 import logging
 import types
 import traceback
 import time
 import itertools
 from paste.response import replace_header
 from pylons.controllers import WSGIController
 from webob.exc import HTTPError
 from kallithea.model.db import User
 from kallithea.model import meta
-from kallithea.lib.compat import izip_longest, json
 from kallithea.lib.compat import json
 from kallithea.lib.auth import AuthUser
 from kallithea.lib.base import _get_ip_addr as _get_ip, _get_access_path
 from kallithea.lib.utils2 import safe_unicode, safe_str
 log = logging.getLogger('JSONRPC')
 class JSONRPCError(BaseException):
     def __init__(self, message):
         self.message = message
         super(JSONRPCError, self).__init__()
     def __str__(self):
         return safe_str(self.message)
 def jsonrpc_error(message, retid=None, code=None):
     """
     Generate a Response object with a JSON-RPC error body
     :param code:
     :param retid:
     :param message:
     """
     from pylons.controllers.util import Response
     return Response(
         body=json.dumps(dict(id=retid, result=None, error=message)),
         status=code,
         content_type='application/json'
+    )
 class JSONRPCController(WSGIController):
     """
      A WSGI-speaking JSON-RPC controller class
      See the specification:
      <http://json-rpc.org/wiki/specification>`.
      Valid controller return values should be json-serializable objects.
      Sub-classes should catch their exceptions and raise JSONRPCError
      if they want to pass meaningful errors to the client.
      """
     def _get_ip_addr(self, environ):
         return _get_ip(environ)
     def _get_method_args(self):
         """
         Return `self._rpc_args` to dispatched controller method
         chosen by __call__
         """
         return self._rpc_args
     def __call__(self, environ, start_response):
         """
         Parse the request body as JSON, look up the method on the
         controller and if it exists, dispatch to it.
         """
         try:
             return self._handle_request(environ, start_response)
         finally:
             meta.Session.remove()
     def _handle_request(self, environ, start_response):
         start = time.time()
         ip_addr = self.ip_addr = self._get_ip_addr(environ)
         self._req_id = None
         if 'CONTENT_LENGTH' not in environ:
             log.debug("No Content-Length")
             return jsonrpc_error(retid=self._req_id,
                                  message="No Content-Length in request")
         else:
             length = environ['CONTENT_LENGTH'] or 0
             length = int(environ['CONTENT_LENGTH'])
             log.debug('Content-Length: %s', length)
         if length == 0:
             return jsonrpc_error(retid=self._req_id,
                                  message="Content-Length is 0")
         raw_body = environ['wsgi.input'].read(length)
         try:
             json_body = json.loads(raw_body)
         except ValueError as e:
             # catch JSON errors Here
             return jsonrpc_error(retid=self._req_id,
                                  message="JSON parse error ERR:%s RAW:%r"
                                  % (e, raw_body))
         # check AUTH based on API key
         try:
             self._req_api_key = json_body['api_key']
             self._req_id = json_body['id']
             self._req_method = json_body['method']
             self._request_params = json_body['args']
             if not isinstance(self._request_params, dict):
                 self._request_params = {}
             log.debug(
                 'method: %s, params: %s', self._req_method,
                                             self._request_params
+            )
         except KeyError as e:
             return jsonrpc_error(retid=self._req_id,
                                  message='Incorrect JSON query missing %s' % e)
         # check if we can find this session using api_key
         try:
             u = User.get_by_api_key(self._req_api_key)
             if u is None:
                 return jsonrpc_error(retid=self._req_id,
                                      message='Invalid API key')
             auth_u = AuthUser(dbuser=u)
             if not AuthUser.check_ip_allowed(auth_u, ip_addr):
                 return jsonrpc_error(retid=self._req_id,
                         message='request from IP:%s not allowed' % (ip_addr,))
             else:
                 log.info('Access for IP:%s allowed', ip_addr)
         except Exception as e:
             return jsonrpc_error(retid=self._req_id,
                                  message='Invalid API key')
         self._error = None
         try:
             self._func = self._find_method()
         except AttributeError as e:
             return jsonrpc_error(retid=self._req_id,
                                  message=str(e))
         # now that we have a method, add self._req_params to
         # self.kargs and dispatch control to WGIController
         argspec = inspect.getargspec(self._func)
         arglist = argspec[0][1:]
         defaults = map(type, argspec[3] or [])
         default_empty = types.NotImplementedType
         # kw arguments required by this method
         func_kwargs = dict(izip_longest(reversed(arglist), reversed(defaults),
+        func_kwargs = dict(itertools.izip_longest(reversed(arglist), reversed(defaults),
                                         fillvalue=default_empty))
         # this is little trick to inject logged in user for
         # perms decorators to work they expect the controller class to have
         # authuser attribute set
         self.authuser = auth_u
         # This attribute will need to be first param of a method that uses
         # api_key, which is translated to instance of user at that name
         USER_SESSION_ATTR = 'apiuser'
         if USER_SESSION_ATTR not in arglist:
             return jsonrpc_error(
                 retid=self._req_id,
                 message='This method [%s] does not support '
                          'authentication (missing %s param)' % (
                                     self._func.__name__, USER_SESSION_ATTR)
+            )
         # get our arglist and check if we provided them as args
         for arg, default in func_kwargs.iteritems():
             if arg == USER_SESSION_ATTR:
                 # USER_SESSION_ATTR is something translated from API key and
                 # this is checked before so we don't need validate it
                 continue
             # skip the required param check if it's default value is
             # NotImplementedType (default_empty)
             if default == default_empty and arg not in self._request_params:
                 return jsonrpc_error(
                     retid=self._req_id,
                     message=(
                         'Missing non optional `%s` arg in JSON DATA' % arg
+                    )
+                )
         self._rpc_args = {USER_SESSION_ATTR: u}
         self._rpc_args.update(self._request_params)
         self._rpc_args['action'] = self._req_method
         self._rpc_args['environ'] = environ
         self._rpc_args['start_response'] = start_response
         status = []
         headers = []
         exc_info = []
         def change_content(new_status, new_headers, new_exc_info=None):
             status.append(new_status)
             headers.extend(new_headers)
             exc_info.append(new_exc_info)
         output = WSGIController.__call__(self, environ, change_content)
         output = list(output) # expand iterator - just to ensure exact timing
         replace_header(headers, 'Content-Type', 'application/json')
         start_response(status[0], headers, exc_info[0])
         log.info('IP: %s Request to %s time: %.3fs' % (
             self._get_ip_addr(environ),
             safe_unicode(_get_access_path(environ)), time.time() - start)
+        )
         return output
     def _dispatch_call(self):
         """
         Implement dispatch interface specified by WSGIController
         """
         raw_response = ''
         try:
             raw_response = self._inspect_call(self._func)
             if isinstance(raw_response, HTTPError):
                 self._error = str(raw_response)
         except JSONRPCError as e:
             self._error = safe_str(e)
         except Exception as e:
             log.error('Encountered unhandled exception: %s',
                       traceback.format_exc(),)
             json_exc = JSONRPCError('Internal server error')
             self._error = safe_str(json_exc)
         if self._error is not None:
             raw_response = None
         response = dict(id=self._req_id, result=raw_response, error=self._error)
         try:
             return json.dumps(response)
         except TypeError as e:
             log.error('API FAILED. Error encoding response: %s', e)
             return json.dumps(
                 dict(
                     id=self._req_id,
                     result=None,
                     error="Error encoding response"
+                )
+            )

kallithea/controllers/summary.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.summary
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import traceback
 import calendar
 import logging
 import itertools
 from time import mktime
 from datetime import timedelta, date
 from pylons import tmpl_context as c, request
 from pylons.i18n.translation import _
 from webob.exc import HTTPBadRequest
 from beaker.cache import cache_region, region_invalidate
 from kallithea.lib.compat import product
 from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeDoesNotExistError
 from kallithea.config.conf import ALL_READMES, ALL_EXTS, LANGUAGES_EXTENSIONS_MAP
 from kallithea.model.db import Statistics, CacheInvalidation, User
 from kallithea.lib.utils import jsonify
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator, \
     NotAnonymous
 from kallithea.lib.base import BaseRepoController, render
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.markup_renderer import MarkupRenderer
 from kallithea.lib.celerylib.tasks import get_commits_stats
 from kallithea.lib.compat import json
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.controllers.changelog import _load_changelog_summary
 log = logging.getLogger(__name__)
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
+                    sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 class SummaryController(BaseRepoController):
     def __before__(self):
         super(SummaryController, self).__before__()
     def __get_readme_data(self, db_repo):
         repo_name = db_repo.repo_name
         log.debug('Looking for README file')
         @cache_region('long_term', '_get_readme_from_cache')
         def _get_readme_from_cache(key, kind):
             readme_data = None
             readme_file = None
             try:
                 # gets the landing revision! or tip if fails
                 cs = db_repo.get_landing_changeset()
                 if isinstance(cs, EmptyChangeset):
                     raise EmptyRepositoryError()
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         if not isinstance(readme, FileNode):
                             continue
                         readme_file = f
                         log.debug('Found README file `%s` rendering...',
                                   readme_file)
                         readme_data = renderer.render(readme.content,
                                                       filename=f)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 log.error(traceback.format_exc())
                 pass
             except EmptyRepositoryError:
                 pass
             return readme_data, readme_file
         kind = 'README'
         valid = CacheInvalidation.test_and_set_valid(repo_name, kind)
         if not valid:
             region_invalidate(_get_readme_from_cache, None, '_get_readme_from_cache', repo_name, kind)
         return _get_readme_from_cache(repo_name, kind)
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name):
         _load_changelog_summary()
         if self.authuser.is_default_user:
             username = ''
         else:
             username = safe_str(self.authuser.username)
         _def_clone_uri = _def_clone_uri_by_id = c.clone_uri_tmpl
         if '{repo}' in _def_clone_uri:
             _def_clone_uri_by_id = _def_clone_uri.replace('{repo}', '_{repoid}')
         elif '{repoid}' in _def_clone_uri:
             _def_clone_uri_by_id = _def_clone_uri.replace('_{repoid}', '{repo}')
         c.clone_repo_url = c.db_repo.clone_url(user=username,
                                                 uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = c.db_repo.clone_url(user=username,
                                                 uri_tmpl=_def_clone_uri_by_id)
         if c.db_repo.enable_statistics:
             c.show_stats = True
         else:
             c.show_stats = False
         stats = self.sa.query(Statistics) \
             .filter(Statistics.repository == c.db_repo) \
             .scalar()
         c.stats_percentage = 0
         if stats and stats.languages:
             c.no_data = False is c.db_repo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
             c.trending_languages = json.dumps(
                 sorted(lang_stats, reverse=True, key=lambda k: k[1])[:10]
+            )
         else:
             c.no_data = True
             c.trending_languages = json.dumps([])

kallithea/lib/auth_modules/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Authentication modules
 """
 import logging
 import traceback
 import importlib
 from kallithea.lib.compat import importlib
 from kallithea.lib.utils2 import str2bool
 from kallithea.lib.compat import formatted_json, hybrid_property
 from kallithea.lib.auth import PasswordGenerator
 from kallithea.model.user import UserModel
 from kallithea.model.db import Setting, User
 from kallithea.model.meta import Session
 from kallithea.model.user_group import UserGroupModel
 log = logging.getLogger(__name__)
 class LazyFormencode(object):
     def __init__(self, formencode_obj, *args, **kwargs):
         self.formencode_obj = formencode_obj
         self.args = args
         self.kwargs = kwargs
     def __call__(self, *args, **kwargs):
         from inspect import isfunction
         formencode_obj = self.formencode_obj
         if isfunction(formencode_obj):
             #case we wrap validators into functions
             formencode_obj = self.formencode_obj(*args, **kwargs)
         return formencode_obj(*self.args, **self.kwargs)
 class KallitheaAuthPluginBase(object):
     auth_func_attrs = {
         "username": "unique username",
         "firstname": "first name",
         "lastname": "last name",
         "email": "email address",
         "groups": '["list", "of", "groups"]',
         "extern_name": "name in external source of record",
         "admin": 'True|False defines if user should be Kallithea admin',
         "active": 'True|False defines active state of user in Kallithea',
         "active_from_extern": "True|False|None, active state from the external auth, "
                               "None means use value from the auth plugin"
+    }
     @property
     def validators(self):
         """
         Exposes Kallithea validators modules
         """
         # this is a hack to overcome issues with pylons threadlocals and
         # translator object _() not being registered properly.
         class LazyCaller(object):
             def __init__(self, name):
                 self.validator_name = name
             def __call__(self, *args, **kwargs):
                 from kallithea.model import validators as v
                 obj = getattr(v, self.validator_name)
                 #log.debug('Initializing lazy formencode object: %s', obj)
                 return LazyFormencode(obj, *args, **kwargs)
         class ProxyGet(object):
             def __getattribute__(self, name):
                 return LazyCaller(name)
         return ProxyGet()
     @hybrid_property
     def name(self):
         """
         Returns the name of this authentication plugin.
         :returns: string
         """
         raise NotImplementedError("Not implemented in base class")
     @hybrid_property
     def is_container_auth(self):
         """
         Returns bool if this module uses container auth.
         This property will trigger an automatic call to authenticate on
         a visit to the website or during a push/pull.
         :returns: bool
         """
         return False
     def accepts(self, user, accepts_empty=True):
         """
         Checks if this authentication module should accept a request for
         the current user.
         :param user: user object fetched using plugin's get_user() method.
         :param accepts_empty: if True accepts don't allow the user to be empty
         :returns: boolean
         """
         plugin_name = self.name
         if not user and not accepts_empty:

kallithea/lib/compat.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.compat
 ~~~~~~~~~~~~~~~~~~~~
 Python backward compatibility functions and common libs
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Oct 7, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import sys
 import os
 import functools
 import importlib
 from kallithea import __py_version__, is_windows
 #==============================================================================
 # json
 #==============================================================================
 from kallithea.lib.ext_json import json
 # alias for formatted json
 formatted_json = functools.partial(json.dumps, indent=4, sort_keys=True)
 if __py_version__ >= (2, 7):
 #==============================================================================
 # unittest
 #==============================================================================
 if sys.version_info >= (2, 7):
     import unittest
 else:
     import unittest2 as unittest
 #==============================================================================
 # izip_longest
 #==============================================================================
 try:
     from itertools import izip_longest
 except ImportError:
     import itertools
     def izip_longest(*args, **kwds):
         fillvalue = kwds.get("fillvalue")
         def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
             yield counter()  # yields the fillvalue, or raises IndexError
         fillers = itertools.repeat(fillvalue)
         iters = [itertools.chain(it, sentinel(), fillers)
                     for it in args]
         try:
             for tup in itertools.izip(*iters):
                 yield tup
         except IndexError:
             pass
 #==============================================================================
 # OrderedDict
+# OrderedDict - Python 2.7 could perhaps use collections.OrderedDict
 #==============================================================================
 # Python Software Foundation License
 # XXX: it feels like using the class with "is" and "is not" instead of "==" and
 # "!=" should be faster.
 class _Nil(object):
     def __repr__(self):
         return "nil"
     def __eq__(self, other):
         if (isinstance(other, _Nil)):
             return True
         else:
             return NotImplemented
     def __ne__(self, other):
         if (isinstance(other, _Nil)):
             return False
         else:
             return NotImplemented
 _nil = _Nil()
 class _odict(object):
     """Ordered dict data structure, with O(1) complexity for dict operations
     that modify one element.
     Overwriting values doesn't change their original sequential order.
     """
     def _dict_impl(self):
         return None
     def __init__(self, data=(), **kwds):
         """This doesn't accept keyword initialization as normal dicts to avoid
         a trap - inside a function or method the keyword args are accessible
         only as a dict, without a defined order, so their original order is
         lost.
         """
         if kwds:
             raise TypeError("__init__() of ordered dict takes no keyword "
                             "arguments to avoid an ordering trap.")
         self._dict_impl().__init__(self)
         # If you give a normal dict, then the order of elements is undefined
         if hasattr(data, "iteritems"):
             for key, val in data.iteritems():
                 self[key] = val
         else:
             for key, val in data:
                 self[key] = val
     # Double-linked list header
     def _get_lh(self):
         dict_impl = self._dict_impl()
         if not hasattr(self, '_lh'):
             dict_impl.__setattr__(self, '_lh', _nil)
         return dict_impl.__getattribute__(self, '_lh')
     def _set_lh(self, val):
         self._dict_impl().__setattr__(self, '_lh', val)
     lh = property(_get_lh, _set_lh)
     # Double-linked list tail
     def _get_lt(self):
         dict_impl = self._dict_impl()
         if not hasattr(self, '_lt'):
             dict_impl.__setattr__(self, '_lt', _nil)
         return dict_impl.__getattribute__(self, '_lt')
     def _set_lt(self, val):
         self._dict_impl().__setattr__(self, '_lt', val)
     lt = property(_get_lt, _set_lt)
     def __getitem__(self, key):
         return self._dict_impl().__getitem__(self, key)[1]
     def __setitem__(self, key, val):
         dict_impl = self._dict_impl()
         try:
             dict_impl.__getitem__(self, key)[1] = val
         except KeyError:
             new = [dict_impl.__getattribute__(self, 'lt'), val, _nil]
             dict_impl.__setitem__(self, key, new)
             if dict_impl.__getattribute__(self, 'lt') == _nil:
                 dict_impl.__setattr__(self, 'lh', key)
             else:
                 dict_impl.__getitem__(
                     self, dict_impl.__getattribute__(self, 'lt'))[2] = key
             dict_impl.__setattr__(self, 'lt', key)
     def __delitem__(self, key):
@@ @@ -284,287 +264,113 @@ class _odict(object): @@
             return x
     def popitem(self):
         try:
             dict_impl = self._dict_impl()
             key = dict_impl.__getattribute__(self, 'lt')
             return key, self.pop(key)
         except KeyError:
             raise KeyError("'popitem(): ordered dictionary is empty'")
     def riterkeys(self):
         """To iterate on keys in reversed order.
         """
         dict_impl = self._dict_impl()
         curr_key = dict_impl.__getattribute__(self, 'lt')
         while curr_key != _nil:
             yield curr_key
             curr_key = dict_impl.__getitem__(self, curr_key)[0]
     __reversed__ = riterkeys
     def rkeys(self):
         """List of the keys in reversed order.
         """
         return list(self.riterkeys())
     def ritervalues(self):
         """To iterate on values in reversed order.
         """
         dict_impl = self._dict_impl()
         curr_key = dict_impl.__getattribute__(self, 'lt')
         while curr_key != _nil:
             curr_key, val, _ = dict_impl.__getitem__(self, curr_key)
             yield val
     def rvalues(self):
         """List of the values in reversed order.
         """
         return list(self.ritervalues())
     def riteritems(self):
         """To iterate on (key, value) in reversed order.
         """
         dict_impl = self._dict_impl()
         curr_key = dict_impl.__getattribute__(self, 'lt')
         while curr_key != _nil:
             pred_key, val, _ = dict_impl.__getitem__(self, curr_key)
             yield curr_key, val
             curr_key = pred_key
     def ritems(self):
         """List of the (key, value) in reversed order.
         """
         return list(self.riteritems())
     def firstkey(self):
         if self:
             return self._dict_impl().__getattribute__(self, 'lh')
         else:
             raise KeyError("'firstkey(): ordered dictionary is empty'")
     def lastkey(self):
         if self:
             return self._dict_impl().__getattribute__(self, 'lt')
         else:
             raise KeyError("'lastkey(): ordered dictionary is empty'")
     def as_dict(self):
         return self._dict_impl()(self.items())
     def _repr(self):
         """_repr(): low level repr of the whole data contained in the odict.
         Useful for debugging.
         """
         dict_impl = self._dict_impl()
         form = "odict low level repr lh,lt,data: %r, %r, %s"
         return form % (dict_impl.__getattribute__(self, 'lh'),
                        dict_impl.__getattribute__(self, 'lt'),
                        dict_impl.__repr__(self))
 class OrderedDict(_odict, dict):
     def _dict_impl(self):
         return dict
 #==============================================================================
 # OrderedSet
 #==============================================================================
 from sqlalchemy.util import OrderedSet
 #==============================================================================
 # Hybrid property/method
 #==============================================================================
-from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
 from sqlalchemy.ext.hybrid import hybrid_property
 #==============================================================================
-# kill FUNCTIONS
 # kill
 #==============================================================================
-if is_windows:
+if os.name == 'nt': # Windows
     import ctypes
     def kill(pid, sig):
         """kill function for Win32"""
         kernel32 = ctypes.windll.kernel32
         handle = kernel32.OpenProcess(1, 0, pid)
         return (0 != kernel32.TerminateProcess(handle, 0))
 else:
     kill = os.kill
 #==============================================================================
 # itertools.product
 #==============================================================================
 try:
     from itertools import product
 except ImportError:
     def product(*args, **kwds):
         # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
         # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
         pools = map(tuple, args) * kwds.get('repeat', 1)
         result = [[]]
         for pool in pools:
             result = [x + [y] for x in result for y in pool]
         for prod in result:
             yield tuple(prod)
 #==============================================================================
 # BytesIO
 #==============================================================================
 try:
     from io import BytesIO
 except ImportError:
     from cStringIO import StringIO as BytesIO
 #==============================================================================
 # bytes
 #==============================================================================
 if __py_version__ >= (2, 6):
     _bytes = bytes
 else:
     # in py2.6 bytes is a synonym for str
     _bytes = str
 if __py_version__ >= (2, 6):
     _bytearray = bytearray
 else:
     import array
     # no idea if this is correct but all integration tests are passing
     # i think we never use bytearray anyway
     _bytearray = array
 #==============================================================================
 # deque
 #==============================================================================
 if __py_version__ >= (2, 6):
     from collections import deque
 else:
     #need to implement our own deque with maxlen
     class deque(object):
         def __init__(self, iterable=(), maxlen= -1):
             if not hasattr(self, 'data'):
                 self.left = self.right = 0
                 self.data = {}
             self.maxlen = maxlen or -1
             self.extend(iterable)
         def append(self, x):
             self.data[self.right] = x
             self.right += 1
             if self.maxlen != -1 and len(self) > self.maxlen:
                 self.popleft()
         def appendleft(self, x):
             self.left -= 1
             self.data[self.left] = x
             if self.maxlen != -1 and len(self) > self.maxlen:
                 self.pop()
         def pop(self):
             if self.left == self.right:
                 raise IndexError('cannot pop from empty deque')
             self.right -= 1
             elem = self.data[self.right]
             del self.data[self.right]
             return elem
         def popleft(self):
             if self.left == self.right:
                 raise IndexError('cannot pop from empty deque')
             elem = self.data[self.left]
             del self.data[self.left]
             self.left += 1
             return elem
         def clear(self):
             self.data.clear()
             self.left = self.right = 0
         def extend(self, iterable):
             for elem in iterable:
                 self.append(elem)
         def extendleft(self, iterable):
             for elem in iterable:
                 self.appendleft(elem)
         def rotate(self, n=1):
             if self:
                 n %= len(self)
                 for i in xrange(n):
                     self.appendleft(self.pop())
         def __getitem__(self, i):
             if i < 0:
                 i += len(self)
             try:
                 return self.data[i + self.left]
             except KeyError:
                 raise IndexError
         def __setitem__(self, i, value):
             if i < 0:
                 i += len(self)
             try:
                 self.data[i + self.left] = value
             except KeyError:
                 raise IndexError
         def __delitem__(self, i):
             size = len(self)
             if not (-size <= i < size):
                 raise IndexError
             data = self.data
             if i < 0:
                 i += size
             for j in xrange(self.left + i, self.right - 1):
                 data[j] = data[j + 1]
             self.pop()
         def __len__(self):
             return self.right - self.left
         def __cmp__(self, other):
             if type(self) != type(other):
                 return cmp(type(self), type(other))
             return cmp(list(self), list(other))
         def __repr__(self, _track=None):
             _track = _track or []
             if id(self) in _track:
                 return '...'
             _track.append(id(self))
             r = 'deque(%r, maxlen=%s)' % (list(self), self.maxlen)
             _track.remove(id(self))
             return r
         def __getstate__(self):
             return (tuple(self),)
         def __setstate__(self, s):
             self.__init__(s[0])
         def __hash__(self):
             raise TypeError
         def __copy__(self):
             return self.__class__(self)
         def __deepcopy__(self, memo=None):
             from copy import deepcopy
             memo = memo or {}
             result = self.__class__()
             memo[id(self)] = result
             result.__init__(deepcopy(tuple(self), memo))
             return result

kallithea/lib/vcs/subprocessio.py

➞

Show inline comments

 """
 Module provides a class allowing to wrap communication over subprocess.Popen
 input, output, error streams into a meaningful, non-blocking, concurrent
 stream processor exposing the output data as an iterator fitting to be a
 return value passed by a WSGI application to a WSGI server per PEP 3333.
 Copyright (c) 2011  Daniel Dotsenko <dotsa[at]hotmail.com>
 This file is part of git_http_backend.py Project.
 git_http_backend.py Project is free software: you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public License as
 published by the Free Software Foundation, either version 2.1 of the License,
 or (at your option) any later version.
 git_http_backend.py Project is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 GNU Lesser General Public License for more details.
 You should have received a copy of the GNU Lesser General Public License
 along with git_http_backend.py Project.
 If not, see <http://www.gnu.org/licenses/>.
 """
 import os
 import subprocess
 from kallithea.lib.vcs.utils.compat import deque, Event, Thread, _bytes, _bytearray
 import collections
 import threading
 class StreamFeeder(Thread):
+class StreamFeeder(threading.Thread):
     """
     Normal writing into pipe-like is blocking once the buffer is filled.
     This thread allows a thread to seep data from a file-like into a pipe
     without blocking the main thread.
     We close inpipe once the end of the source stream is reached.
     """
     def __init__(self, source):
         super(StreamFeeder, self).__init__()
         self.daemon = True
         filelike = False
         self.bytes = _bytes()
         if type(source) in (type(''), _bytes, _bytearray):  # string-like
             self.bytes = _bytes(source)
         self.bytes = bytes()
         if type(source) in (type(''), bytes, bytearray):  # string-like
             self.bytes = bytes(source)
         else:  # can be either file pointer or file-like
             if type(source) in (int, long):  # file pointer it is
                 ## converting file descriptor (int) stdin into file-like
                 source = os.fdopen(source, 'rb', 16384)
             # let's see if source is file-like by now
             filelike = hasattr(source, 'read')
         if not filelike and not self.bytes:
             raise TypeError("StreamFeeder's source object must be a readable "
                             "file-like, a file descriptor, or a string-like.")
         self.source = source
         self.readiface, self.writeiface = os.pipe()
     def run(self):
         t = self.writeiface
         if self.bytes:
             os.write(t, self.bytes)
         else:
             s = self.source
             b = s.read(4096)
             while b:
                 os.write(t, b)
                 b = s.read(4096)
         os.close(t)
     @property
     def output(self):
         return self.readiface
 class InputStreamChunker(Thread):
+class InputStreamChunker(threading.Thread):
     def __init__(self, source, target, buffer_size, chunk_size):
         super(InputStreamChunker, self).__init__()
         self.daemon = True  # die die die.
         self.source = source
         self.target = target
         self.chunk_count_max = int(buffer_size / chunk_size) + 1
         self.chunk_size = chunk_size
         self.data_added = Event()
+        self.data_added = threading.Event()
         self.data_added.clear()
         self.keep_reading = Event()
+        self.keep_reading = threading.Event()
         self.keep_reading.set()
         self.EOF = Event()
+        self.EOF = threading.Event()
         self.EOF.clear()
         self.go = Event()
+        self.go = threading.Event()
         self.go.set()
     def stop(self):
         self.go.clear()
         self.EOF.set()
         try:
             # this is not proper, but is done to force the reader thread let
             # go of the input because, if successful, .close() will send EOF
             # down the pipe.
             self.source.close()
         except:
             pass
     def run(self):
         s = self.source
         t = self.target
         cs = self.chunk_size
         ccm = self.chunk_count_max
         kr = self.keep_reading
         da = self.data_added
         go = self.go
         try:
             b = s.read(cs)
         except ValueError:
             b = ''
         while b and go.is_set():
             if len(t) > ccm:
                 kr.clear()
                 kr.wait(2)
                 # # this only works on 2.7.x and up
                 # if not kr.wait(10):
                 #     raise Exception("Timed out while waiting for input to be read.")
                 # instead we'll use this
                 if len(t) > ccm + 3:
                     raise IOError(
                         "Timed out while waiting for input from subprocess.")
             t.append(b)
             da.set()
             try:
                 b = s.read(cs)
             except ValueError: # probably "I/O operation on closed file"
                 b = ''
         self.EOF.set()
         da.set()  # for cases when done but there was no input.
 class BufferedGenerator(object):
     """
     Class behaves as a non-blocking, buffered pipe reader.
     Reads chunks of data (through a thread)
     from a blocking pipe, and attaches these to an array (Deque) of chunks.
     Reading is halted in the thread when max chunks is internally buffered.
     The .next() may operate in blocking or non-blocking fashion by yielding
     '' if no data is ready
     to be sent or by not returning until there is some data to send
     When we get EOF from underlying source pipe we raise the marker to raise
     StopIteration after the last chunk of data is yielded.
     """
     def __init__(self, source, buffer_size=65536, chunk_size=4096,
                  starting_values=None, bottomless=False):
         starting_values = starting_values or []
         if bottomless:
             maxlen = int(buffer_size / chunk_size)
         else:
             maxlen = None
         self.data = deque(starting_values, maxlen)
+        self.data = collections.deque(starting_values, maxlen)
         self.worker = InputStreamChunker(source, self.data, buffer_size,
                                          chunk_size)
         if starting_values:
             self.worker.data_added.set()
         self.worker.start()
     ####################
     # Generator's methods
     ####################
     def __iter__(self):
         return self
     def next(self):
         while not len(self.data) and not self.worker.EOF.is_set():
             self.worker.data_added.clear()
             self.worker.data_added.wait(0.2)
         if len(self.data):
             self.worker.keep_reading.set()
-            return _bytes(self.data.popleft())
+            return bytes(self.data.popleft())
         elif self.worker.EOF.is_set():
             raise StopIteration
     def throw(self, type, value=None, traceback=None):
         if not self.worker.EOF.is_set():
             raise type(value)
     def start(self):
         self.worker.start()
     def stop(self):
         self.worker.stop()
     def close(self):
         try:
             self.worker.stop()
             self.throw(GeneratorExit)
         except (GeneratorExit, StopIteration):
             pass
     def __del__(self):
         self.close()
     ####################
     # Threaded reader's infrastructure.
     ####################
     @property
     def input(self):
         return self.worker.w
     @property
     def data_added_event(self):
         return self.worker.data_added
     @property
     def data_added(self):
         return self.worker.data_added.is_set()
     @property
     def reading_paused(self):
         return not self.worker.keep_reading.is_set()
     @property
     def done_reading_event(self):
         """
         Done_reading does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
         :returns: An Event class instance.
+        :returns: An threading.Event class instance.
         """
         return self.worker.EOF
     @property
     def done_reading(self):
         """
         Done_reading does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
         :returns: An Bool value.
         """
         return self.worker.EOF.is_set()
     @property
     def length(self):
         """
         returns int.
         This is the length of the queue of chunks, not the length of
         the combined contents in those chunks.
         __len__() cannot be meaningfully implemented because this
         reader is just flying through a bottomless pit content and
         can only know the length of what it already saw.
         If __len__() on WSGI server per PEP 3333 returns a value,
         the response's length will be set to that. In order not to
         confuse WSGI PEP3333 servers, we will not implement __len__
         at all.
         """
         return len(self.data)
     def prepend(self, x):
         self.data.appendleft(x)
     def append(self, x):
         self.data.append(x)
     def extend(self, o):
         self.data.extend(o)
     def __getitem__(self, i):
         return self.data[i]
 class SubprocessIOChunker(object):
     """
     Processor class wrapping handling of subprocess IO.
     In a way, this is a "communicate()" replacement with a twist.
     - We are multithreaded. Writing in and reading out, err are all sep threads.
     - We support concurrent (in and out) stream processing.
     - The output is not a stream. It's a queue of read string (bytes, not unicode)
       chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
     - We are non-blocking in more respects than communicate()
       (reading from subprocess out pauses when internal buffer is full, but
        does not block the parent calling code. On the flip side, reading from
        slow-yielding subprocess may block the iteration until data shows up. This
        does not block the parallel inpipe reading occurring parallel thread.)
     The purpose of the object is to allow us to wrap subprocess interactions into
     an iterable that can be passed to a WSGI server as the application's return
     value. Because of stream-processing-ability, WSGI does not have to read ALL
     of the subprocess's output and buffer it, before handing it to WSGI server for
     HTTP response. Instead, the class initializer reads just a bit of the stream
     to figure out if error occurred or likely to occur and if not, just hands the
     further iteration over subprocess output to the server for completion of HTTP
     response.
     The real or perceived subprocess error is trapped and raised as one of
     EnvironmentError family of exceptions
     Example usage:
     #    try:
     #        answer = SubprocessIOChunker(
     #            cmd,
     #            input,
     #            buffer_size = 65536,
     #            chunk_size = 4096
     #            )
     #    except (EnvironmentError) as e:
     #        print str(e)
     #        raise e
+    #
     #    return answer
     """
     def __init__(self, cmd, inputstream=None, buffer_size=65536,
                  chunk_size=4096, starting_values=None, **kwargs):
         """
         Initializes SubprocessIOChunker

kallithea/lib/vcs/utils/compat.py

➞

Show inline comments

 """
 Various utilities to work with Python < 2.7.
 Those utilities may be deleted once ``vcs`` stops support for older Python
 versions.
 """
 import sys
 import array
 if sys.version_info >= (2, 7):
     unittest = __import__('unittest')
 else:
     unittest = __import__('unittest2')
 if sys.version_info >= (2, 6):
     _bytes = bytes
 else:
     # in py2.6 bytes is a synonym for str
     _bytes = str
 if sys.version_info >= (2, 6):
     _bytearray = bytearray
 else:
     # no idea if this is correct but all integration tests are passing
     # i think we never use bytearray anyway
     _bytearray = array
 if sys.version_info >= (2, 6):
     from collections import deque
 else:
     #need to implement our own deque with maxlen
     class deque(object):
         def __init__(self, iterable=(), maxlen= -1):
             if not hasattr(self, 'data'):
                 self.left = self.right = 0
                 self.data = {}
             self.maxlen = maxlen or -1
             self.extend(iterable)
         def append(self, x):
             self.data[self.right] = x
             self.right += 1
             if self.maxlen != -1 and len(self) > self.maxlen:
                 self.popleft()
         def appendleft(self, x):
             self.left -= 1
             self.data[self.left] = x
             if self.maxlen != -1 and len(self) > self.maxlen:
                 self.pop()
         def pop(self):
             if self.left == self.right:
                 raise IndexError('cannot pop from empty deque')
             self.right -= 1
             elem = self.data[self.right]
             del self.data[self.right]
             return elem
         def popleft(self):
             if self.left == self.right:
                 raise IndexError('cannot pop from empty deque')
             elem = self.data[self.left]
             del self.data[self.left]
             self.left += 1
             return elem
         def clear(self):
             self.data.clear()
             self.left = self.right = 0
         def extend(self, iterable):
             for elem in iterable:
                 self.append(elem)
         def extendleft(self, iterable):
             for elem in iterable:
                 self.appendleft(elem)
         def rotate(self, n=1):
             if self:
                 n %= len(self)
                 for i in xrange(n):
                     self.appendleft(self.pop())
         def __getitem__(self, i):
             if i < 0:
                 i += len(self)
             try:
                 return self.data[i + self.left]
             except KeyError:
                 raise IndexError
         def __setitem__(self, i, value):
             if i < 0:
                 i += len(self)
             try:
                 self.data[i + self.left] = value
             except KeyError:
                 raise IndexError
         def __delitem__(self, i):
             size = len(self)
             if not (-size <= i < size):
                 raise IndexError
             data = self.data
             if i < 0:
                 i += size
             for j in xrange(self.left + i, self.right - 1):
                 data[j] = data[j + 1]
             self.pop()
         def __len__(self):
             return self.right - self.left
         def __cmp__(self, other):
             if type(self) != type(other):
                 return cmp(type(self), type(other))
             return cmp(list(self), list(other))
         def __repr__(self, _track=None):
             _track = _track or []
             if id(self) in _track:
                 return '...'
             _track.append(id(self))
             r = 'deque(%r, maxlen=%s)' % (list(self), self.maxlen)
             _track.remove(id(self))
             return r
         def __getstate__(self):
             return (tuple(self),)
         def __setstate__(self, s):
             self.__init__(s[0])
         def __hash__(self):
             raise TypeError
         def __copy__(self):
             return self.__class__(self)
         def __deepcopy__(self, memo=None):
             from copy import deepcopy
             memo = memo or {}
             result = self.__class__()
             memo[id(self)] = result
             result.__init__(deepcopy(tuple(self), memo))
             return result
 #==============================================================================
 # threading.Event
 #==============================================================================
 if sys.version_info >= (2, 6):
     from threading import Event, Thread
 else:
     from threading import _Verbose, Lock, Thread, _time, \
         _allocate_lock, RLock, _sleep
     def Condition(*args, **kwargs):
         return _Condition(*args, **kwargs)
     class _Condition(_Verbose):
         def __init__(self, lock=None, verbose=None):
             _Verbose.__init__(self, verbose)
             if lock is None:
                 lock = RLock()
             self.__lock = lock
             # Export the lock's acquire() and release() methods
             self.acquire = lock.acquire
             self.release = lock.release
             # If the lock defines _release_save() and/or _acquire_restore(),
             # these override the default implementations (which just call
             # release() and acquire() on the lock).  Ditto for _is_owned().
             try:
                 self._release_save = lock._release_save
             except AttributeError:
                 pass
             try:
                 self._acquire_restore = lock._acquire_restore
             except AttributeError:
                 pass
             try:
                 self._is_owned = lock._is_owned
             except AttributeError:
                 pass
             self.__waiters = []
         def __enter__(self):
             return self.__lock.__enter__()
         def __exit__(self, *args):
             return self.__lock.__exit__(*args)
         def __repr__(self):
             return "<Condition(%s, %d)>" % (self.__lock, len(self.__waiters))
         def _release_save(self):
             self.__lock.release()           # No state to save
         def _acquire_restore(self, x):
             self.__lock.acquire()           # Ignore saved state
         def _is_owned(self):
             # Return True if lock is owned by current_thread.
             # This method is called only if __lock doesn't have _is_owned().
             if self.__lock.acquire(0):
                 self.__lock.release()
                 return False
             else:
                 return True
         def wait(self, timeout=None):
             if not self._is_owned():
                 raise RuntimeError("cannot wait on un-acquired lock")
             waiter = _allocate_lock()
             waiter.acquire()
             self.__waiters.append(waiter)
             saved_state = self._release_save()
             try:    # restore state no matter what (e.g., KeyboardInterrupt)
                 if timeout is None:
                     waiter.acquire()
                     if __debug__:
                         self._note("%s.wait(): got it", self)
                 else:
                     # Balancing act:  We can't afford a pure busy loop, so we
                     # have to sleep; but if we sleep the whole timeout time,
                     # we'll be unresponsive.  The scheme here sleeps very
                     # little at first, longer as time goes on, but never longer
                     # than 20 times per second (or the timeout time remaining).
                     endtime = _time() + timeout
                     delay = 0.0005 # 500 us -> initial delay of 1 ms
                     while True:
                         gotit = waiter.acquire(0)
                         if gotit:
                             break
                         remaining = endtime - _time()
                         if remaining <= 0:
                             break
                         delay = min(delay * 2, remaining, .05)
                         _sleep(delay)
                     if not gotit:
                         if __debug__:
                             self._note("%s.wait(%s): timed out", self, timeout)
                         try:
                             self.__waiters.remove(waiter)
                         except ValueError:
                             pass
                     else:
                         if __debug__:
                             self._note("%s.wait(%s): got it", self, timeout)
             finally:
                 self._acquire_restore(saved_state)
         def notify(self, n=1):
             if not self._is_owned():
                 raise RuntimeError("cannot notify on un-acquired lock")
             __waiters = self.__waiters
             waiters = __waiters[:n]
             if not waiters:
                 if __debug__:
                     self._note("%s.notify(): no waiters", self)
                 return
             self._note("%s.notify(): notifying %d waiter%s", self, n,
                        n != 1 and "s" or "")
             for waiter in waiters:
                 waiter.release()
                 try:
                     __waiters.remove(waiter)
                 except ValueError:
                     pass
         def notifyAll(self):
             self.notify(len(self.__waiters))
         notify_all = notifyAll
     def Event(*args, **kwargs):
         return _Event(*args, **kwargs)
     class _Event(_Verbose):
         # After Tim Peters' event class (without is_posted())
         def __init__(self, verbose=None):
             _Verbose.__init__(self, verbose)
             self.__cond = Condition(Lock())
             self.__flag = False
         def isSet(self):
             return self.__flag
         is_set = isSet
         def set(self):
             self.__cond.acquire()
             try:
                 self.__flag = True
                 self.__cond.notify_all()
             finally:
                 self.__cond.release()
         def clear(self):
             self.__cond.acquire()
             try:
                 self.__flag = False
             finally:
                 self.__cond.release()
         def wait(self, timeout=None):
             self.__cond.acquire()
             try:
                 if not self.__flag:
                     self.__cond.wait(timeout)
             finally:
                 self.__cond.release()

0 comments (0 inline, 0 general)