kallithea Changeset - 0ebcc88f1280

Changeset - 0ebcc88f1280

Parent rev.

Child rev.

[Not reviewed]

default

0 4 0

Mads Kiilerich - 7 years ago 2018-08-08 02:23:11
mads@kiilerich.com

cache: move cache invalidation from web handler to post push hook

We need the post push hook anyway ... and having the cache invalidation here
will also work for ssh pushes in the future.

The name log_push_action is thus no longer spot-on. That might change later,
but requires some care as it also is used directly as hook name.

Note that having cache invalidation in the hook will do that debug logging no
longer will appear in the server log.

Based on a patch by Dominik Ruf.

4 files changed with 14 insertions and 39 deletions:

kallithea/lib/base.py

kallithea/lib/hooks.py

kallithea/lib/middleware/simplegit.py

kallithea/lib/middleware/simplehg.py

0 comments (0 inline, 0 general)

kallithea/lib/base.py

➞

Show inline comments

@@ @@ -302,56 +302,48 @@ class BaseVCSController(object): @@
             return None, webob.exc.HTTPForbidden()
         return user, None
     def _handle_request(self, environ, start_response):
         raise NotImplementedError()
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> permanent URLs
         :param repo_name:
         """
         data = repo_name.split('/')
         if len(data) >= 2:
             from kallithea.lib.utils import get_repo_by_id
             by_id_match = get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = safe_str(by_id_match)
         return '/'.join(data)
     def _invalidate_cache(self, repo_name):
         """
         Sets cache for this repository for invalidation on next access
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
     def _check_permission(self, action, user, repo_name, ip_addr=None):
         """
         Checks permissions using action (push/pull) user and repository
         name
         :param action: push or pull action
         :param user: `User` instance
         :param repo_name: repository name
         """
         # check IP
         ip_allowed = AuthUser.check_ip_allowed(user, ip_addr)
         if ip_allowed:
             log.info('Access for IP:%s allowed', ip_addr)
         else:
             return False
         if action == 'push':
             if not HasPermissionAnyMiddleware('repository.write',
                                               'repository.admin')(user,
                                                                   repo_name):
                 return False
         else:
             #any other action need at least read permission
@@ @@ -609,64 +601,47 @@ class BaseRepoController(BaseController) @@
     def _get_ref_rev(repo, ref_type, ref_name, returnempty=False):
         """
         Safe way to get changeset. If error occurs show error.
         """
         from kallithea.lib import helpers as h
         try:
             return repo.scm_instance.get_ref_revision(ref_type, ref_name)
         except EmptyRepositoryError as e:
             if returnempty:
                 return repo.scm_instance.EMPTY_CHANGESET
             h.flash(h.literal(_('There are no changesets yet')),
                     category='error')
             raise webob.exc.HTTPNotFound()
         except ChangesetDoesNotExistError as e:
             h.flash(h.literal(_('Changeset for %s %s not found in %s') %
                               (ref_type, ref_name, repo.repo_name)),
                     category='error')
             raise webob.exc.HTTPNotFound()
         except RepositoryError as e:
             log.error(traceback.format_exc())
             h.flash(safe_str(e), category='error')
             raise webob.exc.HTTPBadRequest()
 class WSGIResultCloseCallback(object):
     """Wrap a WSGI result and let close call close after calling the
     close method on the result.
     """
     def __init__(self, result, close):
         self._result = result
         self._close = close
     def __iter__(self):
         return iter(self._result)
     def close(self):
         if hasattr(self._result, 'close'):
             self._result.close()
         self._close()
 @decorator.decorator
 def jsonify(func, *args, **kwargs):
     """Action decorator that formats output for JSON
     Given a function that will return content, this decorator will turn
     the result into JSON, with a content-type of 'application/json' and
     output it.
     """
     response.headers['Content-Type'] = 'application/json; charset=utf-8'
     data = func(*args, **kwargs)
     if isinstance(data, (list, tuple)):
         # A JSON list response is syntactically valid JavaScript and can be
         # loaded and executed as JavaScript by a malicious third-party site
         # using <script>, which can lead to cross-site data leaks.
         # JSON responses should therefore be scalars or objects (i.e. Python
         # dicts), because a JSON object is a syntax error if intepreted as JS.
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "https://web.archive.org/web/20120519231904/http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
     return json.dumps(data, encoding='utf-8')

kallithea/lib/hooks.py

➞

Show inline comments

@@ @@ -127,80 +127,88 @@ def log_pull_action(ui, repo, **kwargs): @@
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
     if ex.make_lock is not None and ex.make_lock:
         Repository.lock(Repository.get_by_repo_name(ex.repository), user.user_id)
         #msg = 'Made lock on repo `%s`' % repository
         #ui.status(msg)
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             ui.status(safe_str(_http_ret.title))
     return 0
 def log_push_action(ui, repo, **kwargs):
     """
     Register that changes have been pushed.
     Register that changes have been pushed - log it *and* invalidate caches.
     Note: It is not only logging, but also the side effect invalidating cahes!
     The function should perhaps be renamed.
     Called as Mercurial hook changegroup.push_logger or from the Git post-receive hook calling handle_git_post_receive ... or from scm _handle_push
     Called as Mercurial hook changegroup.push_logger or from the Git
     post-receive hook calling handle_git_post_receive ... or from scm _handle_push.
     Revisions are passed in different hack-ish ways.
     """
     ex = _extract_extras()
     action_tmpl = ex.action + ':%s'
     revs = []
     if ex.scm == 'hg':
         node = kwargs['node']
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
         stop, start = get_revs(repo, [node + ':'])
         _h = binascii.hexlify
         revs = [_h(repo[r].node()) for r in xrange(start, stop + 1)]
     elif ex.scm == 'git':
         revs = kwargs.get('_git_revs', [])
         if '_git_revs' in kwargs:
             kwargs.pop('_git_revs')
     action = action_tmpl % ','.join(revs)
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
     from kallithea.model.scm import ScmModel
     ScmModel().mark_for_invalidation(ex.repository)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
     if callable(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
     if ex.make_lock is not None and not ex.make_lock:
         Repository.unlock(Repository.get_by_repo_name(ex.repository))
         ui.status(safe_str('Released lock on repo `%s`\n' % ex.repository))
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             ui.status(safe_str(_http_ret.title))
     return 0
 def log_create_repository(repository_dict, created_by, **kwargs):
     """

kallithea/lib/middleware/simplegit.py

➞

Show inline comments

@@ @@ -19,49 +19,49 @@ SimpleGit middleware for handling git pr @@
 It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import re
 import logging
 import traceback
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
     HTTPNotAcceptable
 from kallithea.model.db import Ui
 from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
     _set_extras
-from kallithea.lib.base import BaseVCSController, WSGIResultCloseCallback, check_locking_state
 from kallithea.lib.base import BaseVCSController, check_locking_state
 from kallithea.lib.utils import make_ui, is_valid_repo
 from kallithea.lib.exceptions import HTTPLockedRC
 from kallithea.lib.hooks import pull_lock_handling
 from kallithea.lib import auth_modules
 log = logging.getLogger(__name__)
 GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
 def is_git(environ):
     path_info = environ['PATH_INFO']
     isgit_path = GIT_PROTO_PAT.match(path_info)
     log.debug('pathinfo: %s detected as Git %s',
         path_info, isgit_path is not None
+    )
     return isgit_path
 class SimpleGit(BaseVCSController):
     def _handle_request(self, environ, start_response):
         if not is_git(environ):
@@ @@ -118,53 +118,49 @@ class SimpleGit(BaseVCSController): @@
         #===================================================================
         # GIT REQUEST HANDLING
         #===================================================================
         repo_path = os.path.join(safe_str(self.basepath),str_repo_name)
         log.debug('Repository path is %s', repo_path)
         # CHECK LOCKING only if it's not ANONYMOUS USER
         if not user.is_default_user:
             log.debug('Checking locking on repository')
             make_lock, locked, locked_by = check_locking_state(action, repo_name, user)
             # store the make_lock for later evaluation in hooks
             extras.update({'make_lock': make_lock,
                            'locked_by': locked_by})
         fix_PATH()
         log.debug('HOOKS extras is %s', extras)
         baseui = make_ui('db')
         _set_extras(extras or {})
         try:
             self._handle_githooks(repo_name, action, baseui, environ)
             log.info('%s action on Git repo "%s" by "%s" from %s',
                      action, str_repo_name, safe_str(user.username), ip_addr)
             app = self.__make_app(repo_name, repo_path, extras)
             result = app(environ, start_response)
             if action == 'push':
                 result = WSGIResultCloseCallback(result,
                     lambda: self._invalidate_cache(repo_name))
             return result
             return app(environ, start_response)
         except HTTPLockedRC as e:
             log.debug('Locked, response %s: %s', e.code, e.title)
             return e(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
     def __make_app(self, repo_name, repo_path, extras):
         """
         Make an wsgi application using dulserver
         :param repo_name: name of the repository
         :param repo_path: full path to the repository
         """
         from kallithea.lib.middleware.pygrack import make_wsgi_app
         app = make_wsgi_app(
             repo_root=safe_str(self.basepath),
             repo_name=safe_unicode(repo_name),
             extras=extras,
+        )
         return app
     def __get_repository(self, environ):

kallithea/lib/middleware/simplehg.py

➞

Show inline comments

@@ @@ -16,49 +16,49 @@ kallithea.lib.middleware.simplehg @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 SimpleHg middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import logging
 import traceback
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
     HTTPNotAcceptable, HTTPBadRequest
 from kallithea.lib.utils2 import safe_str, safe_unicode, fix_PATH, get_server_url, \
     _set_extras
-from kallithea.lib.base import BaseVCSController, WSGIResultCloseCallback, check_locking_state
 from kallithea.lib.base import BaseVCSController, check_locking_state
 from kallithea.lib.utils import make_ui, is_valid_repo, ui_sections
 from kallithea.lib.vcs.utils.hgcompat import RepoError, hgweb_mod
 from kallithea.lib.exceptions import HTTPLockedRC
 log = logging.getLogger(__name__)
 def is_mercurial(environ):
     """
     Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     path_info = environ['PATH_INFO']
     if http_accept and http_accept.startswith('application/mercurial'):
         ishg_path = True
     else:
         ishg_path = False
     log.debug('pathinfo: %s detected as Mercurial %s',
         path_info, ishg_path
+    )
     return ishg_path
@@ @@ -128,53 +128,49 @@ class SimpleHg(BaseVCSController): @@
         # A Mercurial HTTP server will see listkeys operations (bookmarks,
         # phases and obsolescence marker) in a different request - we don't
         # want to check locking on those
         if environ['QUERY_STRING'] == 'cmd=listkeys':
             pass
         # CHECK LOCKING only if it's not ANONYMOUS USER
         elif not user.is_default_user:
             log.debug('Checking locking on repository')
             make_lock, locked, locked_by = check_locking_state(action, repo_name, user)
             # store the make_lock for later evaluation in hooks
             extras.update({'make_lock': make_lock,
                            'locked_by': locked_by})
         fix_PATH()
         log.debug('HOOKS extras is %s', extras)
         baseui = make_ui('db')
         self._augment_hgrc(repo_path, baseui)
         _set_extras(extras or {})
         try:
             log.info('%s action on Mercurial repo "%s" by "%s" from %s',
                      action, str_repo_name, safe_str(user.username), ip_addr)
             app = self.__make_app(repo_path, baseui, extras)
             result = app(environ, start_response)
             if action == 'push':
                 result = WSGIResultCloseCallback(result,
                     lambda: self._invalidate_cache(repo_name))
             return result
             return app(environ, start_response)
         except RepoError as e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except HTTPLockedRC as e:
             # Before Mercurial 3.6, lock exceptions were caught here
             log.debug('Locked, response %s: %s', e.code, e.title)
             return e(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
     def __make_app(self, repo_name, baseui, extras):
         """
         Make an wsgi application using hgweb, and inject generated baseui
         instance, additionally inject some extras into ui object
         """
         class HgWebWrapper(hgweb_mod.hgweb):
             # Work-around for Mercurial 3.6+ causing lock exceptions to be
             # thrown late
             def _runwsgi(self, *args):
                 try:
                     return super(HgWebWrapper, self)._runwsgi(*args)
                 except HTTPLockedRC as e:
                     log.debug('Locked, response %s: %s', e.code, e.title)

0 comments (0 inline, 0 general)