Changeset - a67bcc6f9118
[Not reviewed]
default
1 11 0
Mads Kiilerich - 6 years ago 2019-10-20 22:06:26
mads@kiilerich.com
Grafted from: 3939a1c407b4
db: drop SA caching_query and FromCache, and thus sql_cache_short beaker cache

It is not a good idea to have dead ORM objects. If we want caching, we should
do it explicit.

It is unknown how much this cache helps, but we can profile and introduce
better caching of simple data where relevant.
12 files changed with 27 insertions and 329 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -272,26 +272,22 @@ celery.task_always_eager = false
 
###         BEAKER CACHE        ####
 
####################################
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short,long_term_file
 
beaker.cache.regions = short_term,long_term,long_term_file
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.long_term_file.type = file
 
beaker.cache.long_term_file.expire = 604800
 
beaker.cache.long_term_file.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
kallithea/lib/auth.py
Show inline comments
 
@@ -37,13 +37,12 @@ from sqlalchemy.orm import joinedload
 
from sqlalchemy.orm.exc import ObjectDeletedError
 
from tg import request
 
from tg.i18n import ugettext as _
 
from webob.exc import HTTPForbidden, HTTPFound
 

	
 
from kallithea.config.routing import url
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.utils import conditional_cache, get_repo_group_slug, get_repo_slug, get_user_group_slug
 
from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes
 
from kallithea.lib.vcs.utils.lazy import LazyProperty
 
from kallithea.model.db import (Permission, User, UserApiKeys, UserGroup, UserGroupMember, UserGroupRepoGroupToPerm, UserGroupRepoToPerm, UserGroupToPerm,
 
                                UserGroupUserGroupToPerm, UserIpMap, UserToPerm)
 
from kallithea.model.meta import Session
 
@@ -136,13 +135,13 @@ def _cached_perms_data(user_id, user_is_
 
        if new_perm_val > cur_perm_val:
 
            permissions[kind][key] = new_perm
 

	
 
    #======================================================================
 
    # fetch default permissions
 
    #======================================================================
 
    default_user = User.get_by_username('default', cache=True)
 
    default_user = User.get_by_username('default')
 
    default_user_id = default_user.user_id
 

	
 
    default_repo_perms = Permission.get_default_perms(default_user_id)
 
    default_repo_groups_perms = Permission.get_default_group_perms(default_user_id)
 
    default_user_group_perms = Permission.get_default_user_group_perms(default_user_id)
 

	
 
@@ -387,13 +386,13 @@ class AuthUser(object):
 
        if dbuser is None:
 
            log.info('No db user for authentication')
 
            return None
 
        if not dbuser.active:
 
            log.info('Db user %s not active', dbuser.username)
 
            return None
 
        allowed_ips = AuthUser.get_allowed_ips(dbuser.user_id, cache=True)
 
        allowed_ips = AuthUser.get_allowed_ips(dbuser.user_id)
 
        if not check_ip_access(source_ip=ip_addr, allowed_ips=allowed_ips):
 
            log.info('Access for %s from %s forbidden - not in %s', dbuser.username, ip_addr, allowed_ips)
 
            return None
 
        return cls(dbuser=dbuser, is_external_auth=is_external_auth)
 

	
 
    def __init__(self, user_id=None, dbuser=None, is_external_auth=False):
 
@@ -547,32 +546,26 @@ class AuthUser(object):
 
            dbuser=UserModel().get(cookie.get('user_id')),
 
            is_external_auth=cookie.get('is_external_auth', False),
 
            ip_addr=ip_addr,
 
        )
 

	
 
    @classmethod
 
    def get_allowed_ips(cls, user_id, cache=False):
 
    def get_allowed_ips(cls, user_id):
 
        _set = set()
 

	
 
        default_ips = UserIpMap.query().filter(UserIpMap.user_id ==
 
                                        User.get_default_user(cache=True).user_id)
 
        if cache:
 
            default_ips = default_ips.options(FromCache("sql_cache_short",
 
                                              "get_user_ips_default"))
 
                                        User.get_default_user().user_id)
 
        for ip in default_ips:
 
            try:
 
                _set.add(ip.ip_addr)
 
            except ObjectDeletedError:
 
                # since we use heavy caching sometimes it happens that we get
 
                # deleted objects here, we just skip them
 
                pass
 

	
 
        user_ips = UserIpMap.query().filter(UserIpMap.user_id == user_id)
 
        if cache:
 
            user_ips = user_ips.options(FromCache("sql_cache_short",
 
                                                  "get_user_ips_%s" % user_id))
 
        for ip in user_ips:
 
            try:
 
                _set.add(ip.ip_addr)
 
            except ObjectDeletedError:
 
                # since we use heavy caching sometimes it happens that we get
 
                # deleted objects here, we just skip them
kallithea/lib/base.py
Show inline comments
 
@@ -220,13 +220,13 @@ class BaseVCSController(object):
 
        authentication to the web server.
 

	
 
        Returns (user, None) on successful authentication and authorization.
 
        Returns (None, wsgi_app) to send the wsgi_app response to the client.
 
        """
 
        # Use anonymous access if allowed for action on repo.
 
        default_user = User.get_default_user(cache=True)
 
        default_user = User.get_default_user()
 
        default_authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
 
        if default_authuser is None:
 
            log.debug('No anonymous access at all') # move on to proper user auth
 
        else:
 
            if self._check_permission(action, default_authuser, repo_name):
 
                return default_authuser, None
 
@@ -451,13 +451,13 @@ class BaseController(TGController):
 
                if user_info is not None:
 
                    username = user_info['username']
 
                    user = User.get_by_username(username, case_insensitive=True)
 
                    return log_in_user(user, remember=False, is_external_auth=True, ip_addr=ip_addr)
 

	
 
        # User is default user (if active) or anonymous
 
        default_user = User.get_default_user(cache=True)
 
        default_user = User.get_default_user()
 
        authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
 
        if authuser is None: # fall back to anonymous
 
            authuser = AuthUser(dbuser=default_user) # TODO: somehow use .make?
 
        return authuser
 

	
 
    @staticmethod
kallithea/lib/caching_query.py
Show inline comments
 
deleted file
kallithea/lib/helpers.py
Show inline comments
 
@@ -565,13 +565,13 @@ def is_hg(repository):
 
def user_attr_or_none(author, show_attr):
 
    """Try to match email part of VCS committer string with a local user and return show_attr
 
    - or return None if user not found"""
 
    email = author_email(author)
 
    if email:
 
        from kallithea.model.db import User
 
        user = User.get_by_email(email, cache=True) # cache will only use sql_cache_short
 
        user = User.get_by_email(email)
 
        if user is not None:
 
            return getattr(user, show_attr)
 
    return None
 

	
 

	
 
def email_or_none(author):
kallithea/lib/paster_commands/template.ini.mako
Show inline comments
 
@@ -378,26 +378,22 @@ celery.task_always_eager = false
 
<%text>###         BEAKER CACHE        ####</%text>
 
<%text>####################################</%text>
 

	
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short,long_term_file
 
beaker.cache.regions = short_term,long_term,long_term_file
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.long_term_file.type = file
 
beaker.cache.long_term_file.expire = 604800
 
beaker.cache.long_term_file.key_length = 256
 

	
 
<%text>####################################</%text>
 
<%text>###       BEAKER SESSION        ####</%text>
kallithea/model/db.py
Show inline comments
 
@@ -43,13 +43,12 @@ from sqlalchemy.ext.hybrid import hybrid
 
from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
 
from tg.i18n import lazy_ugettext as _
 
from webob.exc import HTTPNotFound
 

	
 
import kallithea
 
from kallithea.lib import ext_json
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.exceptions import DefaultUserException
 
from kallithea.lib.utils2 import (Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, str2bool,
 
                                  urlreadable)
 
from kallithea.lib.vcs import get_backend
 
from kallithea.lib.vcs.backends.base import EmptyChangeset
 
from kallithea.lib.vcs.utils.helpers import get_scm
 
@@ -275,39 +274,35 @@ class Setting(Base, BaseDbModel):
 
            if not isinstance(type, Optional):
 
                # update if set
 
                res.app_settings_type = type
 
        return res
 

	
 
    @classmethod
 
    def get_app_settings(cls, cache=False):
 
    def get_app_settings(cls):
 

	
 
        ret = cls.query()
 

	
 
        if cache:
 
            ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 

	
 
        if ret is None:
 
            raise Exception('Could not get application settings !')
 
        settings = {}
 
        for each in ret:
 
            settings[each.app_settings_name] = \
 
                each.app_settings_value
 

	
 
        return settings
 

	
 
    @classmethod
 
    def get_auth_settings(cls, cache=False):
 
    def get_auth_settings(cls):
 
        ret = cls.query() \
 
                .filter(cls.app_settings_name.startswith('auth_')).all()
 
        fd = {}
 
        for row in ret:
 
            fd[row.app_settings_name] = row.app_settings_value
 
        return fd
 

	
 
    @classmethod
 
    def get_default_repo_settings(cls, cache=False, strip_prefix=False):
 
    def get_default_repo_settings(cls, strip_prefix=False):
 
        ret = cls.query() \
 
                .filter(cls.app_settings_name.startswith('default_')).all()
 
        fd = {}
 
        for row in ret:
 
            key = row.app_settings_name
 
            if strip_prefix:
 
@@ -545,50 +540,39 @@ class User(Base, BaseDbModel):
 
        user = super(User, cls).get_or_404(id_)
 
        if not allow_default and user.is_default_user:
 
            raise DefaultUserException()
 
        return user
 

	
 
    @classmethod
 
    def get_by_username_or_email(cls, username_or_email, case_insensitive=True, cache=False):
 
    def get_by_username_or_email(cls, username_or_email, case_insensitive=True):
 
        """
 
        For anything that looks like an email address, look up by the email address (matching
 
        case insensitively).
 
        For anything else, try to look up by the user name.
 

	
 
        This assumes no normal username can have '@' symbol.
 
        """
 
        if '@' in username_or_email:
 
            return User.get_by_email(username_or_email, cache=cache)
 
            return User.get_by_email(username_or_email)
 
        else:
 
            return User.get_by_username(username_or_email, case_insensitive=case_insensitive, cache=cache)
 
            return User.get_by_username(username_or_email, case_insensitive=case_insensitive)
 

	
 
    @classmethod
 
    def get_by_username(cls, username, case_insensitive=False, cache=False):
 
    def get_by_username(cls, username, case_insensitive=False):
 
        if case_insensitive:
 
            q = cls.query().filter(sqlalchemy.func.lower(cls.username) == sqlalchemy.func.lower(username))
 
        else:
 
            q = cls.query().filter(cls.username == username)
 

	
 
        if cache:
 
            q = q.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_user_%s" % _hash_key(username)
 
                          )
 
            )
 
        return q.scalar()
 

	
 
    @classmethod
 
    def get_by_api_key(cls, api_key, cache=False, fallback=True):
 
    def get_by_api_key(cls, api_key, fallback=True):
 
        if len(api_key) != 40 or not api_key.isalnum():
 
            return None
 

	
 
        q = cls.query().filter(cls.api_key == api_key)
 

	
 
        if cache:
 
            q = q.options(FromCache("sql_cache_short",
 
                                    "get_api_key_%s" % api_key))
 
        res = q.scalar()
 

	
 
        if fallback and not res:
 
            # fallback to additional keys
 
            _res = UserApiKeys.query().filter_by(api_key=api_key, is_expired=False).first()
 
            if _res:
 
@@ -597,26 +581,18 @@ class User(Base, BaseDbModel):
 
            return None
 
        return res
 

	
 
    @classmethod
 
    def get_by_email(cls, email, cache=False):
 
        q = cls.query().filter(sqlalchemy.func.lower(cls.email) == sqlalchemy.func.lower(email))
 

	
 
        if cache:
 
            q = q.options(FromCache("sql_cache_short",
 
                                    "get_email_key_%s" % email))
 

	
 
        ret = q.scalar()
 
        if ret is None:
 
            q = UserEmailMap.query()
 
            # try fetching in alternate email map
 
            q = q.filter(sqlalchemy.func.lower(UserEmailMap.email) == sqlalchemy.func.lower(email))
 
            q = q.options(joinedload(UserEmailMap.user))
 
            if cache:
 
                q = q.options(FromCache("sql_cache_short",
 
                                        "get_email_map_key_%s" % email))
 
            ret = getattr(q.scalar(), 'user', None)
 

	
 
        return ret
 

	
 
    @classmethod
 
    def get_from_cs_author(cls, author):
 
@@ -648,14 +624,14 @@ class User(Base, BaseDbModel):
 
        user = User.query().filter(User.admin == True).first()
 
        if user is None:
 
            raise Exception('Missing administrative account!')
 
        return user
 

	
 
    @classmethod
 
    def get_default_user(cls, cache=False):
 
        user = User.get_by_username(User.DEFAULT_USER, cache=cache)
 
    def get_default_user(cls):
 
        user = User.get_by_username(User.DEFAULT_USER)
 
        if user is None:
 
            raise Exception('Missing default account!')
 
        return user
 

	
 
    def get_api_data(self, details=False):
 
        """
 
@@ -848,32 +824,22 @@ class UserGroup(Base, BaseDbModel):
 

	
 
    @classmethod
 
    def guess_instance(cls, value):
 
        return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
 

	
 
    @classmethod
 
    def get_by_group_name(cls, group_name, cache=False,
 
                          case_insensitive=False):
 
    def get_by_group_name(cls, group_name, case_insensitive=False):
 
        if case_insensitive:
 
            q = cls.query().filter(sqlalchemy.func.lower(cls.users_group_name) == sqlalchemy.func.lower(group_name))
 
        else:
 
            q = cls.query().filter(cls.users_group_name == group_name)
 
        if cache:
 
            q = q.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_group_%s" % _hash_key(group_name)
 
                          )
 
            )
 
        return q.scalar()
 

	
 
    @classmethod
 
    def get(cls, user_group_id, cache=False):
 
    def get(cls, user_group_id):
 
        user_group = cls.query()
 
        if cache:
 
            user_group = user_group.options(FromCache("sql_cache_short",
 
                                    "get_users_group_%s" % user_group_id))
 
        return user_group.get(user_group_id)
 

	
 
    def get_api_data(self, with_members=True):
 
        user_group = self
 

	
 
        data = dict(
 
@@ -1477,26 +1443,20 @@ class RepoGroup(Base, BaseDbModel):
 

	
 
    @classmethod
 
    def guess_instance(cls, value):
 
        return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
 

	
 
    @classmethod
 
    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
 
    def get_by_group_name(cls, group_name, case_insensitive=False):
 
        group_name = group_name.rstrip('/')
 
        if case_insensitive:
 
            gr = cls.query() \
 
                .filter(sqlalchemy.func.lower(cls.group_name) == sqlalchemy.func.lower(group_name))
 
        else:
 
            gr = cls.query() \
 
                .filter(cls.group_name == group_name)
 
        if cache:
 
            gr = gr.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_group_%s" % _hash_key(group_name)
 
                            )
 
            )
 
        return gr.scalar()
 

	
 
    @property
 
    def parents(self):
 
        groups = []
 
        group = self.parent_group
kallithea/model/meta.py
Show inline comments
 
@@ -15,35 +15,28 @@
 
SQLAlchemy Metadata and Session object
 
"""
 
from beaker import cache
 
from sqlalchemy.ext.declarative import declarative_base
 
from sqlalchemy.orm import scoped_session, sessionmaker
 

	
 
from kallithea.lib import caching_query
 

	
 

	
 
# Beaker CacheManager.  A home base for cache configurations.
 
cache_manager = cache.CacheManager()
 

	
 
__all__ = ['Base', 'Session']
 

	
 
#
 
# SQLAlchemy session manager.
 
#
 
session_factory = sessionmaker(
 
    query_cls=caching_query.query_callable(cache_manager),
 
    expire_on_commit=True)
 
session_factory = sessionmaker(expire_on_commit=True)
 
Session = scoped_session(session_factory)
 

	
 
# The base class for declarative schemas in db.py
 
# Engine is injected when model.__init__.init_model() sets meta.Base.metadata.bind
 
Base = declarative_base()
 

	
 
# to use cache use this in query:
 
#   .options(FromCache("sqlalchemy_cache_type", "cachekey"))
 

	
 

	
 
# Define naming conventions for foreign keys, primary keys, indexes,
 
# check constraints, and unique constraints, respectively.
 
Base.metadata.naming_convention = {
 
    'fk': 'fk_%(table_name)s_%(column_0_name)s',
 
    'pk': 'pk_%(table_name)s',
kallithea/model/repo.py
Show inline comments
 
@@ -80,22 +80,20 @@ class RepoModel(object):
 
        q = Ui.query().filter(Ui.ui_key == '/').one()
 
        return q.ui_value
 

	
 
    def get(self, repo_id):
 
        repo = Repository.query() \
 
            .filter(Repository.repo_id == repo_id)
 

	
 
        return repo.scalar()
 

	
 
    def get_repo(self, repository):
 
        return Repository.guess_instance(repository)
 

	
 
    def get_by_repo_name(self, repo_name):
 
        repo = Repository.query() \
 
            .filter(Repository.repo_name == repo_name)
 

	
 
        return repo.scalar()
 

	
 
    def get_all_user_repos(self, user):
 
        """
 
        Gets all repositories that user have at least read access
 

	
kallithea/model/user.py
Show inline comments
 
@@ -33,30 +33,26 @@ import time
 
import traceback
 

	
 
from sqlalchemy.exc import DatabaseError
 
from tg import config
 
from tg.i18n import ugettext as _
 

	
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException
 
from kallithea.lib.utils2 import generate_api_key, get_current_authuser
 
from kallithea.model.db import Permission, User, UserEmailMap, UserIpMap, UserToPerm
 
from kallithea.model.meta import Session
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class UserModel(object):
 
    password_reset_token_lifetime = 86400 # 24 hours
 

	
 
    def get(self, user_id, cache=False):
 
    def get(self, user_id):
 
        user = User.query()
 
        if cache:
 
            user = user.options(FromCache("sql_cache_short",
 
                                          "get_user_%s" % user_id))
 
        return user.get(user_id)
 

	
 
    def get_user(self, user):
 
        return User.guess_instance(user)
 

	
 
    def create(self, form_data, cur_user=None):
 
@@ -200,13 +196,13 @@ class UserModel(object):
 
                                   type_=NotificationModel.TYPE_REGISTRATION,
 
                                   email_kwargs=email_kwargs)
 

	
 
    def update(self, user_id, form_data, skip_attrs=None):
 
        from kallithea.lib.auth import get_crypt_password
 
        skip_attrs = skip_attrs or []
 
        user = self.get(user_id, cache=False)
 
        user = self.get(user_id)
 
        if user.is_default_user:
 
            raise DefaultUserException(
 
                            _("You can't edit this user since it's "
 
                              "crucial for entire application"))
 

	
 
        for k, v in form_data.items():
kallithea/model/user_group.py
Show inline comments
 
@@ -91,14 +91,14 @@ class UserGroupModel(object):
 
    def get(self, user_group_id):
 
        return UserGroup.get(user_group_id)
 

	
 
    def get_group(self, user_group):
 
        return UserGroup.guess_instance(user_group)
 

	
 
    def get_by_name(self, name, cache=False, case_insensitive=False):
 
        return UserGroup.get_by_group_name(name, cache=cache, case_insensitive=case_insensitive)
 
    def get_by_name(self, name, case_insensitive=False):
 
        return UserGroup.get_by_group_name(name, case_insensitive=case_insensitive)
 

	
 
    def create(self, name, description, owner, active=True, group_data=None):
 
        try:
 
            new_user_group = UserGroup()
 
            new_user_group.owner = User.guess_instance(owner)
 
            new_user_group.users_group_name = name
kallithea/tests/conftest.py
Show inline comments
 
@@ -42,13 +42,12 @@ def pytest_configure():
 
            'ssh_enabled': 'true',
 
            # Mainly to safeguard against accidentally overwriting the real one:
 
            'ssh_authorized_keys': os.path.join(TESTS_TMP_PATH, 'authorized_keys'),
 
            #'ssh_locale': 'C',
 
            'app_instance_uuid': 'test',
 
            'show_revision_number': 'true',
 
            'beaker.cache.sql_cache_short.expire': '1',
 
            'session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
 
            #'i18n.lang': '',
 
        },
 
        '[handler_console]': {
 
            'formatter': 'color_formatter',
 
        },
0 comments (0 inline, 0 general)