Changeset - a67bcc6f9118
[Not reviewed]
default
1 11 0
Mads Kiilerich - 6 years ago 2019-10-20 22:06:26
mads@kiilerich.com
Grafted from: 3939a1c407b4
db: drop SA caching_query and FromCache, and thus sql_cache_short beaker cache

It is not a good idea to have dead ORM objects. If we want caching, we should
do it explicit.

It is unknown how much this cache helps, but we can profile and introduce
better caching of simple data where relevant.
12 files changed with 27 insertions and 329 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -275,7 +275,7 @@ celery.task_always_eager = false
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short,long_term_file
 
beaker.cache.regions = short_term,long_term,long_term_file
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
@@ -285,10 +285,6 @@ beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.long_term_file.type = file
 
beaker.cache.long_term_file.expire = 604800
 
beaker.cache.long_term_file.key_length = 256
kallithea/lib/auth.py
Show inline comments
 
@@ -40,7 +40,6 @@ from tg.i18n import ugettext as _
 
from webob.exc import HTTPForbidden, HTTPFound
 

	
 
from kallithea.config.routing import url
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.utils import conditional_cache, get_repo_group_slug, get_repo_slug, get_user_group_slug
 
from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes
 
from kallithea.lib.vcs.utils.lazy import LazyProperty
 
@@ -139,7 +138,7 @@ def _cached_perms_data(user_id, user_is_
 
    #======================================================================
 
    # fetch default permissions
 
    #======================================================================
 
    default_user = User.get_by_username('default', cache=True)
 
    default_user = User.get_by_username('default')
 
    default_user_id = default_user.user_id
 

	
 
    default_repo_perms = Permission.get_default_perms(default_user_id)
 
@@ -390,7 +389,7 @@ class AuthUser(object):
 
        if not dbuser.active:
 
            log.info('Db user %s not active', dbuser.username)
 
            return None
 
        allowed_ips = AuthUser.get_allowed_ips(dbuser.user_id, cache=True)
 
        allowed_ips = AuthUser.get_allowed_ips(dbuser.user_id)
 
        if not check_ip_access(source_ip=ip_addr, allowed_ips=allowed_ips):
 
            log.info('Access for %s from %s forbidden - not in %s', dbuser.username, ip_addr, allowed_ips)
 
            return None
 
@@ -550,14 +549,11 @@ class AuthUser(object):
 
        )
 

	
 
    @classmethod
 
    def get_allowed_ips(cls, user_id, cache=False):
 
    def get_allowed_ips(cls, user_id):
 
        _set = set()
 

	
 
        default_ips = UserIpMap.query().filter(UserIpMap.user_id ==
 
                                        User.get_default_user(cache=True).user_id)
 
        if cache:
 
            default_ips = default_ips.options(FromCache("sql_cache_short",
 
                                              "get_user_ips_default"))
 
                                        User.get_default_user().user_id)
 
        for ip in default_ips:
 
            try:
 
                _set.add(ip.ip_addr)
 
@@ -567,9 +563,6 @@ class AuthUser(object):
 
                pass
 

	
 
        user_ips = UserIpMap.query().filter(UserIpMap.user_id == user_id)
 
        if cache:
 
            user_ips = user_ips.options(FromCache("sql_cache_short",
 
                                                  "get_user_ips_%s" % user_id))
 
        for ip in user_ips:
 
            try:
 
                _set.add(ip.ip_addr)
kallithea/lib/base.py
Show inline comments
 
@@ -223,7 +223,7 @@ class BaseVCSController(object):
 
        Returns (None, wsgi_app) to send the wsgi_app response to the client.
 
        """
 
        # Use anonymous access if allowed for action on repo.
 
        default_user = User.get_default_user(cache=True)
 
        default_user = User.get_default_user()
 
        default_authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
 
        if default_authuser is None:
 
            log.debug('No anonymous access at all') # move on to proper user auth
 
@@ -454,7 +454,7 @@ class BaseController(TGController):
 
                    return log_in_user(user, remember=False, is_external_auth=True, ip_addr=ip_addr)
 

	
 
        # User is default user (if active) or anonymous
 
        default_user = User.get_default_user(cache=True)
 
        default_user = User.get_default_user()
 
        authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
 
        if authuser is None: # fall back to anonymous
 
            authuser = AuthUser(dbuser=default_user) # TODO: somehow use .make?
kallithea/lib/caching_query.py
Show inline comments
 
deleted file
kallithea/lib/helpers.py
Show inline comments
 
@@ -568,7 +568,7 @@ def user_attr_or_none(author, show_attr)
 
    email = author_email(author)
 
    if email:
 
        from kallithea.model.db import User
 
        user = User.get_by_email(email, cache=True) # cache will only use sql_cache_short
 
        user = User.get_by_email(email)
 
        if user is not None:
 
            return getattr(user, show_attr)
 
    return None
kallithea/lib/paster_commands/template.ini.mako
Show inline comments
 
@@ -381,7 +381,7 @@ celery.task_always_eager = false
 
beaker.cache.data_dir = %(here)s/data/cache/data
 
beaker.cache.lock_dir = %(here)s/data/cache/lock
 

	
 
beaker.cache.regions = short_term,long_term,sql_cache_short,long_term_file
 
beaker.cache.regions = short_term,long_term,long_term_file
 

	
 
beaker.cache.short_term.type = memory
 
beaker.cache.short_term.expire = 60
 
@@ -391,10 +391,6 @@ beaker.cache.long_term.type = memory
 
beaker.cache.long_term.expire = 36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type = memory
 
beaker.cache.sql_cache_short.expire = 10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.long_term_file.type = file
 
beaker.cache.long_term_file.expire = 604800
 
beaker.cache.long_term_file.key_length = 256
kallithea/model/db.py
Show inline comments
 
@@ -46,7 +46,6 @@ from webob.exc import HTTPNotFound
 

	
 
import kallithea
 
from kallithea.lib import ext_json
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.exceptions import DefaultUserException
 
from kallithea.lib.utils2 import (Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, str2bool,
 
                                  urlreadable)
 
@@ -278,13 +277,9 @@ class Setting(Base, BaseDbModel):
 
        return res
 

	
 
    @classmethod
 
    def get_app_settings(cls, cache=False):
 
    def get_app_settings(cls):
 

	
 
        ret = cls.query()
 

	
 
        if cache:
 
            ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 

	
 
        if ret is None:
 
            raise Exception('Could not get application settings !')
 
        settings = {}
 
@@ -295,7 +290,7 @@ class Setting(Base, BaseDbModel):
 
        return settings
 

	
 
    @classmethod
 
    def get_auth_settings(cls, cache=False):
 
    def get_auth_settings(cls):
 
        ret = cls.query() \
 
                .filter(cls.app_settings_name.startswith('auth_')).all()
 
        fd = {}
 
@@ -304,7 +299,7 @@ class Setting(Base, BaseDbModel):
 
        return fd
 

	
 
    @classmethod
 
    def get_default_repo_settings(cls, cache=False, strip_prefix=False):
 
    def get_default_repo_settings(cls, strip_prefix=False):
 
        ret = cls.query() \
 
                .filter(cls.app_settings_name.startswith('default_')).all()
 
        fd = {}
 
@@ -548,7 +543,7 @@ class User(Base, BaseDbModel):
 
        return user
 

	
 
    @classmethod
 
    def get_by_username_or_email(cls, username_or_email, case_insensitive=True, cache=False):
 
    def get_by_username_or_email(cls, username_or_email, case_insensitive=True):
 
        """
 
        For anything that looks like an email address, look up by the email address (matching
 
        case insensitively).
 
@@ -557,35 +552,24 @@ class User(Base, BaseDbModel):
 
        This assumes no normal username can have '@' symbol.
 
        """
 
        if '@' in username_or_email:
 
            return User.get_by_email(username_or_email, cache=cache)
 
            return User.get_by_email(username_or_email)
 
        else:
 
            return User.get_by_username(username_or_email, case_insensitive=case_insensitive, cache=cache)
 
            return User.get_by_username(username_or_email, case_insensitive=case_insensitive)
 

	
 
    @classmethod
 
    def get_by_username(cls, username, case_insensitive=False, cache=False):
 
    def get_by_username(cls, username, case_insensitive=False):
 
        if case_insensitive:
 
            q = cls.query().filter(sqlalchemy.func.lower(cls.username) == sqlalchemy.func.lower(username))
 
        else:
 
            q = cls.query().filter(cls.username == username)
 

	
 
        if cache:
 
            q = q.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_user_%s" % _hash_key(username)
 
                          )
 
            )
 
        return q.scalar()
 

	
 
    @classmethod
 
    def get_by_api_key(cls, api_key, cache=False, fallback=True):
 
    def get_by_api_key(cls, api_key, fallback=True):
 
        if len(api_key) != 40 or not api_key.isalnum():
 
            return None
 

	
 
        q = cls.query().filter(cls.api_key == api_key)
 

	
 
        if cache:
 
            q = q.options(FromCache("sql_cache_short",
 
                                    "get_api_key_%s" % api_key))
 
        res = q.scalar()
 

	
 
        if fallback and not res:
 
@@ -600,20 +584,12 @@ class User(Base, BaseDbModel):
 
    @classmethod
 
    def get_by_email(cls, email, cache=False):
 
        q = cls.query().filter(sqlalchemy.func.lower(cls.email) == sqlalchemy.func.lower(email))
 

	
 
        if cache:
 
            q = q.options(FromCache("sql_cache_short",
 
                                    "get_email_key_%s" % email))
 

	
 
        ret = q.scalar()
 
        if ret is None:
 
            q = UserEmailMap.query()
 
            # try fetching in alternate email map
 
            q = q.filter(sqlalchemy.func.lower(UserEmailMap.email) == sqlalchemy.func.lower(email))
 
            q = q.options(joinedload(UserEmailMap.user))
 
            if cache:
 
                q = q.options(FromCache("sql_cache_short",
 
                                        "get_email_map_key_%s" % email))
 
            ret = getattr(q.scalar(), 'user', None)
 

	
 
        return ret
 
@@ -651,8 +627,8 @@ class User(Base, BaseDbModel):
 
        return user
 

	
 
    @classmethod
 
    def get_default_user(cls, cache=False):
 
        user = User.get_by_username(User.DEFAULT_USER, cache=cache)
 
    def get_default_user(cls):
 
        user = User.get_by_username(User.DEFAULT_USER)
 
        if user is None:
 
            raise Exception('Missing default account!')
 
        return user
 
@@ -851,26 +827,16 @@ class UserGroup(Base, BaseDbModel):
 
        return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
 

	
 
    @classmethod
 
    def get_by_group_name(cls, group_name, cache=False,
 
                          case_insensitive=False):
 
    def get_by_group_name(cls, group_name, case_insensitive=False):
 
        if case_insensitive:
 
            q = cls.query().filter(sqlalchemy.func.lower(cls.users_group_name) == sqlalchemy.func.lower(group_name))
 
        else:
 
            q = cls.query().filter(cls.users_group_name == group_name)
 
        if cache:
 
            q = q.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_group_%s" % _hash_key(group_name)
 
                          )
 
            )
 
        return q.scalar()
 

	
 
    @classmethod
 
    def get(cls, user_group_id, cache=False):
 
    def get(cls, user_group_id):
 
        user_group = cls.query()
 
        if cache:
 
            user_group = user_group.options(FromCache("sql_cache_short",
 
                                    "get_users_group_%s" % user_group_id))
 
        return user_group.get(user_group_id)
 

	
 
    def get_api_data(self, with_members=True):
 
@@ -1480,7 +1446,7 @@ class RepoGroup(Base, BaseDbModel):
 
        return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
 

	
 
    @classmethod
 
    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
 
    def get_by_group_name(cls, group_name, case_insensitive=False):
 
        group_name = group_name.rstrip('/')
 
        if case_insensitive:
 
            gr = cls.query() \
 
@@ -1488,12 +1454,6 @@ class RepoGroup(Base, BaseDbModel):
 
        else:
 
            gr = cls.query() \
 
                .filter(cls.group_name == group_name)
 
        if cache:
 
            gr = gr.options(FromCache(
 
                            "sql_cache_short",
 
                            "get_group_%s" % _hash_key(group_name)
 
                            )
 
            )
 
        return gr.scalar()
 

	
 
    @property
kallithea/model/meta.py
Show inline comments
 
@@ -18,8 +18,6 @@ from beaker import cache
 
from sqlalchemy.ext.declarative import declarative_base
 
from sqlalchemy.orm import scoped_session, sessionmaker
 

	
 
from kallithea.lib import caching_query
 

	
 

	
 
# Beaker CacheManager.  A home base for cache configurations.
 
cache_manager = cache.CacheManager()
 
@@ -29,18 +27,13 @@ __all__ = ['Base', 'Session']
 
#
 
# SQLAlchemy session manager.
 
#
 
session_factory = sessionmaker(
 
    query_cls=caching_query.query_callable(cache_manager),
 
    expire_on_commit=True)
 
session_factory = sessionmaker(expire_on_commit=True)
 
Session = scoped_session(session_factory)
 

	
 
# The base class for declarative schemas in db.py
 
# Engine is injected when model.__init__.init_model() sets meta.Base.metadata.bind
 
Base = declarative_base()
 

	
 
# to use cache use this in query:
 
#   .options(FromCache("sqlalchemy_cache_type", "cachekey"))
 

	
 

	
 
# Define naming conventions for foreign keys, primary keys, indexes,
 
# check constraints, and unique constraints, respectively.
kallithea/model/repo.py
Show inline comments
 
@@ -83,7 +83,6 @@ class RepoModel(object):
 
    def get(self, repo_id):
 
        repo = Repository.query() \
 
            .filter(Repository.repo_id == repo_id)
 

	
 
        return repo.scalar()
 

	
 
    def get_repo(self, repository):
 
@@ -92,7 +91,6 @@ class RepoModel(object):
 
    def get_by_repo_name(self, repo_name):
 
        repo = Repository.query() \
 
            .filter(Repository.repo_name == repo_name)
 

	
 
        return repo.scalar()
 

	
 
    def get_all_user_repos(self, user):
kallithea/model/user.py
Show inline comments
 
@@ -36,7 +36,6 @@ from sqlalchemy.exc import DatabaseError
 
from tg import config
 
from tg.i18n import ugettext as _
 

	
 
from kallithea.lib.caching_query import FromCache
 
from kallithea.lib.exceptions import DefaultUserException, UserOwnsReposException
 
from kallithea.lib.utils2 import generate_api_key, get_current_authuser
 
from kallithea.model.db import Permission, User, UserEmailMap, UserIpMap, UserToPerm
 
@@ -49,11 +48,8 @@ log = logging.getLogger(__name__)
 
class UserModel(object):
 
    password_reset_token_lifetime = 86400 # 24 hours
 

	
 
    def get(self, user_id, cache=False):
 
    def get(self, user_id):
 
        user = User.query()
 
        if cache:
 
            user = user.options(FromCache("sql_cache_short",
 
                                          "get_user_%s" % user_id))
 
        return user.get(user_id)
 

	
 
    def get_user(self, user):
 
@@ -203,7 +199,7 @@ class UserModel(object):
 
    def update(self, user_id, form_data, skip_attrs=None):
 
        from kallithea.lib.auth import get_crypt_password
 
        skip_attrs = skip_attrs or []
 
        user = self.get(user_id, cache=False)
 
        user = self.get(user_id)
 
        if user.is_default_user:
 
            raise DefaultUserException(
 
                            _("You can't edit this user since it's "
kallithea/model/user_group.py
Show inline comments
 
@@ -94,8 +94,8 @@ class UserGroupModel(object):
 
    def get_group(self, user_group):
 
        return UserGroup.guess_instance(user_group)
 

	
 
    def get_by_name(self, name, cache=False, case_insensitive=False):
 
        return UserGroup.get_by_group_name(name, cache=cache, case_insensitive=case_insensitive)
 
    def get_by_name(self, name, case_insensitive=False):
 
        return UserGroup.get_by_group_name(name, case_insensitive=case_insensitive)
 

	
 
    def create(self, name, description, owner, active=True, group_data=None):
 
        try:
kallithea/tests/conftest.py
Show inline comments
 
@@ -45,7 +45,6 @@ def pytest_configure():
 
            #'ssh_locale': 'C',
 
            'app_instance_uuid': 'test',
 
            'show_revision_number': 'true',
 
            'beaker.cache.sql_cache_short.expire': '1',
 
            'session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
 
            #'i18n.lang': '',
 
        },
0 comments (0 inline, 0 general)