kallithea Changeset - f37d7514e7ab

Changeset - f37d7514e7ab

Parent rev.

Child rev.

[Not reviewed]

beta

0 4 0

Marcin Kuzminski - 13 years ago 2013-04-07 19:55:30
marcin@python-works.com

always use json from compat module

4 files changed with 10 insertions and 3 deletions:

rhodecode/bin/ldap_sync.py

rhodecode/controllers/admin/repos_groups.py

rhodecode/controllers/home.py

rhodecode/lib/utils.py

0 comments (0 inline, 0 general)

rhodecode/bin/ldap_sync.py

➞

Show inline comments

 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import ldap
 import urllib2
 import uuid
 try:
     from rhodecode.lib.compat import json
 except ImportError:
     try:
         import simplejson as json
     except ImportError:
 import json
 from ConfigParser import ConfigParser
 config = ConfigParser()
 config.read('ldap_sync.conf')
 class InvalidResponseIDError(Exception):
     """ Request and response don't have the same UUID. """
 class RhodecodeResponseError(Exception):
     """ Response has an error, something went wrong with request execution. """
 class UserAlreadyInGroupError(Exception):
     """ User is already a member of the target group. """
 class UserNotInGroupError(Exception):
     """ User is not a member of the target group. """
 class RhodecodeAPI():
     def __init__(self, url, key):
         self.url = url
         self.key = key
     def get_api_data(self, uid, method, args):
         """Prepare dict for API post."""
         return {
             "id": uid,
             "api_key": self.key,
             "method": method,
             "args": args
+        }
     def rhodecode_api_post(self, method, args):
         """Send a generic API post to Rhodecode.
         This will generate the UUID for validation check after the
         response is returned. Handle errors and get the result back.
         """
         uid = str(uuid.uuid1())
         data = self.get_api_data(uid, method, args)
         data = json.dumps(data)
         headers = {'content-type': 'text/plain'}
         req = urllib2.Request(self.url, data, headers)
         response = urllib2.urlopen(req)
         response = json.load(response)
         if uid != response["id"]:
             raise InvalidResponseIDError("UUID does not match.")
         if response["error"] != None:
             raise RhodecodeResponseError(response["error"])
         return response["result"]
     def create_group(self, name, active=True):
         """Create the Rhodecode user group."""
         args = {
             "group_name": name,
             "active": str(active)
+        }
         self.rhodecode_api_post("create_users_group", args)
     def add_membership(self, group, username):
         """Add specific user to a group."""
         args = {
             "usersgroupid": group,
             "userid": username
+        }
         result = self.rhodecode_api_post("add_user_to_users_group", args)
         if not result["success"]:
             raise UserAlreadyInGroupError("User %s already in group %s." %
                                           (username, group))
     def remove_membership(self, group, username):
         """Remove specific user from a group."""
         args = {
             "usersgroupid": group,
             "userid": username
+        }
         result = self.rhodecode_api_post("remove_user_from_users_group", args)
         if not result["success"]:
             raise UserNotInGroupError("User %s not in group %s." %
                                       (username, group))
     def get_group_members(self, name):
         """Get the list of member usernames from a user group."""
         args = {"usersgroupid": name}
         members = self.rhodecode_api_post("get_users_group", args)['members']
         member_list = []
         for member in members:
             member_list.append(member["username"])
         return member_list
     def get_group(self, name):
         """Return group info."""
         args = {"usersgroupid": name}
         return self.rhodecode_api_post("get_users_group", args)
     def get_user(self, username):
         """Return user info."""
         args = {"userid": username}
         return self.rhodecode_api_post("get_user", args)
 class LdapClient():
     def __init__(self, uri, user, key, base_dn):
         self.client = ldap.initialize(uri, trace_level=0)
         self.client.set_option(ldap.OPT_REFERRALS, 0)
         self.client.simple_bind(user, key)
         self.base_dn = base_dn
     def __del__(self):
         self.client.unbind()
     def get_groups(self):
         """Get all the groups in form of dict {group_name: group_info,...}."""
         searchFilter = "objectClass=groupOfUniqueNames"
         result = self.client.search_s(self.base_dn, ldap.SCOPE_SUBTREE,
                                       searchFilter)
         groups = {}
         for group in result:
             groups[group[1]['cn'][0]] = group[1]
         return groups
     def get_group_users(self, groups, group):
         """Returns all the users belonging to a single group.
         Based on the list of groups and memberships, returns all the
         users belonging to a single group, searching recursively.
         """
         users = []
         for member in groups[group]["uniqueMember"]:
             member = self.parse_member_string(member)
             if member[0] == "uid":
                 users.append(member[1])
             elif member[0] == "cn":
                 users += self.get_group_users(groups, member[1])
         return users
     def parse_member_string(self, member):
         """Parses the member string and returns a touple of type and name.
         Unique member can be either user or group. Users will have 'uid' as
         prefix while groups will have 'cn'.
         """
         member = member.split(",")[0]
         return member.split('=')
 class LdapSync(object):
     def __init__(self):
         self.ldap_client = LdapClient(config.get("default", "ldap_uri"),
                                       config.get("default", "ldap_user"),
                                       config.get("default", "ldap_key"),
                                       config.get("default", "base_dn"))
         self.rhodocode_api = RhodecodeAPI(config.get("default", "api_url"),
                                           config.get("default", "api_key"))
     def update_groups_from_ldap(self):
         """Add all the groups from LDAP to Rhodecode."""
         added = existing = 0
         groups = self.ldap_client.get_groups()
         for group in groups:
             try:
                 self.rhodecode_api.create_group(group)
                 added += 1
             except Exception:
                 existing += 1
         return added, existing
     def update_memberships_from_ldap(self, group):
         """Update memberships in rhodecode based on the LDAP groups."""
         groups = self.ldap_client.get_groups()
         group_users = self.ldap_client.get_group_users(groups, group)
         # Delete memberships first from each group which are not part
         # of the group any more.
         rhodecode_members = self.rhodecode_api.get_group_members(group)
         for rhodecode_member in rhodecode_members:
             if rhodecode_member not in group_users:
                 try:
                     self.rhodocode_api.remove_membership(group,
                                                          rhodecode_member)
                 except UserNotInGroupError:
                     pass
         # Add memberships.
         for member in group_users:
             try:
                 self.rhodecode_api.add_membership(group, member)
             except UserAlreadyInGroupError:
                 # TODO: handle somehow maybe..
                 pass
 if __name__ == '__main__':
     sync = LdapSync()
     print sync.update_groups_from_ldap()
     for gr in sync.ldap_client.get_groups():
         # TODO: exception when user does not exist during add membership...
         # How should we handle this.. Either sync users as well at this step,
         # or just ignore those who don't exist. If we want the second case,
         # we need to find a way to recognize the right exception (we always get
         # RhodecodeResponseError with no error code so maybe by return msg (?)
         sync.update_memberships_from_ldap(gr)

rhodecode/controllers/admin/repos_groups.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.admin.repos_groups
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Repository groups controller for RhodeCode
     :created_on: Mar 23, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import traceback
 import formencode
 from formencode import htmlfill
 from pylons import request, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
 from sqlalchemy.exc import IntegrityError
 import rhodecode
 from rhodecode.lib import helpers as h
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.compat import json
 from rhodecode.lib.auth import LoginRequired, HasPermissionAnyDecorator,\
     HasReposGroupPermissionAnyDecorator, HasReposGroupPermissionAll,\
     HasPermissionAll
 from rhodecode.lib.base import BaseController, render
 from rhodecode.model.db import RepoGroup, Repository
 from rhodecode.model.repos_group import ReposGroupModel
 from rhodecode.model.forms import ReposGroupForm
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from webob.exc import HTTPInternalServerError, HTTPNotFound
 from rhodecode.lib.utils2 import str2bool, safe_int
 from sqlalchemy.sql.expression import func
 from rhodecode.model.scm import GroupList
 log = logging.getLogger(__name__)
 class ReposGroupsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('repos_group', 'repos_groups')
     @LoginRequired()
     def __before__(self):
         super(ReposGroupsController, self).__before__()
     def __load_defaults(self, allow_empty_group=False, exclude_group_ids=[]):
         if HasPermissionAll('hg.admin')('group edit'):
             #we're global admin, we're ok and we can create TOP level groups
             allow_empty_group = True
         #override the choices for this form, we need to filter choices
         #and display only those we have ADMIN right
         groups_with_admin_rights = GroupList(RepoGroup.query().all(),
                                              perm_set=['group.admin'])
         c.repo_groups = RepoGroup.groups_choices(groups=groups_with_admin_rights,
                                                  show_empty_group=allow_empty_group)
         # exclude filtered ids
         c.repo_groups = filter(lambda x: x[0] not in exclude_group_ids,
                                c.repo_groups)
         c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
         repo_model = RepoModel()
         c.users_array = repo_model.get_users_js()
         c.users_groups_array = repo_model.get_users_groups_js()
     def __load_data(self, group_id):
         """
         Load defaults settings for edit, and update
         :param group_id:
         """
         repo_group = RepoGroup.get_or_404(group_id)
         data = repo_group.get_dict()
         data['group_name'] = repo_group.name
         # fill repository users
         for p in repo_group.repo_group_to_perm:
             data.update({'u_perm_%s' % p.user.username:
                              p.permission.permission_name})
         # fill repository groups
         for p in repo_group.users_group_to_perm:
             data.update({'g_perm_%s' % p.users_group.users_group_name:
                              p.permission.permission_name})
         return data
     def _revoke_perms_on_yourself(self, form_result):
         _up = filter(lambda u: c.rhodecode_user.username == u[0],
                      form_result['perms_updates'])
         _new = filter(lambda u: c.rhodecode_user.username == u[0],
                       form_result['perms_new'])
         if _new and _new[0][1] != 'group.admin' or _up and _up[0][1] != 'group.admin':
             return True
         return False
     def index(self, format='html'):
         """GET /repos_groups: All items in the collection"""
         # url('repos_groups')
         group_iter = GroupList(RepoGroup.query().all(), perm_set=['group.admin'])
         sk = lambda g: g.parents[0].group_name if g.parents else g.group_name
         c.groups = sorted(group_iter, key=sk)
         return render('admin/repos_groups/repos_groups_show.html')
     def create(self):
         """POST /repos_groups: Create a new item"""
         # url('repos_groups')
         self.__load_defaults()
         # permissions for can create group based on parent_id are checked
         # here in the Form
         repos_group_form = ReposGroupForm(available_groups=
                                 map(lambda k: unicode(k[0]), c.repo_groups))()
         try:
             form_result = repos_group_form.to_python(dict(request.POST))
             ReposGroupModel().create(
                     group_name=form_result['group_name'],
                     group_description=form_result['group_description'],
                     parent=form_result['group_parent_id'],
                     owner=self.rhodecode_user.user_id
+            )
             Session().commit()
             h.flash(_('Created repository group %s') \
                     % form_result['group_name'], category='success')
             #TODO: in futureaction_logger(, '', '', '', self.sa)
         except formencode.Invalid, errors:
             return htmlfill.render(
                 render('admin/repos_groups/repos_groups_add.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('Error occurred during creation of repository group %s') \
                     % request.POST.get('group_name'), category='error')
         parent_group_id = form_result['group_parent_id']
         #TODO: maybe we should get back to the main view, not the admin one
         return redirect(url('repos_groups', parent_group=parent_group_id))
     def new(self, format='html'):
         """GET /repos_groups/new: Form to create a new item"""
         # url('new_repos_group')
         if HasPermissionAll('hg.admin')('group create'):
             #we're global admin, we're ok and we can create TOP level groups
             pass
         else:
             # we pass in parent group into creation form, thus we know
             # what would be the group, we can check perms here !
             group_id = safe_int(request.GET.get('parent_group'))
             group = RepoGroup.get(group_id) if group_id else None
             group_name = group.group_name if group else None
             if HasReposGroupPermissionAll('group.admin')(group_name, 'group create'):
                 pass
             else:
                 return abort(403)
         self.__load_defaults()
         return render('admin/repos_groups/repos_groups_add.html')
     @HasReposGroupPermissionAnyDecorator('group.admin')
     def update(self, group_name):
         """PUT /repos_groups/group_name: Update an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="PUT" />
         # Or using helpers:
         #    h.form(url('repos_group', group_name=GROUP_NAME),
         #           method='put')
         # url('repos_group', group_name=GROUP_NAME)
         c.repos_group = ReposGroupModel()._get_repos_group(group_name)
         if HasPermissionAll('hg.admin')('group edit'):
             #we're global admin, we're ok and we can create TOP level groups
             allow_empty_group = True
         elif not c.repos_group.parent_group:
             allow_empty_group = True
         else:
             allow_empty_group = False
         self.__load_defaults(allow_empty_group=allow_empty_group,
                              exclude_group_ids=[c.repos_group.group_id])
         repos_group_form = ReposGroupForm(
             edit=True,
             old_data=c.repos_group.get_dict(),
             available_groups=c.repo_groups_choices,
             can_create_in_root=allow_empty_group,
         )()
         try:
             form_result = repos_group_form.to_python(dict(request.POST))
             if not c.rhodecode_user.is_admin:
                 if self._revoke_perms_on_yourself(form_result):
                     msg = _('Cannot revoke permission for yourself as admin')
                     h.flash(msg, category='warning')
                     raise Exception('revoke admin permission on self')
             new_gr = ReposGroupModel().update(group_name, form_result)
             Session().commit()
             h.flash(_('Updated repository group %s') \
                     % form_result['group_name'], category='success')
             # we now have new name !
             group_name = new_gr.group_name
             #TODO: in future action_logger(, '', '', '', self.sa)
         except formencode.Invalid, errors:
             return htmlfill.render(
                 render('admin/repos_groups/repos_groups_edit.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('Error occurred during update of repository group %s') \
                     % request.POST.get('group_name'), category='error')
         return redirect(url('edit_repos_group', group_name=group_name))
     @HasReposGroupPermissionAnyDecorator('group.admin')
     def delete(self, group_name):
         """DELETE /repos_groups/group_name: Delete an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="DELETE" />
         # Or using helpers:
         #    h.form(url('repos_group', group_name=GROUP_NAME),
         #           method='delete')
         # url('repos_group', group_name=GROUP_NAME)
         gr = c.repos_group = ReposGroupModel()._get_repos_group(group_name)
         repos = gr.repositories.all()
         if repos:
             h.flash(_('This group contains %s repositores and cannot be '
                       'deleted') % len(repos), category='warning')
             return redirect(url('repos_groups'))
         children = gr.children.all()
         if children:
             h.flash(_('This group contains %s subgroups and cannot be deleted'
                       % (len(children))), category='warning')
             return redirect(url('repos_groups'))
         try:
             ReposGroupModel().delete(group_name)
             Session().commit()
             h.flash(_('Removed repository group %s') % group_name,
                     category='success')
             #TODO: in future action_logger(, '', '', '', self.sa)
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('Error occurred during deletion of repos '
                       'group %s') % group_name, category='error')
         return redirect(url('repos_groups'))
     @HasReposGroupPermissionAnyDecorator('group.admin')
     def delete_repos_group_user_perm(self, group_name):
         """
         DELETE an existing repository group permission user
         :param group_name:
         """
         try:
             if not c.rhodecode_user.is_admin:
                 if c.rhodecode_user.user_id == safe_int(request.POST['user_id']):
                     msg = _('Cannot revoke permission for yourself as admin')
                     h.flash(msg, category='warning')
                     raise Exception('revoke admin permission on self')
             recursive = str2bool(request.POST.get('recursive', False))
             ReposGroupModel().delete_permission(
                 repos_group=group_name, obj=request.POST['user_id'],
                 obj_type='user', recursive=recursive
+            )
             Session().commit()
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('An error occurred during deletion of group user'),
                     category='error')
             raise HTTPInternalServerError()
     @HasReposGroupPermissionAnyDecorator('group.admin')
     def delete_repos_group_users_group_perm(self, group_name):
         """
         DELETE an existing repository group permission user group
         :param group_name:
         """
         try:
             recursive = str2bool(request.POST.get('recursive', False))
             ReposGroupModel().delete_permission(
                 repos_group=group_name, obj=request.POST['users_group_id'],
                 obj_type='users_group', recursive=recursive
+            )
             Session().commit()
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('An error occurred during deletion of group'
                       ' user groups'),
                     category='error')
             raise HTTPInternalServerError()
     def show_by_name(self, group_name):
         """
         This is a proxy that does a lookup group_name -> id, and shows
         the group by id view instead
         """
         group_name = group_name.rstrip('/')
         id_ = RepoGroup.get_by_group_name(group_name)
         if id_:
             return self.show(id_.group_id)
         raise HTTPNotFound
     @HasReposGroupPermissionAnyDecorator('group.read', 'group.write',
                                          'group.admin')
     def show(self, group_name, format='html'):
         """GET /repos_groups/group_name: Show a specific item"""
         # url('repos_group', group_name=GROUP_NAME)
         c.group = c.repos_group = ReposGroupModel()._get_repos_group(group_name)
         c.group_repos = c.group.repositories.all()
         #overwrite our cached list with current filter
         gr_filter = c.group_repos
         c.repo_cnt = 0
         groups = RepoGroup.query().order_by(RepoGroup.group_name)\
             .filter(RepoGroup.group_parent_id == c.group.group_id).all()
         c.groups = self.scm_model.get_repos_groups(groups)
         if not c.visual.lightweight_dashboard:
             c.repos_list = self.scm_model.get_repos(all_repos=gr_filter)
         ## lightweight version of dashboard
         else:
             c.repos_list = Repository.query()\
                             .filter(Repository.group_id == c.group.group_id)\
                             .order_by(func.lower(Repository.repo_name))\
                             .all()
             repos_data = RepoModel().get_repos_as_dict(repos_list=c.repos_list,
                                                        admin=False)
             #json used to render the grid
             c.data = json.dumps(repos_data)
         return render('admin/repos_groups/repos_groups.html')
     @HasReposGroupPermissionAnyDecorator('group.admin')
     def edit(self, group_name, format='html'):
         """GET /repos_groups/group_name/edit: Form to edit an existing item"""
         # url('edit_repos_group', group_name=GROUP_NAME)
         c.repos_group = ReposGroupModel()._get_repos_group(group_name)
         #we can only allow moving empty group if it's already a top-level
         #group, ie has no parents, or we're admin
         if HasPermissionAll('hg.admin')('group edit'):
             #we're global admin, we're ok and we can create TOP level groups
             allow_empty_group = True
         elif not c.repos_group.parent_group:
             allow_empty_group = True
         else:
             allow_empty_group = False
         self.__load_defaults(allow_empty_group=allow_empty_group,
                              exclude_group_ids=[c.repos_group.group_id])
         defaults = self.__load_data(c.repos_group.group_id)
         return htmlfill.render(
             render('admin/repos_groups/repos_groups_edit.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
+        )

rhodecode/controllers/home.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.home
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
     Home controller for Rhodecode
     :created_on: Feb 18, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 from pylons import tmpl_context as c, request
 from pylons.i18n.translation import _
 from webob.exc import HTTPBadRequest
 from sqlalchemy.sql.expression import func
 import rhodecode
 from rhodecode.lib import helpers as h
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.compat import json
 from rhodecode.lib.auth import LoginRequired
 from rhodecode.lib.base import BaseController, render
 from rhodecode.model.db import Repository
 from rhodecode.model.repo import RepoModel
 log = logging.getLogger(__name__)
 class HomeController(BaseController):
     @LoginRequired()
     def __before__(self):
         super(HomeController, self).__before__()
     def index(self):
         c.groups = self.scm_model.get_repos_groups()
         c.group = None
         if not c.visual.lightweight_dashboard:
             c.repos_list = self.scm_model.get_repos()
         ## lightweight version of dashboard
         else:
             c.repos_list = Repository.query()\
                             .filter(Repository.group_id == None)\
                             .order_by(func.lower(Repository.repo_name))\
                             .all()
             repos_data = RepoModel().get_repos_as_dict(repos_list=c.repos_list,
                                                        admin=False)
             #json used to render the grid
             c.data = json.dumps(repos_data)
         return render('/index.html')
     def repo_switcher(self):
         if request.is_xhr:
             all_repos = Repository.query().order_by(Repository.repo_name).all()
             c.repos_list = self.scm_model.get_repos(all_repos,
                                                     sort_key='name_sort',
                                                     simple=True)
             return render('/repo_switcher_list.html')
         else:
             raise HTTPBadRequest()
     def branch_tag_switcher(self, repo_name):
         if request.is_xhr:
             c.rhodecode_db_repo = Repository.get_by_repo_name(c.repo_name)
             if c.rhodecode_db_repo:
                 c.rhodecode_repo = c.rhodecode_db_repo.scm_instance
                 return render('/switch_to_list.html')
         raise HTTPBadRequest()

rhodecode/lib/utils.py

➞

Show inline comments

@@ @@ -384,396 +384,396 @@ def map_groups(path): @@
         if group is None:
             log.debug('creating group level: %s group_name: %s' % (lvl,
                                                                    group_name))
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             sa.add(group)
             rgm._create_default_perms(group)
             sa.flush()
         parent = group
     return group
 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
                    install_git_hook=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hook: if this is True, also check and install githook
         for a repo if missing
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.scm import ScmModel
     sa = meta.Session()
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
     if user is None:
         raise Exception('Missing administrative account!')
     added = []
     ##creation defaults
     defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         db_repo = rm.get_by_repo_name(name)
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository %s not found, creating now' % name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
             new_repo = rm.create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repos_group=getattr(group, 'group_id', None),
                 owner=user,
                 just_db=True,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private
+            )
             # we added that repo just now, and make sure it has githook
             # installed
             if new_repo.repo_type == 'git':
                 ScmModel().install_git_hook(new_repo.scm_instance)
             new_repo.update_changeset_cache()
         elif install_git_hook:
             if db_repo.repo_type == 'git':
                 ScmModel().install_git_hook(db_repo.scm_instance)
         # during starting install all cache keys for all repositories in the
         # system, this will register all repos and multiple instances
         cache_key = CacheInvalidation._get_cache_key(name)
         log.debug("Creating invalidation cache key for %s: %s", name, cache_key)
         CacheInvalidation.invalidate(name)
     sa.commit()
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non-existing repository found in db `%s`" %
                           repo.repo_name)
                 try:
                     removed.append(repo.repo_name)
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                 except Exception:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
     return added, removed
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
 ))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = rhodecode.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...' % rcext)
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
         #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
         if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:
             log.debug('settings custom INDEX_EXTENSIONS')
             conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
         #ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         from rhodecode.config import rcextensions
         for k in dir(rcextensions):
             if not k.startswith('_') and not hasattr(EXT, k):
                 setattr(EXT, k, getattr(rcextensions, k))
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it's defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
     #check if we didn't define this extension as other lexer
     if rhodecode.EXTENSIONS and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
     :param config: test config
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     repo_location = repo_location
     index_location = os.path.join(config['app_conf']['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
     try:
         l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 def create_test_env(repos_test_path, config):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s' % dbconf)
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s' % repos_test_path)
         os.makedirs(repos_test_path)
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
     idx_path = config['app_conf']['index_dir']
     data_path = config['app_conf']['cache_dir']
     #clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s' % idx_path)
         shutil.rmtree(idx_path)
     if data_path and os.path.exists(data_path):
         log.debug('remove %s' % data_path)
         shutil.rmtree(data_path)
     #CREATE DEFAULT TEST REPOS
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_git.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, GIT_REPO))
     tar.close()
     #LOAD VCS test stuff
     from rhodecode.tests.vcs import setup_package
     setup_package()
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
     def notify_msg(self, msg, log=False):
         """Make a notification to user, additionally if logger is passed
         it logs this action using given logger
         :param msg: message that will be printed to user
         :param log: logging instance, to use to additionally log this message
         """
         if log and isinstance(log, logging):
             log(msg)
     def run(self, args):
         """
         Overrides Command.run
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
         self.path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)
     def _init_session(self):
         """
         Inits SqlAlchemy Session
         """
         logging.config.fileConfig(self.path_to_ini_file)
         from pylons import config
         from rhodecode.model import init_model
         from rhodecode.lib.utils2 import engine_from_config
         #get to remove repos !!
         add_cache(config)
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
 def check_git_version():
     """
     Checks what version of git is installed in system, and issues a warning
     if it's too old for RhodeCode to properly work.
     """
     from rhodecode import BACKENDS
     from rhodecode.lib.vcs.backends.git.repository import GitRepository
     from distutils.version import StrictVersion
     stdout, stderr = GitRepository._run_git_command('--version', _bare=True,
                                                     _safe=True)
     ver = (stdout.split(' ')[-1] or '').strip() or '0.0.0'
     if len(ver.split('.')) > 3:
         #StrictVersion needs to be only 3 element type
         ver = '.'.join(ver.split('.')[:3])
     try:
         _ver = StrictVersion(ver)
     except Exception:
         _ver = StrictVersion('0.0.0')
         stderr = traceback.format_exc()
     req_ver = '1.7.4'
     to_old_git = False
     if  _ver < StrictVersion(req_ver):
         to_old_git = True
     if 'git' in BACKENDS:
         log.debug('GIT version detected: %s' % stdout)
         if stderr:
             log.warning('Unable to detect git version, org error was: %r' % stderr)
         elif to_old_git:
             log.warning('RhodeCode detected git version %s, which is too old '
                         'for the system to function properly. Make sure '
                         'its version is at least %s' % (ver, req_ver))
     return _ver
 @decorator.decorator
 def jsonify(func, *args, **kwargs):
     """Action decorator that formats output for JSON
     Given a function that will return content, this decorator will turn
     the result into JSON, with a content-type of 'application/json' and
     output it.
     """
     from pylons.decorators.util import get_pylons
-    from rhodecode.lib.ext_json import json
+    from rhodecode.lib.compat import json
     pylons = get_pylons(args)
     pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
     data = func(*args, **kwargs)
     if isinstance(data, (list, tuple)):
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
     return json.dumps(data, encoding='utf-8')

0 comments (0 inline, 0 general)