kallithea Changeset - af7b367f6b5a

Changeset - af7b367f6b5a

Parent rev.

Child rev.

[Not reviewed]

default

0 1 1

Thomas De Schampheleire - 5 years ago 2020-06-15 12:37:55
thomas.de_schampheleire@nokia.com

db: introduce constraint ensuring no duplicate (reviewer, pullrequest) combinations

A reviewer should only be added once to a review.

Previously, this was not ensured by the database itself, although that the
controller would try to not add duplicate reviewers. But there was no hard
guarantee: e.g. simultaneous adding of the same reviewer to the same review
by a review owner and admin, a framework bug that sends the same request
twice, ... could still trigger duplicate addition. Additionally, code
changes (e.g. a new API) could introduce bugs at the controller level.

Existing production databases were found to contain such duplicate entries.
Nevertheless, as the code displaying reviewers in a pull request filtered
out duplicates, this never showed in the UI, and never was a 'real' problem.

Add a UniqueConstraint in the database to prevent such entries, with a
database migration step that will first find and remove existing duplicates.

2 files changed with 74 insertions and 0 deletions:

kallithea/alembic/versions/f62826179f39_add_unique_constraint_on_.py

kallithea/model/db.py

0 comments (0 inline, 0 general)

kallithea/alembic/versions/f62826179f39_add_unique_constraint_on_.py

➞

Show inline comments

@@ new file 100644 @@
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """add unique constraint on PullRequestReviewer
 Revision ID: f62826179f39
 Revises: a0a1bf09c143
 Create Date: 2020-06-15 12:30:37.420321
 """
 # The following opaque hexadecimal identifiers ("revisions") are used
 # by Alembic to track this migration script and its relations to others.
 revision = 'f62826179f39'
 down_revision = 'a0a1bf09c143'
 branch_labels = None
 depends_on = None
 import sqlalchemy as sa
 from alembic import op
 from kallithea.model.db import PullRequestReviewer
 def upgrade():
     session = sa.orm.session.Session(bind=op.get_bind())
     # there may be existing duplicates in the database, remove them first
     seen = set()
     # duplicate_values contains one copy of each duplicated pair
     duplicate_values = (
         session
         .query(PullRequestReviewer.pull_request_id, PullRequestReviewer.user_id)
         .group_by(PullRequestReviewer.pull_request_id, PullRequestReviewer.user_id)
         .having(sa.func.count(PullRequestReviewer.pull_request_reviewers_id) > 1)
+    )
     for pull_request_id, user_id in duplicate_values:
         # duplicate_occurrences contains all db records of the duplicate_value
         # currently being processed
         duplicate_occurrences = (
             session
             .query(PullRequestReviewer)
             .filter(PullRequestReviewer.pull_request_id == pull_request_id)
             .filter(PullRequestReviewer.user_id == user_id)
+        )
         for prr in duplicate_occurrences:
             if (pull_request_id, user_id) in seen:
                 session.delete(prr)
             else:
                 seen.add((pull_request_id, user_id))
     session.commit()
     # after deleting all duplicates, add the unique constraint
     with op.batch_alter_table('pull_request_reviewers', schema=None) as batch_op:
         batch_op.create_unique_constraint(batch_op.f('uq_pull_request_reviewers_pull_request_id'), ['pull_request_id', 'user_id'])
 def downgrade():
     with op.batch_alter_table('pull_request_reviewers', schema=None) as batch_op:
         batch_op.drop_constraint(batch_op.f('uq_pull_request_reviewers_pull_request_id'), type_='unique')

kallithea/model/db.py

➞

Show inline comments

@@ @@ -1782,531 +1782,532 @@ class UserGroupUserGroupToPerm(Base, Bas @@
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id')
     user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
     permission = relationship('Permission')
     @classmethod
     def create(cls, target_user_group, user_group, permission):
         n = cls()
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.user_group, self.target_user_group, self.permission)
 class UserGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         _table_args_default_dict,
+    )
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 class UserRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
     @classmethod
     def create(cls, user, repository_group, permission):
         n = cls()
         n.user = user
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class UserGroupRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         _table_args_default_dict,
+    )
     users_group_repo_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
     @classmethod
     def create(cls, user_group, repository_group, permission):
         n = cls()
         n.users_group = user_group
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class Statistics(Base, BaseDbModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          _table_args_default_dict,
+    )
     stat_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True)
     stat_on_revision = Column(Integer(), nullable=False)
     commit_activity = Column(LargeBinary(1000000), nullable=False) # JSON data
     commit_activity_combined = Column(LargeBinary(), nullable=False) # JSON data
     languages = Column(LargeBinary(1000000), nullable=False) # JSON data
     repository = relationship('Repository', single_parent=True)
 class UserFollowing(Base, BaseDbModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id', name='uq_user_followings_user_repo'),
         UniqueConstraint('user_id', 'follows_user_id', name='uq_user_followings_user_user'),
         _table_args_default_dict,
+    )
     user_following_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     follows_repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     follows_user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     follows_from = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by=lambda: sqlalchemy.func.lower(Repository.repo_name))
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repository_id == repo_id)
 class ChangesetComment(Base, BaseDbModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         Index('cc_pull_request_id_idx', 'pull_request_id'),
         _table_args_default_dict,
+    )
     comment_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     line_no = Column(Unicode(10), nullable=True)
     f_path = Column(Unicode(1000), nullable=True)
     author_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column(UnicodeText(), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     author = relationship('User')
     repo = relationship('Repository')
     # status_change is frequently used directly in templates - make it a lazy
     # join to avoid fetching each related ChangesetStatus on demand.
     # There will only be one ChangesetStatus referencing each comment so the join will not explode.
     status_change = relationship('ChangesetStatus',
                                  cascade="all, delete-orphan", lazy='joined')
     pull_request = relationship('PullRequest')
     def url(self):
         anchor = "comment-%s" % self.comment_id
         import kallithea.lib.helpers as h
         if self.revision:
             return h.url('changeset_home', repo_name=self.repo.repo_name, revision=self.revision, anchor=anchor)
         elif self.pull_request_id is not None:
             return self.pull_request.url(anchor=anchor)
     def __json__(self):
         return dict(
             comment_id=self.comment_id,
             username=self.author.username,
             text=self.text,
+        )
     def deletable(self):
         return self.created_on > datetime.datetime.now() - datetime.timedelta(minutes=5)
 class ChangesetStatus(Base, BaseDbModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         Index('cs_pull_request_id_idx', 'pull_request_id'),
         Index('cs_changeset_comment_id_idx', 'changeset_comment_id'),
         Index('cs_pull_request_id_user_id_version_idx', 'pull_request_id', 'user_id', 'version'),
         Index('cs_repo_id_pull_request_id_idx', 'repo_id', 'pull_request_id'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         _table_args_default_dict,
+    )
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected' # is shown as "Not approved" - TODO: change database content / scheme
     STATUS_UNDER_REVIEW = 'under_review'
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not reviewed")),  # (no icon) and default
         (STATUS_UNDER_REVIEW, _("Under review")),
         (STATUS_REJECTED, _("Not approved")),
         (STATUS_APPROVED, _("Approved")),
+    ]
     STATUSES_DICT = dict(STATUSES)
     changeset_status_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     status = Column(String(128), nullable=False, default=DEFAULT)
     comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)
     modified_at = Column(DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column(Integer(), nullable=False, default=0)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     author = relationship('User')
     repo = relationship('Repository')
     comment = relationship('ChangesetComment')
     pull_request = relationship('PullRequest')
     def __repr__(self):
         return "<%s %r by %r>" % (
             self.__class__.__name__,
             self.status, self.author
+        )
     @classmethod
     def get_status_lbl(cls, value):
         return cls.STATUSES_DICT.get(value)
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
     def __json__(self):
         return dict(
             status=self.status,
             modified_at=self.modified_at.replace(microsecond=0),
             reviewer=self.author.username,
+            )
 class PullRequest(Base, BaseDbModel):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         Index('pr_org_repo_id_idx', 'org_repo_id'),
         Index('pr_other_repo_id_idx', 'other_repo_id'),
         _table_args_default_dict,
+    )
     # values for .status
     STATUS_NEW = 'new'
     STATUS_CLOSED = 'closed'
     pull_request_id = Column(Integer(), primary_key=True)
     title = Column(Unicode(255), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     status = Column(Unicode(255), nullable=False, default=STATUS_NEW) # only for closedness, not approve/reject/etc
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     _revisions = Column('revisions', UnicodeText(), nullable=False)
     org_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     org_ref = Column(Unicode(255), nullable=False)
     other_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     other_ref = Column(Unicode(255), nullable=False)
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':')
     @revisions.setter
     def revisions(self, val):
         self._revisions = ':'.join(val)
     @property
     def org_ref_parts(self):
         return self.org_ref.split(':')
     @property
     def other_ref_parts(self):
         return self.other_ref.split(':')
     owner = relationship('User')
     reviewers = relationship('PullRequestReviewer',
                              cascade="all, delete-orphan")
     org_repo = relationship('Repository', primaryjoin='PullRequest.org_repo_id==Repository.repo_id')
     other_repo = relationship('Repository', primaryjoin='PullRequest.other_repo_id==Repository.repo_id')
     statuses = relationship('ChangesetStatus', order_by='ChangesetStatus.changeset_status_id')
     comments = relationship('ChangesetComment', order_by='ChangesetComment.comment_id',
                              cascade="all, delete-orphan")
     @classmethod
     def query(cls, reviewer_id=None, include_closed=True, sorted=False):
         """Add PullRequest-specific helpers for common query constructs.
         reviewer_id: only PRs with the specified user added as reviewer.
         include_closed: if False, do not include closed PRs.
         sorted: if True, apply the default ordering (newest first).
         """
         q = super(PullRequest, cls).query()
         if reviewer_id is not None:
             q = q.join(PullRequestReviewer).filter(PullRequestReviewer.user_id == reviewer_id)
         if not include_closed:
             q = q.filter(PullRequest.status != PullRequest.STATUS_CLOSED)
         if sorted:
             q = q.order_by(PullRequest.created_on.desc())
         return q
     def get_reviewer_users(self):
         """Like .reviewers, but actually returning the users"""
         return User.query() \
             .join(PullRequestReviewer) \
             .filter(PullRequestReviewer.pull_request == self) \
             .order_by(PullRequestReviewer.pull_request_reviewers_id) \
             .all()
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
     def user_review_status(self, user_id):
         """Return the user's latest status votes on PR"""
         # note: no filtering on repo - that would be redundant
         status = ChangesetStatus.query() \
             .filter(ChangesetStatus.pull_request == self) \
             .filter(ChangesetStatus.user_id == user_id) \
             .order_by(ChangesetStatus.version) \
             .first()
         return str(status.status) if status else ''
     @classmethod
     def make_nice_id(cls, pull_request_id):
         '''Return pull request id nicely formatted for displaying'''
         return '#%s' % pull_request_id
     def nice_id(self):
         '''Return the id of this pull request, nicely formatted for displaying'''
         return self.make_nice_id(self.pull_request_id)
     def get_api_data(self):
         return self.__json__()
     def __json__(self):
         clone_uri_tmpl = kallithea.CONFIG.get('clone_uri_tmpl') or Repository.DEFAULT_CLONE_URI
         return dict(
             pull_request_id=self.pull_request_id,
             url=self.url(),
             reviewers=self.reviewers,
             revisions=self.revisions,
             owner=self.owner.username,
             title=self.title,
             description=self.description,
             org_repo_url=self.org_repo.clone_url(clone_uri_tmpl=clone_uri_tmpl),
             org_ref_parts=self.org_ref_parts,
             other_ref_parts=self.other_ref_parts,
             status=self.status,
             comments=self.comments,
             statuses=self.statuses,
             created_on=self.created_on.replace(microsecond=0),
             updated_on=self.updated_on.replace(microsecond=0),
+        )
     def url(self, **kwargs):
         canonical = kwargs.pop('canonical', None)
         import kallithea.lib.helpers as h
         b = self.org_ref_parts[1]
         if b != self.other_ref_parts[1]:
             s = '/_/' + b
         else:
             s = '/_/' + self.title
         kwargs['extra'] = urlreadable(s)
         if canonical:
             return h.canonical_url('pullrequest_show', repo_name=self.other_repo.repo_name,
                                    pull_request_id=self.pull_request_id, **kwargs)
         return h.url('pullrequest_show', repo_name=self.other_repo.repo_name,
                      pull_request_id=self.pull_request_id, **kwargs)
 class PullRequestReviewer(Base, BaseDbModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         Index('pull_request_reviewers_user_id_idx', 'user_id'),
         UniqueConstraint('pull_request_id', 'user_id'),
         _table_args_default_dict,
+    )
     def __init__(self, user=None, pull_request=None):
         self.user = user
         self.pull_request = pull_request
     pull_request_reviewers_id = Column('pull_requests_reviewers_id', Integer(), primary_key=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     user = relationship('User')
     pull_request = relationship('PullRequest')
     def __json__(self):
         return dict(
             username=self.user.username if self.user else None,
+        )
 class Notification(object):
     __tablename__ = 'notifications'
 class UserNotification(object):
     __tablename__ = 'user_to_notification'
 class Gist(Base, BaseDbModel):
     __tablename__ = 'gists'
     __table_args__ = (
         Index('g_gist_access_id_idx', 'gist_access_id'),
         Index('g_created_on_idx', 'created_on'),
         _table_args_default_dict,
+    )
     GIST_PUBLIC = 'public'
     GIST_PRIVATE = 'private'
     DEFAULT_FILENAME = 'gistfile1.txt'
     gist_id = Column(Integer(), primary_key=True)
     gist_access_id = Column(Unicode(250), nullable=False)
     gist_description = Column(UnicodeText(), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     gist_expires = Column(Float(53), nullable=False)
     gist_type = Column(Unicode(128), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner = relationship('User')
     @hybrid_property
     def is_expired(self):
         return (self.gist_expires != -1) & (time.time() > self.gist_expires)
     def __repr__(self):
         return "<%s %s %s>" % (
             self.__class__.__name__,
             self.gist_type, self.gist_access_id)
     @classmethod
     def guess_instance(cls, value):
         return super(Gist, cls).guess_instance(value, Gist.get_by_access_id)
     @classmethod
     def get_or_404(cls, id_):
         res = cls.query().filter(cls.gist_access_id == id_).scalar()
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def get_by_access_id(cls, gist_access_id):
         return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
     def gist_url(self):
         alias_url = kallithea.CONFIG.get('gist_alias_url')
         if alias_url:
             return alias_url.replace('{gistid}', self.gist_access_id)
         import kallithea.lib.helpers as h
         return h.canonical_url('gist', gist_id=self.gist_access_id)
     def get_api_data(self):
         """
         Common function for generating gist related data for API
         """
         gist = self
         data = dict(
             gist_id=gist.gist_id,
             type=gist.gist_type,
             access_id=gist.gist_access_id,
             description=gist.gist_description,
             url=gist.gist_url(),
             expires=gist.gist_expires,
             created_on=gist.created_on,
+        )
         return data
     def __json__(self):
         data = dict(
+        )
         data.update(self.get_api_data())
         return data
     ## SCM functions
     @property
     def scm_instance(self):
         from kallithea.lib.vcs import get_repo
         from kallithea.model.gist import GIST_STORE_LOC
         gist_base_path = os.path.join(kallithea.CONFIG['base_path'], GIST_STORE_LOC)
         return get_repo(os.path.join(gist_base_path, self.gist_access_id))
 class UserSshKeys(Base, BaseDbModel):
     __tablename__ = 'user_ssh_keys'
     __table_args__ = (
         Index('usk_fingerprint_idx', 'fingerprint'),
         _table_args_default_dict
+    )
     __mapper_args__ = {}
     user_ssh_key_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     _public_key = Column('public_key', UnicodeText(), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     fingerprint = Column(String(255), nullable=False, unique=True)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     last_seen = Column(DateTime(timezone=False), nullable=True)
     user = relationship('User')
     @property
     def public_key(self):
         return self._public_key
     @public_key.setter
     def public_key(self, full_key):
         """The full public key is too long to be suitable as database key.
         Instead, as a side-effect of setting the public key string, compute the
         fingerprints according to https://tools.ietf.org/html/rfc4716#section-4
         BUT using sha256 instead of md5, similar to 'ssh-keygen -E sha256 -lf
         ~/.ssh/id_rsa.pub' .
         """
         keytype, key_bytes, comment = ssh.parse_pub_key(full_key)
         self._public_key = full_key
         self.fingerprint = base64.b64encode(hashlib.sha256(key_bytes).digest()).replace(b'\n', b'').rstrip(b'=').decode()

0 comments (0 inline, 0 general)