# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.db_manage

~~~~~~~~~~~~~~~~~~~~~~~

Database creation, and setup module for Kallithea. Used for creation

of database as well as for migration operations

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 10, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import time

import uuid

import logging

from kallithea import __dbversion__, __py_version__, EXTERN_TYPE_INTERNAL, DB_MIGRATIONS

from kallithea.model.user import UserModel

from kallithea.lib.utils import ask_ok

from kallithea.model import init_model

from kallithea.model.db import User, Permission, Ui, \

    Setting, UserToPerm, DbMigrateVersion, RepoGroup, \

    UserRepoGroupToPerm, CacheInvalidation, Repository

from sqlalchemy.engine import create_engine

from kallithea.model.repo_group import RepoGroupModel

#from kallithea.model import meta

from kallithea.model.meta import Session, Base

from kallithea.model.repo import RepoModel

from kallithea.model.permission import PermissionModel

log = logging.getLogger(__name__)

def notify(msg):

    """

    Notification for migrations messages

    """

        """

        Upgrades given database schema to given revision following

        all needed steps, to perform the upgrade

        """

        from kallithea.lib.dbmigrate.migrate.versioning import api

        from kallithea.lib.dbmigrate.migrate.exceptions import \

            DatabaseNotControlledError

        if 'sqlite' in self.dburi:

            print (

               '********************** WARNING **********************\n'

               'Make sure your version of sqlite is at least 3.7.X.  \n'

               'Earlier versions are known to fail on some migrations\n'

               '*****************************************************\n')

        upgrade = ask_ok('You are about to perform database upgrade, make '

                         'sure You backed up your database before. '

                         'Continue ? [y/n]')

        if not upgrade:

            print 'No upgrade performed'

            sys.exit(0)

                                       'kallithea', 'lib', 'dbmigrate')

        db_uri = self.dburi

        try:

            curr_version = api.db_version(db_uri, repository_path)

            msg = ('Found current database under version '

                   'control with version %s' % curr_version)

        except (RuntimeError, DatabaseNotControlledError):

            curr_version = 1

            msg = ('Current database is not under version control. Setting '

                   'as version %s' % curr_version)

            api.version_control(db_uri, repository_path, curr_version)

        notify(msg)

        if curr_version == __dbversion__:

            print 'This database is already at the newest version'

            sys.exit(0)

        # clear cache keys

        log.info("Clearing cache keys now...")

        CacheInvalidation.clear_cache()

        upgrade_steps = range(curr_version + 1, __dbversion__ + 1)

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers

~~~~~~~~~~~~~~~~~~~~~~

Whoosh indexing module for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import logging

# Add location of top level folder to sys.path

from whoosh.analysis import RegexTokenizer, LowercaseFilter

from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME

from whoosh.formats import Characters

from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter

from kallithea.lib.utils2 import LazyProperty

log = logging.getLogger(__name__)

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers.daemon

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

A daemon will read from task table and run tasks

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jan 26, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import logging

import traceback

from shutil import rmtree

from time import mktime

# Add location of top level folder to sys.path

sys.path.append(project_path)

from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES

from kallithea.model.scm import ScmModel

from kallithea.model.db import Repository

from kallithea.lib.utils2 import safe_unicode, safe_str

from kallithea.lib.indexers import SCHEMA, IDX_NAME, CHGSETS_SCHEMA, \

    CHGSET_IDX_NAME

from kallithea.lib.vcs.exceptions import ChangesetError, RepositoryError, \

    NodeDoesNotExistError

from whoosh.index import create_in, open_dir, exists_in

from whoosh.query import *

from whoosh.qparser import QueryParser

log = logging.getLogger('whoosh_indexer')

class WhooshIndexingDaemon(object):

    """

    Daemon for atomic indexing jobs

    """

"""

kallithea.lib.paster_commands.cache_keys

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cleanup-keys paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: mar 27, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from kallithea.model.meta import Session

from kallithea.lib.utils import BasePasterCommand

from kallithea.model.db import CacheInvalidation

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Cache keys utils"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        _caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all()

        if self.options.show:

            for c_obj in _caches:

                print 'key:%s active:%s' % (c_obj.cache_key, c_obj.cache_active)

        elif self.options.cleanup:

            for c_obj in _caches:

cleanup-repos paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jul 14, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import re

import shutil

import datetime

from kallithea.lib.utils import BasePasterCommand, ask_ok, REMOVED_REPO_PAT

from kallithea.lib.utils2 import safe_str

from kallithea.model.db import Ui

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Cleanup deleted repos"

    def _parse_older_than(self, val):

        regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')

        parts = regex.match(val)

        if not parts:

            return

        parts = parts.groupdict()

        time_params = {}

        for (name, param) in parts.iteritems():

            if param:

                time_params[name] = int(param)

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.install_iis

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

IIS installation tools for Kallithea

"""

import os

import sys

from paste.script.appinstall import AbstractInstallCommand

from paste.script.command import BadCommand

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(AbstractInstallCommand):

    default_verbosity = 1

    max_args = 1

    min_args = 1

    summary = 'Setup IIS given a config file'

    usage = 'CONFIG_FILE'

    description = '''

    Script for installing into IIS using isapi-wsgi.

    '''

    parser = AbstractInstallCommand.standard_parser(

        simulate=True, quiet=True, interactive=True)

    parser.add_option('--virtualdir',

                      action='store',

                      dest='virtualdir',

                      default='/',

                      help='The virtual folder to install into on IIS')

    def command(self):

        config_spec = self.args[0]

        if not config_spec.startswith('config:'):

            config_spec = 'config:' + config_spec

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.ishell

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

interactive shell paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 4, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from kallithea.lib.utils import BasePasterCommand

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Interactive shell"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        # imports, used in IPython shell

        import os

        import sys

        import time

        import shutil

        import datetime

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_index

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-index paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from string import strip

from kallithea.model.repo import RepoModel

from kallithea.lib.utils import BasePasterCommand, load_rcextensions

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Creates or updates full text search index"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        from pylons import config

        index_location = config['index_dir']

        load_rcextensions(config['here'])

        repo_location = self.options.repo_location \

            if self.options.repo_location else RepoModel().repos_path

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        repo_update_list = map(strip, self.options.repo_update_list.split(',')) \

            if self.options.repo_update_list else None

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from kallithea.lib.pidlock import LockHeld, DaemonLock

        from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

        try:

                                              'make_index.lock'))

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list,

                                 repo_update_list=repo_update_list) \

                .run(full_index=self.options.full_index)

            l.release()

        except LockHeld:

            sys.exit(1)

    def update_parser(self):

        self.parser.add_option('--repo-location',

                          action='store',

                          dest='repo_location',

                          help="Specifies repositories location to index OPTIONAL",

                          )

        self.parser.add_option('--index-only',

                          action='store',

                          dest='repo_list',

                          help="Specifies a comma separated list of repositories "

                                "to build index on. If not given all repositories "

                                "are scanned for indexing. OPTIONAL",

                          )

        self.parser.add_option('--update-only',

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_rcextensions

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-rcext paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Mar 6, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import pkg_resources

from kallithea.lib.utils import BasePasterCommand, ask_ok

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Write template file for extending Kallithea in Python."

    usage = "CONFIG_FILE"

    description = '''\

        A rcextensions directory with a __init__.py file will be created next to

        the ini file. Local customizations in that file will survive upgrades.

        The file contains instructions on how it can be customized.

        '''

    def command(self):

        from pylons import config

        here = config['here']

        content = pkg_resources.resource_string(

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.repo_scan

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

repo-scan paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Feb 9, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from kallithea.model.scm import ScmModel

from kallithea.lib.utils import BasePasterCommand, repo2db_mapper

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Rescan default location for new repositories"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        rm_obsolete = self.options.delete_obsolete

        print 'Now scanning root location for new repos ...'

        added, removed = repo2db_mapper(ScmModel().repo_scan(),

                                        remove_obsolete=rm_obsolete)

        added = ', '.join(added) or '-'

        removed = ', '.join(removed) or '-'

        print 'Scan completed added: %s removed: %s' % (added, removed)

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.setup_db

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Databaset setup paster command for Kallithea

"""

import os

import sys

from paste.script.appinstall import AbstractInstallCommand

from paste.script.command import BadCommand

from paste.deploy import appconfig

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(AbstractInstallCommand):

    default_verbosity = 1

    max_args = 1

    min_args = 1

    summary = "Setup an application, given a config file"

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    description = """\

        Setup Kallithea according to its configuration file.  This is

        the second part of a two-phase web application installation

        process (the first phase is prepare-app).  The setup process

        consist of things like setting up databases, creating super user

    """

    parser = AbstractInstallCommand.standard_parser(

        simulate=True, quiet=True, interactive=True)

    parser.add_option('--user',

                      action='store',

                      dest='username',

kallithea.lib.paster_commands.update_repoinfo

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

update-repoinfo paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jul 14, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import string

from kallithea.lib.utils import BasePasterCommand

from kallithea.model.db import Repository

from kallithea.model.repo import RepoModel

from kallithea.model.meta import Session

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Updates repositories caches for last changeset"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        repo_update_list = map(string.strip,

                               self.options.repo_update_list.split(',')) \

                               if self.options.repo_update_list else None

        if repo_update_list is not None:

            repo_list = list(Repository.query() \

~~~~~~~~~~~~~~~~~~~

Utilities library for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 18, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import re

import logging

import datetime

import traceback

import paste

import beaker

import tarfile

import shutil

import decorator

import warnings

from os.path import abspath

from paste.script.command import Command, BadCommand

from webhelpers.text import collapse, remove_formatting, strip_tags

from beaker.cache import _cache_decorate

from kallithea import BRAND

from kallithea.lib.vcs.utils.hgcompat import ui, config

from kallithea.lib.vcs.utils.helpers import get_scm

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.model import meta

from kallithea.model.db import Repository, User, Ui, \

    UserLog, RepoGroup, Setting, UserGroup

from kallithea.model.meta import Session

from kallithea.model.repo_group import RepoGroupModel

from kallithea.lib.utils2 import safe_str, safe_unicode, get_current_authuser

from kallithea.lib.vcs.utils.fakemod import create_module

log = logging.getLogger(__name__)

REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')

        return lexers.get_lexer_by_name(_lexer_name)

#==============================================================================

# TEST FUNCTIONS AND CREATORS

#==============================================================================

def create_test_index(repo_location, config, full_index):

    """

    Makes default test index

    :param config: test config

    :param full_index:

    """

    from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

    from kallithea.lib.pidlock import DaemonLock, LockHeld

    repo_location = repo_location

    index_location = os.path.join(config['app_conf']['index_dir'])

    if not os.path.exists(index_location):

        os.makedirs(index_location)

    try:

        WhooshIndexingDaemon(index_location=index_location,

                             repo_location=repo_location) \

            .run(full_index=full_index)

        l.release()

    except LockHeld:

        pass

def create_test_env(repos_test_path, config):

    """

    Makes a fresh database and

    install test repository into tmp dir

    """

    from kallithea.lib.db_manage import DbManage

    from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH

    # PART ONE create db

    dbconf = config['sqlalchemy.db1.url']

    log.debug('making test db %s', dbconf)

    # create test dir if it doesn't exist

    if not os.path.isdir(repos_test_path):

        log.debug('Creating testdir %s', repos_test_path)

        os.makedirs(repos_test_path)