# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.db_manage

~~~~~~~~~~~~~~~~~~~~~~~

Database creation, and setup module for Kallithea. Used for creation

of database as well as for migration operations

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 10, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import time

import uuid

import logging

from kallithea import __dbversion__, __py_version__, EXTERN_TYPE_INTERNAL, DB_MIGRATIONS

from kallithea.model.user import UserModel

from kallithea.lib.utils import ask_ok

from kallithea.model import init_model

from kallithea.model.db import User, Permission, Ui, \

    Setting, UserToPerm, DbMigrateVersion, RepoGroup, \

    UserRepoGroupToPerm, CacheInvalidation, Repository

from sqlalchemy.engine import create_engine

from kallithea.model.repo_group import RepoGroupModel

#from kallithea.model import meta

from kallithea.model.meta import Session, Base

from kallithea.model.repo import RepoModel

from kallithea.model.permission import PermissionModel

log = logging.getLogger(__name__)

def notify(msg):

    """

    Notification for migrations messages

    """

    ml = len(msg) + (4 * 2)

    print('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()

class DbManage(object):

    def __init__(self, log_sql, dbconf, root, tests=False, SESSION=None, cli_args=None):

        self.dbname = dbconf.split('/')[-1]

        self.tests = tests

        self.root = root

        self.dburi = dbconf

        self.log_sql = log_sql

        self.db_exists = False

        self.cli_args = cli_args or {}

        self.init_db(SESSION=SESSION)

        force_ask = self.cli_args.get('force_ask')

        if force_ask is not None:

            global ask_ok

            ask_ok = lambda *args, **kwargs: force_ask

    def init_db(self, SESSION=None):

        if SESSION:

            self.sa = SESSION

        else:

        log.info("Any existing database is going to be destroyed")

        if self.tests:

            destroy = True

        else:

            destroy = ask_ok('Are you sure to destroy old database ? [y/n]')

        if not destroy:

            print 'Nothing done.'

            sys.exit(0)

        if destroy:

            Base.metadata.drop_all()

        checkfirst = not override

        Base.metadata.create_all(checkfirst=checkfirst)

        log.info('Created tables for %s', self.dbname)

    def set_db_version(self):

        ver = DbMigrateVersion()

        ver.version = __dbversion__

        ver.repository_id = DB_MIGRATIONS

        ver.repository_path = 'versions'

        self.sa.add(ver)

        log.info('db version set to: %s', __dbversion__)

    def upgrade(self):

        """

        Upgrades given database schema to given revision following

        all needed steps, to perform the upgrade

        """

        from kallithea.lib.dbmigrate.migrate.versioning import api

        from kallithea.lib.dbmigrate.migrate.exceptions import \

            DatabaseNotControlledError

        if 'sqlite' in self.dburi:

            print (

               '********************** WARNING **********************\n'

               'Make sure your version of sqlite is at least 3.7.X.  \n'

               'Earlier versions are known to fail on some migrations\n'

               '*****************************************************\n')

        upgrade = ask_ok('You are about to perform database upgrade, make '

                         'sure You backed up your database before. '

                         'Continue ? [y/n]')

        if not upgrade:

            print 'No upgrade performed'

            sys.exit(0)

                                       'kallithea', 'lib', 'dbmigrate')

        db_uri = self.dburi

        try:

            curr_version = api.db_version(db_uri, repository_path)

            msg = ('Found current database under version '

                   'control with version %s' % curr_version)

        except (RuntimeError, DatabaseNotControlledError):

            curr_version = 1

            msg = ('Current database is not under version control. Setting '

                   'as version %s' % curr_version)

            api.version_control(db_uri, repository_path, curr_version)

        notify(msg)

        if curr_version == __dbversion__:

            print 'This database is already at the newest version'

            sys.exit(0)

        # clear cache keys

        log.info("Clearing cache keys now...")

        CacheInvalidation.clear_cache()

        upgrade_steps = range(curr_version + 1, __dbversion__ + 1)

        notify('attempting to do database upgrade from '

               'version %s to version %s' % (curr_version, __dbversion__))

        # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE

        _step = None

        for step in upgrade_steps:

            notify('performing upgrade step %s' % step)

            time.sleep(0.5)

            api.upgrade(db_uri, repository_path, step)

            notify('schema upgrade for step %s completed' % (step,))

            _step = step

        notify('upgrade to version %s successful' % _step)

    def fix_repo_paths(self):

        """

        Fixes a old kallithea version path into new one without a '*'

        """

        paths = self.sa.query(Ui) \

                .filter(Ui.ui_key == '/') \

                .scalar()

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers

~~~~~~~~~~~~~~~~~~~~~~

Whoosh indexing module for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import logging

# Add location of top level folder to sys.path

from whoosh.analysis import RegexTokenizer, LowercaseFilter

from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME

from whoosh.formats import Characters

from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter

from kallithea.lib.utils2 import LazyProperty

log = logging.getLogger(__name__)

# CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(

    fileid=ID(unique=True),

    owner=TEXT(),

    repository=TEXT(stored=True),

    path=TEXT(stored=True),

    content=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    modtime=STORED(),

    extension=TEXT(stored=True)

)

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = ContextFragmenter(200)

CHGSETS_SCHEMA = Schema(

    raw_id=ID(unique=True, stored=True),

    date=NUMERIC(stored=True),

    last=BOOLEAN(),

    owner=TEXT(),

    repository=ID(unique=True, stored=True),

    author=TEXT(stored=True),

    message=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    parents=TEXT(),

    added=TEXT(),

    removed=TEXT(),

    changed=TEXT(),

)

CHGSET_IDX_NAME = 'CHGSET_INDEX'

# used only to generate queries in journal

JOURNAL_SCHEMA = Schema(

    username=TEXT(),

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers.daemon

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

A daemon will read from task table and run tasks

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jan 26, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import logging

import traceback

from shutil import rmtree

from time import mktime

# Add location of top level folder to sys.path

sys.path.append(project_path)

from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES

from kallithea.model.scm import ScmModel

from kallithea.model.db import Repository

from kallithea.lib.utils2 import safe_unicode, safe_str

from kallithea.lib.indexers import SCHEMA, IDX_NAME, CHGSETS_SCHEMA, \

    CHGSET_IDX_NAME

from kallithea.lib.vcs.exceptions import ChangesetError, RepositoryError, \

    NodeDoesNotExistError

from whoosh.index import create_in, open_dir, exists_in

from whoosh.query import *

from whoosh.qparser import QueryParser

log = logging.getLogger('whoosh_indexer')

class WhooshIndexingDaemon(object):

    """

    Daemon for atomic indexing jobs

    """

    def __init__(self, indexname=IDX_NAME, index_location=None,

                 repo_location=None, sa=None, repo_list=None,

                 repo_update_list=None):

        self.indexname = indexname

        self.index_location = index_location

        if not index_location:

            raise Exception('You have to provide index location')

        self.repo_location = repo_location

        if not repo_location:

            raise Exception('You have to provide repositories location')

        self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)

        #filter repo list

        if repo_list:

            #Fix non-ascii repo names to unicode

            repo_list = map(safe_unicode, repo_list)

            self.filtered_repo_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_list:

                    self.filtered_repo_paths[repo_name] = repo

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.cache_keys

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cleanup-keys paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: mar 27, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from kallithea.model.meta import Session

from kallithea.lib.utils import BasePasterCommand

from kallithea.model.db import CacheInvalidation

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Cache keys utils"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        _caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all()

        if self.options.show:

            for c_obj in _caches:

                print 'key:%s active:%s' % (c_obj.cache_key, c_obj.cache_active)

        elif self.options.cleanup:

            for c_obj in _caches:

                Session().delete(c_obj)

                print 'Removing key: %s' % (c_obj.cache_key)

                Session().commit()

        else:

            print 'Nothing done, exiting...'

    def update_parser(self):

        self.parser.add_option(

            '--show',

            action='store_true',

            dest='show',

            help=("show existing cache keys with together with status")

        )

        self.parser.add_option(

            '--cleanup',

            action="store_true",

            dest="cleanup",

            help="cleanup existing cache keys"

        )

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.cleanup

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cleanup-repos paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jul 14, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import re

import shutil

import datetime

from kallithea.lib.utils import BasePasterCommand, ask_ok, REMOVED_REPO_PAT

from kallithea.lib.utils2 import safe_str

from kallithea.model.db import Ui

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Cleanup deleted repos"

    def _parse_older_than(self, val):

        regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')

        parts = regex.match(val)

        if not parts:

            return

        parts = parts.groupdict()

        time_params = {}

        for (name, param) in parts.iteritems():

            if param:

                time_params[name] = int(param)

        return datetime.timedelta(**time_params)

    def _extract_date(self, name):

        """

        Extract the date part from rm__<date> pattern of removed repos,

        and convert it to datetime object

        :param name:

        """

        date_part = name[4:19]  # 4:19 since we don't parse milliseconds

        return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        repos_location = Ui.get_repos_location()

        to_remove = []

        for dn_, dirs, f in os.walk(safe_str(repos_location)):

            alldirs = list(dirs)

            del dirs[:]

            if ('.hg' in alldirs or

                'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):

                continue

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.install_iis

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

IIS installation tools for Kallithea

"""

import os

import sys

from paste.script.appinstall import AbstractInstallCommand

from paste.script.command import BadCommand

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(AbstractInstallCommand):

    default_verbosity = 1

    max_args = 1

    min_args = 1

    summary = 'Setup IIS given a config file'

    usage = 'CONFIG_FILE'

    description = '''

    Script for installing into IIS using isapi-wsgi.

    '''

    parser = AbstractInstallCommand.standard_parser(

        simulate=True, quiet=True, interactive=True)

    parser.add_option('--virtualdir',

                      action='store',

                      dest='virtualdir',

                      default='/',

                      help='The virtual folder to install into on IIS')

    def command(self):

        config_spec = self.args[0]

        if not config_spec.startswith('config:'):

            config_spec = 'config:' + config_spec

        config_file = config_spec[len('config:'):].split('#', 1)[0]

        config_file = os.path.join(os.getcwd(), config_file)

        try:

            import isapi_wsgi

        except ImportError:

            raise BadCommand('missing requirement: isapi-wsgi not installed')

        file = '''\

# Created by Kallithea install_iis

import sys

if hasattr(sys, "isapidllhandle"):

    import win32traceutil

import isapi_wsgi

import os

def __ExtensionFactory__():

    from paste.deploy import loadapp

    from paste.script.util.logging_config import fileConfig

    fileConfig('%(inifile)s')

    application = loadapp('config:%(inifile)s')

    def app(environ, start_response):

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.ishell

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

interactive shell paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 4, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from kallithea.lib.utils import BasePasterCommand

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Interactive shell"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        # imports, used in IPython shell

        import os

        import sys

        import time

        import shutil

        import datetime

        from kallithea.model.db import *

        try:

            from IPython import embed

            from IPython.config.loader import Config

            cfg = Config()

            cfg.InteractiveShellEmbed.confirm_exit = False

            embed(config=cfg, banner1="Kallithea IShell.")

        except ImportError:

            print 'IPython installation is required for ishell'

            sys.exit(-1)

    def update_parser(self):

        pass

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_index

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-index paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

from string import strip

from kallithea.model.repo import RepoModel

from kallithea.lib.utils import BasePasterCommand, load_rcextensions

# Add location of top level folder to sys.path

sys.path.append(rc_path)

class Command(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    group_name = "Kallithea"

    takes_config_file = -1

    parser = BasePasterCommand.standard_parser(verbose=True)

    summary = "Creates or updates full text search index"

    def command(self):

        #get SqlAlchemy session

        self._init_session()

        from pylons import config

        index_location = config['index_dir']

        load_rcextensions(config['here'])

        repo_location = self.options.repo_location \

            if self.options.repo_location else RepoModel().repos_path

        repo_list = map(strip, self.options.repo_list.split(',')) \

            if self.options.repo_list else None

        repo_update_list = map(strip, self.options.repo_update_list.split(',')) \

            if self.options.repo_update_list else None

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from kallithea.lib.pidlock import LockHeld, DaemonLock

        from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

        try:

                                              'make_index.lock'))

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list,

                                 repo_update_list=repo_update_list) \

                .run(full_index=self.options.full_index)

            l.release()

        except LockHeld:

            sys.exit(1)

    def update_parser(self):

        self.parser.add_option('--repo-location',

                          action='store',

                          dest='repo_location',

                          help="Specifies repositories location to index OPTIONAL",

                          )

        self.parser.add_option('--index-only',

                          action='store',

                          dest='repo_list',

                          help="Specifies a comma separated list of repositories "

                                "to build index on. If not given all repositories "

                                "are scanned for indexing. OPTIONAL",

                          )

        self.parser.add_option('--update-only',

                          action='store',

                          dest='repo_update_list',

                          help="Specifies a comma separated list of repositories "

                                "to re-build index on. OPTIONAL",

                          )

        self.parser.add_option('-f',

                          action='store_true',

                          dest='full_index',

                          help="Specifies that index should be made full i.e"

                                " destroy old and build from scratch",

                          default=False)

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.paster_commands.make_rcextensions

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

make-rcext paster command for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Mar 6, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import pkg_resources

from kallithea.lib.utils import BasePasterCommand, ask_ok

# Add location of top level folder to sys.path