def createvirtualenv = ''

def activatevirtualenv = ''

node {

    properties([[$class: 'BuildDiscarderProperty',

                  strategy: [$class: 'LogRotator',

                              artifactDaysToKeepStr: '',

                              artifactNumToKeepStr: '10',

                              daysToKeepStr: '',

                              numToKeepStr: '']]]);

    if (isUnix()) {

        createvirtualenv = 'rm -r $JENKINS_HOME/venv/$JOB_NAME || true && virtualenv $JENKINS_HOME/venv/$JOB_NAME'

        activatevirtualenv = '. $JENKINS_HOME/venv/$JOB_NAME/bin/activate'

    } else {

        createvirtualenv = 'rmdir /s /q %JENKINS_HOME%\\venv\\%JOB_NAME% || true && virtualenv %JENKINS_HOME%\\venv\\%JOB_NAME%'

        activatevirtualenv = 'call %JENKINS_HOME%\\venv\\%JOB_NAME%\\Scripts\\activate.bat'

    }

    stage('checkout') {

        checkout scm

        if (isUnix()) {

            sh 'hg --config extensions.purge= purge --all'

        } else {

            bat 'hg --config extensions.purge= purge --all'

        }

    }

    stage('virtual env') {

        def virtualenvscript = """$createvirtualenv

            $activatevirtualenv

            python -m pip install --upgrade pip

            pip install --upgrade setuptools

            pip install --upgrade pylint

            pip install --upgrade pytest-cov

            """

        if (isUnix()) {

            virtualenvscript += """

                pip install --upgrade python-ldap

                pip install --upgrade python-pam

                """

            sh virtualenvscript

        } else {

            bat virtualenvscript

        }

    }

    stage('setup') {

        def virtualenvscript = """$activatevirtualenv

            python setup.py compile_catalog

            """

        if (isUnix()) {

            sh virtualenvscript

        } else {

            bat virtualenvscript

        }

        stash name: 'kallithea', useDefaultExcludes: false

    }

    stage('pylint') {

        sh script: """$activatevirtualenv

            pylint -j 0 --disable=C -f parseable kallithea > pylint.out

            """, returnStatus: true

        archiveArtifacts 'pylint.out'

        try {

            step([$class: 'WarningsPublisher', canComputeNew: false, canResolveRelativePaths: false, defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', parserConfigurations: [[parserName: 'PyLint', pattern: 'pylint.out']], unHealthy: ''])

        } catch (java.lang.IllegalArgumentException exc) {

            echo "You need to install the 'Warnings Plug-in' to display the pylint report."

            currentBuild.result = 'UNSTABLE'

            echo "Caught: ${exc}"

        }

    }

}

def pytests = [:]

pytests['sqlite'] = {

    node {

        ws {

            deleteDir()

            unstash name: 'kallithea'

            if (isUnix()) {

                sh script: """$activatevirtualenv

                    py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_sqlite.xml --cov=kallithea

                    """, returnStatus: true

            } else {

                bat script: """$activatevirtualenv

                    py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_sqlite.xml --cov=kallithea

                    """, returnStatus: true

            }

            sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1SQLITE./g" pytest_sqlite.xml'

            archiveArtifacts 'pytest_sqlite.xml'

            junit 'pytest_sqlite.xml'

            writeFile(file: '.coverage.sqlite', text: readFile('.coverage'))

            stash name: 'coverage.sqlite', includes: '.coverage.sqlite'

        }

    }

}

pytests['de'] = {

    node {

        if (isUnix()) {

            ws {

                deleteDir()

                unstash name: 'kallithea'

                withEnv(['LANG=de_DE.UTF-8',

                    'LANGUAGE=de',

                    'LC_ADDRESS=de_DE.UTF-8',

                    'LC_IDENTIFICATION=de_DE.UTF-8',

                    'LC_MEASUREMENT=de_DE.UTF-8',

                    'LC_MONETARY=de_DE.UTF-8',

                    'LC_NAME=de_DE.UTF-8',

                    'LC_NUMERIC=de_DE.UTF-8',

                    'LC_PAPER=de_DE.UTF-8',

                    'LC_TELEPHONE=de_DE.UTF-8',

                    'LC_TIME=de_DE.UTF-8',

                ]) {

                    sh script: """$activatevirtualenv

                        py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_de.xml --cov=kallithea

                        """, returnStatus: true

                }

                sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1DE./g" pytest_de.xml'

                archiveArtifacts 'pytest_de.xml'

                junit 'pytest_de.xml'

                writeFile(file: '.coverage.de', text: readFile('.coverage'))

                stash name: 'coverage.de', includes: '.coverage.de'

            }

        }

    }

}

pytests['mysql'] = {

    node {

        if (isUnix()) {

            ws {

                deleteDir()

                unstash name: 'kallithea'

                sh """$activatevirtualenv

                    pip install --upgrade MySQL-python

                    """

                withEnv(['TEST_DB=mysql://kallithea:kallithea@jenkins_mysql/kallithea_test?charset=utf8']) {

                    if (isUnix()) {

                        sh script: """$activatevirtualenv

                            py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_mysql.xml --cov=kallithea

                            """, returnStatus: true

                    } else {

                        bat script: """$activatevirtualenv

                            py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_mysql.xml --cov=kallithea

                            """, returnStatus: true

                    }

                }

                sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1MYSQL./g" pytest_mysql.xml'

                archiveArtifacts 'pytest_mysql.xml'

                junit 'pytest_mysql.xml'

                writeFile(file: '.coverage.mysql', text: readFile('.coverage'))

                stash name: 'coverage.mysql', includes: '.coverage.mysql'

            }

        }

    }

}

pytests['postgresql'] = {

    node {

        if (isUnix()) {

            ws {

                deleteDir()

                unstash name: 'kallithea'

                sh """$activatevirtualenv

                    pip install --upgrade psycopg2

                    """

                withEnv(['TEST_DB=postgresql://kallithea:kallithea@jenkins_postgresql/kallithea_test']) {

                    if (isUnix()) {

                        sh script: """$activatevirtualenv

                            py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_postgresql.xml --cov=kallithea

                            """, returnStatus: true

                    } else {

                        bat script: """$activatevirtualenv

                            py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_postgresql.xml --cov=kallithea

                            """, returnStatus: true

                    }

                }

                sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1POSTGRES./g" pytest_postgresql.xml'

                archiveArtifacts 'pytest_postgresql.xml'

                junit 'pytest_postgresql.xml'

                writeFile(file: '.coverage.postgresql', text: readFile('.coverage'))

                stash name: 'coverage.postgresql', includes: '.coverage.postgresql'

            }

        }

    }

}

stage('Tests') {

    parallel pytests

    node {

        unstash 'coverage.sqlite'

        unstash 'coverage.de'

        unstash 'coverage.mysql'

        unstash 'coverage.postgresql'

        if (isUnix()) {

            sh script: """$activatevirtualenv

                coverage combine .coverage.sqlite .coverage.de .coverage.mysql .coverage.postgresql

                coverage xml

                """, returnStatus: true

        } else {

            bat script: """$activatevirtualenv

                coverage combine .coverage.sqlite .coverage.de .coverage.mysql .coverage.postgresql

                coverage xml

                """, returnStatus: true

        }

        try {

            step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, zoomCoverageChart: false])

        } catch (java.lang.IllegalArgumentException exc) {

            echo "You need to install the pipeline compatible 'CoberturaPublisher Plug-in' to display the coverage report."

            currentBuild.result = 'UNSTABLE'

            echo "Caught: ${exc}"

        }

    }

}

.. _contributing:

=========================

Contributing to Kallithea

=========================

Kallithea is developed and maintained by its users. Please join us and scratch

your own itch.

Infrastructure

--------------

The main repository is hosted on Our Own Kallithea (aka OOK) at

https://kallithea-scm.org/repos/kallithea/, our self-hosted instance

of Kallithea.

For now, we use Bitbucket_ for `pull requests`_ and `issue tracking`_. The

issue tracker is for tracking bugs, not for support, discussion, or ideas --

please use the `mailing list`_ or :ref:`IRC <readme>` to reach the community.

We use Weblate_ to translate the user interface messages into languages other

than English. Join our project on `Hosted Weblate`_ to help us.

To register, you can use your Bitbucket or GitHub account. See :ref:`translations`

for more details.

Getting started

---------------

To get started with Kallithea development::

        hg clone https://kallithea-scm.org/repos/kallithea

        cd kallithea

        virtualenv ../kallithea-venv

        source ../kallithea-venv/bin/activate

        pip install --upgrade pip setuptools

        kallithea-cli config-create my.ini

        kallithea-cli db-create -c my.ini --user=user --email=user@example.com --password=password --repos=/tmp

        kallithea-cli front-end-build

        gearbox serve -c my.ini --reload &

        firefox http://127.0.0.1:5000/

If you plan to use Bitbucket_ for sending contributions, you can also fork

Kallithea on Bitbucket_ first (https://bitbucket.org/conservancy/kallithea) and

then replace the clone step above by a clone of your fork. In this case, please

see :ref:`contributing-guidelines` below for configuring your fork correctly.

Contribution flow

-----------------

Starting from an existing Kallithea clone, make sure it is up to date with the

latest upstream changes::

        hg pull

        hg update

Review the :ref:`contributing-guidelines` and :ref:`coding-guidelines`.

If you are new to Mercurial, refer to Mercurial `Quick Start`_ and `Beginners

Guide`_ on the Mercurial wiki.

Now, make some changes and test them (see :ref:`contributing-tests`). Don't

forget to add new tests to cover new functionality or bug fixes.

For documentation changes, run ``make html`` from the ``docs`` directory to

generate the HTML result, then review them in your browser.

Before submitting any changes, run the cleanup script::

        ./scripts/run-all-cleanup

When you are completely ready, you can send your changes to the community for

review and inclusion. Most commonly used methods are sending patches to the

mailing list (via ``hg email``) or by creating a pull request on Bitbucket_.

.. _contributing-tests:

Running tests

-------------

After finishing your changes make sure all tests pass cleanly. Run the testsuite

by invoking ``py.test`` from the project root::

    py.test

Note that on unix systems, the temporary directory (``/tmp`` or where

``$TMPDIR`` points) must allow executable files; Git hooks must be executable,

and the test suite creates repositories in the temporary directory. Linux

systems with /tmp mounted noexec will thus fail.

You can also use ``tox`` to run the tests with all supported Python versions

(currently only Python 2.7).

When running tests, Kallithea generates a `test.ini` based on template values

in `kallithea/tests/conftest.py` and populates the SQLite database specified

there.

It is possible to avoid recreating the full test database on each invocation of

the tests, thus eliminating the initial delay. To achieve this, run the tests as::

    gearbox serve -c /tmp/kallithea-test-XXX/test.ini --pid-file=test.pid --daemon

    KALLITHEA_WHOOSH_TEST_DISABLE=1 KALLITHEA_NO_TMP_PATH=1 py.test

    kill -9 $(cat test.pid)

In these commands, the following variables are used::

    KALLITHEA_WHOOSH_TEST_DISABLE=1 - skip whoosh index building and tests

    KALLITHEA_NO_TMP_PATH=1 - disable new temp path for tests, used mostly for testing_vcs_operations

You can run individual tests by specifying their path as argument to py.test.

py.test also has many more options, see `py.test -h`. Some useful options

are::

    -k EXPRESSION         only run tests which match the given substring

                          expression. An expression is a python evaluable

                          expression where all names are substring-matched

                          against test names and their parent classes. Example:

    -x, --exitfirst       exit instantly on first error or failed test.

    --lf                  rerun only the tests that failed at the last run (or

                          all if none failed)

    --ff                  run all tests but run the last failures first. This

                          may re-order tests and thus lead to repeated fixture

                          setup/teardown

    --pdb                 start the interactive Python debugger on errors.

    -s, --capture=no      don't capture stdout (any stdout output will be

                          printed immediately)

Performance tests

^^^^^^^^^^^^^^^^^

A number of performance tests are present in the test suite, but they are

not run in a standard test run. These tests are useful to

evaluate the impact of certain code changes with respect to performance.

To run these tests::

    env TEST_PERFORMANCE=1 py.test kallithea/tests/performance

To analyze performance, you could install pytest-profiling_, which enables the

--profile and --profile-svg options to py.test.

.. _pytest-profiling: https://github.com/manahl/pytest-plugins/tree/master/pytest-profiling

.. _contributing-guidelines:

Contribution guidelines

-----------------------

Kallithea is GPLv3 and we assume all contributions are made by the

committer/contributor and under GPLv3 unless explicitly stated. We do care a

lot about preservation of copyright and license information for existing code

that is brought into the project.

Contributions will be accepted in most formats -- such as pull requests on

Bitbucket, something hosted on your own Kallithea instance, or patches sent by

email to the `kallithea-general`_ mailing list.

When contributing via Bitbucket, please make your fork of

https://bitbucket.org/conservancy/kallithea/ `non-publishing`_ -- it is one of

the settings on "Repository details" page. This ensures your commits are in

"draft" phase and makes it easier for you to address feedback and for project

maintainers to integrate your changes.

.. _non-publishing: https://www.mercurial-scm.org/wiki/Phases#Publishing_Repository

Make sure to test your changes both manually and with the automatic tests

before posting.

We care about quality and review and keeping a clean repository history. We

might give feedback that requests polishing contributions until they are

"perfect". We might also rebase and collapse and make minor adjustments to your

changes when we apply them.

We try to make sure we have consensus on the direction the project is taking.

Everything non-sensitive should be discussed in public -- preferably on the

mailing list.  We aim at having all non-trivial changes reviewed by at least

one other core developer before pushing. Obvious non-controversial changes will

be handled more casually.

There is a main development branch ("default") which is generally stable so that

it can be (and is) used in production. There is also a "stable" branch that is

almost exclusively reserved for bug fixes or trivial changes. Experimental

changes should live elsewhere (for example in a pull request) until they are

ready.

.. _coding-guidelines:

Coding guidelines

-----------------

We don't have a formal coding/formatting standard. We are currently using a mix

of Mercurial's (https://www.mercurial-scm.org/wiki/CodingStyle), pep8, and

consistency with existing code. Run ``scripts/run-all-cleanup`` before

committing to ensure some basic code formatting consistency.

We currently only support Python 2.7.x and nothing else. For now we don't care

about Python 3 compatibility.

We try to support the most common modern web browsers. IE9 is still supported

to the extent it is feasible, IE8 is not.

We primarily support Linux and OS X on the server side but Windows should also work.

HTML templates should use 2 spaces for indentation ... but be pragmatic. We

should use templates cleverly and avoid duplication. We should use reasonable

semantic markup with element classes and IDs that can be used for styling and testing.

We should only use inline styles in places where it really is semantic (such as

``display: none``).

JavaScript must use ``;`` between/after statements. Indentation 4 spaces. Inline

multiline functions should be indented two levels -- one for the ``()`` and one for

``{}``.

Variables holding jQuery objects should be named with a leading ``$``.

Commit messages should have a leading short line summarizing the changes. For

bug fixes, put ``(Issue #123)`` at the end of this line.

Use American English grammar and spelling overall. Use `English title case`_ for

page titles, button labels, headers, and 'labels' for fields in forms.

.. _English title case: https://en.wikipedia.org/wiki/Capitalization#Title_case

Template helpers (that is, everything in ``kallithea.lib.helpers``)

should only be referenced from templates. If you need to call a

#!/bin/bash

set -e

set -x

cleanup()

{

  echo "Removing venv $venv"

  rm  -rf "$venv"

}

echo "Checking that you are NOT inside a virtualenv"

[ -z "$VIRTUAL_ENV" ]

venv=$(mktemp -d --tmpdir kallithea-release-XXXXX)

trap cleanup EXIT

echo "Setting up a fresh virtualenv in $venv"

virtualenv -p python2 "$venv"

. "$venv/bin/activate"

echo "Install/verify tools needed for building and uploading stuff"

echo "Cleanup and update copyrights ... and clean checkout"

scripts/run-all-cleanup

scripts/update-copyrights.py

hg up -cr .

echo "Make release build from clean checkout in build/"

rm -rf build dist

hg archive build

cd build

echo "Check that each entry in MANIFEST.in match something"

sed -e 's/[^ ]*[ ]*\([^ ]*\).*/\1/g' MANIFEST.in | xargs ls -lad

echo "Build dist"

python2 setup.py compile_catalog

python2 setup.py sdist

echo "Verify VERSION from kallithea/__init__.py"

namerel=$(cd dist && echo Kallithea-*.tar.gz)

namerel=${namerel%.tar.gz}

version=${namerel#Kallithea-}

ls -l $(pwd)/dist/$namerel.tar.gz

echo "Releasing Kallithea $version in directory $namerel"

echo "Verify dist file content"

diff -u <((hg mani | grep -v '^\.hg') | LANG=C sort) <(tar tf dist/Kallithea-$version.tar.gz | sed "s|^$namerel/||" | grep . | grep -v '^kallithea/i18n/.*/LC_MESSAGES/kallithea.mo$\|^Kallithea.egg-info/\|^PKG-INFO$\|/$' | LANG=C sort)

echo "Verify docs build"

python2 setup.py build_sphinx # the results are not actually used, but we want to make sure it builds

echo "Shortlog for inclusion in the release announcement"

scripts/shortlog.py "only('.', branch('stable') & tagged() & public() & not '.')"

cat - << EOT

Now, make sure

* all tests are passing

* release note is ready

* announcement is ready

* source has been pushed to https://kallithea-scm.org/repos/kallithea

EOT

echo "Verify current revision is tagged for $version"

hg log -r "'$version'&." | grep .

echo -n "Enter \"pypi\" to upload Kallithea $version to pypi: "

read answer

[ "$answer" = "pypi" ]

echo "Rebuild readthedocs for docs.kallithea-scm.org"

xdg-open https://readthedocs.org/projects/kallithea/

curl -X POST http://readthedocs.org/build/kallithea

xdg-open https://readthedocs.org/builds/kallithea/

xdg-open http://docs.kallithea-scm.org/en/latest/ # or whatever the branch is

twine upload dist/*

xdg-open https://pypi.python.org/pypi/Kallithea

#!/usr/bin/env bash

# Validate the specified commits against test suite and other checks.

if [ -n "$VIRTUAL_ENV" ]; then

    echo "Please run this script from outside a virtualenv."

    exit 1

fi

if ! hg update --check -q .; then

    echo "Working dir is not clean, please commit/revert changes first."

    exit 1

fi

venv=$(mktemp -d kallithea-validatecommits-env-XXXXXX)

resultfile=$(mktemp kallithea-validatecommits-result-XXXXXX)

echo > "$resultfile"

cleanup()

{

    rm -rf /tmp/kallithea-test*

    rm -rf "$venv"

}

finish()

{

    cleanup

    # print (possibly intermediate) results

    cat "$resultfile"

    rm "$resultfile"

}

trap finish EXIT

for rev in $(hg log -r "$1" -T '{node}\n'); do

    hg log -r "$rev"

    hg update "$rev"

    cleanup

    virtualenv -p "$(command -v python2)" "$venv"

    source "$venv/bin/activate"

    pip install --upgrade pip setuptools

    # run-all-cleanup

    scripts/run-all-cleanup

    if ! hg update --check -q .; then

        echo "run-all-cleanup did not give clean results!"

        result="NOK"

        hg diff

        hg revert -a

    else

        result=" OK"

    fi

    echo "$result: $rev (run-all-cleanup)" >> "$resultfile"

    # pytest

    if py.test; then

        result=" OK"

    else

        result="NOK"

    fi

    echo "$result: $rev (pytest)" >> "$resultfile"

    deactivate

    echo

done