[pkg] unify client and server into a single python package

We have been discussing about this merge for a while.
Its main goal is to simplify things: code navigation, but also
packaging.

The rationale is that the code is more cohesive in this way, and there's
only one source package to install.

Dependencies that are only for the server or the client will not be
installed by default, and they are expected to be provided by the
environment. There are setuptools extras defined for the client and the
server.

Debianization is still expected to split the single source package into
3 binaries.

Another avantage is that the documentation can now install a single
package with a single step, and therefore include the docstrings into
the generated docs.

- Resolves: #8896

8 files changed, 227 insertions, 0 deletions

diff --git a/pkg/common/generate_wheels.sh b/pkg/common/generate_wheels.sh
new file mode 100755
index 00000000..a13e2c7a
--- /dev/null
+++ b/pkg/common/generate_wheels.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Generate wheels for dependencies
+# Use at your own risk.
+
+if [ "$WHEELHOUSE" = "" ]; then
+    WHEELHOUSE=$HOME/wheelhouse
+fi
+
+pip wheel --wheel-dir $WHEELHOUSE pip
+pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip
+if [ -f pkg/requirements-testing.pip ]; then
+   pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip
+fi
diff --git a/pkg/common/pip_install_requirements.sh b/pkg/common/pip_install_requirements.sh
new file mode 100755
index 00000000..f4b5f67a
--- /dev/null
+++ b/pkg/common/pip_install_requirements.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+# Update pip and install LEAP base/testing requirements.
+# For convenience, $insecure_packages are allowed with insecure flags enabled.
+# Use at your own risk.
+# See $usage for help
+
+insecure_packages=""
+leap_wheelhouse=https://lizard.leap.se/wheels
+
+show_help() {
+    usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n
+\t\t\tOtherwise, it will install requirements.pip\n
+--use-leap-wheels\tUse wheels from leap.se"
+    echo -e $usage
+
+    exit 1
+}
+
+process_arguments() {
+    testing=false
+    while [ "$#" -gt 0 ]; do
+	# From http://stackoverflow.com/a/31443098
+	case "$1" in
+	    --help) show_help;;
+	    --testing) testing=true; shift 1;;
+	    --use-leap-wheels) use_leap_wheels=true; shift 1;;
+
+	    -h) show_help;;
+	    -*) echo "unknown option: $1" >&2; exit 1;;
+	esac
+    done
+}
+
+return_wheelhouse() {
+    if $use_leap_wheels ; then
+	WHEELHOUSE=$leap_wheelhouse
+    elif [ "$WHEELHOUSE" = "" ]; then
+	WHEELHOUSE=$HOME/wheelhouse
+    fi
+
+    # Tested with bash and zsh
+    if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then
+	    mkdir $WHEELHOUSE
+    fi
+
+    echo "$WHEELHOUSE"
+}
+
+return_install_options() {
+    wheelhouse=`return_wheelhouse`
+    install_options="-U --find-links=$wheelhouse"
+    if $use_leap_wheels ; then
+	install_options="$install_options --trusted-host lizard.leap.se"
+    fi
+
+    echo $install_options
+}
+
+return_insecure_flags() {
+    for insecure_package in $insecure_packages; do
+	flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package"
+    done
+
+    echo $flags
+}
+
+return_packages() {
+    if $testing ; then
+	packages="-r pkg/requirements-testing.pip"
+    else
+	packages="-r pkg/requirements.pip"
+    fi
+
+    echo $packages
+}
+
+process_arguments $@
+install_options=`return_install_options`
+insecure_flags=`return_insecure_flags`
+packages=`return_packages`
+
+pip install -U wheel
+pip install -U pip
+pip install $install_options $insecure_flags $packages
diff --git a/pkg/common/requirements-latest.pip b/pkg/common/requirements-latest.pip
new file mode 100644
index 00000000..852f2433
--- /dev/null
+++ b/pkg/common/requirements-latest.pip
@@ -0,0 +1,4 @@
+--index-url https://pypi.python.org/simple/
+
+-e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common'
+-e .
diff --git a/pkg/common/requirements-leap.pip b/pkg/common/requirements-leap.pip
new file mode 100644
index 00000000..b311859e
--- /dev/null
+++ b/pkg/common/requirements-leap.pip
@@ -0,0 +1 @@
+leap.common>=0.4.0
diff --git a/pkg/common/requirements.pip b/pkg/common/requirements.pip
new file mode 100644
index 00000000..29506cf0
--- /dev/null
+++ b/pkg/common/requirements.pip
@@ -0,0 +1,3 @@
+paste
+routes
+six
\ No newline at end of file
diff --git a/pkg/common/tools/get_authors.sh b/pkg/common/tools/get_authors.sh
new file mode 100755
index 00000000..0169bb17
--- /dev/null
+++ b/pkg/common/tools/get_authors.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2-
diff --git a/pkg/common/tools/with_venvwrapper.sh b/pkg/common/tools/with_venvwrapper.sh
new file mode 100755
index 00000000..b62bc10f
--- /dev/null
+++ b/pkg/common/tools/with_venvwrapper.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#Wraps a command in a virtualenwrapper passed as first argument.
+#Example:
+#with_virtualenvwrapper.sh leap-bitmask ./run_tests.sh
+
+wd=`pwd`
+alias pyver='python -c "import $1;print $1.__path__[0]; print $1.__version__;"'
+
+source `which virtualenvwrapper.sh`
+echo "Activating virtualenv " $1
+echo "------------------------------------"
+workon $1
+cd $wd
+echo "running version: "
+echo `pyver leap.bitmask`
+echo `pyver leap.soledad.common`
+echo `pyver leap.keymanager`
+$2 $3 $4 $5
diff --git a/pkg/common/utils.py b/pkg/common/utils.py
new file mode 100644
index 00000000..d1680102
--- /dev/null
+++ b/pkg/common/utils.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# utils.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Utils to help in the setup process
+"""
+import os
+import re
+import sys
+
+
+def is_develop_mode():
+    """
+    Returns True if we're calling the setup script using the argument for
+    setuptools development mode.
+
+    This avoids messing up with dependency pinning and order, the
+    responsibility of installing the leap dependencies is left to the
+    developer.
+    """
+    args = sys.argv
+    devflags = "setup.py", "develop"
+    if (args[0], args[1]) == devflags:
+        return True
+    return False
+
+
+def get_reqs_from_files(reqfiles):
+    """
+    Returns the contents of the top requirement file listed as a
+    string list with the lines.
+
+    @param reqfiles: requirement files to parse
+    @type reqfiles: list of str
+    """
+    for reqfile in reqfiles:
+        if os.path.isfile(reqfile):
+            return open(reqfile, 'r').read().split('\n')
+
+
+def parse_requirements(reqfiles=['requirements.txt',
+                                 'requirements.pip',
+                                 'pkg/requirements.pip']):
+    """
+    Parses the requirement files provided.
+
+    The passed reqfiles list is a list of possible locations to try, the
+    function will return the contents of the first path found.
+
+    Checks the value of LEAP_VENV_SKIP_PYSIDE to see if it should
+    return PySide as a dep or not. Don't set, or set to 0 if you want
+    to install it through pip.
+
+    @param reqfiles: requirement files to parse
+    @type reqfiles: list of str
+    """
+
+    requirements = []
+    skip_pyside = os.getenv("LEAP_VENV_SKIP_PYSIDE", "0") != "0"
+    for line in get_reqs_from_files(reqfiles):
+        # -e git://foo.bar/baz/master#egg=foobar
+        if re.match(r'\s*-e\s+', line):
+            pass
+            # do not try to do anything with externals on vcs
+            # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
+            # line))
+            # http://foo.bar/baz/foobar/zipball/master#egg=foobar
+        elif re.match(r'\s*https?:', line):
+            requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
+                                line))
+        # -f lines are for index locations, and don't get used here
+        elif re.match(r'\s*-f\s+', line):
+            pass
+
+        # argparse is part of the standard library starting with 2.7
+        # adding it to the requirements list screws distro installs
+        elif line == 'argparse' and sys.version_info >= (2, 7):
+            pass
+        elif line == 'PySide' and skip_pyside:
+            pass
+        # do not include comments
+        elif line.lstrip().startswith('#'):
+            pass
+        else:
+            if line != '':
+                requirements.append(line)
+
+    return requirements