[pkg] unify client and server into a single python package

We have been discussing about this merge for a while.
Its main goal is to simplify things: code navigation, but also
packaging.

The rationale is that the code is more cohesive in this way, and there's
only one source package to install.

Dependencies that are only for the server or the client will not be
installed by default, and they are expected to be provided by the
environment. There are setuptools extras defined for the client and the
server.

Debianization is still expected to split the single source package into
3 binaries.

Another avantage is that the documentation can now install a single
package with a single step, and therefore include the docstrings into
the generated docs.

- Resolves: #8896

9 files changed, 314 insertions, 0 deletions

diff --git a/pkg/server/create-user-db b/pkg/server/create-user-db
new file mode 100755
index 00000000..5e0ef5e2
--- /dev/null
+++ b/pkg/server/create-user-db
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# create-user-db
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import os
+import sys
+import netrc
+import argparse
+from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch.state import is_db_name_valid
+from leap.soledad.common.couch import list_users_dbs
+from leap.soledad.server._config import get_config
+
+
+BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False)
+
+
+description = """
+Creates a user database.
+This is meant to be used by Soledad Server.
+"""
+parser = argparse.ArgumentParser(description=description)
+parser.add_argument('dbname', metavar='user-d34db33f', type=str,
+                    default='', nargs='?',
+                    help='database name on the format user-{uuid4}')
+parser.add_argument('--migrate-all', action='store_true',
+                    help="recreate all design docs for all existing account")
+CONF = get_config()
+DBCONF = get_config(section='database-security')
+NETRC_PATH = CONF['admin_netrc']
+
+
+def url_for_db(dbname):
+    if BYPASS_AUTH:
+        login = ''
+        password = ''
+        host = 'localhost'
+        url = 'http://localhost:5984/%(dbname)s' % {
+            'dbname': dbname}
+    else:
+        if not os.path.exists(NETRC_PATH):
+            print ('netrc not found in %s' % NETRC_PATH)
+            sys.exit(1)
+        parsed_netrc = netrc.netrc(NETRC_PATH)
+        host, (login, _, password) = parsed_netrc.hosts.items()[0]
+        url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % {
+               'login': login,
+               'password': password,
+               'host': host,
+               'dbname': dbname})
+    return url
+
+
+def ensure_database(dbname):
+    """
+    This method will ensure that a database named `dbname` will exist
+    or created if it doesn't. Calling it twice will ensure that design
+    documents are present and updated.
+    The database name has to match this criteria to be considered valid:
+    user-[a-f0-9]+
+
+    :param dbname: name of the user database
+    :type dbname: str
+    """
+    if not is_db_name_valid(dbname):
+        print ("Invalid name! %s" % dbname)
+        sys.exit(1)
+    url = url_for_db(dbname)
+    db_security = DBCONF
+    db = CouchDatabase.open_database(url=url, create=True,
+                                     replica_uid=None,
+                                     database_security=db_security)
+    print ('success! Ensured that database %s exists, with replica_uid: %s' %
+           (db._dbname, db.replica_uid))
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if args.migrate_all:
+        couch_url = url_for_db('')
+        for dbname in list_users_dbs(couch_url):
+            ensure_database(dbname)
+    else:
+        ensure_database(args.dbname)
diff --git a/pkg/server/generate_wheels.sh b/pkg/server/generate_wheels.sh
new file mode 100755
index 00000000..a13e2c7a
--- /dev/null
+++ b/pkg/server/generate_wheels.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Generate wheels for dependencies
+# Use at your own risk.
+
+if [ "$WHEELHOUSE" = "" ]; then
+    WHEELHOUSE=$HOME/wheelhouse
+fi
+
+pip wheel --wheel-dir $WHEELHOUSE pip
+pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip
+if [ -f pkg/requirements-testing.pip ]; then
+   pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip
+fi
diff --git a/pkg/server/pip_install_requirements.sh b/pkg/server/pip_install_requirements.sh
new file mode 100755
index 00000000..f4b5f67a
--- /dev/null
+++ b/pkg/server/pip_install_requirements.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+# Update pip and install LEAP base/testing requirements.
+# For convenience, $insecure_packages are allowed with insecure flags enabled.
+# Use at your own risk.
+# See $usage for help
+
+insecure_packages=""
+leap_wheelhouse=https://lizard.leap.se/wheels
+
+show_help() {
+    usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n
+\t\t\tOtherwise, it will install requirements.pip\n
+--use-leap-wheels\tUse wheels from leap.se"
+    echo -e $usage
+
+    exit 1
+}
+
+process_arguments() {
+    testing=false
+    while [ "$#" -gt 0 ]; do
+	# From http://stackoverflow.com/a/31443098
+	case "$1" in
+	    --help) show_help;;
+	    --testing) testing=true; shift 1;;
+	    --use-leap-wheels) use_leap_wheels=true; shift 1;;
+
+	    -h) show_help;;
+	    -*) echo "unknown option: $1" >&2; exit 1;;
+	esac
+    done
+}
+
+return_wheelhouse() {
+    if $use_leap_wheels ; then
+	WHEELHOUSE=$leap_wheelhouse
+    elif [ "$WHEELHOUSE" = "" ]; then
+	WHEELHOUSE=$HOME/wheelhouse
+    fi
+
+    # Tested with bash and zsh
+    if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then
+	    mkdir $WHEELHOUSE
+    fi
+
+    echo "$WHEELHOUSE"
+}
+
+return_install_options() {
+    wheelhouse=`return_wheelhouse`
+    install_options="-U --find-links=$wheelhouse"
+    if $use_leap_wheels ; then
+	install_options="$install_options --trusted-host lizard.leap.se"
+    fi
+
+    echo $install_options
+}
+
+return_insecure_flags() {
+    for insecure_package in $insecure_packages; do
+	flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package"
+    done
+
+    echo $flags
+}
+
+return_packages() {
+    if $testing ; then
+	packages="-r pkg/requirements-testing.pip"
+    else
+	packages="-r pkg/requirements.pip"
+    fi
+
+    echo $packages
+}
+
+process_arguments $@
+install_options=`return_install_options`
+insecure_flags=`return_insecure_flags`
+packages=`return_packages`
+
+pip install -U wheel
+pip install -U pip
+pip install $install_options $insecure_flags $packages
diff --git a/pkg/server/requirements-latest.pip b/pkg/server/requirements-latest.pip
new file mode 100644
index 00000000..d32e1ffa
--- /dev/null
+++ b/pkg/server/requirements-latest.pip
@@ -0,0 +1,5 @@
+--index-url https://pypi.python.org/simple/
+
+-e 'git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common'
+-e '../common'
+-e .
diff --git a/pkg/server/requirements-leap.pip b/pkg/server/requirements-leap.pip
new file mode 100644
index 00000000..93b447e5
--- /dev/null
+++ b/pkg/server/requirements-leap.pip
@@ -0,0 +1 @@
+leap.soledad.common>=0.9.0
diff --git a/pkg/server/requirements.pip b/pkg/server/requirements.pip
new file mode 100644
index 00000000..8354e94c
--- /dev/null
+++ b/pkg/server/requirements.pip
@@ -0,0 +1,8 @@
+configparser
+PyOpenSSL
+twisted>=12.3.0
+Beaker
+couchdb
+# Upstream needs a patch for py3 compatibility
+# Eventually falling back to standard json
+# python-cjson
diff --git a/pkg/server/soledad-server b/pkg/server/soledad-server
new file mode 100644
index 00000000..450f2277
--- /dev/null
+++ b/pkg/server/soledad-server
@@ -0,0 +1,73 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          soledad
+# Required-Start:    $network $named $remote_fs $syslog $time
+# Required-Stop:     $network $named $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start soledad daemon at boot time
+# Description:       Synchronization of locally encrypted data among devices
+### END INIT INFO
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+PIDFILE=/var/run/soledad.pid
+RESOURCE_CLASS=leap.soledad.server.entrypoint.SoledadEntrypoint
+HTTPS_PORT=2424
+CONFDIR=/etc/soledad
+CERT_PATH="${CONFDIR}/soledad-server.pem"
+PRIVKEY_PATH="${CONFDIR}/soledad-server.key"
+TWISTD_PATH=/usr/bin/twistd
+HOME=/var/lib/soledad/
+SSL_METHOD=SSLv23_METHOD
+USER=soledad
+GROUP=soledad
+
+[ -r /etc/default/soledad ] && . /etc/default/soledad
+
+test -r ${CONFDIR} || exit 0
+
+. /lib/lsb/init-functions
+
+
+case "${1}" in
+    start)
+        echo -n "Starting soledad: twistd"
+          start-stop-daemon --start --quiet \
+            --exec ${TWISTD_PATH} -- \
+            --uid=${USER} --gid=${GROUP} \
+            --pidfile=${PIDFILE} \
+	    --syslog \
+	    --prefix=soledad-server \
+            web \
+            --class=${RESOURCE_CLASS} \
+            --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD}
+        echo "."
+    ;;
+
+    stop)
+        echo -n "Stopping soledad: twistd"
+        start-stop-daemon --stop --quiet  \
+            --pidfile ${PIDFILE}
+        echo "."
+    ;;
+
+    restart)
+        ${0} stop
+        ${0} start
+    ;;
+
+    force-reload)
+        ${0} restart
+    ;;
+
+    status)
+        status_of_proc -p ${PIDFILE} ${TWISTD_PATH} soledad && exit 0 || exit ${?}
+    ;;
+
+    *)
+        echo "Usage: /etc/init.d/soledad {start|stop|restart|force-reload|status}" >&2
+        exit 1
+    ;;
+esac
+
+exit 0
diff --git a/pkg/server/soledad-server.service b/pkg/server/soledad-server.service
new file mode 100644
index 00000000..30c4bf88
--- /dev/null
+++ b/pkg/server/soledad-server.service
@@ -0,0 +1,31 @@
+[Unit]
+Description=Soledad Server
+
+[Service]
+Environment=PATH=/sbin:/bin:/usr/sbin:/usr/bin
+Environment=CLASS=leap.soledad.server.entrypoint.SoledadEntrypoint
+Environment=HTTPS_PORT=2424
+Environment=CERT_PATH=/etc/soledad/soledad-server.pem
+Environment=PRIVKEY_PATH=/etc/soledad/soledad-server.key
+Environment=HOME=/var/lib/soledad/
+Environment=SSL_METHOD=SSLv23_METHOD
+EnvironmentFile=-/etc/default/soledad
+
+ExecStart=/usr/bin/twistd \
+  --nodaemon \
+  --pidfile= \
+  --syslog \
+  --prefix=soledad-server \
+  web \
+  --class=${CLASS} \
+  --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD}
+
+WorkingDirectory=/var/lib/soledad/
+
+User=soledad
+Group=soledad
+
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/pkg/server/soledad-sudoers b/pkg/server/soledad-sudoers
new file mode 100644
index 00000000..642497f8
--- /dev/null
+++ b/pkg/server/soledad-sudoers
@@ -0,0 +1,2 @@
+Cmnd_Alias SOLEDAD_CREATE_DB = /usr/bin/create-user-db
+soledad ALL=(soledad-admin) NOPASSWD: SOLEDAD_CREATE_DB