From 7eb87d94b3bd48242e5a456c0051c810e6db8095 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 5 May 2015 12:08:05 -0300 Subject: [refactor] move initscript to more sensible name --- pkg/leap-mx.init | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pkg/leap_mx.init | 66 -------------------------------------------------------- 2 files changed, 66 insertions(+), 66 deletions(-) create mode 100644 pkg/leap-mx.init delete mode 100644 pkg/leap_mx.init (limited to 'pkg') diff --git a/pkg/leap-mx.init b/pkg/leap-mx.init new file mode 100644 index 0000000..60dddc4 --- /dev/null +++ b/pkg/leap-mx.init @@ -0,0 +1,66 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: leap-mx +# Required-Start: $network $named $remote_fs $syslog $time +# Required-Stop: $network $named $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start leap-mx daemon at boot time +# Description: transparently-encrypting remailer +### END INIT INFO + +PATH=/sbin:/bin:/usr/sbin:/usr/bin +PIDFILE=/var/run/leap_mx.pid +RUNDIR=/var/lib/leap_mx/ +FILE=/usr/share/app/leap_mx.tac +LOGFILE=/var/log/leap_mx.log +TWISTD_PATH=/usr/bin/twistd + +[ -r /etc/default/leap_mx ] && . /etc/default/leap_mx + +. /lib/lsb/init-functions + +test -r $file || exit 0 +test -r /etc/leap/ || exit 0 + + +case "$1" in + start) + echo -n "Starting leap_mx: twistd" + HOME="/var/lib/leap_mx/" \ + start-stop-daemon --start --quiet --exec $TWISTD_PATH -- \ + --pidfile=$PIDFILE \ + --rundir=$RUNDIR \ + --python=$FILE \ + --logfile=$LOGFILE \ + --syslog --prefix=leap-mx + echo "." + ;; + + stop) + echo -n "Stopping leap_mx: twistd" + start-stop-daemon --stop --quiet \ + --pidfile $PIDFILE + echo "." + ;; + + restart) + $0 stop + $0 start + ;; + + force-reload) + $0 restart + ;; + + status) + status_of_proc -p $PIDFILE $TWISTD_PATH leap_mx && exit 0 || exit $? + ;; + + *) + echo "Usage: /etc/init.d/leap_mx {start|stop|restart|force-reload|status}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/pkg/leap_mx.init b/pkg/leap_mx.init deleted file mode 100644 index 60dddc4..0000000 --- a/pkg/leap_mx.init +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: leap-mx -# Required-Start: $network $named $remote_fs $syslog $time -# Required-Stop: $network $named $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start leap-mx daemon at boot time -# Description: transparently-encrypting remailer -### END INIT INFO - -PATH=/sbin:/bin:/usr/sbin:/usr/bin -PIDFILE=/var/run/leap_mx.pid -RUNDIR=/var/lib/leap_mx/ -FILE=/usr/share/app/leap_mx.tac -LOGFILE=/var/log/leap_mx.log -TWISTD_PATH=/usr/bin/twistd - -[ -r /etc/default/leap_mx ] && . /etc/default/leap_mx - -. /lib/lsb/init-functions - -test -r $file || exit 0 -test -r /etc/leap/ || exit 0 - - -case "$1" in - start) - echo -n "Starting leap_mx: twistd" - HOME="/var/lib/leap_mx/" \ - start-stop-daemon --start --quiet --exec $TWISTD_PATH -- \ - --pidfile=$PIDFILE \ - --rundir=$RUNDIR \ - --python=$FILE \ - --logfile=$LOGFILE \ - --syslog --prefix=leap-mx - echo "." - ;; - - stop) - echo -n "Stopping leap_mx: twistd" - start-stop-daemon --stop --quiet \ - --pidfile $PIDFILE - echo "." - ;; - - restart) - $0 stop - $0 start - ;; - - force-reload) - $0 restart - ;; - - status) - status_of_proc -p $PIDFILE $TWISTD_PATH leap_mx && exit 0 || exit $? - ;; - - *) - echo "Usage: /etc/init.d/leap_mx {start|stop|restart|force-reload|status}" >&2 - exit 1 - ;; -esac - -exit 0 -- cgit v1.2.3 From f3261e937390fa703392d1234d47c00647285d42 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 8 May 2015 16:38:54 -0300 Subject: [bug] remove --syslog twistd option Currently, we don't want to log to syslog, because the platform has to be modified before we can do that (see #6942). In the future, we might want to log to syslog only, or make it optional depending on the contents of /etc/default/leap_mx. For now we just remove twistd's --syslog option from the initscript. Closes: #6937. --- pkg/leap-mx.init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pkg') diff --git a/pkg/leap-mx.init b/pkg/leap-mx.init index 60dddc4..d38cc2c 100644 --- a/pkg/leap-mx.init +++ b/pkg/leap-mx.init @@ -33,7 +33,7 @@ case "$1" in --rundir=$RUNDIR \ --python=$FILE \ --logfile=$LOGFILE \ - --syslog --prefix=leap-mx + --prefix=leap-mx echo "." ;; -- cgit v1.2.3 From 3215a470f701962fadfd26a996e8eef6f21444b5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jul 2015 16:34:45 -0400 Subject: [pkg] separate leap requirements this is part of a process to make the setup of the development mode less troublesome. from now on, setting up a virtualenv in pure development mode will be as easy as telling pip to just install the external dependencies:: pip install -r pkg/requirements.pip and traversing all the leap repos for the needed leap dependencies doing:: python setup.py develop - Related: #7288 --- pkg/requirements-leap.pip | 3 +++ pkg/requirements.pip | 4 ---- pkg/utils/reqs.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 pkg/requirements-leap.pip (limited to 'pkg') diff --git a/pkg/requirements-leap.pip b/pkg/requirements-leap.pip new file mode 100644 index 0000000..482d1e2 --- /dev/null +++ b/pkg/requirements-leap.pip @@ -0,0 +1,3 @@ +leap.common>=0.3.5 +leap.soledad.common>=0.4.5 +leap.keymanager>=0.3.4 diff --git a/pkg/requirements.pip b/pkg/requirements.pip index ed3ad0d..328b1c3 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -7,7 +7,3 @@ paisley>=0.3.1 # in soledad-common, but we need to declare here # for the time being. couchdb - -leap.common>=0.3.5 -leap.soledad.common>=0.4.5 -leap.keymanager>=0.3.4 diff --git a/pkg/utils/reqs.py b/pkg/utils/reqs.py index 5e2324f..f10051e 100644 --- a/pkg/utils/reqs.py +++ b/pkg/utils/reqs.py @@ -22,6 +22,22 @@ import re import sys +def is_develop_mode(): + """ + Returns True if we're calling the setup script using the argument for + setuptools development mode. + + This avoids messing up with dependency pinning and order, the + responsibility of installing the leap dependencies is left to the + developer. + """ + args = sys.argv + devflags = "setup.py", "develop" + if (args[0], args[1]) == devflags: + return True + return False + + def get_reqs_from_files(reqfiles): """ Returns the contents of the top requirement file listed as a -- cgit v1.2.3 From b409ba866c57a17a5a702f69db0b44a48ee1e584 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jul 2015 22:26:41 -0400 Subject: [pkg] add script to install base requirements - update pip - install base reqs --- pkg/pip_install_requirements.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 pkg/pip_install_requirements.sh (limited to 'pkg') diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh new file mode 100755 index 0000000..29f03f3 --- /dev/null +++ b/pkg/pip_install_requirements.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Update pip and install LEAP base requirements. +pip install -U pip +pip install -r pkg/requirements.pip -- cgit v1.2.3 From 469af72f5ab20c4a8aa120daaf995b68aa97a6a5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jul 2015 22:28:59 -0400 Subject: [pkg] add AUTHORS file + one-liner to generate it --- pkg/utils/get_authors.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 pkg/utils/get_authors.sh (limited to 'pkg') diff --git a/pkg/utils/get_authors.sh b/pkg/utils/get_authors.sh new file mode 100755 index 0000000..0169bb1 --- /dev/null +++ b/pkg/utils/get_authors.sh @@ -0,0 +1,2 @@ +#!/bin/sh +git log --format='%aN <%aE>' | awk '{arr[$0]++} END{for (i in arr){print arr[i], i;}}' | sort -rn | cut -d' ' -f2- -- cgit v1.2.3 From f63284805e0c489400dfa4956f64f80bd1c9d6b4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jul 2015 10:00:21 -0400 Subject: [tests] add pep8 to requirements-testing --- pkg/requirements-testing.pip | 1 + 1 file changed, 1 insertion(+) create mode 100644 pkg/requirements-testing.pip (limited to 'pkg') diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip new file mode 100644 index 0000000..94ab6e8 --- /dev/null +++ b/pkg/requirements-testing.pip @@ -0,0 +1 @@ +pep8 -- cgit v1.2.3 From 6187e0792de6925d22b3956eb3ba8f46646960d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Parm=C3=A9nides=20GV?= Date: Mon, 3 Aug 2015 22:46:30 +0200 Subject: [feat] use wheels to install dependencies generate_wheels uses $WHEELHOUSE to generate and store the wheels for requirements.pip and requirements-testing.pip (if it exists). pip_install_requirements.sh installs requirements.pip from them if possible (if not, then it fetches them from pypi) or, if passed the --testing flag, it installs requirements-testing.pip. - Related: 7327 --- pkg/generate_wheels.sh | 12 +++++++ pkg/pip_install_requirements.sh | 71 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100755 pkg/generate_wheels.sh (limited to 'pkg') diff --git a/pkg/generate_wheels.sh b/pkg/generate_wheels.sh new file mode 100755 index 0000000..e8096af --- /dev/null +++ b/pkg/generate_wheels.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Generate wheels for dependencies + +if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse +fi + +pip wheel --wheel-dir $WHEELHOUSE pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip +if [ -f pkg/requirements-testing.pip ]; then + pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip +fi diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh index 29f03f3..bd44457 100755 --- a/pkg/pip_install_requirements.sh +++ b/pkg/pip_install_requirements.sh @@ -1,4 +1,69 @@ #!/bin/sh -# Update pip and install LEAP base requirements. -pip install -U pip -pip install -r pkg/requirements.pip +# Update pip and install LEAP base/testing requirements. +# For convenience, $insecure_packages are allowed with insecure flags enabled. +# Use at your own risk. +# See $usage for help + +insecure_packages="" + +return_wheelhouse() { + if [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + if [ ! -d "$WHEELHOUSE" ]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +show_help() { + usage="Usage: $0 [--testing]\n --testing\tInstall dependencies from requirements-testing.pip\n +\t\tOtherwise, it will install requirements.pip" + echo $usage + + exit 1 +} + +process_arguments() { + testing=false + while [ "$#" -gt 0 ]; do + # From http://stackoverflow.com/a/31443098 + case "$1" in + --help) show_help;; + --testing) testing=true; shift 1;; + + -h) show_help;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac + done +} + +return_insecure_flags() { + for insecure_package in $insecure_packages; do + flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" + done + + echo $flags +} + +return_packages() { + if $testing ; then + packages="-r pkg/requirements-testing.pip" + else + packages="-r pkg/requirements.pip" + fi + + echo $packages +} + +process_arguments $@ +wheelhouse=`return_wheelhouse` +install_options="-U --find-links=$wheelhouse" +insecure_flags=`return_insecure_flags` +packages=`return_packages` + +pip install -U wheel +pip install $install_options pip +pip install $install_options $insecure_flags $packages -- cgit v1.2.3 From b988fcb56a2548e114f6bbcb3bb43bdfb5f61fdc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 4 Aug 2015 16:02:20 -0400 Subject: [bug] bind TCPServer to localhost currently, mx services are being bound to all the interfaces. - Resolves: #7348 --- pkg/mx.tac | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'pkg') diff --git a/pkg/mx.tac b/pkg/mx.tac index 7da59cf..e85dea5 100755 --- a/pkg/mx.tac +++ b/pkg/mx.tac @@ -68,12 +68,15 @@ cdb = couchdbhelper.ConnectedCouchDB(server, application = service.Application("LEAP MX") # Alias map -alias_map = internet.TCPServer(alias_port, AliasResolverFactory(couchdb=cdb)) +alias_map = internet.TCPServer( + alias_port, AliasResolverFactory(couchdb=cdb), + interface="localhost") alias_map.setServiceParent(application) # Check recipient access -check_recipient = internet.TCPServer(check_recipient_port, - CheckRecipientAccessFactory(couchdb=cdb)) +check_recipient = internet.TCPServer( + check_recipient_port, CheckRecipientAccessFactory(couchdb=cdb), + interface="localhost") check_recipient.setServiceParent(application) # Mail receiver -- cgit v1.2.3 From 8384b006f575ac0b769f9f6e9ce6b2c623ec9fa1 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Fri, 11 Sep 2015 11:45:17 +0200 Subject: [test] add test infrastructure Some refactor on the couchdb usage was needed to be able to mock couchdb. - Resolves: #7435 --- pkg/mx.tac | 7 +------ pkg/requirements-testing.pip | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) (limited to 'pkg') diff --git a/pkg/mx.tac b/pkg/mx.tac index e85dea5..6f7b104 100755 --- a/pkg/mx.tac +++ b/pkg/mx.tac @@ -80,10 +80,6 @@ check_recipient = internet.TCPServer( check_recipient.setServiceParent(application) # Mail receiver -mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, - password, - server, - port) directories = [] for section in config.sections(): if section in ("couchdb", "alias map", "check recipient", "bounce"): @@ -92,6 +88,5 @@ for section in config.sections(): recursive = config.getboolean(section, "recursive") directories.append([to_watch, recursive]) -mr = MailReceiver(mail_couch_url_prefix, cdb, directories, bounce_from, - bounce_subject) +mr = MailReceiver(cdb, directories, bounce_from, bounce_subject) mr.setServiceParent(application) diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip index 94ab6e8..94b8e9c 100644 --- a/pkg/requirements-testing.pip +++ b/pkg/requirements-testing.pip @@ -1 +1,2 @@ pep8 +setuptools-trial -- cgit v1.2.3 From ebc748d6dbe366d82e601c94f4191718b310bfae Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 18 Nov 2015 19:50:02 +0100 Subject: [feat] Use syslog for logging - Closes: #6859 - Related: #6942 --- pkg/leap-mx.init | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'pkg') diff --git a/pkg/leap-mx.init b/pkg/leap-mx.init index d38cc2c..0c66bc5 100644 --- a/pkg/leap-mx.init +++ b/pkg/leap-mx.init @@ -13,7 +13,6 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/leap_mx.pid RUNDIR=/var/lib/leap_mx/ FILE=/usr/share/app/leap_mx.tac -LOGFILE=/var/log/leap_mx.log TWISTD_PATH=/usr/bin/twistd [ -r /etc/default/leap_mx ] && . /etc/default/leap_mx @@ -32,7 +31,7 @@ case "$1" in --pidfile=$PIDFILE \ --rundir=$RUNDIR \ --python=$FILE \ - --logfile=$LOGFILE \ + --syslog \ --prefix=leap-mx echo "." ;; -- cgit v1.2.3 From 7723faea45549999f1a64b6113289c2d7fbe8695 Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Tue, 24 Nov 2015 15:46:40 -0500 Subject: Change initscript to run leap-mx as an unpriviledged user (#7634) --- pkg/leap-mx.init | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'pkg') diff --git a/pkg/leap-mx.init b/pkg/leap-mx.init index 0c66bc5..3878bce 100644 --- a/pkg/leap-mx.init +++ b/pkg/leap-mx.init @@ -14,6 +14,8 @@ PIDFILE=/var/run/leap_mx.pid RUNDIR=/var/lib/leap_mx/ FILE=/usr/share/app/leap_mx.tac TWISTD_PATH=/usr/bin/twistd +USER=leap-mx +GROUP=leap-mx [ -r /etc/default/leap_mx ] && . /etc/default/leap_mx @@ -32,7 +34,9 @@ case "$1" in --rundir=$RUNDIR \ --python=$FILE \ --syslog \ - --prefix=leap-mx + --prefix=leap-mx \ + --uid=$USER \ + --gid=$GROUP echo "." ;; -- cgit v1.2.3 From 4557b215775b9dbc438292449d05806f2e43ae0f Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 25 Nov 2015 01:33:54 +0100 Subject: [feat] WHEELHOUSE can be a url + --use-leap-wheels --use-leap-wheels sets --trusted-host (remove it when we have a proper cert) and WHEELHOUSE to https://ftp.lizard.leap.se Until we get ftp.lizard cname, use lizard as the wheels server. --- pkg/pip_install_requirements.sh | 53 +++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 18 deletions(-) (limited to 'pkg') diff --git a/pkg/pip_install_requirements.sh b/pkg/pip_install_requirements.sh index bd44457..57732e2 100755 --- a/pkg/pip_install_requirements.sh +++ b/pkg/pip_install_requirements.sh @@ -1,38 +1,31 @@ -#!/bin/sh +#!/bin/bash # Update pip and install LEAP base/testing requirements. # For convenience, $insecure_packages are allowed with insecure flags enabled. # Use at your own risk. # See $usage for help insecure_packages="" - -return_wheelhouse() { - if [ "$WHEELHOUSE" = "" ]; then - WHEELHOUSE=$HOME/wheelhouse - fi - - if [ ! -d "$WHEELHOUSE" ]; then - mkdir $WHEELHOUSE - fi - - echo "$WHEELHOUSE" -} +leap_wheelhouse=https://lizard.leap.se/wheels show_help() { - usage="Usage: $0 [--testing]\n --testing\tInstall dependencies from requirements-testing.pip\n -\t\tOtherwise, it will install requirements.pip" - echo $usage + usage="Usage: $0 [--testing] [--use-leap-wheels]\n --testing\t\tInstall dependencies from requirements-testing.pip\n +\t\t\tOtherwise, it will install requirements.pip\n +--use-leap-wheels\tUse wheels from leap.se" + echo -e $usage exit 1 } process_arguments() { testing=false + use_leap_wheels=false + while [ "$#" -gt 0 ]; do # From http://stackoverflow.com/a/31443098 case "$1" in --help) show_help;; --testing) testing=true; shift 1;; + --use-leap-wheels) use_leap_wheels=true; shift 1;; -h) show_help;; -*) echo "unknown option: $1" >&2; exit 1;; @@ -40,6 +33,31 @@ process_arguments() { done } +return_wheelhouse() { + if $use_leap_wheels ; then + WHEELHOUSE=$leap_wheelhouse + elif [ "$WHEELHOUSE" = "" ]; then + WHEELHOUSE=$HOME/wheelhouse + fi + + # Tested with bash and zsh + if [[ $WHEELHOUSE != http* && ! -d "$WHEELHOUSE" ]]; then + mkdir $WHEELHOUSE + fi + + echo "$WHEELHOUSE" +} + +return_install_options() { + wheelhouse=`return_wheelhouse` + install_options="-U --find-links=$wheelhouse" + if $use_leap_wheels ; then + install_options="$install_options --trusted-host lizard.leap.se" + fi + + echo $install_options +} + return_insecure_flags() { for insecure_package in $insecure_packages; do flags="$flags --allow-external $insecure_package --allow-unverified $insecure_package" @@ -59,8 +77,7 @@ return_packages() { } process_arguments $@ -wheelhouse=`return_wheelhouse` -install_options="-U --find-links=$wheelhouse" +install_options=`return_install_options` insecure_flags=`return_insecure_flags` packages=`return_packages` -- cgit v1.2.3 From 5cba5ae32003efd7b068e6646ba4c0ada4e29867 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 25 Nov 2015 23:46:24 +0100 Subject: [style] fix pep8 errors --- pkg/utils/reqs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'pkg') diff --git a/pkg/utils/reqs.py b/pkg/utils/reqs.py index f10051e..251c7e9 100644 --- a/pkg/utils/reqs.py +++ b/pkg/utils/reqs.py @@ -67,8 +67,8 @@ def parse_requirements(reqfiles=['requirements.txt', if re.match(r'\s*-e\s+', line): pass # do not try to do anything with externals on vcs - #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - #line)) + # requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + # line)) # http://foo.bar/baz/foobar/zipball/master#egg=foobar elif re.match(r'\s*https?:', line): requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', -- cgit v1.2.3