[test] refactor of docker scripts

18 files changed, 102 insertions, 95 deletions

diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile
index 36180633..915508ea 100644
--- a/scripts/docker/Dockerfile
+++ b/scripts/docker/Dockerfile
@@ -5,13 +5,18 @@ FROM debian
 EXPOSE 2424
 
 # install dependencies from debian repos
-COPY files/leap.list /etc/apt/sources.list.d/
+COPY files/apt/leap.list /etc/apt/sources.list.d/
 
 RUN apt-get update
 RUN apt-get -y --force-yes install leap-archive-keyring
 
 RUN apt-get update
+
 RUN apt-get -y install git
+RUN apt-get -y install vim
+RUN apt-get -y install python-ipdb
+
+# install python deps
 RUN apt-get -y install libpython2.7-dev
 RUN apt-get -y install libffi-dev
 RUN apt-get -y install libssl-dev
@@ -23,30 +28,24 @@ RUN apt-get -y install python-scrypt
 RUN apt-get -y install leap-keymanager
 RUN apt-get -y install python-tz
 
-# soledad-perf deps
+RUN pip install -U pip
+RUN pip install psutil
+
+# install soledad-perf deps
 RUN pip install klein
-RUN apt-get -y install gnuplot
 RUN apt-get -y install curl
 RUN apt-get -y install httperf
 
-# debugging deps
-RUN apt-get -y install vim
-RUN apt-get -y install python-ipdb
+# clone repositories
+ENV BASEURL "https://github.com/leapcode"
+ENV VARDIR "/var/local"
+ENV REPOS "soledad leap_pycommon soledad-perf"
+RUN for repo in ${REPOS}; do git clone ${BASEURL}/${repo}.git /var/local/${repo}; done
 
 # copy over files to help setup the environment and run soledad
 RUN mkdir -p /usr/local/soledad
-RUN mkdir -p /usr/local/soledad/conf
-
-# setup the enviroment for running soledad client and server
-COPY files/setup-env.sh /usr/local/soledad/
-RUN /usr/local/soledad/setup-env.sh
-
-# copy runtime files for running server, client, tests, etc on a container
-COPY files/client_side_db.py /usr/local/soledad/
-COPY files/start-client-test.sh /usr/local/soledad/
-COPY files/run-perf-test.sh /usr/local/soledad/
-COPY files/start-server.sh /usr/local/soledad/
-COPY files/start-trial-test.sh /usr/local/soledad/
-COPY files/test-env.py /usr/local/soledad/
-COPY files/util.py /usr/local/soledad/
-COPY files/conf/* /usr/local/soledad/conf/
+
+COPY files/build/install-deps-from-repos.sh /usr/local/soledad/
+RUN /usr/local/soledad/install-deps-from-repos.sh
+
+COPY files/bin/ /usr/local/soledad/
diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile
index 080fd16c..9dbe9062 100644
--- a/scripts/docker/Makefile
+++ b/scripts/docker/Makefile
@@ -10,7 +10,7 @@
 # Example usage:
 #
 #   make run-server CONTAINER_ID_FILE=/tmp/container-id.txt
-#   make run-client-test CONTAINER_ID_FILE=/tmp/container-id.txt
+#   make run-client-perf CONTAINER_ID_FILE=/tmp/container-id.txt
 
 #####################################################################
 # Some configurations you might override when calling this makefile #
@@ -50,9 +50,9 @@ run-server:
 	  --cidfile=$(CONTAINER_ID_FILE) \
 	  --detach \
 	  $(IMAGE_NAME) \
-	  /usr/local/soledad/start-server.sh
+	  /usr/local/soledad/run-server.sh # --drop-to-shell
 
-run-client-test:
+run-client-bootstrap:
 	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
 	  echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \
 	  exit 2; \
@@ -65,28 +65,28 @@ run-client-test:
 	  --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \
 	  --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \
 	  $(IMAGE_NAME) \
-	  /usr/local/soledad/start-client-test.sh
+	  /usr/local/soledad/run-client-bootstrap.sh
 
 #################################################
 # Run all trial tests inside a docker container #
 #################################################
 
-run-trial-test:
+run-trial:
 	docker run -t -i \
 	  --memory="$(MEMORY)" \
 	  --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \
 	  --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \
 	  $(IMAGE_NAME) \
-	  /usr/local/soledad/start-trial-test.sh
+	  /usr/local/soledad/run-trial.sh
 
 ############################################
 # Performance tests and graphic generation #
 ############################################
 
-run-perf:
+run-perf-test:
 	helper/run-test.sh perf
 
-run-perf-test:
+run-client-perf:
 	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
 	  echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \
 	  exit 2; \
@@ -105,7 +105,7 @@ run-perf-test:
 	  --env="SOLEDAD_STATS=1" \
 	  --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \
 	  $(IMAGE_NAME) \
-	  /usr/local/soledad/run-perf-test.sh
+	  /usr/local/soledad/run-client-perf.sh # --drop-to-shell
 
 cp-perf-result:
 	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
diff --git a/scripts/docker/files/leap.list b/scripts/docker/files/apt/leap.list
index 7eb474d8..7eb474d8 100644
--- a/scripts/docker/files/leap.list
+++ b/scripts/docker/files/apt/leap.list
diff --git a/scripts/docker/files/client_side_db.py b/scripts/docker/files/bin/client_side_db.py
index 4be33d13..4be33d13 100644
--- a/scripts/docker/files/client_side_db.py
+++ b/scripts/docker/files/bin/client_side_db.py
diff --git a/scripts/docker/files/conf/cert_default.conf b/scripts/docker/files/bin/conf/cert_default.conf
index 8043cea3..8043cea3 100644
--- a/scripts/docker/files/conf/cert_default.conf
+++ b/scripts/docker/files/bin/conf/cert_default.conf
diff --git a/scripts/docker/files/conf/couchdb_default.ini b/scripts/docker/files/bin/conf/couchdb_default.ini
index 5ab72d7b..5ab72d7b 100644
--- a/scripts/docker/files/conf/couchdb_default.ini
+++ b/scripts/docker/files/bin/conf/couchdb_default.ini
diff --git a/scripts/docker/files/conf/soledad-server_default.conf b/scripts/docker/files/bin/conf/soledad-server_default.conf
index 5e286374..5e286374 100644
--- a/scripts/docker/files/conf/soledad-server_default.conf
+++ b/scripts/docker/files/bin/conf/soledad-server_default.conf
diff --git a/scripts/docker/files/start-client-test.sh b/scripts/docker/files/bin/run-client-bootstrap.sh
index 9dec3371..fbbb42e8 100755
--- a/scripts/docker/files/start-client-test.sh
+++ b/scripts/docker/files/bin/run-client-bootstrap.sh
@@ -5,7 +5,7 @@
 # This script is meant to be copied to the docker container and run upon
 # container start.
 
-CMD="/usr/local/soledad/test-env.py"
+CMD="/usr/local/soledad/setup-test-env.py"
 REPO="/var/local/soledad"
 
 if [ ! -z "${SOLEDAD_REMOTE}" ]; then
diff --git a/scripts/docker/files/run-perf-test.sh b/scripts/docker/files/bin/run-client-perf.sh
index ebd54d23..01b27b98 100755
--- a/scripts/docker/files/run-perf-test.sh
+++ b/scripts/docker/files/bin/run-client-perf.sh
@@ -23,7 +23,7 @@
 # This script is meant to be copied to the docker container and run upon
 # container start.
 
-CMD="/usr/local/soledad/test-env.py"
+CMD="/usr/local/soledad/setup-test-env.py"
 REPO="/var/local/soledad"
 TIMEOUT=20
 
@@ -74,6 +74,11 @@ auth_token = an-auth-token
 stats_file = ./out/stats.json
 EOF
 
+if [ "${1}" = "--drop-to-shell" ]; then
+  /bin/bash
+  exit 0
+fi
+
 #-----------------------------------------------------------------------------
 # start the local server and wait for it to come up
 #-----------------------------------------------------------------------------
@@ -109,7 +114,7 @@ fi
 # create docs and run test
 #-----------------------------------------------------------------------------
 
-#set -e
+set -e
 
 # create documents in client
 make trigger-create-docs
@@ -121,4 +126,3 @@ sleep 5  # wait a bit for some data points
 # run a sync and generate a graph
 make trigger-sync
 make trigger-stop
-make graph-image
diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/bin/run-server.sh
index 0980d352..feedee7e 100755
--- a/scripts/docker/files/start-server.sh
+++ b/scripts/docker/files/bin/run-server.sh
@@ -18,7 +18,7 @@
 # This script is meant to be copied to the docker container and run upon
 # container start.
 
-CMD="/usr/local/soledad/test-env.py"
+CMD="/usr/local/soledad/setup-test-env.py"
 
 #---------------------------------------------------------------------------
 # eventually checkout a specific branch from a specific remote
@@ -80,5 +80,10 @@ fi
 # actually run the server
 #---------------------------------------------------------------------------
 
+if [ "${1}" = "--drop-to-shell" ]; then
+  /bin/bash
+  exit 0
+fi
+
 echo "Starting soledad server..."
 ${CMD} soledad-server start --no-daemonize
diff --git a/scripts/docker/files/start-trial-test.sh b/scripts/docker/files/bin/run-trial.sh
index ad139288..f38f3124 100755
--- a/scripts/docker/files/start-trial-test.sh
+++ b/scripts/docker/files/bin/run-trial.sh
@@ -5,7 +5,7 @@
 # This script is meant to be copied to the docker container and run upon
 # container start.
 
-CMD="/usr/local/soledad/test-env.py"
+CMD="/usr/local/soledad/setup-test-env.py"
 REPO="/var/local/soledad"
 
 if [ ! -z "${SOLEDAD_REMOTE}" ]; then
diff --git a/scripts/docker/files/test-env.py b/scripts/docker/files/bin/setup-test-env.py
index 0569b65d..0569b65d 100755
--- a/scripts/docker/files/test-env.py
+++ b/scripts/docker/files/bin/setup-test-env.py
diff --git a/scripts/docker/files/util.py b/scripts/docker/files/bin/util.py
index e7e2ef9a..e7e2ef9a 100644
--- a/scripts/docker/files/util.py
+++ b/scripts/docker/files/bin/util.py
diff --git a/scripts/docker/files/bin/util.sh b/scripts/docker/files/bin/util.sh
new file mode 100644
index 00000000..77287d0d
--- /dev/null
+++ b/scripts/docker/files/bin/util.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+configure_soledad_repos() {
+  if [ ! -z "${SOLEDAD_REMOTE}" ]; then
+    git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE}
+    git -C ${REPO} fetch origin
+  fi
+
+  if [ ! -z "${SOLEDAD_BRANCH}" ]; then
+    git -C ${REPO} checkout ${SOLEDAD_BRANCH}
+  fi
+}
diff --git a/scripts/docker/files/build/install-deps-from-repos.sh b/scripts/docker/files/build/install-deps-from-repos.sh
new file mode 100755
index 00000000..46530c86
--- /dev/null
+++ b/scripts/docker/files/build/install-deps-from-repos.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Install dependencies needed to run client and server in a test environment.
+#
+# In details, this script does the following:
+#
+#   - install dependencies for packages in /var/local from their requirements
+#     files in each of the repositories, using python wheels when possible.
+#
+#   - install the python packages in development mode
+#
+# This script is meant to be copied to the docker container during container
+# build and run after system dependencies have been installed.
+
+BASEDIR="/var/local"
+
+# install dependencies and packages
+install_script="pkg/pip_install_requirements.sh"
+opts="--use-leap-wheels"
+pkgs="leap_pycommon soledad/common soledad/client soledad/server"
+
+for pkg in ${pkgs}; do
+  pkgdir=${BASEDIR}/${pkg}
+  testing=""
+  if [ -f ${pkgdir}/pkg/requirements-testing.pip ]; then
+    testing="--testing"
+  fi
+  (cd ${pkgdir} && ${install_script} ${testing} ${opts})
+  (cd ${pkgdir} && python setup.py develop)
+done
diff --git a/scripts/docker/files/setup-env.sh b/scripts/docker/files/setup-env.sh
deleted file mode 100755
index d5aeab7d..00000000
--- a/scripts/docker/files/setup-env.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# Clone soledad repository and install soledad dependencies needed to run
-# client and server in a test environment.
-#
-# In details, this script does the following:
-#
-#   - clone a series of python package repositories into /var/local/soledad.
-#   - install dependencies for those packages from the requirements files in
-#     each of the repositories, using python wheels when possible.
-#   - install the python packages in development mode
-#
-# The cloned git repositories might have a remote configured and a branch
-# checked out on runtime, before a server, client or test instance is actually
-# run. Check the other scripts in this directory.
-#
-# This script is meant to be copied to the docker container and run after
-# system dependencies have been installed.
-
-BASEDIR="/var/local"
-BASEURL="https://github.com/leapcode"
-
-mkdir -p ${BASEDIR}
-
-# clone repositories
-repos="soledad leap_pycommon soledad-perf"
-
-for repo in ${repos}; do
-  repodir=${BASEDIR}/${repo}
-  if [ ! -d ${repodir} ]; then
-    git clone ${BASEURL}/${repo} ${repodir}
-    git -C ${repodir} fetch origin
-  fi
-done
-
-# use latest pip because the version available in debian jessie doesn't
-# support wheels
-pip install -U pip
-
-pip install psutil
-
-# install dependencies and packages
-install_script="pkg/pip_install_requirements.sh"
-opts="--use-leap-wheels"
-pkgs="leap_pycommon soledad/common soledad/client soledad/server"
-
-for pkg in ${pkgs}; do
-  pkgdir=${BASEDIR}/${pkg}
-  testing=""
-  if [ -f ${pkgdir}/pkg/requirements-testing.pip ]; then
-    testing="--testing"
-  fi
-  (cd ${pkgdir} && ${install_script} ${testing} ${opts})
-  (cd ${pkgdir} && python setup.py develop)
-done
diff --git a/scripts/docker/helper/run-test.sh b/scripts/docker/helper/run-test.sh
index 1b0e3db7..c3bdd00c 100755
--- a/scripts/docker/helper/run-test.sh
+++ b/scripts/docker/helper/run-test.sh
@@ -16,14 +16,14 @@ TIMEOUT=20
 
 # parse command
 if [ ${#} -ne 1 ]; then
-  echo "Usage: ${0} perf|connect"
+  echo "Usage: ${0} perf|bootstrap"
   exit 1
 fi
 
 test=${1}
 
-if [ "${1}" != "perf" -a "${1}" != "perf" ]; then
-  echo "Usage: ${0} perf|connect"
+if [ "${test}" != "perf" -a "${test}" != "bootstrap" ]; then
+  echo "Usage: ${0} perf|bootstrap"
   exit 1
 fi
 
@@ -67,5 +67,5 @@ fi
 set -e
 
 # run the test
-make run-${test}-test CONTAINER_ID_FILE=${tempfile}
+make run-client-${test} CONTAINER_ID_FILE=${tempfile}
 rm -r ${tempfile}
diff --git a/scripts/docker/helper/run-until-error.sh b/scripts/docker/helper/run-until-error.sh
new file mode 100755
index 00000000..a4cab6ec
--- /dev/null
+++ b/scripts/docker/helper/run-until-error.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+status=0
+runs=10
+
+while [ ${status} -eq 0 -a ${runs} -gt 0 ]; do
+  echo "=== RUN ${runs}"
+  make rm-all-containers
+  make run-perf-test
+  status=${?}
+  runs=`expr ${runs} - 1`
+done