[test] add docker perf tests

7 files changed, 323 insertions, 57 deletions

diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile
index 8b7dcb71..36180633 100644
--- a/scripts/docker/Dockerfile
+++ b/scripts/docker/Dockerfile
@@ -22,7 +22,15 @@ RUN apt-get -y install python-srp
 RUN apt-get -y install python-scrypt
 RUN apt-get -y install leap-keymanager
 RUN apt-get -y install python-tz
+
+# soledad-perf deps
+RUN pip install klein
+RUN apt-get -y install gnuplot
 RUN apt-get -y install curl
+RUN apt-get -y install httperf
+
+# debugging deps
+RUN apt-get -y install vim
 RUN apt-get -y install python-ipdb
 
 # copy over files to help setup the environment and run soledad
@@ -34,10 +42,11 @@ COPY files/setup-env.sh /usr/local/soledad/
 RUN /usr/local/soledad/setup-env.sh
 
 # copy runtime files for running server, client, tests, etc on a container
-COPY files/test-env.py /usr/local/soledad/
 COPY files/client_side_db.py /usr/local/soledad/
-COPY files/util.py /usr/local/soledad/
-COPY files/start-server.sh /usr/local/soledad/
 COPY files/start-client-test.sh /usr/local/soledad/
+COPY files/run-perf-test.sh /usr/local/soledad/
+COPY files/start-server.sh /usr/local/soledad/
 COPY files/start-trial-test.sh /usr/local/soledad/
+COPY files/test-env.py /usr/local/soledad/
+COPY files/util.py /usr/local/soledad/
 COPY files/conf/* /usr/local/soledad/conf/
diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile
index 872bdc40..41334142 100644
--- a/scripts/docker/Makefile
+++ b/scripts/docker/Makefile
@@ -12,17 +12,29 @@
 #   make run-server CONTAINER_ID_FILE=/tmp/container-id.txt
 #   make run-client-test CONTAINER_ID_FILE=/tmp/container-id.txt
 
+#####################################################################
+# Some configurations you might override when calling this makefile #
+#####################################################################
 
-IMAGE_NAME        ?= "leap/soledad:1.0"
-SOLEDAD_REMOTE    ?= "https://0xacab.org/leap/soledad.git"
-SOLEDAD_BRANCH    ?= "develop"
+IMAGE_NAME           ?= "leap/soledad:1.0"
+SOLEDAD_REMOTE       ?= "https://0xacab.org/leap/soledad.git"
+SOLEDAD_BRANCH       ?= "develop"
+SOLEDAD_PRELOAD_NUM  ?= "100"
+SOLEDAD_PRELOAD_SIZE ?= "500"
 
+##############################################
+# Docker image generation (main make target) #
+##############################################
 
 all: image
 
 image:
 	docker build -t $(IMAGE_NAME) .
 
+##################################################
+# Run a Soledad Server inside a docker container #
+##################################################
+
 run-server:
 	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
 	  echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \
@@ -31,6 +43,8 @@ run-server:
 	docker run \
 	  --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \
 	  --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \
+	  --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \
+	  --env="SOLEDAD_PRELOAD_SIZE=$(SOLEDAD_PRELOAD_SIZE)" \
 	  --cidfile=$(CONTAINER_ID_FILE) \
 	  --detach \
 	  $(IMAGE_NAME) \
@@ -50,6 +64,10 @@ run-client-test:
 	  $(IMAGE_NAME) \
 	  /usr/local/soledad/start-client-test.sh
 
+#################################################
+# Run all trial tests inside a docker container #
+#################################################
+
 run-trial-test:
 	docker run -t -i \
 	  --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \
@@ -57,8 +75,49 @@ run-trial-test:
 	  $(IMAGE_NAME) \
 	  /usr/local/soledad/start-trial-test.sh
 
+############################################
+# Performance tests and graphic generation #
+############################################
+
+run-perf:
+	helper/run-test.sh perf
+
+run-perf-test:
+	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
+	  echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \
+	  exit 2; \
+	fi
+	container_id=`cat $(CONTAINER_ID_FILE)`; \
+	server_ip=`./helper/get-container-ip.sh $${container_id}`; \
+	docker run -t -i \
+	  --cidfile=$(CONTAINER_ID_FILE)-perf \
+	  --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \
+	  --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \
+	  --env="SOLEDAD_PERF_REMOTE=https://0xacab.org/drebs/soledad-perf.git" \
+	  --env="SOLEDAD_PERF_BRANCH=bug/ensure-events-server" \
+	  --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \
+	  --env="SOLEDAD_PRELOAD_SIZE=$(SOLEDAD_PRELOAD_SIZE)" \
+	  --env="SOLEDAD_STATS=1" \
+	  --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \
+	  $(IMAGE_NAME) \
+	  /usr/local/soledad/run-perf-test.sh
+
+cp-perf-result:
+	@if [ -z "$(CONTAINER_ID_FILE)" ]; then \
+	  echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \
+	  exit 2; \
+	fi
+	perf_id=`cat $(CONTAINER_ID_FILE)-perf`; \
+	docker cp $${perf_id}:/var/local/soledad-perf/out/sync-stats.png /tmp/; \
+	docker cp $${perf_id}:/var/local/soledad-perf/out/series.log /tmp/
+
+########################
+# Other helper targets #
+########################
+
 run-shell: image
 	docker run -t -i $(IMAGE_NAME) /bin/bash
 
 rm-all-containers:
-	docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs docker rm -f
+	containers=`docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs`; \
+	if [ ! -z ${containers} ]; then docker rm -f $${containers}; fi
diff --git a/scripts/docker/TODO b/scripts/docker/TODO
index 75d45a8e..5185d754 100644
--- a/scripts/docker/TODO
+++ b/scripts/docker/TODO
@@ -1,3 +1 @@
 - limit resources of containers (mem and cpu)
-- add a file to run tests inside container
-- use server ip to run test
diff --git a/scripts/docker/files/run-perf-test.sh b/scripts/docker/files/run-perf-test.sh
new file mode 100755
index 00000000..80138b2a
--- /dev/null
+++ b/scripts/docker/files/run-perf-test.sh
@@ -0,0 +1,124 @@
+#!/bin/sh
+
+# Start a soledad-perf test using a remote server.
+#
+# The script does the following:
+#
+#   - configure a remote repository for soledad repo if SOLEDAD_REMOTE is set.
+#
+#   - checkout a specific branch if SOLEDAD_BRANCH is set.
+#
+#   - run the soledad-perf local twisted server that runs the client. Note
+#     that the actual soledad server should be running on another docker
+#     container. This local server is only used to measure responsiveness of
+#     soledad client. The script waits for the server to come up before
+#     continuing, or else times out after TIMEOUT seconds.
+#
+#   - trigger the creation of documents for sync.
+#
+#   - start the measurement of server responsiveness and sync stages.
+#
+#   - stop the test.
+#
+# This script is meant to be copied to the docker container and run upon
+# container start.
+
+CMD="/usr/local/soledad/test-env.py"
+REPO="/var/local/soledad"
+TIMEOUT=20
+
+#-----------------------------------------------------------------------------
+# configure a remote and checkout a branch
+#-----------------------------------------------------------------------------
+
+if [ ! -z "${SOLEDAD_REMOTE}" ]; then
+  git -C ${REPO} remote add test ${SOLEDAD_REMOTE}
+  git -C ${REPO} fetch test
+fi
+
+if [ ! -z "${SOLEDAD_BRANCH}" ]; then
+  git -C ${REPO} checkout ${SOLEDAD_BRANCH}
+fi
+
+if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then
+  git -C /var/local/soledad-perf remote add test ${SOLEDAD_PERF_REMOTE}
+  git -C /var/local/soledad-perf fetch test
+fi
+
+if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then
+  git -C /var/local/soledad-perf checkout ${SOLEDAD_PERF_BRANCH}
+fi
+
+#-----------------------------------------------------------------------------
+# write a configuration file for the perf test
+#-----------------------------------------------------------------------------
+
+cd /var/local/soledad-perf
+
+cat > defaults.conf <<EOF
+[server]
+host = ${SOLEDAD_SERVER_URL}
+
+[client]
+uuid = 1234567890abcdef
+basedir = /tmp/soledad_client_test
+passphrase = 12345678
+
+[sync]
+num_docs = ${SOLEDAD_PRELOAD_NUM}
+payload = /tmp/payload
+payload_size = ${SOLEDAD_PRELOAD_SIZE}
+auth_token = an-auth-token
+
+[test]
+stats_file = ./out/stats.json
+EOF
+
+#-----------------------------------------------------------------------------
+# start the local server and wait for it to come up
+#-----------------------------------------------------------------------------
+
+# start local test server on background
+make soledad-sync-server | grep -v stats | grep -v ping  &
+
+# wait for server until timeout
+start=`date +%s`
+elapsed=0
+
+echo "Waiting for perf server to come up..."
+
+while [ ${elapsed} -lt ${TIMEOUT} ]; do
+  result=`curl -s http://127.0.0.1:8080/ping`
+  if [ ${?} -eq 0 -a "${result}" = "easy!" ]; then
+    echo "Perf server (running soledad client) is up!"
+    break
+  else
+    sleep 1
+  fi
+  now=`date +%s`
+  elapsed=`expr ${now} - ${start}`
+done
+
+# exit with an error code if timed out waiting for server
+if [ ${elapsed} -ge ${TIMEOUT} ]; then
+  echo "Error: server unreachable at http://127.0.0.1:8080 after ${TIMEOUT} seconds."
+  exit 1
+fi
+
+sleep 2
+
+#-----------------------------------------------------------------------------
+# create docs and run test
+#-----------------------------------------------------------------------------
+
+# create documents in client
+make trigger-create-docs
+
+# launch background series measurement
+make measure-series > /dev/null &
+sleep 5  # wait a bit for some data points
+
+# run a sync and generate a graph
+make trigger-sync
+make trigger-stop
+make graph-image
diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh
index 7493930a..b9b5e4ad 100755
--- a/scripts/docker/files/start-server.sh
+++ b/scripts/docker/files/start-server.sh
@@ -2,10 +2,28 @@
 
 # Start a soledad server inside a docker container.
 #
+# This script will:
+#
+#   - eventually checkout a specific branch from a specific soledad remote.
+#
+#   - create everything a soledad server needs to run (certificate, backend
+#     server database, tables, etc.
+#
+#   - eventually preload the server database with a number of documents equal
+#     to SOLEDAD_PRELOAD_NUM, and with payload size equal to
+#     SOLEDAD_PRELOAD_SIZE.
+#
+#   - run the soledad server.
+#
 # This script is meant to be copied to the docker container and run upon
 # container start.
 
 CMD="/usr/local/soledad/test-env.py"
+
+#---------------------------------------------------------------------------
+# eventually checkout a specific branch from a specific remote
+#---------------------------------------------------------------------------
+
 REPO="/var/local/soledad"
 
 if [ ! -z "${SOLEDAD_REMOTE}" ]; then
@@ -17,10 +35,49 @@ if [ ! -z "${SOLEDAD_BRANCH}" ]; then
   git -C ${REPO} checkout ${SOLEDAD_BRANCH}
 fi
 
+#---------------------------------------------------------------------------
+# setup environment for running soledad server
+#---------------------------------------------------------------------------
+
 ${CMD} couch start
 ${CMD} user-db create
 ${CMD} token-db create
 ${CMD} token-db insert-token
 ${CMD} shared-db create
 ${CMD} cert create
+
+#---------------------------------------------------------------------------
+# write a configuration file for the perf test
+#---------------------------------------------------------------------------
+
+if [ "${SOLEDAD_PRELOAD_NUM}" -gt 0 ]; then
+  cd /var/local/soledad-perf
+
+  cat > defaults.conf <<EOF
+[server]
+host = http://127.0.0.1:2424
+
+[client]
+uuid = 1234567890abcdef
+basedir = /tmp/soledad_client_test
+passphrase = 12345678
+
+[sync]
+num_docs = ${SOLEDAD_PRELOAD_NUM}
+payload = /tmp/payload
+payload_size = ${SOLEDAD_PRELOAD_SIZE}
+auth_token = an-auth-token
+
+[test]
+stats_file = ./out/stats.json
+EOF
+
+  echo "Preloading server database..."
+  ./scripts/preload_server_database.py
+fi
+
+#---------------------------------------------------------------------------
+# actually run the server
+#---------------------------------------------------------------------------
+
 ${CMD} soledad-server start --no-daemonize
diff --git a/scripts/docker/helper/run-test.sh b/scripts/docker/helper/run-test.sh
new file mode 100755
index 00000000..5fd6e4f0
--- /dev/null
+++ b/scripts/docker/helper/run-test.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+
+# Run 2 docker images, one with soledad server and another with a soledad
+# client running a test.
+#
+# As there are many possible, tests, you have to pass an argument to the
+# script saying which test you want to run. Currently, possible values are
+# "connect" and "perf".
+#
+# After launching the server container, the script waits for TIMEOUT seconds
+# for it to come up. If we fail to detect the server, the script exits with
+# nonzero status.
+
+# seconds to wait before giving up waiting from server
+TIMEOUT=20
+
+# get a test
+if [ ${#} -ne 1 ]; then
+  "Usage: ${0} [perf|connect]"
+  exit 1
+fi
+
+TEST=${1}
+
+# get script name and path
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+# run the server
+tempfile=`mktemp -u`
+make run-server CONTAINER_ID_FILE=${tempfile}
+
+# get server container info
+container_id=`cat ${tempfile}`
+server_ip=`${SCRIPTPATH}/get-container-ip.sh ${container_id}`
+
+# wait for server until timeout
+start=`date +%s`
+elapsed=0
+
+echo "Waiting for soledad server container to come up..."
+
+while [ ${elapsed} -lt ${TIMEOUT} ]; do
+  result=`curl -s http://${server_ip}:2424`
+  if [ ${?} -eq 0 ]; then
+    echo "Soledad server container is up!"
+    break
+  else
+    sleep 1
+  fi
+  now=`date +%s`
+  elapsed=`expr ${now} - ${start}`
+done
+
+# exit with an error code if timed out waiting for server
+if [ ${elapsed} -ge ${TIMEOUT} ]; then
+  echo "Error: server unreachable at ${server_ip} after ${TIMEOUT} seconds."
+  exit 1
+fi
+
+# run the client
+if [ "${TEST}" = "connect" ]; then
+  make run-client-test CONTAINER_ID_FILE=${tempfile}
+elif [ "${TEST}" = "perf" ]; then
+  make run-perf-test CONTAINER_ID_FILE=${tempfile}
+  make cp-perf-result CONTAINER_ID_FILE=${tempfile}
+fi
diff --git a/scripts/docker/helper/run-tests.sh b/scripts/docker/helper/run-tests.sh
deleted file mode 100755
index bcd7a565..00000000
--- a/scripts/docker/helper/run-tests.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/sh
-
-# Run 2 docker images, one with soledad server and another with a soledad
-# client running the tests.
-#
-# After launching the server, the script waits for TIMEOUT seconds for it to
-# come up. If we fail to detect the server, the script exits with nonzero
-# status.
-
-
-# seconds to wait before giving up waiting from server
-TIMEOUT=20
-
-# some info from this script
-SCRIPT=$(readlink -f "$0")
-SCRIPTPATH=$(dirname "$SCRIPT")
-
-# run the server
-tempfile=`mktemp -u`
-make run-server CONTAINER_ID_FILE=${tempfile}
-
-# get server container info
-container_id=`cat ${tempfile}`
-server_ip=`${SCRIPTPATH}/get-container-ip.sh ${container_id}`
-
-# wait for server until timeout
-start=`date +%s`
-elapsed=0
-
-while [ ${elapsed} -lt ${TIMEOUT} ]; do
-  result=`curl http://${server_ip}:2424`
-  if [ ${?} -eq 0 ]; then
-    break
-  else
-    sleep 1
-  fi
-  now=`date +%s`
-  elapsed=`expr ${now} - ${start}`
-done
-
-# exit with an error code if timed out waiting for server
-if [ ${elapsed} -ge ${TIMEOUT} ]; then
-  echo "Error: server unreacheble at ${server_ip} after ${TIMEOUT} seconds."
-  exit 1
-fi
-
-# run the client
-make run-client-test CONTAINER_ID_FILE=${tempfile}