moved infrastructure tests run by `leap run` to tests/server-tests

19 files changed, 1995 insertions, 0 deletions

diff --git a/tests/server-tests/README.md b/tests/server-tests/README.md
new file mode 100644
index 00000000..29db2e06
--- /dev/null
+++ b/tests/server-tests/README.md
@@ -0,0 +1,44 @@
+Tests for Server
+---------------------------------
+
+The tests in this directory are run against the servers of a live running
+provider.
+
+Usage
+---------------------------------
+
+To run the tests from a local workstation:
+
+    workstation$ cd <my provider directory>
+    workstation$ leap test
+
+To run the tests from the server itself:
+
+    workstation$ leap ssh servername
+    servername# run_tests
+
+Notes
+---------------------------------
+
+server-tests/white-box/
+
+    These tests are run on the server as superuser. They are for
+    troubleshooting any problems with the internal setup of the server.
+
+server-tests/black-box/
+
+    These test are run the user's local machine. They are for troubleshooting
+    any external problems with the service exposed by the server.
+
+Additional Files
+---------------------------------
+
+server-tests/helpers/
+
+    Utility functions made available to all tests.
+
+server-tests/order.rb
+
+    Configuration file to specify which nodes should be tested in which order.
+
+
diff --git a/tests/server-tests/helpers/bonafide_helper.rb b/tests/server-tests/helpers/bonafide_helper.rb
new file mode 100644
index 00000000..5b886228
--- /dev/null
+++ b/tests/server-tests/helpers/bonafide_helper.rb
@@ -0,0 +1,235 @@
+#
+# helper for the communication with the provider API for creating, authenticating, and deleting accounts.
+#
+
+class LeapTest
+
+  def assert_tmp_user
+    user = assert_create_user
+    assert_authenticate_user(user)
+    yield user if block_given?
+    assert_delete_user(user)
+  rescue StandardError, MiniTest::Assertion => exc
+    begin
+      assert_delete_user(user)
+    rescue
+    end
+    raise exc
+  end
+
+  #
+  # attempts to create a user account via the API,
+  # returning the user object if successful.
+  #
+  def assert_create_user(username=nil, auth=nil)
+    user = SRP::User.new(username)
+    url = api_url("/users.json")
+    params = user.to_params
+    if auth
+      options = api_options(:auth => auth)
+    else
+      options = api_options
+      if property('webapp.invite_required')
+        @invite_code = generate_invite_code
+        params['user[invite_code]'] = @invite_code
+      end
+    end
+
+    assert_post(url, params, options) do |body|
+      assert response = JSON.parse(body), 'response should be JSON'
+      assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead."
+      user.ok = true
+      user.id = response['id']
+    end
+    return user
+  end
+
+  # TODO: use the api for this instead.
+  def generate_invite_code
+    `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "")
+  end
+
+  #
+  # attempts to authenticate user. if successful,
+  # user object is updated with id and session token.
+  #
+  def assert_authenticate_user(user)
+    url = api_url("/sessions.json")
+    session = SRP::Session.new(user)
+    params = {'login' => user.username, 'A' => session.aa}
+    assert_post(url, params, api_options) do |body, response|
+      cookie = response['Set-Cookie'].split(';').first
+      assert(response = JSON.parse(body), 'response should be JSON')
+      assert(session.bb = response["B"], 'response should include "B"')
+      url = api_url("/sessions/login.json")
+      params = {'client_auth' => session.m, 'A' => session.aa}
+      assert_put(url, params, api_options('Cookie' => cookie)) do |body|
+        assert(response = JSON.parse(body), 'response should be JSON')
+        assert(response['M2'], 'response should include M2')
+        user.session_token = response['token']
+        user.id = response['id']
+        assert(user.session_token, 'response should include token')
+        assert(user.id, 'response should include user id')
+      end
+    end
+  end
+
+  #
+  # attempts to destroy a user account via the API.
+  #
+  def assert_delete_user(user)
+    if user.is_a? String
+      assert_delete_user_by_login(user)
+    elsif user.is_a? SRP::User
+      assert_delete_srp_user(user)
+    end
+  end
+
+  #
+  # returns true if the identity exists, uses monitor token auth
+  #
+  def identity_exists?(address)
+    url = api_url("/identities/#{URI.encode(address)}.json")
+    options = {:ok_codes => [200, 404]}.merge(
+      api_options(:auth => :monitor)
+    )
+    assert_get(url, nil, options) do |body, response|
+      return response.code == "200"
+    end
+  end
+
+  def upload_public_key(user_id, public_key)
+    url = api_url("/users/#{user_id}.json")
+    params = {"user[public_key]" => public_key}
+    assert_put(url, params, api_options(:auth => :monitor))
+  end
+
+  #
+  # return user document as a Hash. uses monitor token auth
+  #
+  def find_user_by_id(user_id)
+    url = api_url("/users/#{user_id}.json")
+    assert_get(url, nil, api_options(:auth => :monitor)) do |body|
+      return JSON.parse(body)
+    end
+  end
+
+  #
+  # return user document as a Hash. uses monitor token auth
+  # NOTE: this relies on deprecated behavior of the API
+  # and will not work when multi-domain support is added.
+  #
+  def find_user_by_login(login)
+    url = api_url("/users/0.json?login=#{login}")
+    options = {:ok_codes => [200, 404]}.merge(
+      api_options(:auth => :monitor)
+    )
+    assert_get(url, nil, options) do |body, response|
+      if response.code == "200"
+        return JSON.parse(body)
+      else
+        return nil
+      end
+    end
+  end
+
+  private
+
+  def api_url(path)
+    unless path =~ /^\//
+      path = '/' + path
+    end
+    if property('testing.api_uri')
+      return property('testing.api_uri') + path
+    elsif property('api')
+      api = property('api')
+      return "https://%{domain}:%{port}/%{version}#{path}" % {
+        :domain   => api['domain'],
+        :port     => api['port'],
+        :version  => api['version'] || 1
+      }
+    else
+      fail 'This node needs to have either testing.api_url or api.{domain,port} configured.'
+    end
+  end
+
+  #
+  # produces an options hash used for api http requests.
+  #
+  # argument options hash gets added to "headers"
+  # of the http request.
+  #
+  # special :auth key in argument will expand to
+  # add api_token_auth header.
+  #
+  # if you want to try manually:
+  #
+  #   export API_URI=`grep api_uri /etc/leap/hiera.yaml | cut -d\" -f2`
+  #   export TOKEN=`grep monitor_auth_token /etc/leap/hiera.yaml | awk '{print $2}'`
+  #   curl -H "Accept: application/json" -H "Token: $TOKEN" $API_URI
+  #
+  def api_options(options={})
+    # note: must be :headers, not "headers"
+    hsh = {
+      :headers => {
+        "Accept" => "application/json"
+      }
+    }
+    if options[:auth]
+      hsh[:headers].merge!(api_token_auth(options.delete(:auth)))
+    end
+    hsh[:headers].merge!(options)
+    return hsh
+  end
+
+  #
+  # add token authentication to a http request.
+  #
+  # returns a hash suitable for adding to the 'headers' option
+  # of an http function.
+  #
+  def api_token_auth(token)
+    if token.is_a?(Symbol) && property('testing')
+      if token == :monitor
+        token_str = property('testing.monitor_auth_token')
+      else
+        raise ArgumentError.new 'no such token'
+      end
+    else
+      token_str = token
+    end
+    {"Authorization" => "Token token=\"#{token_str}\""}
+  end
+
+  #
+  # not actually used in any test, but useful when
+  # writing new tests.
+  #
+  def assert_delete_user_by_login(login_name)
+    user = find_user_by_login(login_name)
+    url = api_url("/users/#{user['id']}.json")
+    params =  {:identities => 'destroy'}
+    delete(url, params, api_options(:auth => :monitor)) do |body, response, error|
+      assert error.nil?, "Error deleting user: #{error}"
+      assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+      assert(response = JSON.parse(body), 'Delete response should be JSON')
+      assert(response["success"], 'Deleting user should be a success')
+    end
+  end
+
+  def assert_delete_srp_user(user)
+    if user && user.ok && user.id && user.session_token && !user.deleted
+      url = api_url("users/#{user.id}.json")
+      params = {:identities => 'destroy'}
+      user.deleted = true
+      delete(url, params, api_options(:auth => user.session_token)) do |body, response, error|
+        assert error.nil?, "Error deleting user: #{error}"
+        assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+        assert(response = JSON.parse(body), 'Delete response should be JSON')
+        assert(response["success"], 'Deleting user should be a success')
+      end
+    end
+  end
+
+
+end
diff --git a/tests/server-tests/helpers/client_side_db.py b/tests/server-tests/helpers/client_side_db.py
new file mode 100644
index 00000000..2f8c220f
--- /dev/null
+++ b/tests/server-tests/helpers/client_side_db.py
@@ -0,0 +1,167 @@
+import logging
+import os
+import tempfile
+import getpass
+import binascii
+import json
+
+try:
+    import requests
+    import srp._pysrp as srp
+except ImportError:
+    pass
+
+from twisted.internet.defer import inlineCallbacks
+
+from leap.soledad.client import Soledad
+
+
+"""
+Helper functions to give access to client-side Soledad database.
+Copied over from soledad/scripts folder.
+"""
+
+# create a logger
+logger = logging.getLogger(__name__)
+
+# DEBUG: enable debug logs
+# LOG_FORMAT = '%(asctime)s %(message)s'
+# logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG)
+
+
+safe_unhexlify = lambda x: binascii.unhexlify(x) if (
+    len(x) % 2 == 0) else binascii.unhexlify('0' + x)
+
+
+def _fail(reason):
+    logger.error('Fail: ' + reason)
+    exit(2)
+
+
+def get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file,
+                         token):
+    # setup soledad info
+    logger.info('UUID is %s' % uuid)
+    logger.info('Server URL is %s' % server_url)
+    secrets_path = os.path.join(
+        basedir, '%s.secret' % uuid)
+    local_db_path = os.path.join(
+        basedir, '%s.db' % uuid)
+    # instantiate soledad
+    return Soledad(
+        uuid,
+        unicode(passphrase),
+        secrets_path=secrets_path,
+        local_db_path=local_db_path,
+        server_url=server_url,
+        cert_file=cert_file,
+        auth_token=token,
+        defer_encryption=True)
+
+
+def _get_api_info(provider):
+    info = requests.get(
+        'https://'+provider+'/provider.json', verify=False).json()
+    return info['api_uri'], info['api_version']
+
+
+def _login(username, passphrase, provider, api_uri, api_version):
+    usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
+    auth = None
+    try:
+        auth = _authenticate(api_uri, api_version, usr).json()
+    except requests.exceptions.ConnectionError:
+        _fail('Could not connect to server.')
+    if 'errors' in auth:
+        _fail(str(auth['errors']))
+    return api_uri, api_version, auth
+
+
+def _authenticate(api_uri, api_version, usr):
+    api_url = "%s/%s" % (api_uri, api_version)
+    session = requests.session()
+    uname, A = usr.start_authentication()
+    params = {'login': uname, 'A': binascii.hexlify(A)}
+    init = session.post(
+        api_url + '/sessions', data=params, verify=False).json()
+    if 'errors' in init:
+        _fail('test user not found')
+    M = usr.process_challenge(
+        safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
+    return session.put(api_url + '/sessions/' + uname, verify=False,
+                       data={'client_auth': binascii.hexlify(M)})
+
+
+def _get_soledad_info(username, provider, passphrase, basedir):
+    api_uri, api_version = _get_api_info(provider)
+    auth = _login(username, passphrase, provider, api_uri, api_version)
+    # get soledad server url
+    service_url = '%s/%s/config/soledad-service.json' % \
+                  (api_uri, api_version)
+    soledad_hosts = requests.get(service_url, verify=False).json()['hosts']
+    hostnames = soledad_hosts.keys()
+    # allow for choosing the host
+    host = hostnames[0]
+    if len(hostnames) > 1:
+        i = 1
+        print "There are many available hosts:"
+        for h in hostnames:
+            print "  (%d) %s.%s" % (i, h, provider)
+            i += 1
+        choice = raw_input("Choose a host to use (default: 1): ")
+        if choice != '':
+            host = hostnames[int(choice) - 1]
+    server_url = 'https://%s:%d/user-%s' % \
+        (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'],
+         auth[2]['id'])
+    # get provider ca certificate
+    ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text
+    cert_file = os.path.join(basedir, 'ca.crt')
+    with open(cert_file, 'w') as f:
+        f.write(ca_cert)
+    return auth[2]['id'], server_url, cert_file, auth[2]['token']
+
+
+def _get_passphrase(args):
+    passphrase = args.passphrase
+    if passphrase is None:
+        passphrase = getpass.getpass(
+            'Password for %s@%s: ' % (args.username, args.provider))
+    return passphrase
+
+
+def _get_basedir(args):
+    basedir = args.basedir
+    if basedir is None:
+        basedir = tempfile.mkdtemp()
+    elif not os.path.isdir(basedir):
+        os.mkdir(basedir)
+    logger.info('Using %s as base directory.' % basedir)
+    return basedir
+
+
+@inlineCallbacks
+def _export_key(args, km, fname, private=False):
+    address = args.username + "@" + args.provider
+    pkey = yield km.get_key(
+        address, OpenPGPKey, private=private, fetch_remote=False)
+    with open(args.export_private_key, "w") as f:
+        f.write(pkey.key_data)
+
+
+@inlineCallbacks
+def _export_incoming_messages(soledad, directory):
+    yield soledad.create_index("by-incoming", "bool(incoming)")
+    docs = yield soledad.get_from_index("by-incoming", '1')
+    i = 1
+    for doc in docs:
+        with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f:
+            f.write(doc.content["_enc_json"])
+        i += 1
+
+
+@inlineCallbacks
+def _get_all_docs(soledad):
+    _, docs = yield soledad.get_all_docs()
+    for doc in docs:
+        print json.dumps(doc.content, indent=4)
diff --git a/tests/server-tests/helpers/couchdb_helper.rb b/tests/server-tests/helpers/couchdb_helper.rb
new file mode 100644
index 00000000..efb2c2bf
--- /dev/null
+++ b/tests/server-tests/helpers/couchdb_helper.rb
@@ -0,0 +1,143 @@
+class LeapTest
+
+  #
+  # generates a couchdb url for when couchdb is running
+  # remotely and is available via stunnel.
+  #
+  # example properties:
+  #
+  # stunnel:
+  #   clients:
+  #     couch_client:
+  #       couch1_5984:
+  #         accept_port: 4000
+  #         connect: couch1.bitmask.i
+  #         connect_port: 15984
+  #
+  def couchdb_urls_via_stunnel(path="", options=nil)
+    path = path.gsub('"', '%22')
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf|
+      assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+      URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url|
+        remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
+        url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
+      }
+    end
+  end
+
+  #
+  # generates a couchdb url for accessing couchdb via haproxy
+  #
+  # example properties:
+  #
+  # haproxy:
+  #   couch:
+  #     listen_port: 4096
+  #     servers:
+  #       panda:
+  #         backup: false
+  #         host: localhost
+  #         port: 4000
+  #         weight: 100
+  #         writable: true
+  #
+  def couchdb_url_via_haproxy(path="", options=nil)
+    path = path.gsub('"', '%22')
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    port = assert_property('haproxy.couch.listen_port')
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via haproxy)'
+    }
+  end
+
+  #
+  # generates a couchdb url for when couchdb is running locally.
+  #
+  # example properties:
+  #
+  # couch:
+  #   port: 5984
+  #
+  def couchdb_url_via_localhost(path="", options=nil)
+    path = path.gsub('"', '%22')
+    port = (options && options[:port]) || assert_property('couch.port')
+    if options && options[:username]
+      password = property("couch.users.%{username}.password" % options)
+      userpart = "%s:%s@" % [options[:username], password]
+    else
+      userpart = ""
+    end
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via direct localhost connection)'
+    }
+  end
+
+  #
+  # returns a single url for accessing couchdb
+  #
+  def couchdb_url(path="", options=nil)
+    if property('couch.port')
+      couchdb_url_via_localhost(path, options)
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options).first
+    end
+  end
+
+  #
+  # returns an array of urls for accessing couchdb
+  #
+  def couchdb_urls(path="", options=nil)
+    if property('couch.port')
+      [couchdb_url_via_localhost(path, options)]
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options)
+    end
+  end
+
+  def assert_destroy_user_db(user_id, options=nil)
+    db_name = "user-#{user_id}"
+    url = couchdb_url("/#{db_name}", options)
+    http_options = {:ok_codes => [200, 404]} # ignore missing dbs
+    assert_delete(url, nil, http_options)
+  end
+
+  def assert_create_user_db(user_id, options=nil)
+    db_name = "user-#{user_id}"
+    url = couchdb_url("/#{db_name}", options)
+    http_options = {:ok_codes => [200, 404]} # ignore missing dbs
+    assert_put(url, nil, :format => :json) do |body|
+      assert response = JSON.parse(body), "PUT response should be JSON"
+      assert response["ok"], "PUT response should be OK"
+    end
+  end
+
+  #
+  # returns true if the per-user db created by soledad-server exists.
+  #
+  def user_db_exists?(user_id, options=nil)
+    options = {:username => 'admin'}.merge(options || {})
+    db_name = "user-#{user_id}"
+    url = couchdb_url("/#{db_name}", options)
+    get(url) do |body, response, error|
+      if response.nil?
+        fail "could not query couchdb #{url}: #{error}\n#{body}"
+      elsif response.code.to_i == 200
+        return true
+      elsif response.code.to_i == 404
+        return false
+      else
+        fail ["could not query couchdb #{url}: expected response code 200 or 404, but got #{response.code}.", error, body].compact.join("\n")
+      end
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/files_helper.rb b/tests/server-tests/helpers/files_helper.rb
new file mode 100644
index 00000000..d6795889
--- /dev/null
+++ b/tests/server-tests/helpers/files_helper.rb
@@ -0,0 +1,54 @@
+class LeapTest
+
+  #
+  # Matches the regexp in the file, and returns the first matched string (or fails if no match).
+  #
+  def file_match(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures.first
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # Matches the regexp in the file, and returns array of matched strings (or fails if no match).
+  #
+  def file_matches(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # checks to make sure the given property path exists in $node (e.g. hiera.yaml)
+  # and returns the value
+  #
+  def assert_property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      fail "Required node property `#{property}` is missing." if latest.nil?
+    end
+    return latest
+  end
+
+  #
+  # a handy function to get the value of a long property path
+  # without needing to test the existance individually of each part
+  # in the tree.
+  #
+  # e.g. property("stunnel.clients.couch_client")
+  #
+  def property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      return nil if latest.nil?
+    end
+    return latest
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/http_helper.rb b/tests/server-tests/helpers/http_helper.rb
new file mode 100644
index 00000000..0d0bb7d5
--- /dev/null
+++ b/tests/server-tests/helpers/http_helper.rb
@@ -0,0 +1,157 @@
+require 'net/http'
+
+class LeapTest
+
+  #
+  # In order to easily provide detailed error messages, it is useful
+  # to append a memo to a url string that details what this url is for
+  # (e.g. stunnel, haproxy, etc).
+  #
+  # So, the url happens to be a UrlString, the memo field is used
+  # if there is an error in assert_get.
+  #
+  class URLString < String
+    attr_accessor :memo
+  end
+
+  #
+  # aliases for http_send()
+  #
+  def get(url, params=nil, options=nil, &block)
+    http_send("GET", url, params, options, &block)
+  end
+  def delete(url, params=nil, options=nil, &block)
+    http_send("DELETE", url, params, options, &block)
+  end
+  def post(url, params=nil, options=nil, &block)
+    http_send("POST", url, params, options, &block)
+  end
+  def put(url, params=nil, options=nil, &block)
+    http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # send a GET, DELETE, POST, or PUT
+  # yields |body, response, error|
+  #
+  def http_send(method, url, params=nil, options=nil)
+    options ||= {}
+    response = nil
+
+    # build uri
+    uri = URI(url)
+    if params && (method == 'GET' || method == 'DELETE')
+      uri.query = URI.encode_www_form(params)
+    end
+
+    # build http
+    http = Net::HTTP.new uri.host, uri.port
+    if uri.scheme == 'https'
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http.use_ssl = true
+    end
+
+    # build request
+    request = build_request(method, uri, params, options)
+
+    # make http request
+    http.start do |agent|
+      response = agent.request(request)
+      yield response.body, response, nil
+    end
+  rescue => exc
+    yield nil, response, exc
+  end
+
+  #
+  # Aliases for assert_http_send()
+  #
+  def assert_get(url, params=nil, options=nil, &block)
+    assert_http_send("GET", url, params, options, &block)
+  end
+  def assert_delete(url, params=nil, options=nil, &block)
+    assert_http_send("DELETE", url, params, options, &block)
+  end
+  def assert_post(url, params=nil, options=nil, &block)
+    assert_http_send("POST", url, params, options, &block)
+  end
+  def assert_put(url, params=nil, options=nil, &block)
+    assert_http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # calls http_send, yielding results if successful or failing with
+  # descriptive info otherwise.
+  #
+  # options:
+  # - error_msg: custom error message to display.
+  # - ok_codes: in addition to 2xx, codes in this array will not produce an error.
+  #
+  def assert_http_send(method, url, params=nil, options=nil, &block)
+    options ||= {}
+    error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil)
+    http_send(method, url, params, options) do |body, response, error|
+      if response
+        code = response.code.to_i
+        ok = code >= 200 && code < 300
+        if options[:ok_codes]
+          ok ||= options[:ok_codes].include?(code)
+        end
+        if ok
+          if block
+            yield(body) if block.arity == 1
+            yield(body, response) if block.arity == 2
+            yield(body, response, error) if block.arity == 3
+          end
+        else
+          fail ["Expected success code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n")
+        end
+      else
+        fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error
+      end
+    end
+  end
+
+  #
+  # only a warning for now, should be a failure in the future
+  #
+  def assert_auth_fail(url, params)
+    uri = URI(url)
+    get(url, params) do |body, response, error|
+      unless response.code.to_s == "401"
+        warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')."
+        return false
+      end
+    end
+    true
+  end
+
+  private
+
+  def build_request(method, uri, params, options)
+    request = case method
+      when "GET"    then Net::HTTP::Get.new(uri.request_uri)
+      when "DELETE" then Net::HTTP::Delete.new(uri.request_uri)
+      when "POST"   then Net::HTTP::Post.new(uri.request_uri)
+      when "PUT"    then Net::HTTP::Put.new(uri.request_uri)
+    end
+    if uri.user
+      request.basic_auth uri.user, uri.password
+    end
+    if params && (method == 'POST' || method == 'PUT')
+      if options[:format] == :json || options[:format] == 'json'
+        request["Content-Type"] = "application/json"
+        request.body = params.to_json
+      else
+        request.set_form_data(params) if params
+      end
+    end
+    if options[:headers]
+      options[:headers].each do |key, value|
+        request[key] = value
+      end
+    end
+    request
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/network_helper.rb b/tests/server-tests/helpers/network_helper.rb
new file mode 100644
index 00000000..713d57aa
--- /dev/null
+++ b/tests/server-tests/helpers/network_helper.rb
@@ -0,0 +1,79 @@
+class LeapTest
+
+  #
+  # tcp connection helper with timeout
+  #
+  def try_tcp_connect(host, port, timeout = 5)
+    addr     = Socket.getaddrinfo(host, nil)
+    sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+    Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
+      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      begin
+        socket.connect_nonblock(sockaddr)
+      rescue IO::WaitReadable
+        if IO.select([socket], nil, nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      rescue IO::WaitWritable
+        if IO.select(nil, [socket], nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      end
+      return socket
+    end
+  end
+
+  def try_tcp_write(socket, timeout = 5)
+    begin
+      socket.write_nonblock("\0")
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def try_tcp_read(socket, timeout = 5)
+    begin
+      socket.read_nonblock(1)
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def assert_tcp_socket(host, port, msg=nil)
+    begin
+      socket = try_tcp_connect(host, port, 1)
+      #try_tcp_write(socket,1)
+      #try_tcp_read(socket,1)
+    rescue StandardError => exc
+      fail ["Failed to open socket #{host}:#{port}", exc, msg].compact.join("\n")
+    ensure
+      socket.close if socket
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/os_helper.rb b/tests/server-tests/helpers/os_helper.rb
new file mode 100644
index 00000000..9923d5b1
--- /dev/null
+++ b/tests/server-tests/helpers/os_helper.rb
@@ -0,0 +1,41 @@
+class LeapTest
+
+  #
+  # works like pgrep command line
+  # return an array of hashes like so [{:pid => "1234", :process => "ls"}]
+  #
+  def pgrep(match)
+    output = `pgrep --full --list-name '#{match}'`
+    output.each_line.map{|line|
+      pid = line.split(' ')[0]
+      process = line.gsub(/(#{pid} |\n)/, '')
+      # filter out pgrep cmd itself
+      # on wheezy hosts, the "process" var contains the whole cmd including all parameters
+      # on jessie hosts, it only contains the first cmd (which is the default sheel invoked by 'sh')
+      if process =~ /^sh/
+        nil
+      else
+        {:pid => pid, :process => process}
+      end
+    }.compact
+  end
+
+  def assert_running(process, options={})
+    processes = pgrep(process)
+    assert processes.any?, "No running process for #{process}"
+    if options[:single]
+      assert processes.length == 1, "More than one process for #{process}"
+    end
+  end
+
+  #
+  # runs the specified command, failing on a non-zero exit status.
+  #
+  def assert_run(command)
+    output = `#{command} 2>&1`
+    if $?.exitstatus != 0
+      fail "Error running `#{command}`:\n#{output}"
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/smtp_helper.rb b/tests/server-tests/helpers/smtp_helper.rb
new file mode 100644
index 00000000..ea7fb9fa
--- /dev/null
+++ b/tests/server-tests/helpers/smtp_helper.rb
@@ -0,0 +1,45 @@
+require 'net/smtp'
+
+class LeapTest
+
+  TEST_EMAIL_USER = "test_user_email"
+  TEST_BAD_USER = "test_user_bad"
+
+  MSG_BODY = %(Since it seems that any heart which beats for freedom has the right only to a
+lump of lead, I too claim my share. If you let me live, I shall never stop
+crying for revenge and I shall avenge my brothers. I have finished. If you are
+not cowards, kill me!
+
+--Louise Michel)
+
+  def send_email(recipient, options={})
+    sender = options[:sender] || recipient
+    helo_domain = property('domain.full_suffix')
+    headers = {
+      "Date" => Time.now.utc,
+      "From" => sender,
+      "To" => recipient,
+      "Subject" => "Test Message",
+      "X-LEAP-TEST" => "true"
+    }.merge(options[:headers]||{})
+    message = []
+    headers.each do |key, value|
+      message << "#{key}: #{value}"
+    end
+    message << ""
+    message << MSG_BODY
+    Net::SMTP.start('localhost', 25, helo_domain) do |smtp|
+      smtp.send_message message.join("\n"), recipient, sender
+    end
+  end
+
+  def assert_send_email(recipient, options={})
+    begin
+      send_email(recipient, options)
+    rescue IOError, Net::OpenTimeout,
+           Net::ReadTimeout, Net::SMTPError => e
+      fail "Could not send mail to #{recipient} (#{e})"
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/server-tests/helpers/soledad_sync.py b/tests/server-tests/helpers/soledad_sync.py
new file mode 100755
index 00000000..f4fc81ae
--- /dev/null
+++ b/tests/server-tests/helpers/soledad_sync.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+soledad_sync.py
+
+This script exercises soledad synchronization.
+Its exit code is 0 if the sync took place correctly, 1 otherwise.
+
+It takes 5 arguments:
+
+  uuid: uuid of the user to sync
+  token: a valid session token
+  server: the url of the soledad server we should connect to
+  cert_file: the file containing the certificate for the CA that signed the
+             cert for the soledad server.
+  password: the password for the user to sync
+
+__author__: kali@leap.se
+"""
+import os
+import shutil
+import sys
+import tempfile
+
+# This is needed because the twisted shipped with wheezy is too old
+# to do proper ssl verification.
+os.environ['SKIP_TWISTED_SSL_CHECK'] = '1'
+
+from twisted.internet import defer, reactor
+from twisted.python import log
+
+from client_side_db import get_soledad_instance
+from leap.common.events import flags
+
+flags.set_events_enabled(False)
+
+NUMDOCS = 1
+USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0]
+
+
+def bail(msg, exitcode):
+    print "[!] %s" % msg
+    sys.exit(exitcode)
+
+
+def create_docs(soledad):
+    """
+    Populates the soledad database with dummy messages, so we can exercise
+    sending payloads during the sync.
+    """
+    deferreds = []
+    for index in xrange(NUMDOCS):
+        deferreds.append(soledad.create_doc({'payload': 'dummy'}))
+    return defer.gatherResults(deferreds)
+
+# main program
+
+if __name__ == '__main__':
+
+    tempdir = tempfile.mkdtemp()
+
+    def rm_tempdir():
+        shutil.rmtree(tempdir)
+
+    if len(sys.argv) < 6:
+        bail(USAGE, 2)
+
+    uuid, token, server, cert_file, passphrase = sys.argv[1:]
+    s = get_soledad_instance(
+        uuid, passphrase, tempdir, server, cert_file, token)
+
+    def onSyncDone(sync_result):
+        print "SYNC_RESULT:", sync_result
+        s.close()
+        rm_tempdir()
+        reactor.stop()
+
+    def log_and_exit(f):
+        log.err(f)
+        rm_tempdir()
+        reactor.stop()
+
+    def start_sync():
+        d = create_docs(s)
+        d.addCallback(lambda _: s.sync())
+        d.addCallback(onSyncDone)
+        d.addErrback(log_and_exit)
+
+    reactor.callWhenRunning(start_sync)
+    reactor.run()
diff --git a/tests/server-tests/helpers/srp_helper.rb b/tests/server-tests/helpers/srp_helper.rb
new file mode 100644
index 00000000..b30fa768
--- /dev/null
+++ b/tests/server-tests/helpers/srp_helper.rb
@@ -0,0 +1,171 @@
+#
+# Here are some very stripped down helper methods for SRP, useful only for
+# testing the client side.
+#
+
+require 'digest'
+require 'openssl'
+require 'securerandom'
+require 'base64'
+
+module SRP
+
+  ##
+  ## UTIL
+  ##
+
+  module Util
+    PRIME_N = <<-EOS.split.join.hex
+115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3
+    EOS
+    BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N)
+eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657
+6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da
+d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5
+7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb
+06e3
+    EOS
+    GENERATOR = 2 # g
+
+    def hn_xor_hg
+      byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR))
+    end
+
+    # a^n (mod m)
+    def modpow(a, n, m = BIG_PRIME_N)
+      r = 1
+      while true
+        r = r * a % m if n[0] == 1
+        n >>= 1
+        return r if n == 0
+        a = a * a % m
+      end
+    end
+
+    #  Hashes the (long) int args
+    def sha256_int(*args)
+      sha256_hex(*args.map{|a| "%02x" % a})
+    end
+
+    #  Hashes the hex args
+    def sha256_hex(*args)
+      h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('')
+      sha256_str([h].pack('H*'))
+    end
+
+    def sha256_str(s)
+      Digest::SHA2.hexdigest(s)
+    end
+
+    def bigrand(bytes)
+      OpenSSL::Random.random_bytes(bytes).unpack("H*")[0]
+    end
+
+    def multiplier
+      @muliplier ||= calculate_multiplier
+    end
+
+    protected
+
+    def calculate_multiplier
+      sha256_int(BIG_PRIME_N, GENERATOR).hex
+    end
+
+    def byte_xor_hex(a, b)
+      a = [a].pack('H*')
+      b = [b].pack('H*')
+      a.bytes.each_with_index.map do |a_byte, i|
+        (a_byte ^ (b[i].ord || 0)).chr
+      end.join
+    end
+  end
+
+  ##
+  ## SESSION
+  ##
+
+  class Session
+    include SRP::Util
+    attr_accessor :user
+    attr_accessor :bb
+
+    def initialize(user, aa=nil)
+      @user = user
+      @a = bigrand(32).hex
+    end
+
+    def m
+      @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k)
+    end
+
+    def aa
+      @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N)
+    end
+
+    protected
+
+    # client: K = H( (B - kg^x) ^ (a + ux) )
+    def client_secret
+      base = bb.hex
+      base -= modpow(GENERATOR, @user.private_key) * multiplier
+      base = base % BIG_PRIME_N
+      modpow(base, @user.private_key * u.hex + @a)
+    end
+
+    def k
+      @k ||= sha256_int(client_secret)
+    end
+
+    def n_xor_g_long
+      @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join
+    end
+
+    def login_hash
+      @login_hash ||= sha256_str(@user.username)
+    end
+
+    def u
+      @u ||= sha256_hex(aa, bb)
+    end
+  end
+
+  ##
+  ## Dummy USER
+  ##
+
+  class User
+    include SRP::Util
+
+    attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted
+
+    def initialize(username=nil)
+      @username = username || "tmp_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '')
+      @password = "password_" + SecureRandom.urlsafe_base64(10)
+      @salt     = bigrand(4).hex
+      @verifier = modpow(GENERATOR, private_key)
+      @ok       = false
+      @deleted  = false
+    end
+
+    def private_key
+      @private_key ||= calculate_private_key
+    end
+
+    def to_params
+      {
+        'user[login]' => @username,
+        'user[password_verifier]' => @verifier.to_s(16),
+        'user[password_salt]' => @salt.to_s(16)
+      }
+    end
+
+    private
+
+    def calculate_private_key
+      shex = '%x' % [@salt]
+      inner = sha256_str([@username, @password].join(':'))
+      sha256_hex(shex, inner).hex
+    end
+  end
+
+end
diff --git a/tests/server-tests/order.rb b/tests/server-tests/order.rb
new file mode 100644
index 00000000..14aad9be
--- /dev/null
+++ b/tests/server-tests/order.rb
@@ -0,0 +1,22 @@
+class LeapCli::Config::Node
+  #
+  # returns a list of node names that should be tested before this node.
+  # make sure to not return ourselves (please no dependency loops!).
+  #
+  # NOTE: this method determines the order that nodes are tested in. To specify
+  # the order of tests on a particular node, each test can call class method
+  # LeapTest.depends_on().
+  #
+  def test_dependencies
+    dependents = LeapCli::Config::ObjectList.new
+
+    # webapp, mx, and soledad depend on couchdb nodes
+    if services.include?('webapp') || services.include?('mx') || services.include?('soledad')
+      if !services.include?('couchdb')
+        dependents.merge! nodes_like_me[:services => 'couchdb']
+      end
+    end
+
+    dependents.keys.delete_if {|name| self.name == name}
+  end
+end
\ No newline at end of file
diff --git a/tests/server-tests/white-box/couchdb.rb b/tests/server-tests/white-box/couchdb.rb
new file mode 100644
index 00000000..44a2769b
--- /dev/null
+++ b/tests/server-tests/white-box/couchdb.rb
@@ -0,0 +1,169 @@
+raise SkipTest unless service?(:couchdb)
+
+require 'json'
+
+class CouchDB < LeapTest
+  depends_on "Network"
+
+  def setup
+  end
+
+  def test_00_Are_daemons_running?
+    assert_running 'bin/beam'
+    if multimaster?
+      assert_running 'bin/epmd'
+    end
+    pass
+  end
+
+  #
+  # check to make sure we can get welcome response from local couchdb
+  #
+  def test_01_Is_CouchDB_running?
+    assert_get(couchdb_url) do |body|
+      assert_match /"couchdb":"Welcome"/, body, "Could not get welcome message from #{couchdb_url}. Probably couchdb is not running."
+    end
+    pass
+  end
+
+  #
+  # all configured nodes are in 'cluster_nodes'
+  # all nodes online and communicating are in 'all_nodes'
+  #
+  # this seems backward to me, so it might be the other way around.
+  #
+  def test_03_Are_configured_nodes_online?
+    return unless multimaster?
+    url = couchdb_url("/_membership", :username => 'admin')
+    assert_get(url) do |body|
+      response = JSON.parse(body)
+      nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes']
+      nodes_available_but_not_configured = response['all_nodes'] - response['cluster_nodes']
+      if nodes_configured_but_not_available.any?
+        warn "These nodes are configured but not available:", nodes_configured_but_not_available
+      end
+      if nodes_available_but_not_configured.any?
+        warn "These nodes are available but not configured:", nodes_available_but_not_configured
+      end
+      if response['cluster_nodes'] == response['all_nodes']
+        pass
+      end
+    end
+  end
+
+  def test_04_Do_ACL_users_exist?
+    acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication']
+    url = couchdb_backend_url("/_users/_all_docs", :username => 'admin')
+    assert_get(url) do |body|
+      response = JSON.parse(body)
+      assert_equal acl_users.count, response['total_rows']
+      actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') }
+      assert_equal acl_users.sort, actual_users.sort
+    end
+    pass
+  end
+
+  def test_05_Do_required_databases_exist?
+    dbs_that_should_exist = ["customers","identities","keycache","shared","tickets","users", "tmp_users"]
+    dbs_that_should_exist << "tokens_#{rotation_suffix}"
+    dbs_that_should_exist << "sessions_#{rotation_suffix}"
+    dbs_that_should_exist.each do |db_name|
+      url = couchdb_url("/"+db_name, :username => 'admin')
+      assert_get(url) do |body|
+        assert response = JSON.parse(body)
+        assert_equal db_name, response['db_name']
+      end
+    end
+    pass
+  end
+
+  # disable ACL enforcement, because it's a known issue with bigcouch
+  # and will only confuse the user
+  # see https://leap.se/code/issues/6030 for more details
+  #
+  ## for now, this just prints warnings, since we are failing these tests.
+  ##
+
+  #def test_06_Is_ACL_enforced?
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  )
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  ) && ok
+  #  pass if ok
+  #end
+
+  def test_07_Can_records_be_created?
+    record = DummyRecord.new
+    url = couchdb_url("/tokens_#{rotation_suffix}", :username => 'admin')
+    assert_post(url, record, :format => :json) do |body|
+      assert response = JSON.parse(body), "POST response should be JSON"
+      assert response["ok"], "POST response should be OK"
+      assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body|
+        assert response = JSON.parse(body), "DELETE response should be JSON"
+        assert response["ok"], "DELETE response should be OK"
+      end
+    end
+    pass
+  end
+
+  #
+  # This is not really a "test", just an attempt to make sure that
+  # the mx tests that fire off dummy emails don't fill up the
+  # storage db.
+  #
+  # mx tests can't run this because they don't have access to
+  # the storage db.
+  #
+  # This "test" is responsible for both creating the db if it does not
+  # exist, and destroying if it does.
+  #
+  # Yes, this is super hacky. Properly, we should add something to
+  # the soledad api to support create/delete of user storage dbs.
+  #
+  def test_99_Delete_mail_storage_used_in_mx_tests
+    user = find_user_by_login(TEST_EMAIL_USER)
+    if user
+      if user_db_exists?(user["id"])
+        # keep the test email db from filling up:
+        assert_destroy_user_db(user["id"], :username => 'admin')
+      end
+      # either way, make sure we leave a db for the mx tests:
+      assert_create_user_db(user["id"], :username => 'admin')
+    end
+    silent_pass
+  end
+
+  private
+
+  def multimaster?
+    mode == "multimaster"
+  end
+
+  def mode
+    assert_property('couch.mode')
+  end
+
+  # TODO: admin port is hardcoded for now but should be configurable.
+  def couchdb_backend_url(path="", options={})
+    options = {port: multimaster? && "5986"}.merge options
+    couchdb_url(path, options)
+  end
+
+  def rotation_suffix
+    rotation_suffix = Time.now.utc.to_i / 2592000 # monthly
+  end
+
+  require 'securerandom'
+  require 'digest/sha2'
+  class DummyRecord < Hash
+    def initialize
+      self['data'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '')
+      self['_id'] = Digest::SHA512.hexdigest(self['data'])
+    end
+  end
+
+end
diff --git a/tests/server-tests/white-box/dummy.rb b/tests/server-tests/white-box/dummy.rb
new file mode 100644
index 00000000..a3e8ad68
--- /dev/null
+++ b/tests/server-tests/white-box/dummy.rb
@@ -0,0 +1,71 @@
+# only run in the dummy case where there is no hiera.yaml file.
+raise SkipTest unless $node["dummy"]
+
+class Robot
+  def can_shoot_lasers?
+    "OHAI!"
+  end
+
+  def can_fly?
+    "YES!"
+  end
+end
+
+class TestDummy < LeapTest
+  def setup
+    @robot = Robot.new
+  end
+
+  def test_lasers
+    assert_equal "OHAI!", @robot.can_shoot_lasers?
+    pass
+  end
+
+  def test_fly
+    refute_match /^no/i, @robot.can_fly?
+    pass
+  end
+
+  def test_fail
+    fail "fail"
+    pass
+  end
+
+  def test_01_will_be_skipped
+    skip "test this later"
+    pass
+  end
+
+  def test_socket_failure
+    assert_tcp_socket('localhost', 900000)
+    pass
+  end
+
+  def test_warn
+    block_test do
+      warn "not everything", "is a success or failure"
+    end
+  end
+
+  # used to test extracting the proper caller even when in a block
+  def block_test
+    yield
+  end
+
+  def test_socket_success
+    fork {
+      Socket.tcp_server_loop('localhost', 12345) do |sock, client_addrinfo|
+        begin
+          sock.write('hi')
+        ensure
+          sock.close
+          exit
+        end
+      end
+    }
+    sleep 0.2
+    assert_tcp_socket('localhost', 12345)
+    pass
+  end
+
+end
diff --git a/tests/server-tests/white-box/mx.rb b/tests/server-tests/white-box/mx.rb
new file mode 100644
index 00000000..0eeaacd0
--- /dev/null
+++ b/tests/server-tests/white-box/mx.rb
@@ -0,0 +1,271 @@
+raise SkipTest unless service?(:mx)
+
+require 'date'
+require 'json'
+require 'net/smtp'
+
+class Mx < LeapTest
+  depends_on "Network"
+  depends_on "Webapp" if service?(:webapp)
+
+  def setup
+  end
+
+  def test_01_Can_contact_couchdb?
+    dbs = ["identities"]
+    dbs.each do |db_name|
+      couchdb_urls("/"+db_name, couch_url_options).each do |url|
+        assert_get(url) do |body|
+          assert response = JSON.parse(body)
+          assert_equal db_name, response['db_name']
+        end
+      end
+    end
+    pass
+  end
+
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", couch_url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
+    end
+  end
+
+  #
+  # this test picks a random identity document, then queries
+  # using the by_address view for that same document again.
+  #
+  def test_03_Can_query_identities_db?
+    ident = pick_random_identity
+    address = ident['address']
+    url_base = %(/identities/_design/Identity/_view/by_address)
+    params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}")
+    assert_get(couchdb_url(url_base+params, couch_url_options)) do |body|
+      assert response = JSON.parse(body)
+      assert record = response['rows'].first
+      assert_equal address, record['doc']['address']
+      pass
+    end
+  end
+
+  def test_04_Are_MX_daemons_running?
+    assert_running '.*/usr/bin/twistd.*mx.tac'
+    assert_running '^/usr/lib/postfix/master$'
+    assert_running '^/usr/sbin/postfwd'
+    assert_running 'postfwd2::cache$'
+    assert_running 'postfwd2::policy$'
+    assert_running '^/usr/sbin/unbound$'
+    assert_running '^/usr/bin/freshclam'
+    assert_running '^/usr/sbin/opendkim'
+    if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0
+      assert_running '^/usr/sbin/clamd'
+      assert_running '^/usr/sbin/clamav-milter'
+      pass
+    else
+      skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running."
+    end
+  end
+
+  #
+  # TODO: test to make sure postmap returned the right result
+  #
+  def test_05_Can_postfix_query_leapmx?
+    ident = pick_random_identity(10, :with_public_key => true)
+    address = ident["address"]
+
+    #
+    # virtual alias map:
+    #
+    #   user@domain => 41c29a80a44f4775513c64ac9cab91b9@deliver.local
+    #
+    assert_run("postmap -v -q \"#{address}\" tcp:localhost:4242")
+
+    #
+    # recipient access map:
+    #
+    #   user@domain => [OK|REJECT|TEMP_FAIL]
+    #
+    # This map is queried by the mail server before delivery to the mail spool
+    # directory, and should check if the address is able to receive messages.
+    # Examples of reasons for denying delivery would be that the user is out of
+    # quota, is user, or have no pgp public key in the server.
+    #
+    # NOTE: in the future, when we support quota, we need to make sure that
+    # we don't randomly pick a user for this test that happens to be over quota.
+    #
+    assert_run("postmap -v -q \"#{address}\" tcp:localhost:2244")
+
+    #
+    # certificate validity map:
+    #
+    #  fa:2a:70:1f:d8:16:4e:1a:3b:15:c1:67:00:f0 => [200|500]
+    #
+    # Determines whether a particular SMTP client cert is authorized
+    # to relay mail, based on the fingerprint.
+    #
+    if ident["cert_fingerprints"]
+      not_expired = ident["cert_fingerprints"].select {|key, value|
+        Time.now.utc < DateTime.strptime("2016-01-03", "%F").to_time.utc
+      }
+      if not_expired.any?
+        fingerprint = not_expired.first
+        assert_run("postmap -v -q #{fingerprint} tcp:localhost:2424")
+      end
+    end
+
+    pass
+  end
+
+  #
+  # The email sent by this test might get bounced back.
+  # In this case, the test will pass, but the bounce message will
+  # get sent to root, so the sysadmin will still figure out pretty
+  # quickly that something is wrong.
+  #
+  def test_05_Can_deliver_email?
+    if pgrep('^/usr/sbin/clamd').empty? || pgrep('^/usr/sbin/clamav-milter').empty?
+      skip "Mail delivery is being deferred because clamav daemon is not running"
+    else
+      addr = [TEST_EMAIL_USER, property('domain.full_suffix')].join('@')
+      bad_addr = [TEST_BAD_USER, property('domain.full_suffix')].join('@')
+
+      assert !identity_exists?(bad_addr), "the address #{bad_addr} must not exist."
+      if !identity_exists?(addr)
+        user = assert_create_user(TEST_EMAIL_USER, :monitor)
+        upload_public_key(user.id, TEST_EMAIL_PUBLIC_KEY)
+      end
+      assert identity_exists?(addr), "The identity #{addr} should have been created, but it doesn't exist yet."
+      assert_send_email(addr)
+      assert_raises(Net::SMTPError) do
+        send_email(bad_addr)
+      end
+      pass
+    end
+  end
+
+  private
+
+  def couch_url_options
+    {
+      :username => property('couchdb_leap_mx_user.username'),
+      :password => property('couchdb_leap_mx_user.password')
+    }
+  end
+
+  #
+  # returns a random identity record that also has valid address
+  # and destination fields.
+  #
+  # options:
+  #
+  # * :with_public_key -- searches only for identities with public keys
+  #
+  # note to self: for debugging, here is the curl you want:
+  # curl --netrc "127.0.0.1:5984/identities/_design/Identity/_view/by_address?startkey=\"xxxx@leap.se\"&endkey=\"xxxx@leap.se\"&reduce=false&include_docs=true"
+  #
+  def pick_random_identity(tries=5, options={})
+    assert_get(couchdb_url("/identities", couch_url_options)) do |body|
+      assert response = JSON.parse(body)
+      doc_count = response['doc_count'].to_i
+      if doc_count <= 1
+        # the design document counts as one document.
+        skip "There are no identity documents yet."
+      else
+        # try repeatedly to get a valid doc
+        for i in 1..tries
+          offset    = rand(doc_count) # pick a random document
+          url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options)
+          assert_get(url) do |body|
+            assert response = JSON.parse(body)
+            record = response['rows'].first
+            if record['id'] =~ /_design/
+              next
+            elsif record['doc'] && record['doc']['address']
+              next if record['doc']['destination'].nil? || record['doc']['destination'].empty?
+              next if options[:with_public_key] && !record_has_key?(record)
+              return record['doc']
+            else
+              fail "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}"
+            end
+          end
+        end
+        if options[:with_public_key]
+          skip "Could not find an Identity document with a public key for testing."
+        else
+          fail "Failed to find a valid Identity document (with address and destination)."
+        end
+      end
+    end
+  end
+
+  def record_has_key?(record)
+    !record['doc']['keys'].nil? &&
+    !record['doc']['keys'].empty? &&
+    !record['doc']['keys']['pgp'].nil? &&
+    !record['doc']['keys']['pgp'].empty?
+  end
+
+  TEST_EMAIL_PUBLIC_KEY=<<HERE
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+mI0EVvzIKQEEAN4f8FOGntJGTTD+fFUQS6y/ihn6tYLtyGZZbCOd0t/9kHt/raoR
+xEUks8rCOPMqHX+yeHsvDBtDyZYTvyhtfuWrBUbYGW+QZ4Pdvo+7NyLHPW0dKsCB
+Czrx7pxqpq1oq+LpUFqpSfjJTfYaGVDNXrPK144a7Rox2+MCbgq3twnFABEBAAG0
+EiA8dGVzdF91c2VyX2VtYWlsPoi4BBMBAgAiBQJW/MgpAhsvBgsJCAcDAgYVCAIJ
+CgsEFgIDAQIeAQIXgAAKCRAqYf65XmeSk0orBADUXjEiGnjzyBpXqaiVmJr4MyfP
+IfKTK4a+4qvR+2fseD7hteF98m26i1YRI5omLp4/MnxGSpgKFKIuWIdkEiLg7IJc
+pFZVdoDVufEtzbj9gmOHlnteksbCtuESyB0Hytsba4uS9afcTJdGiPNMHeniI/SY
+UKcCcIrQmpNIoOA5OLiNBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD
+7PK4xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6Ok
+NkhfGfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+c
+m3JuZy1k7QARAQABiQE9BBgBAgAJBQJW/MgpAhsuAKgJECph/rleZ5KTnSAEGQEC
+AAYFAlb8yCkACgkQsJSYitQUOv4w1wQAn3atI5EsmRyw6iC6UVWWJv/lKi1Priyt
+DsrdH5xUmHUgp6VU8Pw9Y6G+sv50KLfbVQ1l+8/3B71TjadsOxh+PBPsEyYpK6WX
+TVGy44IDvFWGyOod8tmfcFN9IpU5DmSk/vny9G7RK/nbnta2VnfZOzwm5i3cNkPr
+FGPL1z0K3qs0VwP+M7BXdqBRSFDDBpG1J0TrZioEjvKeOsT/Ul8mbVt7HQpcN93I
+wTO4uky0Woy2nb7SbTQw6wOpU54u7+5dSQ03ltUHg1owy6Y3CMOeFL+e9ALpAZAU
+aMwY7zMFhqlPVZZMfdMLRsdLin67RIM+OJ6A925AM52bEQT1YwkQlP4mvQY=
+=qclE
+-----END PGP PUBLIC KEY BLOCK-----
+HERE
+
+  TEST_EMAIL_PRIVATE_KEY = <<HERE
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+lQHYBFb8yCkBBADeH/BThp7SRk0w/nxVEEusv4oZ+rWC7chmWWwjndLf/ZB7f62q
+EcRFJLPKwjjzKh1/snh7LwwbQ8mWE78obX7lqwVG2BlvkGeD3b6Puzcixz1tHSrA
+gQs68e6caqataKvi6VBaqUn4yU32GhlQzV6zyteOGu0aMdvjAm4Kt7cJxQARAQAB
+AAP8DTFfcE6UG1AioJDU6KZ9oCaGONHLuxmNaArSofDrR/ODA9rLAUlp22N5LEdJ
+46NyOhXrEwHx2aK2k+vbVDbgrP4ZTH7GxIK/2KzmH4zX0fWUNsaRy94Q12lJegXH
+sH2Im8Jjxu16YwGgFNTX1fCPqLB6WdQpf1796s6+/3PnCDcCAOXTCul3N7V5Yl+9
+N2Anupn+qNDXKT/kiKIZLHsMbo7EriGWReG3lLj1cOJPC6Nf0uOEri4ErSjFEadR
+F2TNITsCAPdsZjc5RGppUXyBfxhQkAnZ0r+UT2meCH3g3EVh3W9SBrXNhwipNpW3
+bPzRjUCDtmA8EOvd93oPCZv4/tb50P8B/jC+QIZ3GncP1CFPSVDoIZ7OUU5M1330
+DP77vG1GxeQvYO/hlxL5/KdtTR6m5zlIuooDxUaNJz1w5/oVjlG3NZKpl7QSIDx0
+ZXN0X3VzZXJfZW1haWw+iLgEEwECACIFAlb8yCkCGy8GCwkIBwMCBhUIAgkKCwQW
+AgMBAh4BAheAAAoJECph/rleZ5KTSisEANReMSIaePPIGlepqJWYmvgzJ88h8pMr
+hr7iq9H7Z+x4PuG14X3ybbqLVhEjmiYunj8yfEZKmAoUoi5Yh2QSIuDsglykVlV2
+gNW58S3NuP2CY4eWe16SxsK24RLIHQfK2xtri5L1p9xMl0aI80wd6eIj9JhQpwJw
+itCak0ig4Dk4nQHYBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD7PK4
+xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6OkNkhf
+GfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+cm3Ju
+Zy1k7QARAQABAAP9HrUaGvdpqTwVx3cHyXUhId6GzCuuKyaP4mZoGeBCcaQS2vQR
+YtiykwBwX/AlfwSFJmmHKB6EErWIA+QyaEFR/fO56cHD2TY3Ql0BGcuHIx3+9pkp
+biPBZdiiGz7oa6k6GWsbKSksqwV8poSXV7qbn+Bjm2xCM4VnjNZIrFtL7fkCAMOf
+e9yHBFoXfc175bkNXEUXrNS34kv2ODAlx6KyY+PS77D+nprpHpGCnLn77G+xH1Xi
+qvX1Dr/iSQU5Tzsd+tcCAPkYZulaC/9itwme7wIT3ur+mdqMHymsCzv9193iLgjJ
+9t7fARo18yB845hI9Xv7TwRcoyuSpfvuM05rCMRzydsCAOI1MZeKtZSogXVa9QTX
+sVGZeCkrujSVOgsA3w48OLc2OrwZskDfx5QHfeJnumjQLut5qsnZ+1onj9P2dGdn
+JaChe4kBPQQYAQIACQUCVvzIKQIbLgCoCRAqYf65XmeSk50gBBkBAgAGBQJW/Mgp
+AAoJELCUmIrUFDr+MNcEAJ92rSORLJkcsOogulFVlib/5SotT64srQ7K3R+cVJh1
+IKelVPD8PWOhvrL+dCi321UNZfvP9we9U42nbDsYfjwT7BMmKSull01RsuOCA7xV
+hsjqHfLZn3BTfSKVOQ5kpP758vRu0Sv5257WtlZ32Ts8JuYt3DZD6xRjy9c9Ct6r
+NFcD/jOwV3agUUhQwwaRtSdE62YqBI7ynjrE/1JfJm1bex0KXDfdyMEzuLpMtFqM
+tp2+0m00MOsDqVOeLu/uXUkNN5bVB4NaMMumNwjDnhS/nvQC6QGQFGjMGO8zBYap
+T1WWTH3TC0bHS4p+u0SDPjiegPduQDOdmxEE9WMJEJT+Jr0G
+=hvJM
+-----END PGP PRIVATE KEY BLOCK-----
+HERE
+
+end
diff --git a/tests/server-tests/white-box/network.rb b/tests/server-tests/white-box/network.rb
new file mode 100644
index 00000000..a08cdfbe
--- /dev/null
+++ b/tests/server-tests/white-box/network.rb
@@ -0,0 +1,90 @@
+require 'socket'
+require 'openssl'
+
+raise SkipTest if $node["dummy"]
+
+class Network < LeapTest
+
+  def setup
+  end
+
+  def test_01_Can_connect_to_internet?
+    assert_get('http://www.google.com/images/srpr/logo11w.png')
+    pass
+  end
+
+  #
+  # example properties:
+  #
+  # stunnel:
+  #   ednp_clients:
+  #     elk_9002:
+  #       accept_port: 4003
+  #       connect: elk.dev.bitmask.i
+  #       connect_port: 19002
+  #   couch_server:
+  #     accept: 15984
+  #     connect: "127.0.0.1:5984"
+  #
+  def test_02_Is_stunnel_running?
+    ignore unless $node['stunnel']
+    good_stunnel_pids = []
+    release = `facter lsbmajdistrelease`
+    if release.to_i > 7
+      # on jessie, there is only one stunnel proc running instead of 6
+      expected = 1
+    else
+      expected = 6
+    end
+    $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs|
+      stunnel_configs.each do |stunnel_name, stunnel_conf|
+        config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+        processes = pgrep(config_file_name)
+        assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`"
+        good_stunnel_pids += processes.map{|ps| ps[:pid]}
+        assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+        assert_tcp_socket('localhost', port)
+      end
+    end
+    $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf|
+      config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+      processes = pgrep(config_file_name)
+      assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`"
+      good_stunnel_pids += processes.map{|ps| ps[:pid]}
+      assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', accept_port)
+      assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', connect_port,
+        "The local connect endpoint for stunnel `#{stunnel_name}` is unavailable.\n"+
+        "This is probably caused by a daemon that died or failed to start on\n"+
+        "port `#{connect_port}`, not stunnel itself.")
+    end
+    all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
+    assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
+    pass
+  end
+
+  def test_03_Is_shorewall_running?
+    ignore unless File.exist?('/sbin/shorewall')
+    assert_run('/sbin/shorewall status')
+    pass
+  end
+
+  THIRTY_DAYS = 60*60*24*30
+
+  def test_04_Are_server_certificates_valid?
+    cert_paths = ["/etc/x509/certs/leap_commercial.crt", "/etc/x509/certs/leap.crt"]
+    cert_paths.each do |cert_path|
+      if File.exist?(cert_path)
+        cert = OpenSSL::X509::Certificate.new(File.read(cert_path))
+        if Time.now > cert.not_after
+          fail "The certificate #{cert_path} expired on #{cert.not_after}"
+        elsif Time.now + THIRTY_DAYS > cert.not_after
+          fail "The certificate #{cert_path} will expire soon, on #{cert.not_after}"
+        end
+      end
+    end
+    pass
+  end
+
+end
diff --git a/tests/server-tests/white-box/openvpn.rb b/tests/server-tests/white-box/openvpn.rb
new file mode 100644
index 00000000..170d4503
--- /dev/null
+++ b/tests/server-tests/white-box/openvpn.rb
@@ -0,0 +1,16 @@
+raise SkipTest unless service?(:openvpn)
+
+class OpenVPN < LeapTest
+  depends_on "Network"
+
+  def setup
+  end
+
+  def test_01_Are_daemons_running?
+    assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$'
+    assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$'
+    assert_running '^/usr/sbin/unbound$'
+    pass
+  end
+
+end
diff --git a/tests/server-tests/white-box/soledad.rb b/tests/server-tests/white-box/soledad.rb
new file mode 100644
index 00000000..d41bee58
--- /dev/null
+++ b/tests/server-tests/white-box/soledad.rb
@@ -0,0 +1,17 @@
+raise SkipTest unless service?(:soledad)
+
+require 'json'
+
+class Soledad < LeapTest
+  depends_on "Network"
+  depends_on "CouchDB" if service?(:couchdb)
+
+  def setup
+  end
+
+  def test_00_Is_Soledad_running?
+    assert_running '.*/usr/bin/twistd.*--wsgi=leap.soledad.server.application'
+    pass
+  end
+
+end
diff --git a/tests/server-tests/white-box/webapp.rb b/tests/server-tests/white-box/webapp.rb
new file mode 100644
index 00000000..40c234d6
--- /dev/null
+++ b/tests/server-tests/white-box/webapp.rb
@@ -0,0 +1,114 @@
+raise SkipTest unless service?(:webapp)
+
+require 'json'
+
+class Webapp < LeapTest
+  depends_on "Network"
+
+  def setup
+  end
+
+  def test_01_Can_contact_couchdb?
+    url = couchdb_url("", url_options)
+    assert_get(url) do |body|
+      assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+    end
+    pass
+  end
+
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
+    end
+  end
+
+  def test_03_Are_daemons_running?
+    assert_running '^/usr/sbin/apache2'
+    assert_running '^/usr/bin/ruby /usr/bin/nickserver'
+    pass
+  end
+
+  #
+  # this is technically a black-box test. so, move this when we have support
+  # for black box tests.
+  #
+  def test_04_Can_access_webapp?
+    assert_get('https://' + $node['webapp']['domain'] + '/')
+    pass
+  end
+
+  def test_05_Can_create_and_authenticate_and_delete_user_via_API?
+    if property('webapp.allow_registration')
+      assert_tmp_user
+      pass
+    else
+      skip "New user registrations are disabled."
+    end
+  end
+
+  def test_06_Can_sync_Soledad?
+    return unless property('webapp.allow_registration')
+    soledad_config = property('definition_files.soledad_service')
+    if soledad_config && !soledad_config.empty?
+      soledad_server = pick_soledad_server(soledad_config)
+      if soledad_server
+        assert_tmp_user do |user|
+          command = File.expand_path "../../helpers/soledad_sync.py", __FILE__
+          soledad_url = "https://#{soledad_server}/user-#{user.id}"
+          soledad_cert = "/usr/local/share/ca-certificates/leap_ca.crt"
+          assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url} #{soledad_cert} #{user.password}"
+          assert_user_db_privileges(user)
+          pass
+        end
+      end
+    else
+      skip 'No soledad service configuration'
+    end
+  end
+
+  private
+
+  def url_options
+    {
+      :username => property('webapp.couchdb_webapp_user.username'),
+      :password => property('webapp.couchdb_webapp_user.password')
+    }
+  end
+
+  #
+  # pick a random soledad server.
+  # I am not sure why, but using IP address directly does not work.
+  #
+  def pick_soledad_server(soledad_config_json_str)
+    soledad_config = JSON.parse(soledad_config_json_str)
+    host_name = soledad_config['hosts'].keys.shuffle.first
+    if host_name
+      hostname = soledad_config['hosts'][host_name]['hostname']
+      port = soledad_config['hosts'][host_name]['port']
+      return "#{hostname}:#{port}"
+    else
+      return nil
+    end
+  end
+
+  #
+  # checks if user db exists and is properly protected
+  #
+  def assert_user_db_privileges(user)
+    db_name = "/user-#{user.id}"
+    get(couchdb_url(db_name)) do |body, response, error|
+      code = response.code.to_i
+      assert code != 404, "Could not find user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}"
+      # After moving to couchdb, webapp user is not allowed to Read user dbs,
+      # but the return code for non-existent databases is 404. See #7674
+      # 401 should come as we aren't supposed to have read privileges on it.
+      assert code != 200, "Incorrect security settings (design doc) on user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}"
+      assert code == 401, "Unknown error on user db on user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}"
+    end
+  end
+
+end