Merge branch 'develop'

Conflicts:
	platform.rb

Change-Id: Ic2e08e594d29a585691341c8667ac0b64933a505

16 files changed, 1053 insertions, 97 deletions

diff --git a/tests/README.md b/tests/README.md
index debbf700..814c25b1 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,12 +1,25 @@
-This directory contains to kinds of tests:
+Tests
+---------------------------------
 
-White Box Tests
-================================
+tests/white-box/
 
-These tests are run on the server as superuser. They are for troubleshooting any problems with the internal setup of the server.
+    These tests are run on the server as superuser. They are for
+    troubleshooting any problems with the internal setup of the server.
 
-Black Box Tests
-================================
+tests/black-box/
+
+    These test are run the user's local machine. They are for troubleshooting
+    any external problems with the service exposed by the server.
+
+Additional Files
+---------------------------------
+
+tests/helpers/
+
+    Utility functions made available to all tests.
+
+tests/order.rb
+
+    Configuration file to specify which nodes should be tested in which order.
 
-These test are run the user's local machine. They are for troubleshooting any external problems with the service exposed by the server.
 
diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb
new file mode 100644
index 00000000..9b26eaaf
--- /dev/null
+++ b/tests/helpers/bonafide_helper.rb
@@ -0,0 +1,99 @@
+#
+# helper for the communication with the provider API for creating, authenticating, and deleting accounts.
+#
+
+class LeapTest
+
+  def assert_tmp_user
+    user = assert_create_user
+    assert_authenticate_user(user)
+    yield user if block_given?
+    assert_delete_user(user)
+  rescue StandardError, MiniTest::Assertion => exc
+    begin
+      assert_delete_user(user)
+    rescue
+    end
+    raise exc
+  end
+
+  def api_url(path)
+    api = property('api')
+    "https://%{domain}:%{port}#{path}" % {
+      :domain   => api['domain'],
+      :port     => api['port']
+    }
+  end
+
+  #
+  # attempts to create a user account via the API,
+  # returning the user object if successful.
+  #
+  def assert_create_user
+    user = SRP::User.new
+    url = api_url("/1/users.json")
+    assert_post(url, user.to_params) do |body|
+      assert response = JSON.parse(body), 'response should be JSON'
+      assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead."
+    end
+    user.ok = true
+    return user
+  end
+
+  #
+  # attempts to authenticate user. if successful,
+  # user object is updated with id and session token.
+  #
+  def assert_authenticate_user(user)
+    url = api_url("/1/sessions.json")
+    session = SRP::Session.new(user)
+    params = {'login' => user.username, 'A' => session.aa}
+    assert_post(url, params) do |response, body|
+      cookie = response['Set-Cookie'].split(';').first
+      assert(response = JSON.parse(body), 'response should be JSON')
+      assert(session.bb = response["B"], 'response should include "B"')
+      url = api_url("/1/sessions/login.json")
+      params = {'client_auth' => session.m, 'A' => session.aa}
+      options = {:headers => {'Cookie' => cookie}}
+      assert_put(url, params, options) do |body|
+        assert(response = JSON.parse(body), 'response should be JSON')
+        assert(response['M2'], 'response should include M2')
+        user.session_token = response['token']
+        user.id = response['id']
+        assert(user.session_token, 'response should include token')
+        assert(user.id, 'response should include user id')
+      end
+    end
+  end
+
+  #
+  # attempts to destroy a user account via the API.
+  #
+  def assert_delete_user(user)
+    if user && user.ok && user.id && user.session_token && !user.deleted
+      url = api_url("/1/users/#{user.id}.json")
+      options = {:headers => {
+        "Authorization" => "Token token=\"#{user.session_token}\""
+      }}
+      params = {
+        :identities => 'destroy'
+      }
+      user.deleted = true
+      delete(url, params, options) do |body, response, error|
+        assert error.nil?, "Error deleting user: #{error}"
+        assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+        assert(response = JSON.parse(body), 'Delete response should be JSON')
+        assert(response["success"], 'Deleting user should be a success')
+      end
+      domain = property('domain.full_suffix')
+      identities_url = couchdb_url("/identities/_design/Identity/_view/by_address?key=%22#{user.username}@#{domain}%22")
+      get(identities_url) do |body, response, error|
+        assert error.nil?, "Error checking identities db: #{error}"
+        assert response.code.to_i == 200, "Unable to check that user identity was deleted: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+        assert(response = JSON.parse(body), 'Couch response should be JSON')
+        assert response['rows'].empty?, "Identity should have been deleted for test user #{user.username} (id #{user.id}), but was not! Response was: #{body}."
+      end
+    end
+  end
+
+end
diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb
new file mode 100644
index 00000000..d4d3c0e0
--- /dev/null
+++ b/tests/helpers/couchdb_helper.rb
@@ -0,0 +1,103 @@
+class LeapTest
+
+  #
+  # generates a couchdb url for when couchdb is running
+  # remotely and is available via stunnel.
+  #
+  # example properties:
+  #
+  # stunnel:
+  #   clients:
+  #     couch_client:
+  #       couch1_5984:
+  #         accept_port: 4000
+  #         connect: couch1.bitmask.i
+  #         connect_port: 15984
+  #
+  def couchdb_urls_via_stunnel(path="", options=nil)
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf|
+      assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+      URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url|
+        remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
+        url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
+      }
+    end
+  end
+
+  #
+  # generates a couchdb url for accessing couchdb via haproxy
+  #
+  # example properties:
+  #
+  # haproxy:
+  #   couch:
+  #     listen_port: 4096
+  #     servers:
+  #       panda:
+  #         backup: false
+  #         host: localhost
+  #         port: 4000
+  #         weight: 100
+  #         writable: true
+  #
+  def couchdb_url_via_haproxy(path="", options=nil)
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    port = assert_property('haproxy.couch.listen_port')
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via haproxy)'
+    }
+  end
+
+  #
+  # generates a couchdb url for when couchdb is running locally.
+  #
+  # example properties:
+  #
+  # couch:
+  #   port: 5984
+  #
+  def couchdb_url_via_localhost(path="", options=nil)
+    port = (options && options[:port]) || assert_property('couch.port')
+    if options && options[:username]
+      password = property("couch.users.%{username}.password" % options)
+      userpart = "%s:%s@" % [options[:username], password]
+    else
+      userpart = ""
+    end
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via direct localhost connection)'
+    }
+  end
+
+  #
+  # returns a single url for accessing couchdb
+  #
+  def couchdb_url(path="", options=nil)
+    if property('couch.port')
+      couchdb_url_via_localhost(path, options)
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options).first
+    end
+  end
+
+  #
+  # returns an array of urls for accessing couchdb
+  #
+  def couchdb_urls(path="", options=nil)
+    if property('couch.port')
+      [couchdb_url_via_localhost(path, options)]
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options)
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/files_helper.rb b/tests/helpers/files_helper.rb
new file mode 100644
index 00000000..d6795889
--- /dev/null
+++ b/tests/helpers/files_helper.rb
@@ -0,0 +1,54 @@
+class LeapTest
+
+  #
+  # Matches the regexp in the file, and returns the first matched string (or fails if no match).
+  #
+  def file_match(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures.first
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # Matches the regexp in the file, and returns array of matched strings (or fails if no match).
+  #
+  def file_matches(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # checks to make sure the given property path exists in $node (e.g. hiera.yaml)
+  # and returns the value
+  #
+  def assert_property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      fail "Required node property `#{property}` is missing." if latest.nil?
+    end
+    return latest
+  end
+
+  #
+  # a handy function to get the value of a long property path
+  # without needing to test the existance individually of each part
+  # in the tree.
+  #
+  # e.g. property("stunnel.clients.couch_client")
+  #
+  def property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      return nil if latest.nil?
+    end
+    return latest
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb
new file mode 100644
index 00000000..0b13b754
--- /dev/null
+++ b/tests/helpers/http_helper.rb
@@ -0,0 +1,145 @@
+require 'net/http'
+
+class LeapTest
+
+  #
+  # In order to easily provide detailed error messages, it is useful
+  # to append a memo to a url string that details what this url is for
+  # (e.g. stunnel, haproxy, etc).
+  #
+  # So, the url happens to be a UrlString, the memo field is used
+  # if there is an error in assert_get.
+  #
+  class URLString < String
+    attr_accessor :memo
+  end
+
+  #
+  # aliases for http_send()
+  #
+  def get(url, params=nil, options=nil, &block)
+    http_send("GET", url, params, options, &block)
+  end
+  def delete(url, params=nil, options=nil, &block)
+    http_send("DELETE", url, params, options, &block)
+  end
+  def post(url, params=nil, options=nil, &block)
+    http_send("POST", url, params, options, &block)
+  end
+  def put(url, params=nil, options=nil, &block)
+    http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # send a GET, DELETE, POST, or PUT
+  # yields |body, response, error|
+  #
+  def http_send(method, url, params=nil, options=nil)
+    options ||= {}
+    response = nil
+
+    # build uri
+    uri = URI(url)
+    if params && (method == 'GET' || method == 'DELETE')
+      uri.query = URI.encode_www_form(params)
+    end
+
+    # build http
+    http = Net::HTTP.new uri.host, uri.port
+    if uri.scheme == 'https'
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http.use_ssl = true
+    end
+
+    # build request
+    request = build_request(method, uri, params, options)
+
+    # make http request
+    http.start do |agent|
+      response = agent.request(request)
+      yield response.body, response, nil
+    end
+  rescue => exc
+    yield nil, response, exc
+  end
+
+  #
+  # Aliases for assert_http_send()
+  #
+  def assert_get(url, params=nil, options=nil, &block)
+    assert_http_send("GET", url, params, options, &block)
+  end
+  def assert_delete(url, params=nil, options=nil, &block)
+    assert_http_send("DELETE", url, params, options, &block)
+  end
+  def assert_post(url, params=nil, options=nil, &block)
+    assert_http_send("POST", url, params, options, &block)
+  end
+  def assert_put(url, params=nil, options=nil, &block)
+    assert_http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # calls http_send, yielding results if successful or failing with
+  # descriptive infor otherwise.
+  #
+  def assert_http_send(method, url, params=nil, options=nil, &block)
+    options ||= {}
+    error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil)
+    http_send(method, url, params, options) do |body, response, error|
+      if body && response && response.code.to_i >= 200 && response.code.to_i < 300
+        if block
+          yield(body) if block.arity == 1
+          yield(response, body) if block.arity == 2
+        end
+      elsif response
+        fail ["Expected a 200 status code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n")
+      else
+        fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error
+      end
+    end
+  end
+
+  #
+  # only a warning for now, should be a failure in the future
+  #
+  def assert_auth_fail(url, params)
+    uri = URI(url)
+    get(url, params) do |body, response, error|
+      unless response.code.to_s == "401"
+        warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')."
+        return false
+      end
+    end
+    true
+  end
+
+  private
+
+  def build_request(method, uri, params, options)
+    request = case method
+      when "GET"    then Net::HTTP::Get.new(uri.request_uri)
+      when "DELETE" then Net::HTTP::Delete.new(uri.request_uri)
+      when "POST"   then Net::HTTP::Post.new(uri.request_uri)
+      when "PUT"    then Net::HTTP::Put.new(uri.request_uri)
+    end
+    if uri.user
+      request.basic_auth uri.user, uri.password
+    end
+    if params && (method == 'POST' || method == 'PUT')
+      if options[:format] == :json || options[:format] == 'json'
+        request["Content-Type"] = "application/json"
+        request.body = params.to_json
+      else
+        request.set_form_data(params) if params
+      end
+    end
+    if options[:headers]
+      options[:headers].each do |key, value|
+        request[key] = value
+      end
+    end
+    request
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/network_helper.rb b/tests/helpers/network_helper.rb
new file mode 100644
index 00000000..ff92d382
--- /dev/null
+++ b/tests/helpers/network_helper.rb
@@ -0,0 +1,79 @@
+class LeapTest
+
+  #
+  # tcp connection helper with timeout
+  #
+  def try_tcp_connect(host, port, timeout = 5)
+    addr     = Socket.getaddrinfo(host, nil)
+    sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+    Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
+      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      begin
+        socket.connect_nonblock(sockaddr)
+      rescue IO::WaitReadable
+        if IO.select([socket], nil, nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      rescue IO::WaitWritable
+        if IO.select(nil, [socket], nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      end
+      return socket
+    end
+  end
+
+  def try_tcp_write(socket, timeout = 5)
+    begin
+      socket.write_nonblock("\0")
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def try_tcp_read(socket, timeout = 5)
+    begin
+      socket.read_nonblock(1)
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def assert_tcp_socket(host, port, msg=nil)
+    begin
+      socket = try_tcp_connect(host, port, 1)
+      #try_tcp_write(socket,1)
+      #try_tcp_read(socket,1)
+    rescue StandardError => exc
+      fail ["Failed to open socket #{host}:#{port}", exc].join("\n")
+    ensure
+      socket.close if socket
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb
new file mode 100644
index 00000000..aad67dda
--- /dev/null
+++ b/tests/helpers/os_helper.rb
@@ -0,0 +1,38 @@
+class LeapTest
+
+  #
+  # works like pgrep command line
+  # return an array of hashes like so [{:pid => "1234", :process => "ls"}]
+  #
+  def pgrep(match)
+    output = `pgrep --full --list-name '#{match}'`
+    output.each_line.map{|line|
+      pid = line.split(' ')[0]
+      process = line.gsub(/(#{pid} |\n)/, '')
+      if process =~ /pgrep --full --list-name/
+        nil
+      else
+        {:pid => pid, :process => process}
+      end
+    }.compact
+  end
+
+  def assert_running(process, options={})
+    processes = pgrep(process)
+    assert processes.any?, "No running process for #{process}"
+    if options[:single]
+      assert processes.length == 1, "More than one process for #{process}"
+    end
+  end
+
+  #
+  # runs the specified command, failing on a non-zero exit status.
+  #
+  def assert_run(command)
+    output = `#{command}`
+    if $?.exitstatus != 0
+      fail "Error running `#{command}`:\n#{output}"
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py
new file mode 100755
index 00000000..2fb865fc
--- /dev/null
+++ b/tests/helpers/soledad_sync.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+#
+# Test Soledad sync
+#
+# This script performs a slightly modified U1DB sync to the Soledad server and
+# returns whether that sync was successful or not.
+#
+# It takes three arguments:
+#
+#   uuid   -- uuid of the user to sync
+#   token  -- a valid session token
+#   server -- the url of the soledad server we should connect to
+#
+# For example:
+#
+#   soledad_sync.py f6bef0586fcfdb8705e26a58f2d9e580 uYO-4ucEJFksJ6afjmcYwIyap2vW7bv6uLxk0w_RfCc https://199.119.112.9:2323/user-f6bef0586fcfdb8705e26a58f2d9e580
+#
+
+import os
+import sys
+import traceback
+import tempfile
+import shutil
+import u1db
+
+from u1db.remote.http_target import HTTPSyncTarget
+
+#
+# monkey patch U1DB's HTTPSyncTarget to perform token based auth
+#
+
+def set_token_credentials(self, uuid, token):
+    self._creds = {'token': (uuid, token)}
+
+def _sign_request(self, method, url_query, params):
+    uuid, token = self._creds['token']
+    auth = '%s:%s' % (uuid, token)
+    return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])]
+
+HTTPSyncTarget.set_token_credentials = set_token_credentials
+HTTPSyncTarget._sign_request = _sign_request
+
+#
+# Create a temporary local u1db replica and attempt to sync to it.
+# Returns a failure message if something went wrong.
+#
+
+def soledad_sync(uuid, token, server):
+    tempdir = tempfile.mkdtemp()
+    try:
+        db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
+        creds = {'token': {'uuid': uuid, 'token': token}}
+        db.sync(server, creds=creds, autocreate=False)
+    finally:
+        shutil.rmtree(tempdir)
+
+#
+# exit codes:
+#
+# 0 - OK
+# 1 - WARNING
+# 2 - ERROR
+#
+
+if __name__ == '__main__':
+    try:
+        uuid, token, server = sys.argv[1:]
+        result = soledad_sync(uuid, token, server)
+        if result is None:
+            exit(0)
+        else:
+            print(result)
+            exit(1)
+    except Exception as exc:
+        print(exc.message or str(exc))
+        traceback.print_exc(file=sys.stdout)
+        exit(2)
diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb
new file mode 100644
index 00000000..5d30b459
--- /dev/null
+++ b/tests/helpers/srp_helper.rb
@@ -0,0 +1,171 @@
+#
+# Here are some very stripped down helper methods for SRP, useful only for
+# testing the client side.
+#
+
+require 'digest'
+require 'openssl'
+require 'securerandom'
+require 'base64'
+
+module SRP
+
+  ##
+  ## UTIL
+  ##
+
+  module Util
+    PRIME_N = <<-EOS.split.join.hex
+115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3
+    EOS
+    BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N)
+eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657
+6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da
+d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5
+7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb
+06e3
+    EOS
+    GENERATOR = 2 # g
+
+    def hn_xor_hg
+      byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR))
+    end
+
+    # a^n (mod m)
+    def modpow(a, n, m = BIG_PRIME_N)
+      r = 1
+      while true
+        r = r * a % m if n[0] == 1
+        n >>= 1
+        return r if n == 0
+        a = a * a % m
+      end
+    end
+
+    #  Hashes the (long) int args
+    def sha256_int(*args)
+      sha256_hex(*args.map{|a| "%02x" % a})
+    end
+
+    #  Hashes the hex args
+    def sha256_hex(*args)
+      h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('')
+      sha256_str([h].pack('H*'))
+    end
+
+    def sha256_str(s)
+      Digest::SHA2.hexdigest(s)
+    end
+
+    def bigrand(bytes)
+      OpenSSL::Random.random_bytes(bytes).unpack("H*")[0]
+    end
+
+    def multiplier
+      @muliplier ||= calculate_multiplier
+    end
+
+    protected
+
+    def calculate_multiplier
+      sha256_int(BIG_PRIME_N, GENERATOR).hex
+    end
+
+    def byte_xor_hex(a, b)
+      a = [a].pack('H*')
+      b = [b].pack('H*')
+      a.bytes.each_with_index.map do |a_byte, i|
+        (a_byte ^ (b[i].ord || 0)).chr
+      end.join
+    end
+  end
+
+  ##
+  ## SESSION
+  ##
+
+  class Session
+    include SRP::Util
+    attr_accessor :user
+    attr_accessor :bb
+
+    def initialize(user, aa=nil)
+      @user = user
+      @a = bigrand(32).hex
+    end
+
+    def m
+      @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k)
+    end
+
+    def aa
+      @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N)
+    end
+
+    protected
+
+    # client: K = H( (B - kg^x) ^ (a + ux) )
+    def client_secret
+      base = bb.hex
+      base -= modpow(GENERATOR, @user.private_key) * multiplier
+      base = base % BIG_PRIME_N
+      modpow(base, @user.private_key * u.hex + @a)
+    end
+
+    def k
+      @k ||= sha256_int(client_secret)
+    end
+
+    def n_xor_g_long
+      @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join
+    end
+
+    def login_hash
+      @login_hash ||= sha256_str(@user.username)
+    end
+
+    def u
+      @u ||= sha256_hex(aa, bb)
+    end
+  end
+
+  ##
+  ## Dummy USER
+  ##
+
+  class User
+    include SRP::Util
+
+    attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted
+
+    def initialize
+      @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '')
+      @password = "password_" + SecureRandom.urlsafe_base64(10)
+      @salt     = bigrand(4).hex
+      @verifier = modpow(GENERATOR, private_key)
+      @ok       = false
+      @deleted  = false
+    end
+
+    def private_key
+      @private_key ||= calculate_private_key
+    end
+
+    def to_params
+      {
+        'user[login]' => @username,
+        'user[password_verifier]' => @verifier.to_s(16),
+        'user[password_salt]' => @salt.to_s(16)
+      }
+    end
+
+    private
+
+    def calculate_private_key
+      shex = '%x' % [@salt]
+      inner = sha256_str([@username, @password].join(':'))
+      sha256_hex(shex, inner).hex
+    end
+  end
+
+end
diff --git a/tests/order.rb b/tests/order.rb
index ffa6ae4e..4468686f 100644
--- a/tests/order.rb
+++ b/tests/order.rb
@@ -3,6 +3,10 @@ class LeapCli::Config::Node
   # returns a list of node names that should be tested before this node.
   # make sure to not return ourselves (please no dependency loops!).
   #
+  # NOTE: this method determines the order that nodes are tested in. To specify
+  # the order of tests on a particular node, each test can call class method
+  # LeapTest.depends_on().
+  #
   def test_dependencies
     dependents = LeapCli::Config::ObjectList.new
     unless services.include?('couchdb')
diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb
index 9d5da94f..450c4201 100644
--- a/tests/white-box/couchdb.rb
+++ b/tests/white-box/couchdb.rb
@@ -1,4 +1,4 @@
-raise SkipTest unless $node["services"].include?("couchdb")
+raise SkipTest unless service?(:couchdb)
 
 require 'json'
 
@@ -9,9 +9,11 @@ class CouchDB < LeapTest
   end
 
   def test_00_Are_daemons_running?
-    assert_running 'tapicero'
-    assert_running 'bin/beam'
-    assert_running 'bin/epmd'
+    assert_running '^tapicero', :single => true
+    if multimaster?
+      assert_running 'bin/beam'
+      assert_running 'bin/epmd'
+    end
     pass
   end
 
@@ -29,6 +31,7 @@ class CouchDB < LeapTest
   # compare the configured nodes to the nodes that are actually listed in bigcouch
   #
   def test_02_Is_cluster_membership_ok?
+    return unless multimaster?
     url = couchdb_backend_url("/nodes/_all_docs")
     neighbors = assert_property('couch.bigcouch.neighbors')
     neighbors << assert_property('domain.full')
@@ -48,7 +51,8 @@ class CouchDB < LeapTest
   # this seems backward to me, so it might be the other way around.
   #
   def test_03_Are_configured_nodes_online?
-    url = couchdb_url("/_membership")
+    return unless multimaster?
+    url = couchdb_url("/_membership", :username => 'admin')
     assert_get(url) do |body|
       response = JSON.parse(body)
       nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes']
@@ -66,11 +70,11 @@ class CouchDB < LeapTest
   end
 
   def test_04_Do_ACL_users_exist?
-    acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp']
-    url = couchdb_backend_url("/_users/_all_docs")
+    acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication']
+    url = couchdb_backend_url("/_users/_all_docs", :username => 'admin')
     assert_get(url) do |body|
       response = JSON.parse(body)
-      assert_equal 6, response['total_rows']
+      assert_equal acl_users.count, response['total_rows']
       actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') }
       assert_equal acl_users.sort, actual_users.sort
     end
@@ -80,7 +84,8 @@ class CouchDB < LeapTest
   def test_05_Do_required_databases_exist?
     dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"]
     dbs_that_should_exist.each do |db_name|
-      assert_get(couchdb_url("/"+db_name)) do |body|
+      url = couchdb_url("/"+db_name, :username => 'admin')
+      assert_get(url) do |body|
         assert response = JSON.parse(body)
         assert_equal db_name, response['db_name']
       end
@@ -88,22 +93,63 @@ class CouchDB < LeapTest
     pass
   end
 
-  private
+  # disable ACL enforcement, because it's a known issue with bigcouch
+  # and will only confuse the user
+  # see https://leap.se/code/issues/6030 for more details
+  #
+  ## for now, this just prints warnings, since we are failing these tests.
+  ##
 
-  def couchdb_url(path="", port=nil)
-    @port ||= begin
-      assert_property 'couch.port'
-      $node['couch']['port']
-    end
-    @password ||= begin
-      assert_property 'couch.users.admin.password'
-      $node['couch']['users']['admin']['password']
+  #def test_06_Is_ACL_enforced?
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  )
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  ) && ok
+  #  pass if ok
+  #end
+
+  def test_07_Can_records_be_created?
+    token = Token.new
+    url = couchdb_url("/tokens", :username => 'admin')
+    assert_post(url, token, :format => :json) do |body|
+      assert response = JSON.parse(body), "POST response should be JSON"
+      assert response["ok"], "POST response should be OK"
+      assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body|
+        assert response = JSON.parse(body), "DELETE response should be JSON"
+        assert response["ok"], "DELETE response should be OK"
+      end
     end
-    "http://admin:#{@password}@localhost:#{port || @port}#{path}"
+    pass
+  end
+
+  private
+
+  def multimaster?
+    mode == "multimaster"
+  end
+
+  def mode
+    assert_property('couch.mode')
+  end
+
+  # TODO: admin port is hardcoded for now but should be configurable.
+  def couchdb_backend_url(path="", options={})
+    options = {port: multimaster? && "5986"}.merge options
+    couchdb_url(path, options)
   end
 
-  def couchdb_backend_url(path="")
-    couchdb_url(path, "5986") # TODO: admin port is hardcoded for now but should be configurable.
+  require 'securerandom'
+  require 'digest/sha2'
+  class Token < Hash
+    def initialize
+      self['token'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '')
+      self['_id'] = Digest::SHA512.hexdigest(self['token'])
+      self['last_seen_at'] = Time.now
+    end
   end
 
 end
diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb
new file mode 100644
index 00000000..794a9a41
--- /dev/null
+++ b/tests/white-box/mx.rb
@@ -0,0 +1,50 @@
+raise SkipTest unless service?(:mx)
+
+require 'json'
+
+class Mx < LeapTest
+  depends_on "Network"
+
+  def setup
+  end
+
+  def test_01_Can_contact_couchdb?
+    dbs = ["identities"]
+    dbs.each do |db_name|
+      couchdb_urls("/"+db_name, url_options).each do |url|
+        assert_get(url) do |body|
+          assert response = JSON.parse(body)
+          assert_equal db_name, response['db_name']
+        end
+      end
+    end
+    pass
+  end
+
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
+    end
+  end
+
+  def test_03_Are_MX_daemons_running?
+    assert_running 'leap_mx'
+    assert_running '/usr/lib/postfix/master'
+    assert_running '/usr/sbin/unbound'
+    pass
+  end
+
+  private
+
+  def url_options
+    {
+      :username => property('couchdb_leap_mx_user.username'),
+      :password => property('couchdb_leap_mx_user.password')
+    }
+  end
+
+end
diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb
index e0b0339d..f2041710 100644
--- a/tests/white-box/network.rb
+++ b/tests/white-box/network.rb
@@ -26,38 +26,35 @@ class Network < LeapTest
   #     connect: "127.0.0.1:5984"
   #
   def test_02_Is_stunnel_running?
-    if $node['stunnel']
-      good_stunnel_pids = []
-      $node['stunnel'].each do |stunnel_type, stunnel_configs|
-        if stunnel_type =~ /_clients?$/
-          stunnel_configs.each do |stunnel_name, stunnel_conf|
-            config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
-            processes = pgrep(config_file_name)
-            assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
-            good_stunnel_pids += processes.map{|ps| ps[:pid]}
-            assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
-            assert_tcp_socket('localhost', port)
-          end
-        elsif stunnel_type =~ /_server$/
-          config_file_name = "/etc/stunnel/#{stunnel_type}.conf"
-          processes = pgrep(config_file_name)
-          assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
-          good_stunnel_pids += processes.map{|ps| ps[:pid]}
-          assert accept = stunnel_configs['accept'], "Field `accept` must be present in property `stunnel.#{stunnel_type}`"
-          assert_tcp_socket('localhost', accept)
-          assert connect = stunnel_configs['connect'], "Field `connect` must be present in property `stunnel.#{stunnel_type}`"
-          assert_tcp_socket(*connect.split(':'))
-        else
-          skip "Unknown stunnel type `#{stunnel_type}`"
-        end
+    ignore unless $node['stunnel']
+    good_stunnel_pids = []
+    $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs|
+      stunnel_configs.each do |stunnel_name, stunnel_conf|
+        config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+        processes = pgrep(config_file_name)
+        assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
+        good_stunnel_pids += processes.map{|ps| ps[:pid]}
+        assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+        assert_tcp_socket('localhost', port)
       end
-      all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
-      assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
-      pass
     end
+    $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf|
+      config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+      processes = pgrep(config_file_name)
+      assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
+      good_stunnel_pids += processes.map{|ps| ps[:pid]}
+      assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', accept_port)
+      assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', connect_port)
+    end
+    all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
+    assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
+    pass
   end
 
   def test_03_Is_shorewall_running?
+    ignore unless File.exists?('/sbin/shorewall')
     assert_run('/sbin/shorewall status')
     pass
   end
diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb
index 5eb2bdb5..23a40426 100644
--- a/tests/white-box/openvpn.rb
+++ b/tests/white-box/openvpn.rb
@@ -1,6 +1,6 @@
-raise SkipTest unless $node["services"].include?("openvpn")
+raise SkipTest unless service?(:openvpn)
 
-class Openvpn < LeapTest
+class OpenVPN < LeapTest
   depends_on "Network"
 
   def setup
diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb
new file mode 100644
index 00000000..5a13e4a6
--- /dev/null
+++ b/tests/white-box/soledad.rb
@@ -0,0 +1,17 @@
+raise SkipTest unless service?(:soledad)
+
+require 'json'
+
+class Soledad < LeapTest
+  depends_on "Network"
+  depends_on "CouchDB" if service?(:couchdb)
+
+  def setup
+  end
+
+  def test_00_Is_Soledad_running?
+    assert_running 'soledad'
+    pass
+  end
+
+end
diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb
index 142ac2de..9f104899 100644
--- a/tests/white-box/webapp.rb
+++ b/tests/white-box/webapp.rb
@@ -1,57 +1,29 @@
-raise SkipTest unless $node["services"].include?("webapp")
+raise SkipTest unless service?(:webapp)
 
-require 'socket'
+require 'json'
 
 class Webapp < LeapTest
   depends_on "Network"
 
-  HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg'
-
   def setup
   end
 
-  #
-  # example properties:
-  #
-  # stunnel:
-  #   couch_client:
-  #     couch1_5984:
-  #       accept_port: 4000
-  #       connect: couch1.bitmask.i
-  #       connect_port: 15984
-  #
   def test_01_Can_contact_couchdb?
-    assert_property('stunnel.couch_client')
-    $node['stunnel']['couch_client'].values.each do |stunnel_conf|
-      assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
-      local_stunnel_url = "http://localhost:#{port}"
-      remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
-      msg = "(stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
-      assert_get(local_stunnel_url, nil, error_msg: msg) do |body|
-        assert_match /"couchdb":"Welcome"/, body, "Request to #{local_stunnel_url} should return couchdb welcome message."
-      end
+    url = couchdb_url("", url_options)
+    assert_get(url) do |body|
+      assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
     end
     pass
   end
 
-  #
-  # example properties:
-  #
-  # haproxy:
-  #   servers:
-  #     couch1:
-  #       backup: false
-  #       host: localhost
-  #       port: 4000
-  #       weight: 10
-  #
-  def test_02_Is_haproxy_working?
-    port = file_match(HAPROXY_CONFIG, /^  bind localhost:(\d+)$/)
-    url = "http://localhost:#{port}"
-    assert_get(url) do |body|
-      assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
     end
-    pass
   end
 
   def test_03_Are_daemons_running?
@@ -60,4 +32,94 @@ class Webapp < LeapTest
     pass
   end
 
+  #
+  # this is technically a black-box test. so, move this when we have support
+  # for black box tests.
+  #
+  def test_04_Can_access_webapp?
+    assert_get('https://' + $node['webapp']['domain'] + '/')
+    pass
+  end
+
+  def test_05_Can_create_and_authenticate_and_delete_user_via_API?
+    assert_tmp_user
+    pass
+  end
+
+  def test_06_Can_sync_Soledad?
+    soledad_config = property('definition_files.soledad_service')
+    if soledad_config && !soledad_config.empty?
+      soledad_server = pick_soledad_server(soledad_config)
+      if soledad_server
+        assert_tmp_user do |user|
+          assert_user_db_exists(user)
+          command = File.expand_path "../../helpers/soledad_sync.py", __FILE__
+          soledad_url = "https://#{soledad_server}/user-#{user.id}"
+          assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}"
+          pass
+        end
+      end
+    else
+      skip 'No soledad service configuration'
+    end
+  end
+
+  private
+
+  def url_options
+    {
+      :username => property('couchdb_webapp_user.username'),
+      :password => property('couchdb_webapp_user.password')
+    }
+  end
+
+  #
+  # pick a random soledad server.
+  # I am not sure why, but using IP address directly does not work.
+  #
+  def pick_soledad_server(soledad_config_json_str)
+    soledad_config = JSON.parse(soledad_config_json_str)
+    host_name = soledad_config['hosts'].keys.shuffle.first
+    if host_name
+      hostname = soledad_config['hosts'][host_name]['hostname']
+      port = soledad_config['hosts'][host_name]['port']
+      return "#{hostname}:#{port}"
+    else
+      return nil
+    end
+  end
+
+  #
+  # returns true if the per-user db created by tapicero exists.
+  # we try three times, and give up after that.
+  #
+  def assert_user_db_exists(user)
+    last_body, last_response, last_error = nil
+    3.times do
+      sleep 0.1
+      get(couchdb_url("/user-#{user.id}/_design/docs")) do |body, response, error|
+        last_body, last_response, last_error = body, response, error
+        if response.code.to_i == 200
+          return
+        end
+      end
+      sleep 0.2
+    end
+    assert false, "Could not find user db for test user #{user.username}\nuuid=#{user.id}\nHTTP #{last_response.code} #{last_error} #{last_body}"
+  end
+
+  #
+  # I tried, but couldn't get this working:
+  # #
+  # # get an CSRF authenticity token
+  # #
+  # url = api_url("/")
+  # csrf_token = nil
+  # assert_get(url) do |body|
+  #   lines = body.split("\n").grep(/csrf-token/)
+  #   assert lines.any?, 'failed to find csrf-token'
+  #   csrf_token = lines.first.split('"')[1]
+  #   assert csrf_token, 'failed to find csrf-token'
+  # end
+
 end