Merge branch 'develop'

Conflicts:
	platform.rb

Change-Id: Ic2e08e594d29a585691341c8667ac0b64933a505

153 files changed, 4962 insertions, 1298 deletions

diff --git a/bin/puppet_command b/bin/puppet_command
index a6cd5a69..cdb0b027 100755
--- a/bin/puppet_command
+++ b/bin/puppet_command
@@ -7,12 +7,17 @@
 # (exit codes, lockfile, multiple manifests, etc)
 #
 
+require 'pty'
+require 'yaml'
+
 PUPPET_BIN        = '/usr/bin/puppet'
 PUPPET_DIRECTORY  = '/srv/leap'
 PUPPET_PARAMETERS = '--color=false --detailed-exitcodes --libdir=puppet/lib --confdir=puppet'
 SITE_MANIFEST     = 'puppet/manifests/site.pp'
-SETUP_MANIFEST    = 'puppet/manifests/setup.pp'
+SITE_MODULES      = 'puppet/modules'
+CUSTOM_MODULES    = ':files/puppet/modules'
 DEFAULT_TAGS      = 'leap_base,leap_service'
+HIERA_FILE        = '/etc/leap/hiera.yaml'
 
 def main
   process_command_line_arguments
@@ -54,21 +59,33 @@ def apply
 end
 
 def set_hostname
-  exit_code = puppet_apply(:manifest => SETUP_MANIFEST, :tags => '') do |line|
-    # todo: replace setup.pp with https://github.com/lutter/ruby-augeas
-    # or try this: http://www.puppetcookbook.com/posts/override-a-facter-fact.html
-    if (line !~ /Finished catalog run/ || @verbosity > 2) &&
-       (line !~ /dnsdomainname: Name or service not known/) &&
-       (line !~ /warning: Could not retrieve fact fqdn/)
-      puts line
+  hostname = hiera_file['name']
+  if hostname.nil? || hostname.empty?
+    puts('ERROR: "name" missing from hiera file')
+    exit(1)
+  end
+  current_hostname_file = File.read('/etc/hostname') rescue nil
+  current_hostname = `/bin/hostname`.strip
+
+  # set /etc/hostname
+  if current_hostname_file != hostname
+    File.open('/etc/hostname', 'w', 0611, :encoding => 'ascii') do |f|
+      f.write hostname
+    end
+    if File.read('/etc/hostname') == hostname
+      puts "Changed /etc/hostname to #{hostname}"
+    else
+      puts "ERROR: failed to update /etc/hostname"
     end
   end
-  if exit_code == 2
-    puts "Hostname updated."
-  elsif exit_code == 4 || exit_code == 6
-    puts "ERROR: could not update hostname."
-  elsif exit_code == 0 && @verbosity > 1
-    puts "No change to hostname."
+
+  # call /bin/hostname
+  if current_hostname != hostname
+    if run("/bin/hostname #{hostname}") == 0
+      puts "Changed hostname to #{hostname}"
+    else
+      puts "ERROR: call to `/bin/hostname #{hostname}` returned an error."
+    end
   end
 end
 
@@ -78,11 +95,29 @@ end
 def puppet_apply(options={}, &block)
   options = {:verbosity => @verbosity, :tags => @tags}.merge(options)
   manifest = options[:manifest] || SITE_MANIFEST
+  modulepath = options[:module_path] || SITE_MODULES + CUSTOM_MODULES
+  fqdn = hiera_file['domain']['name']
+  domain = hiera_file['domain']['full_suffix']
   Dir.chdir(PUPPET_DIRECTORY) do
-    return run("#{PUPPET_BIN} apply #{custom_parameters(options)} #{PUPPET_PARAMETERS} #{manifest}", &block)
+    return run("FACTER_fqdn='#{fqdn}' FACTER_domain='#{domain}' #{PUPPET_BIN} apply #{custom_parameters(options)} --modulepath='#{modulepath}' #{PUPPET_PARAMETERS} #{manifest}", &block)
   end
 end
 
+#
+# Return a ruby object representing the contents of the hiera yaml file.
+#
+def hiera_file
+  unless File.exists?(HIERA_FILE)
+    puts("ERROR: hiera file '#{HIERA_FILE}' does not exist.")
+    exit(1)
+  end
+  $hiera_contents ||= YAML.load_file(HIERA_FILE)
+  return $hiera_contents
+rescue Exception => exc
+  puts("ERROR: problem reading hiera file '#{HIERA_FILE}' (#{exc})")
+  exit(1)
+end
+
 def custom_parameters(options)
   params = []
   if options[:tags] && options[:tags].chars.any?
@@ -157,24 +192,18 @@ end
 ## this only works under ruby 1.9
 ##
 
-require "pty"
-
 def run(cmd)
   puts cmd if @verbosity >= 3
   PTY.spawn("#{cmd}") do |output, input, pid|
     begin
       while line = output.gets do
         yield line
-        #$stdout.puts line
-        #$stdout.flush
       end
     rescue Errno::EIO
     end
     Process.wait(pid) # only works in ruby 1.9, required to capture the exit status.
   end
-  status = $?.exitstatus
-  #yield status if block_given?
-  return status
+  return $?.exitstatus
 rescue PTY::ChildExited
 end
 
@@ -188,4 +217,4 @@ Signal.trap("EXIT") do
                   # but only after the child puppet process is also dead (I think).
 end
 
-main()
\ No newline at end of file
+main()
diff --git a/bin/run_tests b/bin/run_tests
index 526aa83a..44384379 100755
--- a/bin/run_tests
+++ b/bin/run_tests
@@ -14,7 +14,6 @@
 require 'minitest/unit'
 require 'yaml'
 require 'tsort'
-require 'net/http'
 
 ##
 ## EXIT CODES
@@ -37,19 +36,28 @@ def bail(code, msg=nil)
 end
 
 ##
+## UTILITY
+##
+
+def service?(service)
+  $node["services"].include?(service.to_s)
+end
+
+##
 ## EXCEPTIONS
 ##
 
 # this class is raised if a test file wants to be skipped entirely.
-class SkipTest < Exception
+# (to skip an individual test, MiniTest::Skip is used instead)
+class SkipTest < StandardError
 end
 
 # raised if --no-continue and there is an error
-class TestError < Exception
+class TestError < StandardError
 end
 
 # raised if --no-continue and there is a failure
-class TestFailure < Exception
+class TestFailure < StandardError
 end
 
 ##
@@ -62,6 +70,8 @@ end
 class LeapTest < MiniTest::Unit::TestCase
   class Pass < MiniTest::Assertion
   end
+  class Ignore < MiniTest::Assertion
+  end
 
   def initialize(name)
     super(name)
@@ -101,10 +111,20 @@ class LeapTest < MiniTest::Unit::TestCase
   end
 
   #
+  # Called when the test should be silently ignored.
+  #
+  def ignore
+    raise LeapTest::Ignore
+  end
+
+  #
   # the default fail() is part of the kernel and it just throws a runtime exception. for tests,
   # we want the same behavior as assert(false)
   #
-  def fail(msg=nil)
+  def fail(msg=nil, exception=nil)
+    if DEBUG && exception && exception.respond_to?(:backtrace)
+      msg += MiniTest::filter_backtrace(exception.backtrace).join "\n"
+    end
     assert(false, msg)
   end
 
@@ -119,182 +139,6 @@ class LeapTest < MiniTest::Unit::TestCase
     :alpha
   end
 
-  #
-  # attempts a http GET on the url, yields |body, response, error|
-  #
-  def get(url, params=nil)
-    uri = URI(url)
-    if params
-      uri.query = URI.encode_www_form(params)
-    end
-    response = Net::HTTP.get_response(uri)
-    if response.is_a?(Net::HTTPSuccess)
-      yield response.body, response, nil
-    else
-      yield nil, response, nil
-    end
-  rescue => exc
-    yield nil, nil, exc
-  end
-
-  def assert_get(url, params=nil, options=nil)
-    options ||= {}
-    get(url, params) do |body, response, error|
-      if body
-        yield body if block_given?
-      elsif response
-        fail ["Expected a 200 status code from #{url}, but got #{response.code} instead.", options[:error_msg]].compact.join("\n")
-      else
-        fail ["Expected a response from #{url}, but got \"#{error}\" instead.", options[:error_msg]].compact.join("\n")
-      end
-    end
-  end
-
-  #
-  # test if a socket can be connected to
-  #
-
-  #
-  # tcp connection helper with timeout
-  #
-  def try_tcp_connect(host, port, timeout = 5)
-    addr     = Socket.getaddrinfo(host, nil)
-    sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
-
-    Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
-      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
-      begin
-        socket.connect_nonblock(sockaddr)
-      rescue IO::WaitReadable
-        if IO.select([socket], nil, nil, timeout) == nil
-          raise "Connection timeout"
-        else
-          socket.connect_nonblock(sockaddr)
-        end
-      rescue IO::WaitWritable
-        if IO.select(nil, [socket], nil, timeout) == nil
-          raise "Connection timeout"
-        else
-          socket.connect_nonblock(sockaddr)
-        end
-      end
-      return socket
-    end
-  end
-
-  def try_tcp_write(socket, timeout = 5)
-    begin
-      socket.write_nonblock("\0")
-    rescue IO::WaitReadable
-      if IO.select([socket], nil, nil, timeout) == nil
-        raise "Write timeout"
-      else
-        retry
-      end
-    rescue IO::WaitWritable
-      if IO.select(nil, [socket], nil, timeout) == nil
-        raise "Write timeout"
-      else
-        retry
-      end
-    end
-  end
-
-  def try_tcp_read(socket, timeout = 5)
-    begin
-      socket.read_nonblock(1)
-    rescue IO::WaitReadable
-      if IO.select([socket], nil, nil, timeout) == nil
-        raise "Read timeout"
-      else
-        retry
-      end
-    rescue IO::WaitWritable
-      if IO.select(nil, [socket], nil, timeout) == nil
-        raise "Read timeout"
-      else
-        retry
-      end
-    end
-  end
-
-  def assert_tcp_socket(host, port, msg=nil)
-    begin
-      socket = try_tcp_connect(host, port, 1)
-      #try_tcp_write(socket,1)
-      #try_tcp_read(socket,1)
-    rescue StandardError => exc
-      fail ["Failed to open socket #{host}:#{port}", exc].join("\n")
-    ensure
-      socket.close if socket
-    end
-  end
-
-  #
-  # Matches the regexp in the file, and returns the first matched string (or fails if no match).
-  #
-  def file_match(filename, regexp)
-    if match = File.read(filename).match(regexp)
-      match.captures.first
-    else
-      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
-    end
-  end
-
-  #
-  # Matches the regexp in the file, and returns array of matched strings (or fails if no match).
-  #
-  def file_matches(filename, regexp)
-    if match = File.read(filename).match(regexp)
-      match.captures
-    else
-      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
-    end
-  end
-
-  #
-  # checks to make sure the given property path exists in $node (e.g. hiera.yaml)
-  # and returns the value
-  #
-  def assert_property(property)
-    latest = $node
-    property.split('.').each do |segment|
-      latest = latest[segment]
-      fail "Required node property `#{property}` is missing." if latest.nil?
-    end
-    return latest
-  end
-
-  #
-  # works like pgrep command line
-  # return an array of hashes like so [{:pid => "1234", :process => "ls"}]
-  #
-  def pgrep(match)
-    output = `pgrep --full --list-name '#{match}'`
-    output.each_line.map{|line|
-      pid = line.split(' ')[0]
-      process = line.gsub(/(#{pid} |\n)/, '')
-      if process =~ /pgrep --full --list-name/
-        nil
-      else
-        {:pid => pid, :process => process}
-      end
-    }.compact
-  end
-end
-
-def assert_running(process)
-  assert pgrep(process).any?, "No running process for #{process}"
-end
-
-#
-# runs the specified command, failing on a non-zero exit status.
-#
-def assert_run(command)
-  output = `#{command}`
-  if $?.exitstatus != 0
-    fail "Error running `#{command}`:\n#{output}"
-  end
 end
 
 #
@@ -307,6 +151,7 @@ class LeapRunner < MiniTest::Unit
   def initialize
     @passes = 0
     @warnings = 0
+    @ignores = 0
     super
   end
 
@@ -347,9 +192,12 @@ class LeapRunner < MiniTest::Unit
     case e
       when MiniTest::Skip then
         @skips += 1
-        #if @verbose
-          report_line("SKIP", klass, meth, e, e.message)
-        #end
+        report_line("SKIP", klass, meth, e, e.message)
+      when LeapTest::Ignore then
+        @ignores += 1
+        if @verbose
+          report_line("IGNORE", klass, meth, e, e.message)
+        end
       when LeapTest::Pass then
         @passes += 1
         report_line("PASS", klass, meth)
@@ -389,7 +237,8 @@ class LeapRunner < MiniTest::Unit
     elsif @failures > 0
       :failure
     elsif @warnings > 0
-      :warning
+      # :warning << warnings don't warrant a non-zero exit code.
+      :success
     else
       :success
     end
@@ -401,7 +250,7 @@ class LeapRunner < MiniTest::Unit
   def report_line(prefix, klass, meth, e=nil, message=nil)
     msg_txt = nil
     if message
-      message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:password@")
+      message = message.sub(/http:\/\/([a-z_]+):([a-zA-Z0-9_]+)@/, "http://\\1:REDACTED@")
       if $output_format == :human
         indent = "\n  "
         msg_txt = indent + message.split("\n").join(indent)
@@ -516,7 +365,8 @@ def print_help
        "  --test TEST      Run only the test with name TEST.",
        "  --list-tests     Prints the names of all available tests and exit.",
        "  --retry COUNT    If the tests don't pass, retry COUNT additional times (default is zero)",
-       "  --wait SECONDS   Wait for SECONDS between retries (default is 5)"].join("\n")
+       "  --wait SECONDS   Wait for SECONDS between retries (default is 5)",
+       "  --debug          Print out full stack trace on errors"].join("\n")
   exit(0)
 end
 
@@ -575,6 +425,9 @@ def main
 
   # load all test classes
   this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+  Dir[File.expand_path('../../tests/helpers/*.rb', this_file)].each do |helper|
+    require helper
+  end
   Dir[File.expand_path('../../tests/white-box/*.rb', this_file)].each do |test_file|
     begin
       require test_file
@@ -596,10 +449,19 @@ def main
       when '--list-tests' then list_tests
       when '--retry' then ARGV.shift; $retry = ARGV.shift.to_i
       when '--wait' then ARGV.shift; $wait = ARGV.shift.to_i
+      when '--debug' then ARGV.shift
+      when '-d' then ARGV.shift
       else break
     end
   end
   run_tests
 end
 
+if ARGV.include?('--debug') || ARGV.include?('-d')
+  DEBUG=true
+  require 'debugger'
+else
+  DEBUG=false
+end
+
 main()
diff --git a/doc/details/couchdb.md b/doc/details/couchdb.md
new file mode 100644
index 00000000..afecf169
--- /dev/null
+++ b/doc/details/couchdb.md
@@ -0,0 +1,57 @@
+@title = "CouchDB"
+
+Rebalance Cluster
+=================
+
+Bigcouch currently does not have automatic rebalancing.
+It will probably be added after merging into couchdb.
+If you add a node, or remove one node from the cluster,
+
+. make sure you have a backup of all DBs !
+
+    /srv/leap/couchdb/scripts/couchdb_dumpall.sh
+
+
+. delete all dbs
+. shut down old node
+. check the couchdb members
+
+    curl -s —netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5986/nodes/_all_docs
+    curl -s —netrc-file /etc/couchdb/couchdb.netrc http://127.0.0.1:5984/_membership
+
+
+. remove bigcouch from all nodes
+
+    apt-get --purge remove bigcouch
+
+
+. deploy to all couch nodes
+
+    leap deploy development +couchdb
+
+. most likely, deploy will fail because bigcouch will complain about not all nodes beeing connected. Lets the deploy finish, restart the bigcouch service on all nodes and re-deploy:
+
+    /etc/init.d/bigcouch restart
+
+
+. restore the backup
+
+     /srv/leap/couchdb/scripts/couchdb_restoreall.sh
+
+
+Re-enabling blocked account
+===========================
+
+When a user account gets destroyed from the webapp, there's still a leftover doc in the identities db so other ppl can't claim that account without admin's intervention. Here's how you delete that doc and therefore enable registration for that particular account again:
+
+. grep the identities db for the email address:
+
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET http://127.0.0.1:5984/identities/_all_docs?include_docs=true|grep test_127@bitmask.net
+
+
+. lookup "id" and "rev" to delete the doc:
+
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X DELETE 'http://127.0.0.1:5984/identities/b25cf10f935b58088f0d547fca823265?rev=2-715a9beba597a2ab01851676f12c3e4a'
+
+
+
diff --git a/doc/development.md b/doc/details/development.md
index 7a761418..97f207ce 100644
--- a/doc/development.md
+++ b/doc/details/development.md
@@ -1,9 +1,10 @@
 @title = "Development Environment"
+@summary = "Setting up an environment for modifying the leap_platform."
 @toc = true
 
-If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch. 
+If you are wanting to make local changes to your provider, or want to contribute some fixes back to LEAP, we recommend that you follow this guide to build up a development environment to test your changes first. Using this method, you can quickly test your changes without deploying them to your production environment, while benefitting from the convenience of reverting to known good states in order to retry things from scratch.
 
-This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots. 
+This page will walk you through setting up nodes using [Vagrant](http://www.vagrantup.com/) for convenient deployment testing, snapshotting known good states, and reverting to previous snapshots.
 
 Requirements
 ============
@@ -11,6 +12,7 @@ Requirements
 * Be a real machine with virtualization support in the CPU (VT-x or AMD-V). In other words, not a virtual machine.
 * Have at least 4gb of RAM.
 * Have a fast internet connection (because you will be downloading a lot of big files, like virtual machine images).
+* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly.
 
 Install prerequisites
 --------------------------------
@@ -29,24 +31,41 @@ Install core prerequisites:
 
     sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make
 
-Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial):
+Install Vagrant in order to be able to test with local virtual machines (typically optional, but required for this tutorial). You probably want a more recent version directly from [vagrant.](https://www.vagrantup.com/downloads.htm)
 
     sudo apt-get install vagrant virtualbox
 
-<!--
-*Mac OS*
 
-1. Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed).
-2. Install Vagrant.dmg from http://downloads.vagrantup.com/
--->
+*Mac OS X 10.9 (Mavericks)*
+
+Install Homebrew package manager from http://brew.sh/ and enable the [System Duplicates Repository](https://github.com/Homebrew/homebrew/wiki/Interesting-Taps-&-Branches) (needed to update old software versions delivered by Apple) with
+
+    brew tap homebrew/dupes
+
+Update OpenSSH to support ECDSA keys. Follow [this guide](http://www.dctrwatson.com/2013/07/how-to-update-openssh-on-mac-os-x/) to let your system use the Homebrew binary.
+
+    brew install openssh --with-brewed-openssl --with-keychain-support
+
+The certtool provided by Apple it's really old, install the one provided by GnuTLS and shadow the system's default.
+
+    sudo brew install gnutls
+    ln -sf /usr/local/bin/gnutls-certtool /usr/local/bin/certool
+
+Install the Vagrant and VirtualBox packages for OS X from their respective Download pages.
+
+* http://www.vagrantup.com/downloads.html
+* https://www.virtualbox.org/wiki/Downloads
+
+
+2. Install
 
 
 Adding development nodes to your provider
 =========================================
 
-Now you will add local-only Vagrant development nodes to your provider. 
+Now you will add local-only Vagrant development nodes to your provider.
 
-You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish.  If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path). 
+You do not need to setup a different provider instance for development, in fact it is more convenient if you do not, but you can if you wish.  If you do not have a provider already, you will need to create one and configure it before continuing (it is recommended you go through the [Quick Start](quick-start) before continuing down this path).
 
 
 Create local development nodes
@@ -62,16 +81,16 @@ While in your provider directory, create a local node, with the service "webapp"
      = created files/nodes/web1/web1.key
      = created files/nodes/web1/web1.crt
 
-This command creates a node configuration file in `nodes/web1.json` with the webapp service. 
+This command creates a node configuration file in `nodes/web1.json` with the webapp service.
 
 Starting local development nodes
 --------------------------------
 
 In order to test the node "web1" we need to start it. Starting a node for the first time will spin up a virtual machine. The first time you do this will take some time because it will need to download a VM image (about 700mb). After you've downloaded the base image, you will not need to download it again, and instead you will re-use the downloaded image (until you need to update the image).
 
-NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please visit the [Vagrant page](vagrant) to troubleshoot your Vagrant install before proceeding.
+NOTE: Many people have difficulties getting Vagrant working. If the following commands do not work, please see the Vagrant section below to troubleshoot your Vagrant install before proceeding.
 
-    $ leap local start web
+    $ leap local start web1
      = created test/
      = created test/Vagrantfile
      = installing vagrant plugin 'sahara'
@@ -123,7 +142,7 @@ You now can follow the normal LEAP process and initialize it and then deploy you
 Useful local development commands
 =================================
 
-There are many useful things you can do with a virtualized development environment. 
+There are many useful things you can do with a virtualized development environment.
 
 Listing what machines are running
 ---------------------------------
@@ -150,21 +169,22 @@ It is not recommended that you leave your virtual machines running when you are
 Connecting to machines
 ----------------------
 
-You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'. 
+You can connect to your local nodes just like you do with normal LEAP nodes, by running 'leap ssh node'.
 
 However, if you cannot connect to your local node, because the networking is not setup properly, or you have deployed a firewall that locks you out, you may need to access the graphical console.
 
-In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to you
-$HOME/.leaprc:
+In order to do that, you will need to configure Vagrant to launch a graphical console and then you can login as root there to diagnose the networking problem. To do this, add the following to your $HOME/.leaprc:
 
-    @custom_vagrant_vm_line = 'config.vm.boot_mode = :gui'
+    @custom_vagrant_vm_line = 'config.vm.provider "virtualbox" do |v|
+      v.gui = true
+    end'
 
 and then start, or restart, your local Vagrant node. You should get a VirtualBox graphical interface presented to you showing you the bootup and eventually the login.
 
 Snapshotting machines
 ---------------------
 
-A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need. 
+A very useful feature of local Vagrant development nodes is the ability to snapshot the current state and then revert to that when you need.
 
 For example, perhaps the base image is a little bit out of date and you want to get the packages updated to the latest before continuing. You can do that simply by starting the node, connecting to it and updating the packages and then snapshotting the node:
 
@@ -190,13 +210,18 @@ Limitations
 Please consult the known issues for vagrant, see the [Known Issues](known-issues), section *Special Environments*
 
 
+Other useful plugins
+====================
+
+. The vagrant-cachier (plugin http://fgrehm.viewdocs.io/vagrant-cachier/) lets you cache .deb packages on your hosts so they are not downloaded by multiple machines over and over again, after resetting to a previous state.
+
 Troubleshooting Vagrant
 =======================
 
 To troubleshoot vagrant issues, try going through these steps:
 
 * Try plain vagrant using the [Getting started guide](http://docs.vagrantup.com/v2/getting-started/index.html).
-* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless). 
+* If that fails, make sure that you can run virtual machines (VMs) in plain virtualbox (Virtualbox GUI or VBoxHeadless).
   We don't suggest a sepecial howto for that, [this one](http://www.thegeekstuff.com/2012/02/virtualbox-install-create-vm/) seems pretty decent, or you follow the [Oracale Virtualbox User Manual](http://www.virtualbox.org/manual/UserManual.html). There's also specific documentation for [Debian](https://wiki.debian.org/VirtualBox) and for [Ubuntu](https://help.ubuntu.com/community/VirtualBox). If you succeeded, try again if you now can start vagrant nodes using plain vagrant (see first step).
 * If plain vagrant works for you, you're very close to using vagrant with leap ! If you encounter any problems now, please [contact us](https://leap.se/en/about-us/contact) or use our [issue tracker](https://leap.se/code)
 
@@ -217,25 +242,37 @@ Ubuntu Raring 13.04
 
 * `virtualbox 4.2.10-dfsg-0ubuntu2.1` from Ubuntu raring and `vagrant 1.2.2` from vagrantup.com
 
+Mac OS X 10.9
+-------------
+
+* `VirtualBox 4.3.10` from virtualbox.org and `vagrant 1.5.4` from vagrantup.com
+
 
 Using Vagrant with libvirt/kvm
 ==============================
 
-Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version (>= 1.3).
+Vagrant can be used with different providers/backends, one of them is [vagrant-libvirt](https://github.com/pradels/vagrant-libvirt). Here are the steps how to use it. Be sure to use a recent vagrant version for the vagrant-libvirt plugin (>= 1.5, which can only be fetched from http://www.vagrantup.com/downloads.html at this moment).
 
 Install vagrant-libvirt plugin and add box
 ------------------------------------------
     sudo apt-get install libvirt-bin libvirt-dev
+    # you need to assign the new 'libvirtd' group to your user in a running x session, or logout and login again:
+    newgrp libvirtd
+    # to build the vagrant-libvirt plugin you need the following packages:
+    sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev
     vagrant plugin install vagrant-libvirt
-    vagrant plugin install sahara 
-    vagrant box add leap-wheezy https://downloads.leap.se/leap-debian-libvirt.box
+    vagrant plugin install sahara
+    vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box --provider libvirt
 
+Remove Virtualbox
+-----------------
+    sudo apt-get remove virtualbox*
 
 Debugging
 ---------
 
 If you get an error in any of the above commands, try to get some debugging information, it will often tell you what is wrong. In order to get debugging logs, you simply need to re-run the command that produced the error but prepend the command with VAGRANT_LOG=info, for example:
-    VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/leap-debian-libvirt.box
+    VAGRANT_LOG=info vagrant box add leap-wheezy https://downloads.leap.se/platform/vagrant/libvirt/leap-wheezy.box
 
 Start it
 --------
@@ -259,7 +296,7 @@ Then:
 
 If everything works, you should export libvirt as the VAGRANT_DEFAULT_PROVIDER:
 
-    export VAGRANT_DEFAULT_PROVIDER="libvirt" 
+    export VAGRANT_DEFAULT_PROVIDER="libvirt"
 
 Now you should be able to use the `leap local` commands.
 
@@ -268,5 +305,10 @@ Known Issues
 
 * 'Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)' - you don't have the libvirtd daemon running or installed, be sure you installed the 'libvirt-bin' package and it is running
 * 'Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied' - you need to be in the libvirt group to access the socket, do 'sudo adduser <user> libvirt' and then re-login to your session
+* if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to https://github.com/pradels/vagrant-libvirt/issues/75 having two virtualization engines installed simultaneously can lead to such weird issues.
 * see the [vagrant-libvirt issue list on github](https://github.com/pradels/vagrant-libvirt/issues)
 * be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with `vagrant plugin install [vagrant-libvirt|sahara]`) for proper libvirt support
+* for shared folder support, you need nfs-kernel-server installed on the host machine and set up sudo to allow unpriviledged users to modify /etc/exports. See [vagrant-libvirt#synced-folders](https://github.com/pradels/vagrant-libvirt#synced-folders)
+
+
+    sudo apt-get install nfs-kernel-server
diff --git a/doc/details/en.haml b/doc/details/en.haml
new file mode 100644
index 00000000..2f59f3f2
--- /dev/null
+++ b/doc/details/en.haml
@@ -0,0 +1,5 @@
+- @title = "Details"
+
+%h1.first Platform Details
+
+= child_summaries
\ No newline at end of file
diff --git a/doc/faq.md b/doc/details/faq.md
index 2654ce80..57afb6c4 100644
--- a/doc/faq.md
+++ b/doc/details/faq.md
@@ -1,7 +1,16 @@
 @title = 'Frequently asked questions'
 @nav_title = 'FAQ'
+@summary = "Frequently Asked Questions"
 @toc = true
 
+APT
+===============
+
+What do I do when unattended upgrades fail?
+--------------------------------------------------
+
+When you receive notification e-mails with a subject of 'unattended-upgrades result for $machinename', that means that some package couldn't be automatically upgraded and needs manual interaction. The reasons vary, so you have to be careful. Most often you can simply login to the affected machine and run `apt-get dist-upgrade`.
+
 Puppet
 ======
 
@@ -23,8 +32,11 @@ Log into your server and issue:
 How can i customize the leap_platform puppet manifests ?
 --------------------------------------------------------
 
-You can create a custom module `site_custom`.  The class `site_custom::setup` will get
-included in the first part of the deploy process, and `site_custom` during the second part. 
+You can create custom puppet modules under `files/puppet`.
+The custom puppet entry point is in class 'custom' which can be put into
+`files/puppet/modules/custom/manifests/init.pp`. This class gets automatically included
+by site_config::default, which is applied to all nodes.
+
 Of cause you can also create a different git branch and change whatever you want, if you are
 familiar wit git.
 
@@ -37,7 +49,7 @@ How can i see custom facts distributed by leap_platform on a node ?
 On the server, export the FACTERLIB env. variable to include the path of the custom fact in question:
 
     export FACTERLIB=/var/lib/puppet/lib/facter:/srv/leap/puppet/modules/stdlib/lib/facter/
-    facter    
+    facter
 
 
 Etc
diff --git a/doc/under-the-hood.md b/doc/details/under-the-hood.md
index 080a153e..dcbddb3e 100644
--- a/doc/under-the-hood.md
+++ b/doc/details/under-the-hood.md
@@ -1,22 +1,11 @@
 @title = "Under the hood"
+@summary = "Various implementation details."
 
 This page contains various details on the how the platform is implemented. You can safely ignore this page, although it may be useful if you plan to make modifications to the platform.
 
 Puppet Details
 ======================================
 
-Run stages
-----------
-
-We use two run stages for resource ordering:
-
-* initial: configure hostname, apt-get update + apt-get dist-upgrade
-* main: everything else
-
-Stage initial is run before stage main.
-
-see http://docs.puppetlabs.com/puppet/2.7/reference/lang_run_stages.html for run stage documentation.
-
 Tags
 ----
 
diff --git a/doc/details/webapp.md b/doc/details/webapp.md
new file mode 100644
index 00000000..2b078af4
--- /dev/null
+++ b/doc/details/webapp.md
@@ -0,0 +1,282 @@
+@title = 'LEAP Web'
+@summary = 'The web component of the LEAP Platform, providing user management, support desk, documentation and more.'
+@toc = true
+
+Introduction
+===================
+
+"LEAP Web" is the webapp component of the LEAP Platform, providing the following services:
+
+* REST API for user registration.
+* Admin interface to manage users.
+* Client certificate distribution and renewal.
+* User support help tickets.
+* Billing
+* Customizable and Localized user documentation
+
+This web application is written in Ruby on Rails 3, using CouchDB as the backend data store.
+
+It is licensed under the GNU Affero General Public License (version 3.0 or higher). See http://www.gnu.org/licenses/agpl-3.0.html for more information.
+
+Known problems
+====================
+
+* Client certificates are generated without a CSR. The problem is that this makes the web
+  application extremely vulnerable to denial of service attacks. This was not an issue until we
+  started to allow the possibility of anonymously fetching a client certificate without
+  authenticating first.
+
+* By its very nature, the user database is vulnerable to enumeration attacks. These are
+  very hard to prevent, because our protocol is designed to allow query of a user database via
+  proxy in order to provide network perspective.
+
+Integration
+===========
+
+LEAP web is part of the leap platform. Most of the time it will be customized and deployed in that context. This section describes the integration of LEAP web in the wider framework. The Development section focusses on development of LEAP web itself.
+
+Configuration & Customization
+------------------------------
+
+The customization of the webapp for a leap provider happens via two means:
+ * configuration settings in services/webapp.json
+ * custom files in files/webapp
+
+### Configuration Settings
+
+The webapp ships with a fairly large set of default settings for all environments. They are stored in config/defaults.yml. During deploy the platform creates config/config.yml from the settings in services/webapp.json. These settings will overwrite the defaults.
+
+### Custom Files
+
+Any file placed in files/webapp in the providers repository will overwrite the content of config/customization in the webapp. These files will override files of the same name.
+
+This mechanism allows customizing basically all aspects of the webapp.
+See files/webapp/README.md in the providers repository for more.
+
+### Provider Information ###
+
+The leap client fetches provider information via json files from the server. The platform prepares that information and stores it in the webapp in public/1/config/*.json. (1 being the current API version).
+
+Provider Documentation
+-------------
+
+LEAP web already comes with a bit of user documentation. It mostly resides in app/views/pages and thus can be overwritten by adding files to files/webapp/views/pages in the provider repository. You probably want to add your own Terms of Services and Privacy Policy here.
+The webapp will render haml, erb and markdown templates and pick translated content from localized files such as privacy_policy.es.md. In order to add or remove languages you have to modify the available_locales setting in the config. (See Configuration Settings above)
+
+Development
+===========
+
+Installation
+---------------------------
+
+Typically, this application is installed automatically as part of the LEAP Platform. To install it manually for testing or development, follow these instructions:
+
+### TL;DR ###
+
+Install git, ruby 1.9, rubygems and couchdb on your system. Then run
+
+    gem install bundler
+    git clone https://leap.se/git/leap_web
+    cd leap_web
+    git submodule update --init
+    bundle install --binstubs
+    bin/rails server
+
+### Install system requirements
+
+First of all you need to install ruby, git and couchdb. On debian based systems this would be achieved by something like
+
+    sudo apt-get install git ruby1.9.3 rubygems couchdb
+
+We install most gems we depend upon through [bundler](http://gembundler.com). So first install bundler
+
+    sudo gem install bundler
+
+On Debian Wheezy or later, there is a Debian package for bundler, so you can alternately run ``sudo apt-get install bundler``.
+
+### Download source
+
+Simply clone the git repository:
+
+    git clone git://leap.se/leap_web
+    cd leap_web
+
+### SRP Submodule
+
+We currently use a git submodule to include srp-js. This will soon be replaced by a ruby gem. but for now you need to run
+
+  git submodule update --init
+
+### Install required ruby libraries
+
+    cd leap_web
+    bundle
+
+Typically, you run ``bundle`` as a normal user and it will ask you for a sudo password when it is time to install the required gems. If you don't have sudo, run ``bundle`` as root.
+
+Configuration
+----------------------------
+
+The configuration file `config/defaults.yml` providers good defaults for most
+values. You can override these defaults by creating a file `config/config.yml`.
+
+There are a few values you should make sure to modify:
+
+    production:
+      admins: ["myusername","otherusername"]
+      domain: example.net
+      force_ssl: true
+      secret_token: "4be2f60fafaf615bd4a13b96bfccf2c2c905898dad34..."
+      client_ca_key: "/etc/ssl/ca.key"
+      client_ca_cert: "/etc/ssl/ca.crt"
+      ca_key_password: nil
+
+* `admins` is an array of usernames that are granted special admin privilege.
+* `domain` is your fully qualified domain name.
+* `force_ssl`, if set to true, will require secure cookies and turn on HSTS. Don't do this if you are using a self-signed server certificate.
+* `secret_token`, used for cookie security, you can create one with `rake secret`. Should be at least 30 characters.
+* `client_ca_key`, the private key of the CA used to generate client certificates.
+* `client_ca_cert`, the public certificate the CA used to generate client certificates.
+* `ca_key_password`, used to unlock the client_ca_key, if needed.
+
+### Provider Settings
+
+The leap client fetches provider information via json files from the server. 
+If you want to use that functionality please add your provider files the public/1/config directory. (1 being the current API version).
+
+Running
+-----------------------------
+
+    cd leap_web
+    bin/rails server
+
+You will find Leap Web running on `localhost:3000`
+
+Testing
+--------------------------------
+
+To run all tests
+
+    rake test
+
+To run an individual test:
+
+    rake test TEST=certs/test/unit/client_certificate_test.rb
+    or
+    ruby -Itest certs/test/unit/client_certificate_test.rb
+
+Engines
+---------------------
+
+Leap Web includes some Engines. All things in `app` will overwrite the engine behaviour. You can clone the leap web repository and add your customizations to the `app` directory. Including leap_web as a gem is currently not supported. It should not require too much work though and we would be happy to include the changes required.
+
+If you have no use for one of the engines you can remove it from the Gemfile. Engines should really be plugins - no other engines should depend upon them. If you need functionality in different engines it should probably go into the toplevel.
+
+# Deployment #
+
+We strongly recommend using the LEAP platform for deploy. Most of the things documented here are automated as part of the platform. If you want to research how the platform deploys or work on your own mechanism this section is for you.
+
+These instructions are targeting a Debian GNU/Linux system. You might need to change the commands to match your own needs.
+
+## Server Preperation ##
+
+### Dependencies ##
+
+The following packages need to be installed:
+
+* git
+* ruby1.9
+* rubygems1.9
+* couchdb (if you want to use a local couch)
+
+### Setup Capistrano ###
+
+We use puppet to deploy. But we also ship an untested config/deploy.rb.example. Edit it to match your needs if you want to use capistrano.
+
+run `cap deploy:setup` to create the directory structure.
+
+run `cap deploy` to deploy to the server.
+
+## Customized Files ##
+
+Please make sure your deploy includes the following files:
+
+* public/1/config/*.json (see Provider Settings section)
+* config/couchdb.yml
+
+## Couch Security ##
+
+We recommend against using an admin user for running the webapp. To avoid this couch design documents need to be created ahead of time and the auto update mechanism needs to be disabled.
+Take a look at test/setup_couch.sh for an example of securing the couch.
+
+## Design Documents ##
+
+After securing the couch design documents need to be deployed with admin permissions. There are two ways of doing this:
+ * rake couchrest:migrate_with_proxies
+ * dump the documents as files with `rake couchrest:dump` and deploy them
+   to the couch by hand or with the platform.
+
+### CouchRest::Migrate ###
+
+The before_script block in .travis.yml illustrates how to do this:
+
+    mv test/config/couchdb.yml.admin config/couchdb.yml  # use admin privileges
+    bundle exec rake couchrest:migrate_with_proxies      # run the migrations
+    bundle exec rake couchrest:migrate_with_proxies      # looks like this needs to run twice
+    mv test/config/couchdb.yml.user config/couchdb.yml   # drop admin privileges
+
+### Deploy design docs from CouchRest::Dump ###
+
+First of all we get the design docs as files:
+
+    # put design docs in /tmp/design
+    bundle exec rake couchrest:dump
+
+Then we add them to files/design in the site_couchdb module in leap_platform so they get deployed with the couch. You could also upload them using curl or sth. similar.
+
+# Troubleshooting #
+
+Here are some less common issues you might run into when installing Leap Web.
+
+## Cannot find Bundler ##
+
+### Error Messages ###
+
+`bundle: command not found`
+
+### Solution ###
+
+Make sure bundler is installed. `gem list bundler` should list `bundler`.
+You also need to be able to access the `bundler` executable in your PATH.
+
+## Outdated version of rubygems ##
+
+### Error Messages ###
+
+`bundler requires rubygems >= 1.3.6`
+
+### Solution ###
+
+`gem update --system` will install the latest rubygems
+
+## Missing development tools ##
+
+Some required gems will compile C extensions. They need a bunch of utils for this.
+
+### Error Messages ###
+
+`make: Command not found`
+
+### Solution ###
+
+Install the required tools. For linux the `build-essential` package provides most of them. For Mac OS you probably want the XCode Commandline tools.
+
+## Missing libraries and headers ##
+
+Some gem dependencies might not compile because they lack the needed c libraries.
+
+### Solution ###
+
+Install the libraries in question including their development files.
+
+
diff --git a/doc/en.md b/doc/en.md
index f1a1fc17..d0dcfcc9 100644
--- a/doc/en.md
+++ b/doc/en.md
@@ -71,7 +71,7 @@ Getting started
 
 We recommend reading the platform documentation in the following order:
 
-1. [Quick start tutorial](platform/quick-start).
+1. [Quick start tutorial](tutorials/quick-start).
 2. [Platform Guide](platform/guide).
 3. [Configuration format](platform/config).
 4. The `leap` [command reference](platform/commands).
diff --git a/doc/commands.md b/doc/guide/commands.md
index b176541f..0cee709a 100644
--- a/doc/commands.md
+++ b/doc/guide/commands.md
@@ -1,73 +1,88 @@
 @title = 'Command Line Reference'
+@summary = "A copy of leap --help"
 
 The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home.
 
 
 # Global Options
 
-* `--log FILE`  
-Override default log file  
-Default Value: None  
+* `--log FILE`
+Override default log file
+Default Value: None
 
-* `-v|--verbose LEVEL`  
-Verbosity level 0..2  
-Default Value: 1  
+* `-v|--verbose LEVEL`
+Verbosity level 0..5
+Default Value: 1
 
-* `--help`  
-Show this message  
+* `--[no-]color`
+Disable colors in output
 
-* `--version`  
-Display version number and exit  
+* `--debug`
+Enable debugging library (leap_cli development only)
 
-* `--yes`  
-Skip prompts and assume "yes"  
+* `--help`
+Show this message
+
+* `--version`
+Display version number and exit
+
+* `--yes`
+Skip prompts and assume "yes"
 
 
 # leap add-user  USERNAME
 
-Adds a new trusted sysadmin
+Adds a new trusted sysadmin by adding public keys to the "users" directory.
 
 
 
 **Options**
 
-* `--pgp-pub-key arg`  
-OpenPGP public key file for this new user  
-Default Value: None  
+* `--pgp-pub-key arg`
+OpenPGP public key file for this new user
+Default Value: None
 
-* `--ssh-pub-key arg`  
-SSH public key file for this new user  
-Default Value: None  
+* `--ssh-pub-key arg`
+SSH public key file for this new user
+Default Value: None
 
-* `--self`  
-lets you choose among your public keys  
+* `--self`
+Add yourself as a trusted sysadin by choosing among the public keys available for the current user.
 
 
-# leap cert 
+# leap cert
 
 Manage X.509 certificates
 
 
 
-## leap cert ca 
+## leap cert ca
 
 Creates two Certificate Authorities (one for validating servers and one for validating clients).
 
 See see what values are used in the generation of the certificates (like name and key size), run `leap inspect provider` and look for the "ca" property. To see the details of the created certs, run `leap inspect <file>`.
 
-## leap cert csr 
+## leap cert csr
 
 Creates a CSR for use in buying a commercial X.509 certificate.
 
-The CSR created is for the for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`.
+Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`.
+
+**Options**
+
+* `--domain DOMAIN`
+Specify what domain to create the CSR for.
+Unless specified, the CSR is created for the provider's primary domain. The properties used for this CSR come from `provider.ca.server_certificates`.
+Default Value: None
 
-## leap cert dh 
+
+## leap cert dh
 
 Creates a Diffie-Hellman parameter file.
 
 
 
-## leap cert update  <node-filter>
+## leap cert update  FILTER
 
 Creates or renews a X.509 certificate/key pair for a single node or all nodes, but only if needed.
 
@@ -75,22 +90,47 @@ This command will a generate new certificate for a node if some value in the nod
 
 **Options**
 
-* `--force`  
-Always generate new certificates  
+* `--force`
+Always generate new certificates
 
 
-# leap clean 
+# leap clean
 
 Removes all files generated with the "compile" command.
 
 
 
-# leap compile 
+# leap compile
+
+Compile generated files.
+
+
+
+## leap compile all  [ENVIRONMENT]
 
 Compiles node configuration files into hiera files used for deployment.
 
 
 
+## leap compile zone
+
+Compile a DNS zone file for your provider.
+
+
+Default Command: all
+
+# leap db
+
+Database commands.
+
+
+
+## leap db destroy  [FILTER]
+
+Destroy all the databases. If present, limit to FILTER nodes.
+
+
+
 # leap deploy  FILTER
 
 Apply recipes to a node or set of nodes.
@@ -99,13 +139,67 @@ The FILTER can be the name of a node, service, or tag.
 
 **Options**
 
-* `--tags TAG[,TAG]`  
-Specify tags to pass through to puppet (overriding the default).  
-Default Value: leap_base,leap_service  
+* `--ip IPADDRESS`
+Override the default SSH IP address.
+Default Value: None
+
+* `--port PORT`
+Override the default SSH port.
+Default Value: None
+
+* `--tags TAG[,TAG]`
+Specify tags to pass through to puppet (overriding the default).
+Default Value: leap_base,leap_service
+
+* `--dev`
+Development mode: don't run 'git submodule update' before deploy.
+
+* `--fast`
+Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.
+
+* `--force`
+Deploy even if there is a lockfile.
+
+* `--[no-]sync`
+Sync files, but don't actually apply recipes.
+
+
+# leap env
+
+Manipulate and query environment information.
+
+The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. A node in one environment will never interact with a node from another environment. Environment pinning works by modifying your ~/.leaprc file and is dependent on the absolute file path of your provider directory (pins don't apply if you move the directory)
+
+## leap env ls
+
+List the available environments. The pinned environment, if any, will be marked with '*'.
+
 
-* `--fast`  
-Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.  
 
+## leap env pin  ENVIRONMENT
+
+Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.
+
+
+
+## leap env unpin
+
+Unpin the environment. All subsequent commands will apply to all nodes.
+
+
+Default Command: ls
+
+# leap facts
+
+Gather information on nodes.
+
+
+
+## leap facts update  FILTER
+
+Query servers to update facts.json.
+
+Queries every node included in FILTER and saves the important information to facts.json
 
 # leap help  command
 
@@ -115,8 +209,8 @@ Gets help for the application or its commands. Can also list the commands in a w
 
 **Options**
 
-* `-c`  
-List commands one per line, to assist with shell completion  
+* `-c`
+List commands one per line, to assist with shell completion
 
 
 # leap inspect  FILE
@@ -125,6 +219,12 @@ Prints details about a file. Alternately, the argument FILE can be the name of a
 
 
 
+**Options**
+
+* `--base`
+Inspect the FILE from the provider_base (i.e. without local inheritance).
+
+
 # leap list  [FILTER]
 
 List nodes and their classifications
@@ -137,12 +237,15 @@ Prints out a listing of nodes, services, or tags. If present, the FILTER can be
 
 **Options**
 
-* `--print arg`  
-What attributes to print (optional)  
-Default Value: None  
+* `--print arg`
+What attributes to print (optional)
+Default Value: None
+
+* `--disabled`
+Include disabled nodes in the list.
 
 
-# leap local 
+# leap local
 
 Manage local virtual machines.
 
@@ -184,6 +287,12 @@ Shuts down the virtual machine(s)
 
 
 
+# leap mosh  NAME
+
+Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).
+
+
+
 # leap new  DIRECTORY
 
 Creates a new provider instance in the specified directory, creating it if necessary.
@@ -192,24 +301,24 @@ Creates a new provider instance in the specified directory, creating it if neces
 
 **Options**
 
-* `--contacts arg`  
-Default email address contacts.  
-Default Value: None  
+* `--contacts arg`
+Default email address contacts.
+Default Value: None
 
-* `--domain arg`  
-The primary domain of the provider.  
-Default Value: None  
+* `--domain arg`
+The primary domain of the provider.
+Default Value: None
 
-* `--name arg`  
-The name of the provider.  
-Default Value: None  
+* `--name arg`
+The name of the provider.
+Default Value: None
 
-* `--platform arg`  
-File path of the leap_platform directory.  
-Default Value: None  
+* `--platform arg`
+File path of the leap_platform directory.
+Default Value: None
 
 
-# leap node 
+# leap node
 
 Node management
 
@@ -231,20 +340,28 @@ Separeate multiple values for a single property with a comma, like so: `leap nod
 
 **Options**
 
-* `--local`  
-Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.  
+* `--local`
+Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.
 
 
 ## leap node init  FILTER
 
 Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages
 
-This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, and installing packages that are required for deploying. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times.
+This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. Node init must be run before deploying to a server, and the server must be running and available via the network. This command only needs to be run once, but there is no harm in running it multiple times.
 
 **Options**
 
-* `--echo`  
-If set, passwords are visible as you type them (default is hidden)  
+* `--ip IPADDRESS`
+Override the default SSH IP address.
+Default Value: None
+
+* `--port PORT`
+Override the default SSH port.
+Default Value: None
+
+* `--echo`
+If set, passwords are visible as you type them (default is hidden)
 
 
 ## leap node mv  OLD_NAME NEW_NAME
@@ -265,21 +382,38 @@ Log in to the specified node with an interactive shell.
 
 
 
-# leap test 
+**Options**
+
+* `--port arg`
+Override ssh port for remote host
+Default Value: None
+
+* `--ssh arg`
+Pass through raw options to ssh (e.g. --ssh '-F ~/sshconfig')
+Default Value: None
+
+
+# leap test
 
 Run tests.
 
 
 
-## leap test init 
+## leap test init
 
 Creates files needed to run tests.
 
 
 
-## leap test run 
+## leap test run
 
 Run tests.
 
 
+
+**Options**
+
+* `--[no-]continue`
+Continue over errors and failures (default is --no-continue).
+
 Default Command: run
diff --git a/doc/config.md b/doc/guide/config.md
index d0b1f6a7..be67e6bd 100644
--- a/doc/config.md
+++ b/doc/guide/config.md
@@ -1,4 +1,39 @@
 @title = "Configuration Files"
+@summary = "How to edit configuration files."
+
+Files
+-------------------------------------------
+
+Here are a list of some of the common files that make up a provider. Except for Leapfile and provider.json, the files are optional. Unless otherwise specified, all file names are relative to the 'provider directory' root (where the Leapfile is).
+
+`Leapfile` -- If present, this file tells `leap` that the directory is a provider directory. This file is usually empty, but can contain global options.
+
+`~/.leaprc` -- Evaluated the same as Leapfile, but not committed to source control.
+
+`provider.json` -- Global options related to this provider.
+
+`provider.ENVIRONMENT.json` -- Global options for the provider that are applied to only a single environment.
+
+`common.json` -- All nodes inherit from this file.
+
+`secrets.json` -- An automatically generated file that contains any randomly generated strings needed in order to deploy. These strings are often secret and should be protected, although any need for a random string or number that is remembered will produce another entry in this file. This file is automatically generated and refreshed each time you run `leap compile` or `leap deploy`. If an entry is no longer needed, it will get removed. If you want to change a secret, you can remove this file and have it regenerated, or remove the particular line item and just those items will be created anew.
+
+`facts.json` -- If some of your servers are running on AWS or OpenStack, you will need to discover certain properties about how networking is configured on these machines in order for a full deploy to work. In these cases, make sure to run `leap facts update` to periodically regenerate the facts.json file.
+
+`nodes/NAME.json` -- The configuration file for node called NAME.
+
+`services/SERVICE.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its `services` property.
+
+`services/SERVICE.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that includes SERVICE in its services and has environment equal to ENVIRONMENT.
+
+`services/TAG.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property.
+
+`services/TAG.ENVIRONMENT.json` -- The properties in this configuration file are applied to any node that has includes TAG in its `tags` property and has `environment` property equal to ENVIRONMENT.
+
+`files/*` -- Various static files used by the platform (e.g. keys, certificates, webapp customization, etc).
+
+`users/USER/` -- A directory that stores the public keys of the sysadmin with name USER. This person will have root access to all the servers.
+
 
 Leapfile
 -------------------------------------------
@@ -13,8 +48,6 @@ Additionally, you can create a `~/.leaprc` file that is loaded after `Leapfile`
 Platform options:
 
 * `@platform_directory_path` (required). This must be set to the path where `leap_platform` lives. The path may be relative.
-* `@platform_branch`. If set, a check is preformed before running any command to ensure that the currently checked out branch of `leap_platform` matches the value set for `@platform_branch`. This is useful if you have a stable branch of your provider that you want to ensure runs off the master branch of `leap_platform`.
-* `@allow_production_deploy`. By default, you can only deploy to production nodes if the current branch is 'master' or if the provider directory is not a git repository. This option allows you to override this behavior.
 
 Vagrant options:
 
@@ -26,7 +59,7 @@ Logging options:
 * `@log`. If set, all command invocation and results are logged to the specified file. This is the same as the switch `--log FILE`, except that the command line switch will override the value in the Leapfile.
 
 
-Configuration files
+JSON format
 -------------------------------------------
 
 All configuration files, other than `Leapfile`, are in the JSON format. For example:
@@ -195,7 +228,8 @@ Access an element by name:
 Create a new hash table by applying filters:
 
     nodes[:public_dns => true] # all nodes where public_dns == true
-    nodes[:services => 'openvpn', :services => 'tor'] # openvpn OR tor
+    nodes[:services => 'openvpn', 'location.country_code' => 'US'] # openvpn service OR in the US.
+    nodes[[:services, 'openvpn'], [:services, 'tor']]  # services equal to openvpn OR tor
     nodes[:services => 'openvpn'][:tags => 'production']  # openvpn AND production
     nodes[:name => "!bob"] # all nodes that are NOT named "bob"
 
diff --git a/doc/guide/en.haml b/doc/guide/en.haml
new file mode 100644
index 00000000..4c9bd69f
--- /dev/null
+++ b/doc/guide/en.haml
@@ -0,0 +1,6 @@
+- @nav_title = "Guide"
+- @title = "Platform Guide"
+
+%h1.first Platform Guide
+
+= child_summaries
\ No newline at end of file
diff --git a/doc/guide.md b/doc/guide/keys-and-certificates.md
index 52c3b2fa..bd7f3495 100644
--- a/doc/guide.md
+++ b/doc/guide/keys-and-certificates.md
@@ -1,84 +1,5 @@
-@title = "LEAP Platform Guide"
-@nav_title = "Guide"
-
-Services
-================================
-
-Every node has one or more services that determines the node's function within your provider's infrastructure.
-
-When adding a new node to your provider, you should ask yourself four questions:
-
-* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes.
-* **required or optional?** Some services are required, while others can be left out.
-* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together.
-* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure.
-
-Brief overview of the services:
-
-* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many.
-* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp` and `mx` nodes.
-* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb. (not currently in stable release)
-* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes. (not currently in stable release)
-* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service.
-* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` nodes.
-* **tor**: Sets up a tor exit node, unconnected to any other service.
-* **dns**: Not yet implemented.
-
-Locations
-================================
-
-All nodes should have a `location.name` specified, and optionally additional information about the location, like the time zone. This location information is used for two things:
-
-* Determine which nodes can, or must, communicate with one another via a local network. The way some virtualization environments work, like OpenStack, requires that nodes communicate via the local network if they are on the same network.
-* Allows the client to prefer connections to nodes that are closer in physical proximity to the user. This is particularly important for OpenVPN nodes.
-
-The location stanza in a node's config file looks like this:
-
-    {
-      "location": {
-        "id": "ankara",
-        "name": "Ankara",
-        "country_code": "TR",
-        "timezone": "+2",
-        "hemisphere": "N"
-      }
-    }
-
-The fields:
-
-* `id`: An internal handle to use for this location. If two nodes have match `location.id`, then they are treated as being on a local network with one another. This value defaults to downcase and underscore of `location.name`.
-* `name`: Can be anything, might be displayed to the user in the client if they choose to manually select a gateway.
-* `country_code`: The [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) two letter country code.
-* `timezone`: The timezone expressed as an offset from UTC (in standard time, not daylight savings). You can look up the timezone using this [handy map](http://www.timeanddate.com/time/map/).
-* `hemisphere`: This should be "S" for all servers in South America, Africa, or Australia. Otherwise, this should be "N".
-
-These location options are very imprecise, but good enough for most usage. The client often does not know its own location precisely either. Instead, the client makes an educated guess at location based on the OS's timezone and locale.
-
-If you have multiple nodes in a single location, it is best to use a tag for the location. For example:
-
-`tags/ankara.json`:
-
-    {
-      "location": {
-        "name": "Ankara",
-        "country_code": "TR",
-        "timezone": "+2",
-        "hemisphere": "N"
-      }
-    }
-
-`nodes/vpngateway.json`:
-
-    {
-      "services": "openvpn",
-      "tags": ["production", "ankara"],
-      "ip_address": "1.1.1.1",
-      "openvpn": {
-        "gateway_address": "1.1.1.2"
-      }
-    }
-
-Unless you are using OpenStack or AWS, setting `location` for nodes is not required. It is, however, highly recommended.
+@title = "Keys and Certificates"
+@summary = "Working with SSH keys, secrets, and X.509 certificates."
 
 Working with SSH
 ================================
@@ -252,15 +173,4 @@ If you want to add additional fields to the CSR, like country, city, or locality
         }
       }
 
-If they are not present, the CSR will be created without them.
-
-Facts
-==============================
-
-There are a few cases when we must gather internal data from a node before we can successfully deploy to other nodes. This is what `facts.json` is for. It stores a snapshot of certain facts about each node, as needed. Entries in `facts.json` are updated automatically when you initialize, rename, or remove a node. To manually force a full update of `facts.json`, run:
-
-    leap facts update FILTER
-
-Run `leap help facts update` for more information.
-
-The file `facts.json` should be committed to source control. You might not have a `facts.json` if one is not required for your provider.
+If they are not present, the CSR will be created without them.
\ No newline at end of file
diff --git a/doc/guide/miscellaneous.md b/doc/guide/miscellaneous.md
new file mode 100644
index 00000000..c38c007c
--- /dev/null
+++ b/doc/guide/miscellaneous.md
@@ -0,0 +1,14 @@
+@title = "Miscellaneous"
+@summary = "Miscellaneous commands you may need to know."
+
+Facts
+==============================
+
+There are a few cases when we must gather internal data from a node before we can successfully deploy to other nodes. This is what `facts.json` is for. It stores a snapshot of certain facts about each node, as needed. Entries in `facts.json` are updated automatically when you initialize, rename, or remove a node. To manually force a full update of `facts.json`, run:
+
+    leap facts update FILTER
+
+Run `leap help facts update` for more information.
+
+The file `facts.json` should be committed to source control. You might not have a `facts.json` if one is not required for your provider.
+
diff --git a/doc/guide/nodes.md b/doc/guide/nodes.md
new file mode 100644
index 00000000..bc48ff32
--- /dev/null
+++ b/doc/guide/nodes.md
@@ -0,0 +1,169 @@
+@title = "Nodes"
+@summary = "Working with nodes, services, tags, and locations."
+
+Node types
+================================
+
+Every node has one or more services that determines the node's function within your provider's infrastructure.
+
+When adding a new node to your provider, you should ask yourself four questions:
+
+* **many or few?** Some services benefit from having many nodes, while some services are best run on only one or two nodes.
+* **required or optional?** Some services are required, while others can be left out.
+* **who does the node communicate with?** Some services communicate very heavily with other particular services. Nodes running these services should be close together.
+* **public or private?** Some services communicate with the public internet, while others only need to communicate with other nodes in the infrastructure.
+
+Brief overview of the services:
+
+* **webapp**: The web application. Runs both webapp control panel for users and admins as well as the REST API that the client uses. Needs to communicate heavily with `couchdb` nodes. You need at least one, good to have two for redundancy. The webapp does not get a lot of traffic, so you will not need many.
+* **couchdb**: The database for users and user data. You can get away with just one, but for proper redundancy you should have at least three. Communicates heavily with `webapp`, `mx`, and `soledad` nodes.
+* **soledad**: Handles the data syncing with clients. Typically combined with `couchdb` service, since it communicates heavily with couchdb.
+* **mx**: Incoming and outgoing MX servers. Communicates with the public internet, clients, and `couchdb` nodes.
+* **openvpn**: OpenVPN gateway for clients. You need at least one, but want as many as needed to support the bandwidth your users are doing. The `openvpn` nodes are autonomous and don't need to communicate with any other nodes. Often combined with `tor` service.
+* **monitor**: Internal service to monitor all the other nodes. Currently, you can have zero or one `monitor` nodes.
+* **tor**: Sets up a tor exit node, unconnected to any other service.
+* **dns**: Not yet implemented.
+
+Webapp
+-----------------------------------
+
+The webapp node is responsible for both the user face web application and the API that the client interacts with.
+
+Some users can be "admins" with special powers to answer tickets and close accounts. To make an account into an administrator, you need to configure the `webapp.admins` property with an array of user names.
+
+For example, to make users `alice` and `bob` into admins, create a file `services/webapp.json` with the following content:
+
+    {
+      "webapp": {
+        "admins": ["bob", "alice"]
+      }
+    }
+
+And then redeploy to all webapp nodes:
+
+    leap deploy webapp
+
+By putting this in `services/webapp.json`, you will ensure that all webapp nodes inherit the value for `webapp.admins`.
+
+Services
+================================
+
+What nodes do you need for a provider that offers particular services?
+
+<table class="table table-striped">
+<tr>
+<th>Node Type</th>
+<th>VPN Service</th>
+<th>Email Service</th>
+</tr>
+<tr>
+<td>webapp</td>
+<td>required</td>
+<td>required</td>
+</tr>
+<tr>
+<td>couchdb</td>
+<td>required</td>
+<td>required</td>
+</tr>
+<tr>
+<td>soledad</td>
+<td>not used</td>
+<td>required</td>
+</tr>
+<tr>
+<td>mx</td>
+<td>not used</td>
+<td>required</td>
+</tr>
+<tr>
+<td>openvpn</td>
+<td>required</td>
+<td>not used</td>
+</tr>
+<tr>
+<td>monitor</td>
+<td>optional</td>
+<td>optional</td>
+</tr>
+<tr>
+<td>tor</td>
+<td>optional</td>
+<td>optional</td>
+</tr>
+<table>
+
+Locations
+================================
+
+All nodes should have a `location.name` specified, and optionally additional information about the location, like the time zone. This location information is used for two things:
+
+* Determine which nodes can, or must, communicate with one another via a local network. The way some virtualization environments work, like OpenStack, requires that nodes communicate via the local network if they are on the same network.
+* Allows the client to prefer connections to nodes that are closer in physical proximity to the user. This is particularly important for OpenVPN nodes.
+
+The location stanza in a node's config file looks like this:
+
+    {
+      "location": {
+        "id": "ankara",
+        "name": "Ankara",
+        "country_code": "TR",
+        "timezone": "+2",
+        "hemisphere": "N"
+      }
+    }
+
+The fields:
+
+* `id`: An internal handle to use for this location. If two nodes have match `location.id`, then they are treated as being on a local network with one another. This value defaults to downcase and underscore of `location.name`.
+* `name`: Can be anything, might be displayed to the user in the client if they choose to manually select a gateway.
+* `country_code`: The [ISO 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) two letter country code.
+* `timezone`: The timezone expressed as an offset from UTC (in standard time, not daylight savings). You can look up the timezone using this [handy map](http://www.timeanddate.com/time/map/).
+* `hemisphere`: This should be "S" for all servers in South America, Africa, or Australia. Otherwise, this should be "N".
+
+These location options are very imprecise, but good enough for most usage. The client often does not know its own location precisely either. Instead, the client makes an educated guess at location based on the OS's timezone and locale.
+
+If you have multiple nodes in a single location, it is best to use a tag for the location. For example:
+
+`tags/ankara.json`:
+
+    {
+      "location": {
+        "name": "Ankara",
+        "country_code": "TR",
+        "timezone": "+2",
+        "hemisphere": "N"
+      }
+    }
+
+`nodes/vpngateway.json`:
+
+    {
+      "services": "openvpn",
+      "tags": ["production", "ankara"],
+      "ip_address": "1.1.1.1",
+      "openvpn": {
+        "gateway_address": "1.1.1.2"
+      }
+    }
+
+Unless you are using OpenStack or AWS, setting `location` for nodes is not required. It is, however, highly recommended.
+
+Disabling Nodes
+=====================================
+
+There are two ways to temporarily disable a node:
+
+**Option 1: enabled == false**
+
+If a node has a property `enabled` set to false, then the `leap` command will skip over the node and pretend that it does not exist. For example:
+
+    {
+      "ip_address": "1.1.1.1",
+      "services": ["openvpn"],
+      "enabled": false
+    }
+
+**Options 2: no-deploy**
+
+If the file `/etc/leap/no-deploy` exists on a node, then when you run the commmand `leap deploy` it will halt and prevent a deploy from going through (if the node was going to be included in the deploy).
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
deleted file mode 100644
index bb2fc4b5..00000000
--- a/doc/troubleshooting.md
+++ /dev/null
@@ -1,147 +0,0 @@
-@title = 'Troubleshooting Guide'
-@nav_title = 'Troubleshooting'
-@toc = true
-
-
-General
-=======
-
-* Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`.
-
-Webapp node
-===========
-
-Places to look for errors
--------------------------
-
-* `/var/log/apache2/error.log`
-* `/srv/leap/webapp/log/production.log`
-* `/var/log/syslog` (watch out for stunnel issues)
-
-Is haproxy ok ? 
----------------
-
-
-    curl -s -X  GET "http://127.0.0.1:4096"
-
-Is couchdb accessible through stunnel ?
----------------------------------------
-
-
-    curl -s -X  GET "http://127.0.0.1:4000"
-
-
-Check couchdb acl
------------------
-
-
-    mkdir /etc/couchdb
-    cat /srv/leap/webapp/config/couchdb.yml.admin  # see username and password
-    echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc
-    chmod 600 /etc/couchdb/couchdb-admin.netrc
-
-    curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096"
-    curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs"
-    
-
-Couchdb node
-============
-
-Places to look for errors
--------------------------
-
-* `/opt/bigcouch/var/log/bigcouch.log`
-* `/var/log/syslog` (watch out for stunnel issues)
-
-
-Bigcouch membership
--------------------
-
-* All nodes configured for the provider should appear here:
-
-
-    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5986/nodes/_all_docs'
-
-* All configured nodes should show up under "cluster_nodes", and the ones online and communicating with each other should appear under "all_nodes". This example output shows the configured cluster nodes `couch1.bitmask.net` and `couch2.bitmask.net`, but `couch2.bitmask.net` is currently not accessible from `couch1.bitmask.net`
-
-
-    curl -s --netrc-file /etc/couchdb/couchdb.netrc 'http://127.0.0.1:5984/_membership'
-    {"all_nodes":["bigcouch@couch1.bitmask.net"],"cluster_nodes":["bigcouch@couch1.bitmask.net","bigcouch@couch2.bitmask.net"]}
-
-
-
-Databases
----------
-
-* Following output shows all neccessary DBs that should be present. Note that the `user-0123456....` DBs are the data stores for a particular user. 
-
-
-    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs' 
-    ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"]
-
-
-
-Design Documents
-----------------
-
-* Is User `_design doc` available ?
-
-
-    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X  GET "http://127.0.0.1:5984/users/_design/User"
-
-
-
-MX node
-=======
-
-Places to look for errors
--------------------------
-
-* `/var/log/mail.log`
-* `/var/log/leap_mx.log`
-* `/var/log/syslog` (watch out for stunnel issues)
-
-
-Query leap-mx
--------------
-
-* for useraccount 
-
-
-    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:2244
-    ...
-    postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net
-    postmap: dict_tcp_lookup: recv: 200 
-    ...
-
-* for mailalias
-
-
-    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:4242
-    ...
-    postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net
-    postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a
-    postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a
-    f01bc1c70de7d7d80bc1ad77d987e73a
-    ...
-
-
-
-Mailspool
----------
-
-* Any file in the mailspool longer for a few seconds ?
-
-
-    ls -la /var/mail/vmail/Maildir/cur/
-
-
-VPN node
-========
-
-Places to look for errors
--------------------------
-
-* `/var/log/syslog` (watch out for openvpn issues)
-
-
diff --git a/doc/troubleshooting/en.haml b/doc/troubleshooting/en.haml
new file mode 100644
index 00000000..a4b44939
--- /dev/null
+++ b/doc/troubleshooting/en.haml
@@ -0,0 +1,5 @@
+- @title = "Troubleshooting"
+
+%h1.first Troubleshooting
+
+= child_summaries
\ No newline at end of file
diff --git a/doc/known-issues.md b/doc/troubleshooting/known-issues.md
index 960eaad7..b924fa4b 100644
--- a/doc/known-issues.md
+++ b/doc/troubleshooting/known-issues.md
@@ -5,23 +5,32 @@
 
 Here you can find documentation about known issues and potential work-arounds in the current Leap Platform release.
 
-0.2.2
+0.6.0
 =====
 
-In this release the following issues are known, work-arounds are noted when available.
+openvpn
+-------
+. On deployment to a openvpn node, if the following happens:
 
-General Issues
---------------
+    - err: /Stage[main]/Site_openvpn/Service[openvpn]/ensure: change from stopped to running failed: Could not start Service[openvpn]: Execution of '/etc/init.d/openvpn start' returned 1:  at /srv/leap/puppet/modules/site_openvpn/manifests/init.pp:189
 
-. This release does *not* anonymize your logs (see: https://leap.se/code/issues/1897)
+this is likely the result of a kernel upgrade that happened during the deployment, requiring that the machine be restarted before this service can start. To confirm this, login to the node (leap ssh <nodename>) and look at the end of the /var/log/daemon.log:
 
-. This release does *not* setup email relaying, so admins will not receive important email notifications. Email service will be part of the next release (see: https://leap.se/code/issues/1683 https://leap.se/code/issues/1905)
+    # tail /var/log/daemon.log
+    Nov 22 19:04:15 snail ovpn-udp_config[16173]: ERROR: Cannot open TUN/TAP dev /dev/net/tun: No such device (errno=19)
+    Nov 22 19:04:15 snail ovpn-udp_config[16173]: Exiting due to fatal error
 
-. Your openvpn gateway address will be added on the /24 network, and is not configurable in this release (see: https://leap.se/code/issues/1863)
+if you see this error, simply restart the node.
 
-. You must not add a node with an underscore in the name, you also cannot use a hyphen for a vagrant node (see: https://leap.se/code/issues/3087) 
+CouchDB
+-------
+. You can't deploy new couchdb nodes after one or more have been deployed. Make *sure* that you configure and deploy all your couchdb nodes when starting the provider. The problem is that we dont not have a clean way of adding couch nodes after initial creation of the databases, so any nodes added after result in improperly synchronized data. See Bug [#5601](https://leap.se/code/issues/5601) for more information.
 
-. The nagios website check reports success when the webapp is not functioning but apache is up (see: https://leap.se/code/issues/1629)
+. In some scenarios, such as when certain components are unavailable, the couchdb syncing will be broken. When things are brought back to normal, shortly after restart, the nodes will attempt to resync all their data, and can fail to complete this process because they run out of file descriptors. A symptom of this is the webapp wont allow you to register or login, the /opt/bigcouch/var/log/bigcouch.log is huge with a lot of errors that include (over multiple lines): {error,  emfile}}. We have raised the limits for available file descriptors to bigcouch to try and accommodate for this situation, but if you still experience it, you may need to increase your /etc/sv/bigcouch/run ulimit values and restart bigcouch while monitoring the open file descriptors. We hope that in the next platform release, a newer couchdb will be better at handling these resources.
+
+You can also see the number of file descriptors in use by doing:
+
+    # watch -n1 -d lsof -p `pidof beam`|wc -l
 
 User setup and ssh
 ------------------
@@ -34,8 +43,6 @@ User setup and ssh
 
 . If the ssh host key changes, you need to run node init again (see: https://leap.se/en/docs/platform/guide#Working.with.SSH)
 
-. At the moment, only ECDSA ssh host keys are supported. If you get the following error: `= FAILED ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)` then you should confirm that you have the following line defined in your server's **/etc/ssh/sshd_config**: `HostKey /etc/ssh/ssh_host_ecdsa_key`. If that file doesn't exist, run `ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ""` in order to create it. If you made a change to your sshd_config, then you need to run `/etc/init.d/ssh restart` (see: https://leap.se/code/issues/2373)
-
 . To remove an admin's access to your servers, please remove the directory for that user under the `users/` subdirectory in your provider directory and then remove that user's ssh keys from files/ssh/authorized_keys. When finished you *must* run a `leap deploy` to update that information on the servers. 
 
 . At the moment, it is only possible to add an admin who will have access to all LEAP servers (see: https://leap.se/code/issues/2280)
@@ -56,6 +63,12 @@ Deploying
 
 . When running a deploy at a verbosity level of 2 and above, you will notice puppet deprecation warnings, these are known and we are working on fixing them
 
+IPv6
+----
+
+As of this release, IPv6 is not supported by the VPN configuration. If IPv6 is detected on your network as a client, it is blocked and instead it should revert to IPv4. We plan on adding IPv6 support in an upcoming release.
+
+
 Special Environments
 --------------------
 
diff --git a/doc/troubleshooting/tests.md b/doc/troubleshooting/tests.md
new file mode 100644
index 00000000..84064043
--- /dev/null
+++ b/doc/troubleshooting/tests.md
@@ -0,0 +1,33 @@
+@title = 'Tests and Monitoring'
+@summary = 'Testing and monitoring your infrastructure.'
+@toc = true
+
+## Troubleshooting Tests
+
+At any time, you can run troubleshooting tests on the nodes of your provider infrastructure to check to see if things seem to be working correctly. If there is a problem, these tests should help you narrow down precisely where the problem is.
+
+To run tests on FILTER node list:
+
+    leap test run FILTER
+
+Alternately, you can run test on all nodes (probably only useful if you have pinned the environment):
+
+    leap test
+
+## Monitoring
+
+In order to set up a monitoring node, you simply add a `monitor` service tag to the node configuration file. It could be combined with any other service, but we propose that you add it to the webapp node, as this already is public accessible via HTTPS.
+
+After deploying, this node will regularly poll every node to ask for the status of various health checks. These health checks include the checks run with `leap test`, plus many others.
+
+We use [Nagios](http://www.nagios.org/) together with [Check MK agent](https://en.wikipedia.org/wiki/Check_MK) for running checks on remote hosts.
+
+You can log into the monitoring web interface via [https://MONITORNODE/nagios3/](https://MONITORNODE/nagios3/). The username is `nagiosadmin` and the password is found in the secrets.json file in your provider directory.
+
+### Log Monitoring
+
+At the moment, we use [check-mk-agent-logwatch](https://mathias-kettner.de/checkmk_check_logwatch.html) for searching logs for irregularities.
+Logs are parsed for patterns using a blacklist, and are stored in `/var/lib/check_mk/logwatch/<Nodename>`.
+
+In order to "acknowledge" a log warning, you need to log in to the monitoring server, and delete the corresponding file in `/var/lib/check_mk/logwatch/<Nodename>`. This should be done via the nagios webinterface in the future.
+
diff --git a/doc/troubleshooting/where-to-look.md b/doc/troubleshooting/where-to-look.md
new file mode 100644
index 00000000..fbd95931
--- /dev/null
+++ b/doc/troubleshooting/where-to-look.md
@@ -0,0 +1,249 @@
+@title = 'Where to look for errors'
+@nav_title = 'Where to look'
+@toc = true
+
+
+General
+=======
+
+* Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. `-v 5` after the `leap` cmd, i.e. `leap -v 2 deploy`.
+
+Webapp
+======
+
+Places to look for errors
+-------------------------
+
+* `/var/log/apache2/error.log`
+* `/srv/leap/webapp/log/production.log`
+* `/var/log/syslog` (watch out for stunnel issues)
+* `/var/log/leap/*`
+
+Is haproxy ok ?
+---------------
+
+
+    curl -s -X  GET "http://127.0.0.1:4096"
+
+Is couchdb accessible through stunnel ?
+---------------------------------------
+
+* Depending on how many couch nodes you have, increase the port for every test
+  (see /etc/haproxy/haproxy.cfg for the server/port mapping):
+
+
+    curl -s -X  GET "http://127.0.0.1:4000"
+    curl -s -X  GET "http://127.0.0.1:4001"
+    ...
+
+
+Check couchdb acl as admin
+--------------------------
+
+    mkdir /etc/couchdb
+    cat /srv/leap/webapp/config/couchdb.yml.admin  # see username and password
+    echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc
+    chmod 600 /etc/couchdb/couchdb-admin.netrc
+
+    curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096"
+    curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs"
+
+Check couchdb acl as unpriviledged user
+---------------------------------------
+
+    cat /srv/leap/webapp/config/couchdb.yml  # see username and password
+    echo "machine 127.0.0.1 login webapp password <PASSWORD>" > /etc/couchdb/couchdb-webapp.netrc
+    chmod 600 /etc/couchdb/couchdb-webapp.netrc
+
+    curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096"
+    curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs"
+
+
+Check client config files
+-------------------------
+
+    https://example.net/provider.json
+    https://example.net/1/config/smtp-service.json
+    https://example.net/1/config/soledad-service.json
+    https://example.net/1/config/eip-service.json
+
+
+Soledad
+=======
+
+    /var/log/soledad.log
+
+
+Couchdb
+=======
+
+Places to look for errors
+-------------------------
+
+* `/opt/bigcouch/var/log/bigcouch.log`
+* `/var/log/syslog` (watch out for stunnel issues)
+
+
+
+Bigcouch membership
+-------------------
+
+* All nodes configured for the provider should appear here:
+
+<pre>
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5986/nodes/_all_docs'
+</pre>
+
+* All configured nodes should show up under "cluster_nodes", and the ones online and communicating with each other should appear under "all_nodes". This example output shows the configured cluster nodes `couch1.bitmask.net` and `couch2.bitmask.net`, but `couch2.bitmask.net` is currently not accessible from `couch1.bitmask.net`
+
+
+<pre>
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc 'http://127.0.0.1:5984/_membership'
+    {"all_nodes":["bigcouch@couch1.bitmask.net"],"cluster_nodes":["bigcouch@couch1.bitmask.net","bigcouch@couch2.bitmask.net"]}
+</pre>
+
+* Sometimes a `/etc/init.d/bigcouch restart` on all nodes is needed, to register new nodes
+
+Databases
+---------
+
+* Following output shows all neccessary DBs that should be present. Note that the `user-0123456....` DBs are the data stores for a particular user.
+
+<pre>
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs'
+    ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"]
+</pre>
+
+
+
+
+Design Documents
+----------------
+
+* Is User `_design doc` available ?
+
+
+<pre>
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X  GET "http://127.0.0.1:5984/users/_design/User"
+</pre>
+
+Is couchdb cluster backend accessible through stunnel ?
+-------------------------------------------------------
+
+* Find out how many connections are set up for the couchdb cluster backend:
+
+<pre>
+    grep "accept = 127.0.0.1" /etc/stunnel/*
+</pre>
+
+
+* Now connect to all of those local endpoints to see if they up. All these tests should return "localhost [127.0.0.1] 4000 (?) open"
+
+<pre>
+    nc -v 127.0.0.1 4000
+    nc -v 127.0.0.1 4001
+    ...
+</pre>
+
+
+MX
+==
+
+Places to look for errors
+-------------------------
+
+* `/var/log/mail.log`
+* `/var/log/leap_mx.log`
+* `/var/log/syslog` (watch out for stunnel issues)
+
+Is couchdb accessible through stunnel ?
+---------------------------------------
+
+* Depending on how many couch nodes you have, increase the port for every test
+  (see /etc/haproxy/haproxy.cfg for the server/port mapping):
+
+
+    curl -s -X  GET "http://127.0.0.1:4000"
+    curl -s -X  GET "http://127.0.0.1:4001"
+    ...
+
+Query leap-mx
+-------------
+
+* for useraccount
+
+
+<pre>
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:2244
+    ...
+    postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200
+    ...
+</pre>
+
+* for mailalias
+
+
+<pre>
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:4242
+    ...
+    postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a
+    postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a
+    f01bc1c70de7d7d80bc1ad77d987e73a
+    ...
+</pre>
+
+
+Check couchdb acl as unpriviledged user
+---------------------------------------
+
+
+
+    cat /etc/leap/mx.conf  # see username and password
+    echo "machine 127.0.0.1 login leap_mx password <PASSWORD>" > /etc/couchdb/couchdb-leap_mx.netrc
+    chmod 600 /etc/couchdb/couchdb-leap_mx.netrc
+
+    curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs"   # pick one "user-<hash>" db
+    curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c"
+
+
+* you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes
+
+
+Mailspool
+---------
+
+* Any file in the leap_mx mailspool longer for a few seconds ?
+
+
+
+<pre>
+    ls -la /var/mail/vmail/Maildir/cur/
+</pre>
+
+* Any mails in postfix mailspool longer than a few seconds ?
+
+<pre>
+    mailq
+</pre>
+
+
+
+Testing mail delivery
+---------------------
+
+    swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 25
+    swaks -f varac@cdev.bitmask.net -t varac@cdev.bitmask.net -s chipmonk.cdev.bitmask.net --port 465 --tlsc
+    swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 587 --tls
+
+
+VPN
+===
+
+Places to look for errors
+-------------------------
+
+* `/var/log/syslog` (watch out for openvpn issues)
+
+
diff --git a/doc/tutorials/en.haml b/doc/tutorials/en.haml
new file mode 100644
index 00000000..72a8a1fc
--- /dev/null
+++ b/doc/tutorials/en.haml
@@ -0,0 +1,5 @@
+- @title = "Tutorials"
+
+%h1.first Platform Tutorials
+
+= child_summaries
\ No newline at end of file
diff --git a/doc/quick-start.md b/doc/tutorials/quick-start.md
index 0bce271a..f45574df 100644
--- a/doc/quick-start.md
+++ b/doc/tutorials/quick-start.md
@@ -1,24 +1,24 @@
 @title = 'LEAP Platform Quick Start'
 @nav_title = 'Quick Start'
+@summary = 'Three node OpenVPN provider.'
 
 Quick Start
 ===========
 
 This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform). This Quick Start guide will guide you through building a three node OpenVPN provider.
-If you are curious how this will look like without trying it out yourself, you can watch our [recorded screencasts](http://shelr.tv/users/524415e69660807910000021).
 
 Our goal
 ------------------
 
-We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity, or more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy).
+We are going to create a minimal LEAP provider offering OpenVPN service. This basic setup can be expanded by adding more OpenVPN nodes to increase capacity, or more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding.
 
 Our goal is something like this:
 
     $ leap list
-           NODES   SERVICES           TAGS
-           couch1  couchdb
-           web1    webapp
-           vpn1    openvpn
+         NODES   SERVICES    TAGS
+       cheetah   couchdb     production
+    wildebeest   webapp      production
+       ostrich   openvpn     production
 
 NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations.
 
@@ -27,13 +27,14 @@ Requirements
 
 In order to complete this Quick Start, you will need a few things:
 
-* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 10G to each node, that should be plenty.
-* You should be able to SSH into them remotely, and know their IP addresses and their SSH host keys
-* You will need four different IPs, one for each node, and a second one for the VPN gateway
+* You will need three real or paravirtualized virtual machines (KVM, Xen, Openstack, Amazon, but not Vagrant - sorry) that have a basic Debian Stable installed. If you allocate 20G of disk space to each node for the system, after this process is completed, you will have used less than 10% of that disk space. If you allocate 2 CPUs and 8G of memory to each node, that should be more than enough to begin with.
+* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys
+* You will need four different IPs. Each node gets a primary IP, and the OpenVPN gateway additionally needs a gateway IP.
 * The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`.
 * You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes
 * Your machines will need to be connected to the internet, and not behind a restrictive firewall.
-* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services.
+* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The `leap` command will login to your servers and configure the services.
+* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly.
 
 All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following:
 
@@ -70,13 +71,13 @@ Install core prerequisites:
 1. Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed).
 -->
 
-NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9.
+NOTE: leap_cli requires ruby 1.9 or later.
 
 
 Install the LEAP command-line utility
----------------------
+-------------------------------------------------
 
-<!--Install the `leap` command as a gem:
+Install the `leap` command from rubygems.org:
 
     $ sudo gem install leap_cli
 
@@ -85,35 +86,24 @@ Alternately, you can install `leap` from source:
     $ git clone https://leap.se/git/leap_cli
     $ cd leap_cli
     $ rake build
--->
-
-Install `leap` command from source:
-
-    $ git clone https://leap.se/git/leap_cli
-    $ cd leap_cli
-    $ rake build
-
-Then, install as root user (recommended):
-
     $ sudo rake install
 
-Or, install as unprivileged user:
+You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this:
 
     $ rake install
     # watch out for the directory leap is installed to, then i.e.
     $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap
 
-With both methods, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH.
+With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH.
 
-If you have successfully installed the LEAP cli, then you should be able to do the following:
+If you have successfully installed the `leap` command, then you should be able to do the following:
 
     $ leap --help
 
-and be presented with the command-line help options. If you receive an error when doing this, please read through the README.md in the LEAP cli source to try and resolve any problems before going forwards.
-
+This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards.
 
 Check out the platform
-----------------------
+--------------------------
 
 The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform.
 
@@ -136,12 +126,12 @@ A provider instance is a directory tree, usually stored in git, that contains ev
 Bootstrap the provider
 -----------------------
 
-Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step. 
+Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step.
 
     $ cd ~/leap/example
     $ leap new .
 
-NOTES: 
+NOTES:
  . make sure you include that trailing dot!
 
 The `leap new` command will ask you for several required values:
@@ -215,25 +205,25 @@ For a full list of possible settings, you can use `leap inspect` to see how prov
 Setup the provider's nodes and services
 ---------------------------------------
 
-A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](developmet) for more information.
+A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information.
 
 Create a node, with the service "webapp":
 
-    $ leap node add web1 ip_address:x.x.x.w services:webapp tags:production
+    $ leap node add wildebeest ip_address:x.x.x.w services:webapp tags:production
 
 NOTE: replace x.x.x.w with the actual IP address of this node
 
-This created a node configuration file in `nodes/web1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure.
+This created a node configuration file in `nodes/wildebeest.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure.
 
 The web application and the VPN nodes require a database, so lets create the database server node:
 
-    $ leap node add couch1 ip_address:x.x.x.x services:couchdb tags:production
+    $ leap node add cheetah ip_address:x.x.x.x services:couchdb tags:production
 
 NOTE: replace x.x.x.x with the actual IP address of this node
 
-Now we need the VPN gateway, so lets create that node:
+Now we need the OpenVPN gateway, so lets create that node:
 
-    $ leap node add vpn1 ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production
+    $ leap node add ostrich ip_address:x.x.x.y openvpn.gateway_address:x.x.x.z services:openvpn tags:production
 
 NOTE: replace x.x.x.y with the IP address of the machine, and x.x.x.z with the second IP. openvpn gateways must be assigned two IP addresses, one for the host itself and one for the openvpn gateway. We do this to prevent incoming and outgoing VPN traffic on the same IP. Without this, the client might send some traffic to other VPN users in the clear, bypassing the VPN.
 
@@ -246,9 +236,9 @@ Now that you have the nodes configured, you should create the DNS entries for th
 Set up your DNS with these hostnames:
 
     $ leap list --print ip_address,domain.full,dns.aliases
-        couch1  x.x.x.w, couch1.example.org, null
-          web1  x.x.x.x, web1.example.org, api.example.org, nicknym.example.org
-          vpn1  x.x.x.y, vpn1.example.org, null
+       cheetah  x.x.x.w, cheetah.example.org, null
+    wildebeest  x.x.x.x, wildebeest.example.org, api.example.org
+       ostrich  x.x.x.y, ostrich.example.org, null
 
 Alternately, you can adapt this zone file snippet:
 
@@ -256,9 +246,9 @@ Alternately, you can adapt this zone file snippet:
 
 If you cannot edit your DNS zone file, you can still test your provider by adding entries to your local resolver hosts file (`/etc/hosts` for linux):
 
-    x.x.x.w couch1.example.org
-    x.x.x.x web1.example.org api.example.org example.org
-    x.x.x.y vpn1.example.org
+    x.x.x.w cheetah.example.org
+    x.x.x.x wildebeest.example.org api.example.org example.org
+    x.x.x.y ostrich.example.org
 
 Please don't forget about these entries, they will override DNS queries if you setup your DNS later.
 
@@ -274,25 +264,26 @@ This will initialize all nodes with the tag "production". When `leap node init`
 
 If you prefer, you can initalize each node, one at a time:
 
-    $ leap node init web1
-    $ leap node init couch1
-    $ leap node init vpn1
+    $ leap node init wildebeest
+    $ leap node init cheetah
+    $ leap node init ostrich
 
 Deploy the LEAP platform to the nodes
 --------------------
 
-Now you should deploy the platform recipes to the nodes. Deployment can take a while to run, especially on the first run, as it needs to update the packages on the new machine:
+Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine.
 
-    $ leap deploy web1
+*Important notes:* currently nodes must be deployed in a certain order. The underlying couch database node(s) must be deployed first, and then all other nodes. Also you need to configure and deploy all of the couchdb nodes that you plan to use at this time, as currently you cannot add more of them later later ([See](https://leap.se/es/docs/platform/known-issues#CouchDB.Sync)).
 
-Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again. 
+    $ leap deploy cheetah
+
+Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again.
 
 However, to deploy our three-node openvpn setup, we need the database and LEAP web application requires a database to run, so let's deploy to the couchdb and openvpn nodes:
 
-    $ leap deploy couch1
-    $ leap deploy vpn1
+    $ leap deploy wildebeest
+    $ leap deploy ostrich
 
-NOTE: the output from deploying can be quite busy, so we often do them each node one by one.
 
 What is going on here?
 --------------------------------------------
@@ -310,19 +301,21 @@ When you run `leap deploy`, a bunch of things happen, in this order:
 
 You can run `leap -v2 deploy` to see exactly what commands are being executed.
 
-<!-- See [under the hood](under-the-hood) for more details. -->
-
 
 Test that things worked correctly
 =================================
 
-You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway. 
+You should now have three machines with the LEAP platform deployed to them, one for the web application, one for the database and one for the OpenVPN gateway.
 
+To run troubleshooting tests:
 
-Access the web application
---------------------------------------------
+    leap test
+
+If you want to confirm for yourself that things are working, you can perform the following manual tests.
+
+### Access the web application
 
-In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named web1 in this example).
+In order to connect to the web application in your browser, you need to point your domain at the IP address of the web application node (named wildebeest in this example).
 
 There are a lot of different ways to do this, but one easy way is to modify your `/etc/hosts` file. First, find the IP address of the webapp node:
 
@@ -336,13 +329,12 @@ Replacing 'leap.example.org' with whatever you specified as the `domain` in the
 
 Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login.
 
-Use the VPN
------------
+### Use the VPN
 
 You should be able to simply test that the OpenVPN gateway works properly by doing the following:
 
     $ leap test init
-    $ sudo openvpn test/openvpn/unlimited.ovpn
+    $ sudo openvpn test/openvpn/production_unlimited.ovpn
 
 Or, you can use the LEAP client (called "bitmask") to connect to your new provider, create a user and then connect to the VPN.
 
@@ -357,7 +349,7 @@ Useful commands
 
 Here are a few useful commands you can run on your new local nodes:
 
-* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first).
+* `leap ssh wildebeest` -- SSH into node wildebeest (requires `leap node init wildebeest` first).
 * `leap list` -- list all nodes.
 * `leap list production` -- list only those nodes with the tag 'production'
 * `leap list --print ip_address` -- list a particular attribute of all nodes.
@@ -381,7 +373,7 @@ Examples:
 * `leap list openvpn` -- list all nodes with service openvpn.
 * `leap list openvpn +production` -- only nodes of service type openvpn AND tag production.
 * `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes.
-* `leap node init vpn1` -- just init the node named vpn1.
+* `leap node init ostrich` -- just init the node named ostrich.
 
 Keep track of your provider configurations
 ------------------------------------------
diff --git a/doc/tutorials/single-node.md b/doc/tutorials/single-node.md
new file mode 100644
index 00000000..02d35c7a
--- /dev/null
+++ b/doc/tutorials/single-node.md
@@ -0,0 +1,343 @@
+@title = 'Single node tutorial'
+@nav_title = 'Single node'
+@summary = 'A single node email provider.'
+
+Quick Start - Single node setup
+===============================
+
+This tutorial walks you through the initial process of creating and deploying a minimal service provider running the [LEAP platform](platform).
+We will guide you through building a single node mail provider.
+
+Our goal
+------------------
+
+We are going to create a minimal LEAP provider offering Email service. This basic setup can be expanded by adding more webapp and couchdb nodes to increase availability (performance wise, a single couchdb and a single webapp are more than enough for most usage, since they are only lightly used, but you might want redundancy). Please note: currently it is not possible to safely add additional couchdb nodes at a later point. They should all be added in the beginning, so please consider carefully if you would like more before proceeding.
+
+Our goal is something like this:
+
+    $ leap list
+        NODES   SERVICES                       TAGS
+        node1   couchdb, mx, soledad, webapp   local
+
+NOTE: You won't be able to run that `leap list` command yet, not until we actually create the node configurations.
+
+Requirements
+------------
+
+In order to complete this Quick Start, you will need a few things:
+
+* You will need one real or paravirtualized virtual machine (Vagrant, KVM, Xen, Openstack, Amazon, …) that have a basic Debian Stable installed.
+* You should be able to SSH into them remotely, and know their root password, IP addresses and their SSH host keys
+* The ability to create/modify DNS entries for your domain is preferable, but not needed. If you don't have access to DNS, you can workaround this by modifying your local resolver, i.e. editing `/etc/hosts`.
+* You need to be aware that this process will make changes to your systems, so please be sure that these machines are a basic install with nothing configured or running for other purposes
+* Your machines will need to be connected to the internet, and not behind a restrictive firewall.
+* You should work locally on your laptop/workstation (one that you trust and that is ideally full-disk encrypted) while going through this guide. This is important because the provider configurations you are creating contain sensitive data that should not reside on a remote machine. The leap cli utility will login to your servers and configure the services.
+* You should do everything described below as an unprivileged user, and only run those commands as root that are noted with *sudo* in front of them. Other than those commands, there is no need for privileged access to your machine, and in fact things may not work correctly.
+
+All the commands in this tutorial are run on your sysadmin machine. In order to complete the tutorial, the sysadmin will do the following:
+
+* Install pre-requisites
+* Install the LEAP command-line utility
+* Check out the LEAP platform
+* Create a provider and its certificates
+* Setup the provider's nodes and the services that will reside on those nodes
+* Initialize the nodes
+* Deploy the LEAP platform to the nodes
+* Test that things worked correctly
+* Some additional commands
+
+We will walk you through each of these steps.
+
+
+Prepare your environment
+========================
+
+There are a few things you need to setup before you can get going. Just some packages, the LEAP cli and the platform.
+
+Install pre-requisites
+--------------------------------
+
+*Debian & Ubuntu*
+
+Install core prerequisites:
+
+    $ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2
+
+<!--
+*Mac OS*
+
+1. Install rubygems from https://rubygems.org/pages/download (unless the `gem` command is already installed).
+-->
+
+NOTE: leap_cli should work with ruby1.8, but has only been tested using ruby1.9.
+
+
+Install the LEAP command-line utility
+-------------------------------------------------
+
+Install the `leap` command from rubygems.org:
+
+    $ sudo gem install leap_cli
+
+Alternately, you can install `leap` from source:
+
+    $ git clone https://leap.se/git/leap_cli
+    $ cd leap_cli
+    $ rake build
+    $ sudo rake install
+
+You can also install from source as an unprivileged user, if you want. For example, instead of `sudo rake install` you can do something like this:
+
+    $ rake install
+    # watch out for the directory leap is installed to, then i.e.
+    $ sudo ln -s ~/.gem/ruby/1.9.1/bin/leap /usr/local/bin/leap
+
+With either `rake install` or `sudo rake install`, you can use now /usr/local/bin/leap, which in most cases will be in your $PATH.
+
+If you have successfully installed the `leap` command, then you should be able to do the following:
+
+    $ leap --help
+
+This will list the command-line help options. If you receive an error when doing this, please read through the README.md in the `leap_cli` source to try and resolve any problems before going forwards.
+
+Check out the platform
+--------------------------
+
+The LEAP Platform is a series of puppet recipes and modules that will be used to configure your provider. You will need a local copy of the platform that will be used to setup your nodes and manage your services. To begin with, you will not need to modify the LEAP Platform.
+Until we have a up to date stable release we recommend using the `develop` branch of the platform and leap_cli for all features of LEAP.
+
+First we'll create a directory for LEAP things, and then we'll check out the platform code and initalize the modules:
+
+    $ mkdir ~/leap
+    $ cd ~/leap
+    $ git clone https://leap.se/git/leap_platform.git
+    $ cd leap_platform
+    $ git checkout develop
+    $ git submodule sync; git submodule update --init
+
+
+Provider Setup
+==============
+
+A provider instance is a directory tree, usually stored in git, that contains everything you need to manage an infrastructure for a service provider. In this case, we create one for example.org and call the instance directory 'example'.
+
+    $ mkdir -p ~/leap/example
+
+Bootstrap the provider
+-----------------------
+
+Now, we will initialize this directory to make it a provider instance. Your provider instance will need to know where it can find the local copy of the git repository leap_platform, which we setup in the previous step.
+
+    $ cd ~/leap/example
+    $ leap new .
+
+NOTES:
+ . make sure you include that trailing dot!
+
+The `leap new` command will ask you for several required values:
+
+* domain: The primary domain name of your service provider. In this tutorial, we will be using "example.org".
+* name: The name of your service provider (we use "Example").
+* contact emails: A comma separated list of email addresses that should be used for important service provider contacts (for things like postmaster aliases, Tor contact emails, etc).
+* platform: The directory where you have a copy of the `leap_platform` git repository checked out.
+
+You could also have passed these configuration options on the command-line, like so:
+
+    $ leap new --contacts your@email.here --domain leap.example.org --name Example --platform=~/leap/leap_platform .
+
+You may want to poke around and see what is in the files we just created. For example:
+
+    $ cat provider.json
+
+Optionally, commit your provider directory using the version control software you fancy. For example:
+
+    $ git init
+    $ git add .
+    $ git commit -m "initial provider commit"
+
+Now add yourself as a privileged sysadmin who will have access to deploy to servers:
+
+    $ leap add-user --self
+
+NOTE: in most cases, `leap` must be run from within a provider instance directory tree (e.g. ~/leap/example).
+
+Create provider certificates
+----------------------------
+
+Create two certificate authorities, one for server certs and one for client
+certs (note: you only need to run this one command to get both):
+
+    $ leap cert ca
+
+Create a temporary cert for your main domain (you should replace with a real commercial cert at some point)
+
+    $ leap cert csr
+
+To see details about the keys and certs that the prior two commands created, you can use `leap inspect` like so:
+
+    $ leap inspect files/ca/ca.crt
+
+NOTE: the files `files/ca/*.key` are extremely sensitive and must be carefully protected. The other key files are much less sensitive and can simply be regenerated if needed.
+
+
+Edit provider.json configuration
+--------------------------------------
+
+There are a few required settings in provider.json. At a minimum, you must have:
+
+    {
+      "domain": "example.org",
+      "name": "Example",
+      "contacts": {
+        "default": "email1@example.org"
+      }
+    }
+
+For a full list of possible settings, you can use `leap inspect` to see how provider.json is evaluated after including the inherited defaults:
+
+    $ leap inspect provider.json
+
+
+Setup the provider's node and services
+--------------------------------------
+
+A "node" is a server that is part of your infrastructure. Every node can have one or more services associated with it. Some nodes are "local" and used only for testing, see [Development](development) for more information.
+
+Create a node, with all the services needed for Email - "couchdb", "mx", "soledad" and "webapp":
+
+    $ leap node add node1 ip_address:x.x.x.w services:couchdb,mx,soledad,webapp tags:production
+
+NOTE: replace x.x.x.w with the actual IP address of this node
+
+This created a node configuration file in `nodes/node1.json`, but it did not do anything else. It also added the 'tag' called 'production' to this node. Tags allow us to conveniently group nodes together. When creating nodes, you should give them the tag 'production' if the node is to be used in your production infrastructure.
+
+Setup DNS
+---------
+
+Now that you have the node configured, you should create the DNS entrie for this node.
+
+Set up your DNS with these hostnames:
+
+    $ leap list --print ip_address,domain.full,dns.aliases
+         node1  x.x.x.w, node1.example.org, example.org, api.example.org, nicknym.example.org
+
+Alternately, you can adapt this zone file snippet:
+
+    $ leap compile zone
+
+If you cannot edit your DNS zone file, you can still test your provider by adding this entry to your local resolver hosts file (`/etc/hosts` for linux):
+
+    x.x.x.w node1.example.org example.org api.example.org nicknym.example.org
+
+Please don't forget about these entries, they will override DNS queries if you setup your DNS later.
+
+
+Initialize the nodes
+--------------------
+
+Node initialization only needs to be done once, but there is no harm in doing it multiple times:
+
+    $ leap node init production
+
+This will initialize the node with the tag "production". When `leap node init` is run, you will be prompted to verify the fingerprint of the SSH host key and to provide the root password of the server. You should only need to do this once.
+
+
+Deploy the LEAP platform to the nodes
+--------------------
+
+Now you should deploy the platform recipes to the nodes. [Deployment can take a while to run](http://xkcd.com/303/), especially on the first run, as it needs to update the packages on the new machine.
+
+    $ leap deploy
+
+Watch the output for any errors (in red), if everything worked fine, you should now have your first running node. If you do have errors, try doing the deploy again.
+
+NOTE: the output from deploying can be quite busy, so we often do them each node one by one.
+
+What is going on here?
+--------------------------------------------
+
+First, some background terminology:
+
+* **puppet**: Puppet is a system for automating deployment and management of servers (called nodes).
+* **hiera files**: In puppet, you can use something called a 'hiera file' to seed a node with a few configuration values. In LEAP, we go all out and put *every* configuration value needed for a node in the hiera file, and automatically compile a custom hiera file for each node.
+
+When you run `leap deploy`, a bunch of things happen, in this order:
+
+1. **Compile hiera files**: The hiera configuration file for each node is compiled in YAML format and saved in the directory `hiera`. The source material for this hiera file consists of all the JSON configuration files imported or inherited by the node's JSON config file.
+* **Copy required files to node**: All the files needed for puppet to run are rsync'ed to each node. This includes the entire leap_platform directory, as well as the node's hiera file and other files needed by puppet to set up the node (keys, binary files, etc).
+* **Puppet is run**: Once the node is ready, leap connects to the node via ssh and runs `puppet apply`. Puppet is applied locally on the node, without a daemon or puppetmaster.
+
+You can run `leap -v2 deploy` to see exactly what commands are being executed.
+
+<!-- See [under the hood](under-the-hood) for more details. -->
+
+
+Test that things worked correctly
+=================================
+
+You should now one machine with the LEAP platform email service deployed to it.
+
+
+Access the web application
+--------------------------------------------
+
+In order to connect to the web application in your browser, you need to point your domain at the IP address of your new node.
+
+Next, you can connect to the web application either using a web browser or via the API using the LEAP client. To use a browser, connect to https://leap.example.org (replacing that with your domain). Your browser will complain about an untrusted cert, but for now just bypass this. From there, you should be able to register a new user and login.
+
+Testing with leap_cli
+---------------------
+
+Use the test command to run a set of different tests:
+
+    leap test
+
+
+Additional information
+======================
+
+It is useful to know a few additional things.
+
+Useful commands
+---------------
+
+Here are a few useful commands you can run on your new local nodes:
+
+* `leap ssh web1` -- SSH into node web1 (requires `leap node init web1` first).
+* `leap list` -- list all nodes.
+* `leap list production` -- list only those nodes with the tag 'production'
+* `leap list --print ip_address` -- list a particular attribute of all nodes.
+* `leap cert update` -- generate new certificates if needed.
+
+See the full command reference for more information.
+
+Node filters
+-------------------------------------------
+
+Many of the `leap` commands take a "node filter". You can use a node filter to target a command at one or more nodes.
+
+A node filter consists of one or more keywords, with an optional "+" before each keyword.
+
+* keywords can be a node name, a service type, or a tag.
+* the "+" before the keyword constructs an AND condition
+* otherwise, multiple keywords together construct an OR condition
+
+Examples:
+
+* `leap list openvpn` -- list all nodes with service openvpn.
+* `leap list openvpn +production` -- only nodes of service type openvpn AND tag production.
+* `leap deploy webapp openvpn` -- deploy to all webapp OR openvpn nodes.
+* `leap node init vpn1` -- just init the node named vpn1.
+
+Keep track of your provider configurations
+------------------------------------------
+
+You should commit your provider changes to your favorite VCS whenever things change. This way you can share your configurations with other admins, all they have to do is to pull the changes to stay up to date. Every time you make a change to your provider, such as adding nodes, services, generating certificates, etc. you should add those to your VCS, commit them and push them to where your repository is hosted.
+
+Note that your provider directory contains secrets! Those secrets include passwords for various services. You do not want to have those passwords readable by the world, so make sure that wherever you are hosting your repository, it is not public for the world to read.
+
+What's next
+-----------------------------------
+
+Read the [LEAP platform guide](guide) to learn about planning and securing your infrastructure.
+
diff --git a/platform.rb b/platform.rb
index 7aebebc8..c37b6d29 100644
--- a/platform.rb
+++ b/platform.rb
@@ -4,8 +4,8 @@
 #
 
 Leap::Platform.define do
-  self.version = "0.5.5"
-  self.compatible_cli = "1.5.5".."1.5.7"
+  self.version = "0.6"
+  self.compatible_cli = "1.6.1".."1.99"
 
   #
   # the facter facts that should be gathered
@@ -13,7 +13,16 @@ Leap::Platform.define do
   self.facts = ["ec2_local_ipv4", "ec2_public_ipv4"]
 
   #
+  # absolute paths on the destination server
+  #
+  self.hiera_path = '/etc/leap/hiera.yaml'
+  self.leap_dir   = '/srv/leap'
+  self.files_dir  = '/srv/leap/files'
+  self.init_path  = '/srv/leap/initialized'
+
+  #
   # the named paths for this platform
+  # (relative to the provider directory)
   #
   self.paths = {
     # directories
@@ -43,6 +52,11 @@ Leap::Platform.define do
     :soledad_service_json_template => 'files/service-definitions/#{arg}/soledad-service.json.erb',
     :smtp_service_json_template    => 'files/service-definitions/#{arg}/smtp-service.json.erb',
 
+    # custom puppet
+    :custom_puppet_dir => 'files/puppet',
+    :custom_puppet_modules_dir => 'files/puppet/modules',
+    :custom_puppet_manifests_dir => 'files/puppet/manifests',
+
     # output files
     :facts            => 'facts.json',
     :user_ssh         => 'users/#{arg}/#{arg}_ssh.pub',
@@ -63,10 +77,12 @@ Leap::Platform.define do
     :vagrantfile      => 'test/Vagrantfile',
 
     # node output files
-    :hiera            => 'hiera/#{arg}.yaml',
-    :node_ssh_pub_key => 'files/nodes/#{arg}/#{arg}_ssh.pub',
-    :node_x509_key    => 'files/nodes/#{arg}/#{arg}.key',
-    :node_x509_cert   => 'files/nodes/#{arg}/#{arg}.crt',
+    :hiera             => 'hiera/#{arg}.yaml',
+    :node_ssh_pub_key  => 'files/nodes/#{arg}/#{arg}_ssh.pub',
+    :node_x509_key     => 'files/nodes/#{arg}/#{arg}.key',
+    :node_x509_cert    => 'files/nodes/#{arg}/#{arg}.crt',
+    :node_tor_priv_key => 'files/nodes/#{arg}/tor.key',
+    :node_tor_pub_key  => 'files/nodes/#{arg}/tor.pub',
 
     # testing files
     :test_client_key     => 'test/cert/client.key',
@@ -85,5 +101,7 @@ Leap::Platform.define do
   self.monitor_username = 'monitor'
 
   self.reserved_usernames = ['monitor']
+
+  self.default_puppet_tags = ['leap_base','leap_service']
 end
 
diff --git a/provider_base/common.json b/provider_base/common.json
index a4d9c5f2..649db0d9 100644
--- a/provider_base/common.json
+++ b/provider_base/common.json
@@ -25,9 +25,13 @@
   "hosts": "=> hosts_file",
   "x509": {
     "use": true,
+    "use_commercial": false,
     "cert": "= x509.use ? file(:node_x509_cert, :missing => 'x509 certificate for node $node. Run `leap cert update`') : nil",
     "key": "= x509.use ? file(:node_x509_key, :missing => 'x509 key for node $node. Run `leap cert update`') : nil",
-    "ca_cert": "= try_file :ca_cert"
+    "ca_cert": "= try_file :ca_cert",
+    "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil",
+    "commercial_key": "= x509.use_commercial ? file([:commercial_key, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil",
+    "commercial_ca_cert": "= x509.use_commercial ? try_file(:commercial_ca_cert) : nil"
   },
   "service_type": "internal_service",
   "development": {
@@ -38,5 +42,13 @@
   "enabled": true,
   "mail": {
     "smarthost": "= nodes_like_me[:services => :mx].exclude(self).field('domain.full')"
+  },
+  "stunnel": {
+    "clients": {},
+    "servers": {}
+  },
+  "platform": {
+    "version": "= Leap::Platform.version.to_s",
+    "major_version": "= Leap::Platform.major_version"
   }
 }
diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb
index 3b8976fd..4bd220df 100644
--- a/provider_base/files/service-definitions/v1/eip-service.json.erb
+++ b/provider_base/files/service-definitions/v1/eip-service.json.erb
@@ -42,8 +42,14 @@
     end
     configuration = node.openvpn.configuration
   end
-  hsh["gateways"] = gateways.compact
-  hsh["locations"] = locations
-  hsh["openvpn_configuration"] = configuration
+  if gateways.any?
+    configuration = configuration.dup
+    if configuration['fragment'] && configuration['fragment'] == 1500
+      configuration.delete('fragment')
+    end
+    hsh["gateways"] = gateways.compact
+    hsh["locations"] = locations
+    hsh["openvpn_configuration"] = configuration
+  end
   JSON.sorted_generate hsh
 %>
\ No newline at end of file
diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb
new file mode 100644
index 00000000..ecc3e6ba
--- /dev/null
+++ b/provider_base/lib/macros.rb
@@ -0,0 +1,15 @@
+#
+# MACROS
+#
+# The methods in these files are available in the context of a .json configuration file.
+# (The module LeapCli::Macro is included in Config::Object)
+#
+
+require_relative 'macros/core'
+require_relative 'macros/files'
+require_relative 'macros/haproxy'
+require_relative 'macros/hosts'
+require_relative 'macros/keys'
+require_relative 'macros/nodes'
+require_relative 'macros/secrets'
+require_relative 'macros/stunnel'
diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb
new file mode 100644
index 00000000..7de50f2f
--- /dev/null
+++ b/provider_base/lib/macros/core.rb
@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+module LeapCli
+  module Macro
+
+    #
+    # return a fingerprint for a x509 certificate
+    #
+    def fingerprint(filename)
+      "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename))
+    end
+
+    #
+    # Creates a hash from the ssh key info in users directory, for use in
+    # updating authorized_keys file. Additionally, the 'monitor' public key is
+    # included, which is used by the monitor nodes to run particular commands
+    # remotely.
+    #
+    def authorized_keys
+      hash = {}
+      keys = Dir.glob(Path.named_path([:user_ssh, '*']))
+      keys.sort.each do |keyfile|
+        ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ")
+        name = File.basename(File.dirname(keyfile))
+        until hash[name].nil?
+          i ||= 1; name = "#{name}#{i+=1}"
+        end
+        hash[name] = {
+          "type" => ssh_type,
+          "key" => ssh_key
+        }
+      end
+      ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ")
+      hash[Leap::Platform.monitor_username] = {
+        "type" => ssh_type,
+        "key" => ssh_key
+      }
+      hash
+    end
+
+    def assert(assertion)
+      if instance_eval(assertion)
+        true
+      else
+        raise AssertionFailed.new(assertion), assertion, caller
+      end
+    end
+
+    def error(msg)
+      raise ConfigError.new(@node, msg), msg, caller
+    end
+
+    #
+    # applies a JSON partial to this node
+    #
+    def apply_partial(partial_path)
+      manager.partials(partial_path).each do |partial_data|
+        self.deep_merge!(partial_data)
+      end
+    end
+
+    #
+    # If at first you don't succeed, then it is time to give up.
+    #
+    # try{} returns nil if anything in the block throws an exception.
+    #
+    # You can wrap something that might fail in `try`, like so.
+    #
+    #   "= try{ nodes[:services => 'tor'].first.ip_address } "
+    #
+    def try(&block)
+      yield
+    rescue NoMethodError
+    rescue ArgumentError
+      nil
+    end
+
+    protected
+
+    #
+    # returns a node list, if argument is not already one
+    #
+    def listify(node_list)
+      if node_list.is_a? Config::ObjectList
+        node_list
+      elsif node_list.is_a? Config::Object
+        Config::ObjectList.new(node_list)
+      else
+        raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller
+      end
+    end
+
+  end
+end
diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb
new file mode 100644
index 00000000..958958bc
--- /dev/null
+++ b/provider_base/lib/macros/files.rb
@@ -0,0 +1,89 @@
+# encoding: utf-8
+
+##
+## FILES
+##
+
+module LeapCli
+  module Macro
+
+    #
+    # inserts the contents of a file
+    #
+    def file(filename, options={})
+      if filename.is_a? Symbol
+        filename = [filename, @node.name]
+      end
+      filepath = Path.find_file(filename)
+      if filepath
+        if filepath =~ /\.erb$/
+          ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding)
+        else
+          File.read(filepath, :encoding => 'UTF-8')
+        end
+      else
+        raise FileMissing.new(Path.named_path(filename), options)
+        ""
+      end
+    end
+
+    #
+    # like #file, but allow missing files
+    #
+    def try_file(filename)
+      return file(filename)
+    rescue FileMissing
+      return nil
+    end
+
+    #
+    # returns what the file path will be, once the file is rsynced to the server.
+    # an internal list of discovered file paths is saved, in order to rsync these files when needed.
+    #
+    # notes:
+    #
+    # * argument 'path' is relative to Path.provider/files or Path.provider_base/files
+    # * the path returned by this method is absolute
+    # * the path stored for use later by rsync is relative to Path.provider
+    # * if the path does not exist locally, but exists in provider_base, then the default file from
+    #   provider_base is copied locally. this is required for rsync to work correctly.
+    #
+    def file_path(path, options={})
+      if path.is_a? Symbol
+        path = [path, @node.name]
+      elsif path.is_a? String
+        # ensure it prefixed with files/
+        unless path =~ /^files\//
+          path = "files/" + path
+        end
+      end
+      actual_path = Path.find_file(path)
+      if actual_path.nil?
+        if options[:missing]
+          raise FileMissing.new(Path.named_path(path), options)
+        else
+          Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file."
+        end
+        nil
+      else
+        if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/
+          # if file is under Path.provider_base, we must copy the default file to
+          # to Path.provider in order for rsync to be able to sync the file.
+          local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider)
+          FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700
+          FileUtils.install actual_path, local_provider_path, :mode => 0600
+          Util.log :created, Path.relative_path(local_provider_path)
+          actual_path = local_provider_path
+        end
+        if File.directory?(actual_path) && actual_path !~ /\/$/
+          actual_path += '/' # ensure directories end with /, important for building rsync command
+        end
+        relative_path = Path.relative_path(actual_path)
+        relative_path.sub!(/^files\//, '') # remove "files/" prefix
+        @node.file_paths << relative_path
+        File.join(Leap::Platform.files_dir, relative_path)
+      end
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb
new file mode 100644
index 00000000..602ae726
--- /dev/null
+++ b/provider_base/lib/macros/haproxy.rb
@@ -0,0 +1,73 @@
+# encoding: utf-8
+
+##
+## HAPROXY
+##
+
+module LeapCli
+  module Macro
+
+    #
+    # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to.
+    #
+    # * node_list - a hash of nodes for the haproxy servers
+    # * stunnel_client - contains the mappings to local ports for each server node.
+    # * non_stunnel_port - in case self is included in node_list, the port to connect to.
+    #
+    # 1000 weight is used for nodes in the same location.
+    # 100 otherwise.
+    #
+    def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil)
+      default_weight = 10
+      local_weight = 100
+
+      # record the hosts_file
+      hostnames(node_list)
+
+      # create a simple map for node name -> local stunnel accept port
+      accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry|
+        name = stunnel_entry.first.sub /_[0-9]+$/, ''
+        hsh[name] = stunnel_entry.last['accept_port']
+        hsh
+      end
+
+      # if one the nodes in the node list is ourself, then there will not be a stunnel to it,
+      # but we need to include it anyway in the haproxy config.
+      if node_list[self.name] && non_stunnel_port
+        accept_ports[self.name] = non_stunnel_port
+      end
+
+      # create the first pass of the servers hash
+      servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node|
+        # make sure we have a port to talk to
+        unless accept_ports[node.name]
+          error "haproxy needs a local port to talk to when connecting to #{node.name}"
+        end
+        weight = default_weight
+        try {
+          weight = local_weight if self.location.name == node.location.name
+        }
+        hsh[node.name] = Config::Object[
+          'backup', false,
+          'host', 'localhost',
+          'port', accept_ports[node.name],
+          'weight', weight
+        ]
+        if node.services.include?('couchdb')
+          hsh[node.name]['writable'] = node.couch.mode != 'mirror'
+        end
+        hsh
+      end
+
+      # if there are some local servers, make the others backup
+      if servers.detect{|k,v| v.weight == local_weight}
+        servers.each do |k,server|
+          server['backup'] = server['weight'] == default_weight
+        end
+      end
+
+      return servers
+    end
+
+  end
+end
diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb
new file mode 100644
index 00000000..8281329f
--- /dev/null
+++ b/provider_base/lib/macros/hosts.rb
@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+module LeapCli
+  module Macro
+
+    ##
+    ## HOSTS
+    ##
+
+    #
+    # records the list of hosts that are encountered for this node
+    #
+    def hostnames(nodes)
+      @referenced_nodes ||= Config::ObjectList.new
+      nodes = listify(nodes)
+      nodes.each_node do |node|
+        @referenced_nodes[node.name] ||= node
+      end
+      return nodes.values.collect {|node| node.domain.name}
+    end
+
+    #
+    # Generates entries needed for updating /etc/hosts on a node (as a hash).
+    #
+    # Argument `nodes` can be nil or a list of nodes. If nil, only include the
+    # IPs of the other nodes this @node as has encountered (plus all mx nodes).
+    #
+    # Also, for virtual machines, we use the local address if this @node is in
+    # the same location as the node in question.
+    #
+    # We include the ssh public key for each host, so that the hash can also
+    # be used to generate the /etc/ssh/known_hosts
+    #
+    def hosts_file(nodes=nil)
+      if nodes.nil?
+        if @referenced_nodes && @referenced_nodes.any?
+          nodes = @referenced_nodes
+          nodes = nodes.merge(nodes_like_me[:services => 'mx'])  # all nodes always need to communicate with mx nodes.
+        end
+      end
+      return {} unless nodes
+      hosts = {}
+      my_location = @node['location'] ? @node['location']['name'] : nil
+      nodes.each_node do |node|
+        hosts[node.name] = {
+          'ip_address' => node.ip_address,
+          'domain_internal' => node.domain.internal,
+          'domain_full' => node.domain.full,
+          'port' => node.ssh.port
+        }
+        node_location = node['location'] ? node['location']['name'] : nil
+        if my_location == node_location
+          if facts = @node.manager.facts[node.name]
+            if facts['ec2_public_ipv4']
+              hosts[node.name]['ip_address'] = facts['ec2_public_ipv4']
+            end
+          end
+        end
+        host_pub_key   = Util::read_file([:node_ssh_pub_key,node.name])
+        if host_pub_key
+          hosts[node.name]['host_pub_key'] = host_pub_key
+        end
+      end
+      hosts
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb
new file mode 100644
index 00000000..0ed7ccd0
--- /dev/null
+++ b/provider_base/lib/macros/keys.rb
@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+#
+# Macro for dealing with cryptographic keys
+#
+
+module LeapCli
+  module Macro
+
+    #
+    # return the path to the tor public key
+    # generating key if it is missing
+    #
+    def tor_public_key_path(path_name, key_type)
+      path = file_path(path_name)
+      if path.nil?
+        generate_tor_key(key_type)
+        file_path(path_name)
+      else
+        path
+      end
+    end
+
+    #
+    # return the path to the tor private key
+    # generating key if it is missing
+    #
+    def tor_private_key_path(path_name, key_type)
+      path = file_path(path_name)
+      if path.nil?
+        generate_tor_key(key_type)
+        file_path(path_name)
+      else
+        path
+      end
+    end
+
+    #
+    # Generates a onion_address from a public RSA key file.
+    #
+    # path_name is the named path of the Tor public key.
+    #
+    # Basically, an onion address is nothing more than a base32 encoding
+    # of the first 10 bytes of a sha1 digest of the public key.
+    #
+    # Additionally, Tor ignores the 22 byte header of the public key
+    # before taking the sha1 digest.
+    #
+    def onion_address(path_name)
+      require 'base32'
+      require 'base64'
+      require 'openssl'
+      path = Path.find_file([path_name, self.name])
+      if path && File.exists?(path)
+        public_key_str = File.readlines(path).grep(/^[^-]/).join
+        public_key     = Base64.decode64(public_key_str)
+        public_key     = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header
+        sha1sum        = Digest::SHA1.new.digest(public_key)
+        Base32.encode(sha1sum.slice(0,10)).downcase
+      else
+        LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path
+      end
+    end
+
+    private
+
+    def generate_tor_key(key_type)
+      if key_type == 'RSA'
+        require 'certificate_authority'
+        keypair = CertificateAuthority::MemoryKeyMaterial.new
+        bit_size = 1024
+        LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do
+          keypair.generate_key(bit_size)
+          LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem
+          LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem
+        end
+      else
+        LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type
+      end
+    end
+
+  end
+end
diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb
new file mode 100644
index 00000000..0c6668a0
--- /dev/null
+++ b/provider_base/lib/macros/nodes.rb
@@ -0,0 +1,88 @@
+# encoding: utf-8
+
+##
+## node related macros
+##
+
+module LeapCli
+  module Macro
+
+    #
+    # the list of all the nodes
+    #
+    def nodes
+      global.nodes
+    end
+
+    #
+    # grab an environment appropriate provider
+    #
+    def provider
+      global.env(@node.environment).provider
+    end
+
+    #
+    # returns a list of nodes that match the same environment
+    #
+    # if @node.environment is not set, we return other nodes
+    # where environment is not set.
+    #
+    def nodes_like_me
+      nodes[:environment => @node.environment]
+    end
+
+    #
+    # returns a list of nodes that match the location name
+    # and environment of @node.
+    #
+    def nodes_near_me
+      if @node['location'] && @node['location']['name']
+        nodes_like_me['location.name' => @node.location.name]
+      else
+        nodes_like_me['location' => nil]
+      end
+    end
+
+    #
+    #
+    # picks a node out from the node list in such a way that:
+    #
+    # (1) which nodes picked which nodes is saved in secrets.json
+    # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node
+    # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly.
+    #
+    # if the node_list is empty, an exception is raised.
+    # if node_list size is 1, then that node is returned and nothing is
+    # memorized via the secrets.json file.
+    #
+    # `label` is needed to distinguish between pools of nodes for different purposes.
+    #
+    # TODO: more evenly balance after all the nodes have been picked.
+    #
+    def pick_node(label, node_list)
+      if node_list.any?
+        if node_list.size == 1
+          return node_list.values.first
+        else
+          secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})"
+          secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {}
+          secrets_value[@node.name] ||= begin
+            node_to_pick = nil
+            node_list.each_node do |node|
+              next if secrets_value.values.include?(node.name)
+              node_to_pick = node.name
+            end
+            node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one.
+            node_to_pick
+          end
+          picked_node_name = secrets_value[@node.name]
+          @manager.secrets.set(secrets_key, secrets_value, @node.environment)
+          return node_list[picked_node_name]
+        end
+      else
+        raise ArgumentError.new('pick_node(node_list): node_list cannot be empty')
+      end
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb
new file mode 100644
index 00000000..51bf3971
--- /dev/null
+++ b/provider_base/lib/macros/secrets.rb
@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+require 'base32'
+
+module LeapCli
+  module Macro
+
+    #
+    # inserts a named secret, generating it if needed.
+    #
+    # manager.export_secrets should be called later to capture any newly generated secrets.
+    #
+    # +length+ is the character length of the generated password.
+    #
+    def secret(name, length=32)
+      @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment])
+    end
+
+    # inserts a base32 encoded secret
+    def base32_secret(name, length=20)
+      @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment])
+    end
+
+    # Picks a random obfsproxy port from given range
+    def rand_range(name, range)
+      @manager.secrets.set(name, rand(range), @node[:environment])
+    end
+
+    #
+    # inserts an hexidecimal secret string, generating it if needed.
+    #
+    # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4)
+    #
+    def hex_secret(name, bit_length=128)
+      @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment])
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb
new file mode 100644
index 00000000..f16308c7
--- /dev/null
+++ b/provider_base/lib/macros/stunnel.rb
@@ -0,0 +1,95 @@
+##
+## STUNNEL
+##
+
+#
+# About stunnel
+# --------------------------
+#
+# The network looks like this:
+#
+#   From the client's perspective:
+#
+#   |------- stunnel client --------------|    |---------- stunnel server -----------------------|
+#    consumer app -> localhost:accept_port  ->  connect:connect_port -> ??
+#
+#   From the server's perspective:
+#
+#   |------- stunnel client --------------|    |---------- stunnel server -----------------------|
+#                                       ??  ->  *:accept_port -> localhost:connect_port -> service
+#
+
+module LeapCli
+  module Macro
+
+    #
+    # stunnel configuration for the client side.
+    #
+    # +node_list+ is a ObjectList of nodes running stunnel servers.
+    #
+    # +port+ is the real port of the ultimate service running on the servers
+    # that the client wants to connect to.
+    #
+    # * accept_port is the port on localhost to which local clients
+    #   can connect. it is auto generated serially.
+    #
+    # * connect_port is the port on the stunnel server to connect to.
+    #   it is auto generated from the +port+ argument.
+    #
+    # generates an entry appropriate to be passed directly to
+    # create_resources(stunnel::service, hiera('..'), defaults)
+    #
+    # local ports are automatically generated, starting at 4000
+    # and incrementing in sorted order (by node name).
+    #
+    def stunnel_client(node_list, port, options={})
+      @next_stunnel_port ||= 4000
+      node_list = listify(node_list)
+      hostnames(node_list) # record the hosts
+      result = Config::ObjectList.new
+      node_list.each_node do |node|
+        if node.name != self.name || options[:include_self]
+          result["#{node.name}_#{port}"] = Config::Object[
+            'accept_port', @next_stunnel_port,
+            'connect', node.domain.internal,
+            'connect_port', stunnel_port(port),
+            'original_port', port
+          ]
+          @next_stunnel_port += 1
+        end
+      end
+      result
+    end
+
+    #
+    # generates a stunnel server entry.
+    #
+    # +port+ is the real port targeted service.
+    #
+    # * `accept_port` is the publicly bound port
+    # * `connect_port` is the port that the local service is running on.
+    #
+    def stunnel_server(port)
+      {
+        "accept_port" => stunnel_port(port),
+        "connect_port" => port
+      }
+    end
+
+    private
+
+    #
+    # maps a real port to a stunnel port (used as the connect_port in the client config
+    # and the accept_port in the server config)
+    #
+    def stunnel_port(port)
+      port = port.to_i
+      if port < 50000
+        return port + 10000
+      else
+        return port - 10000
+      end
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/provider_base/provider.json b/provider_base/provider.json
index 743964ee..77437935 100644
--- a/provider_base/provider.json
+++ b/provider_base/provider.json
@@ -1,6 +1,6 @@
 {
   "domain": "REQUIRED",
-  "domain_internal": "= domain.sub(/\\..*$/,'.i')",
+  "domain_internal": "= domain.sub(/\\.[^\\.]*$/, '.i')",
   "name": {
     "en": "REQUIRED"
   },
@@ -44,7 +44,7 @@
     "digest": "SHA256",
     "life_span": "10y",
     "server_certificates": {
-      "bit_size": 2048,
+      "bit_size": 4096,
       "digest": "SHA256",
       "life_span": "1y"
     },
@@ -56,7 +56,6 @@
       "unlimited_prefix": "UNLIMITED"
     }
   },
-  "hiera_sync_destination": "/etc/leap",
   "client_version": {
     "min": "0.5",
     "max": null
diff --git a/provider_base/services/_couchdb_master.json b/provider_base/services/_couchdb_master.json
new file mode 100644
index 00000000..20c6f99b
--- /dev/null
+++ b/provider_base/services/_couchdb_master.json
@@ -0,0 +1,8 @@
+//
+// Applied to master couchdb node when there is a single master
+//
+{
+  "couch": {
+    "mode": "master"
+  }
+}
\ No newline at end of file
diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json
new file mode 100644
index 00000000..6a3402bd
--- /dev/null
+++ b/provider_base/services/_couchdb_mirror.json
@@ -0,0 +1,21 @@
+//
+// Applied to all non-master couchdb nodes
+//
+{
+  "stunnel": {
+    "clients": {
+      "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)"
+    }
+  },
+  "couch": {
+    "mode": "mirror",
+    "replication": {
+      // for now, pick the first close one, or the first one.
+      // in the future, maybe use haproxy to balance among all the masters
+      "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}",
+      "username": "replication",
+      "password": "= secret :couch_replication_password",
+      "role": "replication"
+    }
+  }
+}
diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json
new file mode 100644
index 00000000..0f340e00
--- /dev/null
+++ b/provider_base/services/_couchdb_multimaster.json
@@ -0,0 +1,24 @@
+//
+// Only applied to master couchdb nodes when there are multiple masters
+//
+{
+  "stunnel": {
+    "servers": {
+      "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)",
+      "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)"
+    },
+    "clients": {
+      "epmd_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.epmd_port)",
+      "ednp_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.ednp_port)"
+    }
+  },
+  "couch": {
+    "mode": "multimaster",
+    "bigcouch": {
+      "epmd_port": 4369,
+      "ednp_port": 9002,
+      "cookie": "= secret :bigcouch_cookie",
+      "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'].exclude(self).field('domain.full')"
+    }
+  }
+}
diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json
index 5f1b5381..8b1386f8 100644
--- a/provider_base/services/couchdb.json
+++ b/provider_base/services/couchdb.json
@@ -3,20 +3,13 @@
         "use": true
     },
     "stunnel": {
-        "couch_server": "= stunnel_server(couch.port)",
-        "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)",
-        "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.epmd_port)",
-        "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)",
-        "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.ednp_port)"
+        "servers": {
+          "couch_server": "= stunnel_server(couch.port)"
+        }
     },
     "couch": {
+        "master": false,
         "port": 5984,
-        "bigcouch": {
-            "epmd_port": 4369,
-            "ednp_port": 9002,
-            "cookie": "= secret :bigcouch_cookie",
-            "neighbors": "= nodes_like_me[:services => :couchdb].exclude(self).field('domain.full')"
-        },
         "users": {
             "admin": {
                 "username": "admin",
@@ -47,10 +40,15 @@
                 "username": "webapp",
                 "password": "= secret :couch_webapp_password",
                 "salt": "= hex_secret :couch_webapp_password_salt, 128"
+            },
+            "replication": {
+                "username": "replication",
+                "password": "= secret :couch_replication_password",
+                "salt": "= hex_secret :couch_replication_password_salt, 128"
             }
         },
-    "webapp": {
-         "nagios_test_pw": "= secret :nagios_test_password"
+        "webapp": {
+            "nagios_test_pw": "= secret :nagios_test_password"
         }
     }
 }
diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb
new file mode 100644
index 00000000..3bee3a67
--- /dev/null
+++ b/provider_base/services/couchdb.rb
@@ -0,0 +1,60 @@
+#######################################################################
+###
+### NOTE!
+###
+###  Currently, mirrors do not work! The only thing that works is all
+###  nodes multimaster or a single master.
+###
+#######################################################################
+#
+# custom logic for couchdb json resolution
+# ============================================
+#
+# There are three modes for a node:
+#
+# Multimaster
+# -----------
+#
+#    Multimaster uses bigcouch (soon to use couchdb in replication mode
+#    similar to bigcouch).
+#
+#    Use "multimaster" mode when:
+#
+#     * multiple nodes are marked couch.master
+#     * OR no nodes are marked couch.master
+#
+# Master
+# ------
+#
+#    Master uses plain couchdb that is readable and writable.
+#
+#    Use "master" mode when:
+#
+#     * Exactly one node, this one, is marked as master.
+#
+# Mirror
+# ------
+#
+#    Mirror creates a read-only copy of the database. It uses plain coucdhb
+#    with legacy couchdb replication (http based).
+#
+#    This does not currently work, because http replication can't handle
+#    the number of user databases.
+#
+#    Use "mirror" mode when:
+#
+#     * some nodes are marked couch.master
+#     * AND this node is not a master
+#
+
+master_count = nodes_like_me['services' => 'couchdb']['couch.master' => true].size
+
+if master_count == 0
+  apply_partial 'services/_couchdb_multimaster.json'
+elsif couch.master && master_count > 1
+  apply_partial 'services/_couchdb_multimaster.json'
+elsif couch.master && master_count == 1
+  apply_partial 'services/_couchdb_master.json'
+else
+  apply_partial 'services/_couchdb_mirror.json'
+end
diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json
index 03f6c6d1..10d5ac81 100644
--- a/provider_base/services/monitor.json
+++ b/provider_base/services/monitor.json
@@ -1,7 +1,9 @@
 {
   "nagios": {
     "nagiosadmin_pw": "= secret :nagios_admin_password",
-    "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')"
+    "domains_internal": "= nagios.hosts.values.map{|h|h['domain_internal_suffix']}.uniq",
+    "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]",
+    "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.internal_suffix', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')"
   },
   "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])",
   "ssh": {
@@ -12,11 +14,9 @@
   },
   "x509": {
     "use": true,
+    "use_commercial": true,
     "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'",
     "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "commercial_cert": "= file [:commercial_cert, domain.full_suffix]",
-    "commercial_key": "= file [:commercial_key, domain.full_suffix]",
-    "commercial_ca_cert": "= try_file :commercial_ca_cert"
+    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'"
   }
 }
diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json
index 731dee9a..11293ae8 100644
--- a/provider_base/services/mx.json
+++ b/provider_base/services/mx.json
@@ -1,9 +1,14 @@
 {
   "stunnel": {
-    "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)"
+    "clients": {
+      "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)"
+    }
   },
   "haproxy": {
-    "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client)"
+    "couch": {
+      "listen_port": 4096,
+      "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)"
+    }
   },
   "couchdb_leap_mx_user": {
     "username": "= global.services[:couchdb].couch.users[:leap_mx].username",
@@ -13,12 +18,10 @@
   "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq",
   "x509": {
     "use": true,
+    "use_commercial": true,
     "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'",
     "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "commercial_cert": "= file [:commercial_cert, domain.full_suffix]",
-    "commercial_key": "= file [:commercial_key, domain.full_suffix]",
-    "commercial_ca_cert": "= try_file :commercial_ca_cert"
+    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'"
   },
   "service_type": "user_service"
 }
diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json
new file mode 100644
index 00000000..979d0ef9
--- /dev/null
+++ b/provider_base/services/obfsproxy.json
@@ -0,0 +1,9 @@
+{
+  "obfsproxy": {
+    "scramblesuit": {
+      "password": "= base32_secret('scramblesuit_password_'+name)",
+      "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)"
+    },
+    "gateway_address": "= try{pick_node(:obfs_gateway,nodes_near_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')} || try{pick_node(:obfs_gateway,nodes_like_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')}"
+  }
+}
diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json
index 090afcd6..11cb0dc2 100644
--- a/provider_base/services/openvpn.json
+++ b/provider_base/services/openvpn.json
@@ -24,7 +24,15 @@
       "auth": "SHA1",
       "cipher": "AES-128-CBC",
       "keepalive": "10 30",
-      "tun-ipv6": true
+      "tun-ipv6": true,
+      "fragment": 1500
     }
+  },
+  "obfsproxy": {
+    "scramblesuit": {
+      "password": "= base32_secret('scramblesuit_password_'+name)",
+      "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)"
+    },
+    "gateway_address": "= openvpn.gateway_address"
   }
 }
diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json
index fc365a19..55d3d2ee 100644
--- a/provider_base/services/tor.json
+++ b/provider_base/services/tor.json
@@ -3,6 +3,13 @@
     "bandwidth_rate": 6550,
     "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten",
     "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]",
-    "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')"
+    "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')",
+    "hidden_service": {
+      "active": null,
+      "key_type": "RSA",
+      "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active",
+      "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active",
+      "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service.active"
+    }
   }
 }
diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json
index bbb52094..67744f99 100644
--- a/provider_base/services/webapp.json
+++ b/provider_base/services/webapp.json
@@ -1,6 +1,7 @@
 {
   "webapp": {
     "admins": [],
+    "forbidden_usernames": ["admin", "administrator", "arin-admin", "certmaster", "contact", "info", "maildrop", "postmaster", "ssladmin", "www-data"],
     "domain": "= domain.full_suffix",
     "modules": ["user", "billing", "help"],
     "couchdb_webapp_user": {
@@ -13,6 +14,7 @@
     "allow_limited_certs": "= provider.service.allow_limited_bandwidth",
     "allow_unlimited_certs": "= provider.service.allow_unlimited_bandwidth",
     "allow_anonymous_certs": "= provider.service.allow_anonymous",
+    "allow_registration": "= provider.service.allow_registration",
     "default_service_level": "= provider.service.default_service_level",
     "service_levels": "= provider.service.levels",
     "secret_token": "= secret :webapp_secret_token",
@@ -20,19 +22,27 @@
     "secure": false,
     "git": {
       "source": "https://leap.se/git/leap_web",
-      "revision": "origin/master"
+      "revision": "origin/version/0.6"
     },
     "client_version": "= provider.client_version",
     "nagios_test_user": {
       "username": "nagios_test",
       "password": "= secret :nagios_test_password"
-    }
+    },
+    "engines": [
+      "support"
+    ]
   },
   "stunnel": {
-    "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)"
+    "clients": {
+      "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)"
+    }
   },
   "haproxy": {
-    "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client, global.services[:couchdb].couch.port)"
+    "couch": {
+      "listen_port": 4096,
+      "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)"
+    }
   },
   "definition_files": {
     "provider": "= file :provider_json_template",
@@ -59,11 +69,9 @@
   },
   "x509": {
     "use": true,
+    "use_commercial": true,
     "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'",
-    "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'",
-    "commercial_cert": "= file [:commercial_cert, webapp.domain]",
-    "commercial_key": "= file [:commercial_key, webapp.domain]",
-    "commercial_ca_cert": "= try_file :commercial_ca_cert"
+    "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'",
+    "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'"
   }
 }
diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp
index 9afa5dfd..57942d99 100644
--- a/puppet/manifests/site.pp
+++ b/puppet/manifests/site.pp
@@ -10,6 +10,7 @@ notice("Services for ${fqdn}: ${services_str}")
 
 if member($services, 'openvpn') {
   include site_openvpn
+  include site_obfsproxy
 }
 
 if member($services, 'couchdb') {
@@ -42,4 +43,8 @@ if member($services, 'static') {
   include site_static
 }
 
+if member($services, 'obfsproxy') {
+  include site_obfsproxy
+}
+
 include site_config::packages::uninstall
diff --git a/puppet/modules/check_mk b/puppet/modules/check_mk
-Subproject 5c11597a055858b5ddc1ce8f7f8db249f5f1b33
+Subproject 205859d87884ac4ceee6d1365548e7dc55640bf
diff --git a/puppet/modules/couchdb b/puppet/modules/couchdb
-Subproject c8f5443e0998d3d3d43505ff5a6fdf8c438d6c2
+Subproject 4c0d5673df02fe42e1bbadfee7d4ea1ca1f88e9
diff --git a/puppet/modules/obfsproxy/files/obfsproxy_init b/puppet/modules/obfsproxy/files/obfsproxy_init
new file mode 100755
index 00000000..01c8013a
--- /dev/null
+++ b/puppet/modules/obfsproxy/files/obfsproxy_init
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+### BEGIN INIT INFO
+# Provides:          obfsproxy daemon
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: obfsproxy daemon
+# Description:       obfsproxy daemon
+### END INIT INFO
+
+. /lib/lsb/init-functions
+
+DAEMON=/usr/bin/obfsproxy
+NAME=obfsproxy
+DESC="obfsproxy daemon"
+USER=obfsproxy
+DATDIR=/etc/obfsproxy
+PIDFILE=/var/run/obfsproxy.pid
+CONF=$DATDIR/obfsproxy.conf
+LOGFILE=/var/log/obfsproxy.log
+
+# If the daemon is not there, then exit.
+test -x $DAEMON || exit 0
+
+if [ -f $CONF ] ; then
+  . $CONF
+else
+  echo "Obfsproxy configuration file is missing, aborting..."
+  exit 2
+fi
+
+DAEMONARGS=" --log-min-severity=$LOG --log-file=$LOGFILE --data-dir=$DATDIR \
+            $TRANSPORT $PARAM --dest=$DEST_IP:$DEST_PORT server $BINDADDR:$PORT"
+
+start_obfsproxy() {
+  start-stop-daemon --start --quiet --oknodo -m --pidfile $PIDFILE \
+		-b -c $USER --startas $DAEMON --$DAEMONARGS
+}
+
+stop_obfsproxy() {
+  start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+}
+
+status_obfsproxy() {
+  status_of_proc -p $PIDFILE $DAEMON $NAME
+}
+
+case $1 in
+ start)
+  if [ -e $PIDFILE ]; then
+   status_obfsproxy
+   if [ $? = "0" ]; then
+    exit
+   fi
+  fi
+  log_begin_msg "Starting $DESC"
+  start_obfsproxy
+  log_end_msg $?
+  ;;
+ stop)
+  if [ -e $PIDFILE ]; then
+   status_obfsproxy
+   if [ $? = "0" ]; then
+     log_begin_msg "Stopping $DESC"
+     stop_obfsproxy
+     rm -f $PIDFILE
+     log_end_msg $?
+   fi
+  else
+     status_obfsproxy
+  fi
+  ;;
+ restart)
+  $0 stop && sleep 2 && $0 start
+  ;;
+ status)
+  status_obfsproxy
+  ;;
+ reload)
+  if [ -e $PIDFILE ]; then
+   start-stop-daemon --stop --signal USR1 --quiet --pidfile $PIDFILE --name $NAME
+   log_success_msg "$DESC reloaded successfully"
+  else
+   log_failure_msg "$PIDFILE does not exist"
+  fi
+  ;;
+ *)
+  echo "Usage: $0 {start|stop|restart|reload|status}"
+  exit 2
+  ;;
+esac
diff --git a/puppet/modules/obfsproxy/files/obfsproxy_logrotate b/puppet/modules/obfsproxy/files/obfsproxy_logrotate
new file mode 100644
index 00000000..e5679d0c
--- /dev/null
+++ b/puppet/modules/obfsproxy/files/obfsproxy_logrotate
@@ -0,0 +1,14 @@
+/var/log/obfsproxy.log {
+  daily
+  missingok
+  rotate 3
+  compress
+  delaycompress
+  notifempty
+  create 600 obfsproxy obfsproxy
+  postrotate
+    if [ -f /var/run/obfsproxy.pid ]; then
+      /etc/init.d/obfsproxy restart > /dev/null
+    fi
+  endscript
+}
diff --git a/puppet/modules/obfsproxy/manifests/init.pp b/puppet/modules/obfsproxy/manifests/init.pp
new file mode 100644
index 00000000..61714fdf
--- /dev/null
+++ b/puppet/modules/obfsproxy/manifests/init.pp
@@ -0,0 +1,86 @@
+class obfsproxy (
+  $transport,
+  $bind_address,
+  $port,
+  $param,
+  $dest_ip,
+  $dest_port,
+  $log_level = 'info'
+){
+
+  $user = 'obfsproxy'
+  $conf = '/etc/obfsproxy/obfsproxy.conf'
+
+  user { $user:
+    ensure => present,
+    system => true,
+    gid    => $user,
+  }
+
+  group { $user:
+    ensure => present,
+    system => true,
+  }
+
+  file { '/etc/init.d/obfsproxy':
+    path      => '/etc/init.d/obfsproxy',
+    ensure    => present,
+    source    => 'puppet:///modules/obfsproxy/obfsproxy_init',
+    owner     => 'root',
+    group     => 'root',
+    mode      => '0750',
+    require   => File[$conf],
+  }
+
+  file { $conf :
+    path    => $conf,
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0600',
+    content => template('obfsproxy/etc_conf.erb'),
+  }
+
+  file { '/etc/obfsproxy':
+    ensure  => directory,
+    owner   => $user,
+    group   => $user,
+    mode    => '0700',
+    require => User[$user],
+  }
+
+  file { '/var/log/obfsproxy.log':
+    ensure  => present,
+    owner   => $user,
+    group   => $user,
+    mode    => '0640',
+    require => User[$user],
+  }
+
+  file { '/etc/logrotate.d/obfsproxy':
+    ensure  => present,
+    source  => 'puppet:///modules/obfsproxy/obfsproxy_logrotate',
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    require => File['/var/log/obfsproxy.log'],
+  }
+
+  package { 'obfsproxy':
+    ensure  => present,
+    require => Class['site_apt::preferences::obfsproxy'],
+  }
+
+  service { 'obfsproxy':
+    ensure    => running,
+    subscribe => File[$conf],
+    require   => [
+      Package['obfsproxy'],
+      File['/etc/init.d/obfsproxy'],
+      User[$user],
+      Group[$user]]
+  }
+
+
+}
+
diff --git a/puppet/modules/obfsproxy/templates/etc_conf.erb b/puppet/modules/obfsproxy/templates/etc_conf.erb
new file mode 100644
index 00000000..8959ef78
--- /dev/null
+++ b/puppet/modules/obfsproxy/templates/etc_conf.erb
@@ -0,0 +1,11 @@
+TRANSPORT=<%= @transport %>
+PORT=<%= @port %>
+DEST_IP=<%= @dest_ip %>
+DEST_PORT=<%= @dest_port %>
+<% if @transport == "scramblesuit" -%>
+PARAM=--password=<%= @param %>
+<% else -%>
+PARAM=<%= @param %>
+<% end -%>
+LOG=<%= @log_level %>
+BINDADDR=<%= @bind_address %>
diff --git a/puppet/modules/rsyslog b/puppet/modules/rsyslog
-Subproject 20fbda6b91472e656331a9c64630fb207e9f578
+Subproject b8ef11c23949d12732ad5cdaebb3023ff39a297
diff --git a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb
index 182e476d..e4732289 100644
--- a/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb
+++ b/puppet/modules/site_apache/templates/vhosts.d/api.conf.erb
@@ -2,12 +2,14 @@
   ServerName <%= api_domain %>
   RewriteEngine On
   RewriteRule ^.*$ https://<%= api_domain -%>:<%= api_port -%>%{REQUEST_URI} [R=permanent,L]
+  CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common
 </VirtualHost>
 
 Listen 0.0.0.0:<%= api_port %>
 
 <VirtualHost *:<%= api_port -%>>
   ServerName <%= api_domain %>
+  CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common
 
   SSLEngine on
   SSLProtocol all -SSLv2 -SSLv3
diff --git a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb
index 5423ed71..a9733a97 100644
--- a/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb
+++ b/puppet/modules/site_apache/templates/vhosts.d/common.conf.erb
@@ -3,12 +3,14 @@
   ServerAlias www.<%= domain %>
   RewriteEngine On
   RewriteRule ^.*$ https://<%= domain -%>%{REQUEST_URI} [R=permanent,L]
+  CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common
 </VirtualHost>
 
 <VirtualHost *:443>
   ServerName <%= domain_name %>
   ServerAlias <%= domain %>
   ServerAlias www.<%= domain %>
+  CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log common
 
   SSLEngine on
   SSLProtocol all -SSLv2 -SSLv3
diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb
new file mode 100644
index 00000000..0c6f3b8e
--- /dev/null
+++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb
@@ -0,0 +1,33 @@
+<VirtualHost 127.0.0.1:80>
+  ServerName <%= tor_domain %>
+
+  <IfModule mod_headers.c>
+    Header always unset X-Powered-By
+    Header always unset X-Runtime
+  </IfModule>
+
+<% if (defined? @services) and (@services.include? 'webapp') -%>
+  DocumentRoot /srv/leap/webapp/public
+
+  RewriteEngine On
+  # Check for maintenance file and redirect all requests
+  RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f
+  RewriteCond %{SCRIPT_FILENAME} !maintenance.html
+  RewriteCond %{REQUEST_URI} !/images/maintenance.jpg
+  RewriteRule ^.*$ %{DOCUMENT_ROOT}/system/maintenance.html [L]
+
+  # http://www.modrails.com/documentation/Users%20guide%20Apache.html#_passengerallowencodedslashes_lt_on_off_gt
+  AllowEncodedSlashes on
+  PassengerAllowEncodedSlashes on
+  PassengerFriendlyErrorPages off
+  SetEnv TMPDIR /var/tmp
+
+  # Allow rails assets to be cached for a very long time (since the URLs change whenever the content changes)
+  <Location /assets/>
+    Header unset ETag
+    FileETag None
+    ExpiresActive On
+    ExpiresDefault "access plus 1 year"
+  </Location>
+<% end -%>
+</VirtualHost>
diff --git a/puppet/modules/site_apt/files/Debian/50unattended-upgrades b/puppet/modules/site_apt/files/Debian/50unattended-upgrades
new file mode 100644
index 00000000..f2f574fc
--- /dev/null
+++ b/puppet/modules/site_apt/files/Debian/50unattended-upgrades
@@ -0,0 +1,16 @@
+// this file is managed by puppet !
+
+Unattended-Upgrade::Allowed-Origins {
+        "${distro_id}:stable";
+        "${distro_id}:${distro_codename}-security";
+        "${distro_id}:${distro_codename}-updates";
+        "${distro_id} Backports:${distro_codename}-backports";
+        "leap.se:stable";
+};
+
+APT::Periodic::Update-Package-Lists "1";
+APT::Periodic::Download-Upgradeable-Packages "1";
+APT::Periodic::Unattended-Upgrade "1";
+
+Unattended-Upgrade::Mail "root";
+Unattended-Upgrade::MailOnlyOnError "true";
diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp
index 9facf4cc..633ccf1e 100644
--- a/puppet/modules/site_apt/manifests/init.pp
+++ b/puppet/modules/site_apt/manifests/init.pp
@@ -1,4 +1,4 @@
-class site_apt  {
+class site_apt {
 
   class { 'apt':
     custom_key_dir     => 'puppet:///modules/site_apt/keys'
@@ -11,7 +11,7 @@ class site_apt  {
     content => 'Acquire::PDiffs "false";';
   }
 
-  include ::apt::unattended_upgrades
+  include ::site_apt::unattended_upgrades
 
   apt::sources_list { 'secondary.list.disabled':
     content => template('site_apt/secondary.list');
diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp
index 6b3d9919..2d4ba0e1 100644
--- a/puppet/modules/site_apt/manifests/leap_repo.pp
+++ b/puppet/modules/site_apt/manifests/leap_repo.pp
@@ -1,6 +1,9 @@
 class site_apt::leap_repo {
+  $platform = hiera_hash('platform')
+  $major_version = $platform['major_version']
+
   apt::sources_list { 'leap.list':
-    content => 'deb http://deb.leap.se/debian stable main',
+    content => "deb http://deb.leap.se/${major_version} wheezy main\n",
     before  => Exec[refresh_apt]
   }
 
diff --git a/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp
new file mode 100644
index 00000000..75b01956
--- /dev/null
+++ b/puppet/modules/site_apt/manifests/preferences/obfsproxy.pp
@@ -0,0 +1,9 @@
+class site_apt::preferences::obfsproxy {
+
+  apt::preferences_snippet { 'obfsproxy':
+    package  => 'obfsproxy',
+    release  => 'wheezy-backports',
+    priority => 999;
+  }
+
+}
diff --git a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp
index 132a6e24..bfeaa7da 100644
--- a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp
+++ b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp
@@ -1,9 +1,13 @@
 class site_apt::preferences::rsyslog {
 
-  apt::preferences_snippet { 'rsyslog_anon_depends':
-    package  => 'libestr0 librelp0 rsyslog*',
-    priority => '999',
-    pin      => 'release a=wheezy-backports',
-    before   => Class['rsyslog::install']
+  apt::preferences_snippet {
+    'rsyslog_anon_depends':
+      package  => 'libestr0 librelp0 rsyslog*',
+      priority => '999',
+      pin      => 'release a=wheezy-backports',
+      before   => Class['rsyslog::install'];
+
+    'fixed_rsyslog_anon_package':
+      ensure => absent;
   }
 }
diff --git a/puppet/modules/site_apt/manifests/unattended_upgrades.pp b/puppet/modules/site_apt/manifests/unattended_upgrades.pp
new file mode 100644
index 00000000..daebffab
--- /dev/null
+++ b/puppet/modules/site_apt/manifests/unattended_upgrades.pp
@@ -0,0 +1,10 @@
+class site_apt::unattended_upgrades inherits apt::unattended_upgrades {
+  # override unattended-upgrades package resource to make sure
+  # that it is upgraded on every deploy (#6245)
+
+  include ::apt::unattended_upgrades
+
+  Package['unattended-upgrades'] {
+    ensure => latest
+  }
+}
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg
index 28f333b0..95ddd2ca 100644
--- a/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg
+++ b/puppet/modules/site_check_mk/files/agent/logwatch/bigcouch.cfg
@@ -6,11 +6,19 @@
  I 127.0.0.1 localhost:5984 .* ok
  # https://leap.se/code/issues/5246
  I Shutting down group server
+ # ignore bigcouch conflict errors, mainly coming from tapicero creating new users
+ I Error in process.*{{nocatch,conflict}
  # ignore "Uncaught error in HTTP request: {exit, normal}" error
  # it's suppressed in later versions of bigcouch anhow
  # see https://leap.se/code/issues/5226
  I Uncaught error in HTTP request: {exit,normal}
  I Uncaught error in HTTP request: {exit,
+ # Ignore rexi_EXIT bigcouch error (Bug #6512) 
+ I Error in process <[0-9.]+> on node .* with exit value: {{rexi_EXIT,{(killed|noproc|shutdown),\[{couch_db,collect_results
+ # Ignore "Generic server terminating" bigcouch message (Feature #6544)
+ I Generic server <.*> terminating
+ I {error_report,<.*>,
+ I {error_info,
  C Uncaught error in HTTP request: {error,
  C Response abnormally terminated: {nodedown,
  C rexi_DOWN,noproc
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg
index 623d1e46..3af5045b 100644
--- a/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg
+++ b/puppet/modules/site_check_mk/files/agent/logwatch/soledad.cfg
@@ -2,4 +2,5 @@
  C WSGI application error
  C Error
  C error
- W Timing out client:
+# Removed this line because we determined it was better to ignore it (#6566)
+# W Timing out client: 
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg
index d58e876d..ac17c0ca 100644
--- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg
+++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/openvpn.cfg
@@ -2,6 +2,12 @@
 # suddenly hangup before properly establishing
 # a tls connection
  I ovpn-.*TLS Error: Unroutable control packet received from
- I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds (check your network connectivity)
+ I ovpn-.*TLS Error: TLS key negotiation failed to occur within 60 seconds \(check your network connectivity\)
  I ovpn-.*TLS Error: TLS handshake failed
+ I ovpn-.*TLS Error: TLS object -> incoming plaintext read error
+ I ovpn-.*Fatal TLS error \(check_tls_errors_co\), restarting
+ I ovpn-.*TLS_ERROR: BIO read tls_read_plaintext error: error:140890B2:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:no certificate
+
+ I ovpn-.*SIGUSR1\[soft,tls-error\] received, client-instance restarting
+ I ovpn-.*VERIFY ERROR: depth=0, error=certificate has expired
 
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg
index 93ce0311..e5721eea 100644
--- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg
+++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog/tapicero.cfg
@@ -1,3 +1,5 @@
+# Ignore transient Tapicero errors when creating a db (#6511)
+ I tapicero.*(Creating database|Checking security of|Writing security to|Uploading design doc to) user-.* failed (\(trying again soon\)|(twice )?due to): (RestClient::Resource Not Found|RestClient::InternalServerError): (404 Resource Not Found|500 Internal Server Error)
  C tapicero.*RestClient::InternalServerError:
 # possible race condition between multiple tapicero
 # instances, so we ignore it
diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg
index 450b9e90..71395c50 100644
--- a/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg
+++ b/puppet/modules/site_check_mk/files/agent/logwatch/syslog_tail.cfg
@@ -1,8 +1,14 @@
 # some general patterns
+ I Error: Driver 'pcspkr' is already registered, aborting...
+# ignore postfix errors on lost connection (Bug #6476)
+ I postfix/smtpd.*SSL_accept error from.*lost connection
+# ignore postfix too many errors after DATA (#6545)
+ I postfix/smtpd.*too many errors after DATA from
  C panic
  C Oops
- I Error: Driver 'pcspkr' is already registered, aborting...
  C Error
+# ignore ipv6 icmp errors for now (Bug #6540)
+ I kernel: .*icmpv6_send: no reply to icmp error
  C error
  W generic protection rip
  W .*Unrecovered read error - auto reallocate failed
diff --git a/puppet/modules/site_check_mk/files/extra_host_conf.mk b/puppet/modules/site_check_mk/files/extra_host_conf.mk
new file mode 100644
index 00000000..2c96f97a
--- /dev/null
+++ b/puppet/modules/site_check_mk/files/extra_host_conf.mk
@@ -0,0 +1,6 @@
+# retry 3 times before setting a host into a hard state
+# and send out notification
+extra_host_conf["max_check_attempts"] = [ 
+  ("4", ALL_HOSTS ) 
+]
+
diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk
new file mode 100644
index 00000000..03d1ea76
--- /dev/null
+++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk
@@ -0,0 +1,13 @@
+# retry 3 times before setting a service into a hard state
+# and send out notification
+extra_service_conf["max_check_attempts"] = [ 
+  ("4", ALL_HOSTS , ALL_SERVICES ) 
+]
+
+# run check_mk_agent every 2 minutes if it terminates
+# successfully.
+# see https://leap.se/code/issues/6539 for the rationale
+extra_service_conf["normal_check_interval"] = [
+  ("2", ALL_HOSTS , "Check_MK" )
+]
+
diff --git a/puppet/modules/site_check_mk/files/ignored_services.mk b/puppet/modules/site_check_mk/files/ignored_services.mk
new file mode 100644
index 00000000..35dc4433
--- /dev/null
+++ b/puppet/modules/site_check_mk/files/ignored_services.mk
@@ -0,0 +1,3 @@
+ignored_services = [
+  ( ALL_HOSTS, [ "NTP Time" ] )
+]
diff --git a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
index 01e2b886..ee0268a3 100644
--- a/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/couchdb.pp
@@ -29,7 +29,7 @@ class site_check_mk::agent::couchdb {
   }
   file_line {
     'Bigcouch_open_files':
-      line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 750,750 -c 1000,1000',
+      line => 'Bigcouch_open_files /srv/leap/nagios/plugins/check_unix_open_fds.pl -a beam -w 28672,28672 -c 30720,30720',
       path => '/etc/check_mk/mrpe.cfg';
   }
 
diff --git a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
index 369ed00b..ffd11100 100644
--- a/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/tapicero.pp
@@ -1,5 +1,7 @@
 class site_check_mk::agent::tapicero {
 
+  include ::site_nagios::plugins
+
   concat::fragment { 'syslog_tapicero':
     source  => 'puppet:///modules/site_check_mk/agent/logwatch/syslog/tapicero.cfg',
     target  => '/etc/check_mk/logwatch.d/syslog.cfg',
@@ -11,6 +13,10 @@ class site_check_mk::agent::tapicero {
     'Tapicero_Procs':
       line => 'Tapicero_Procs  /usr/lib/nagios/plugins/check_procs -w 1:1 -c 1:1 -a tapicero',
       path => '/etc/check_mk/mrpe.cfg';
+
+    'Tapicero_Heartbeat':
+      line => 'Tapicero_Heartbeat  /usr/local/lib/nagios/plugins/check_last_regex_in_log -f /var/log/syslog -r "tapicero" -w 300 -c 600',
+      path => '/etc/check_mk/mrpe.cfg';
   }
 
 }
diff --git a/puppet/modules/site_check_mk/manifests/agent/webapp.pp b/puppet/modules/site_check_mk/manifests/agent/webapp.pp
index 64f5ea6d..88c3da30 100644
--- a/puppet/modules/site_check_mk/manifests/agent/webapp.pp
+++ b/puppet/modules/site_check_mk/manifests/agent/webapp.pp
@@ -1,20 +1,11 @@
 class site_check_mk::agent::webapp {
 
-  # check webapp login + soledad sync
-  package { [ 'python-srp', 'python-requests', 'python-yaml', 'python-u1db' ]:
-    ensure => installed
+  # remove leftovers of webapp python checks
+  file {
+    [ '/usr/lib/check_mk_agent/local/nagios-webapp_login.py',
+      '/usr/lib/check_mk_agent/local/soledad_sync.py' ]:
+    ensure  => absent
   }
-  file { '/usr/lib/check_mk_agent/local/nagios-webapp_login.py':
-    ensure  => link,
-    target  => '/srv/leap/webapp/test/nagios/webapp_login.py',
-    require => Package['check_mk-agent']
-  }
-  file { '/usr/lib/check_mk_agent/local/soledad_sync.py':
-    ensure  => link,
-    target  => '/srv/leap/webapp/test/nagios/soledad_sync.py',
-    require => Package['check_mk-agent']
-  }
-
 
   # check syslog
   concat::fragment { 'syslog_webapp':
diff --git a/puppet/modules/site_check_mk/manifests/server.pp b/puppet/modules/site_check_mk/manifests/server.pp
index e544ef0d..171f1576 100644
--- a/puppet/modules/site_check_mk/manifests/server.pp
+++ b/puppet/modules/site_check_mk/manifests/server.pp
@@ -5,11 +5,13 @@ class site_check_mk::server {
   $type     = $ssh_hash['authorized_keys']['monitor']['type']
   $seckey   = $ssh_hash['monitor']['private_key']
 
-  $nagios_hiera   = hiera_hash('nagios')
-  $nagios_hosts   = $nagios_hiera['hosts']
+  $nagios_hiera     = hiera_hash('nagios')
+  $nagios_hosts     = $nagios_hiera['hosts']
 
-  $hosts          = hiera_hash('hosts')
-  $all_hosts      = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>')
+  $hosts            = hiera_hash('hosts')
+  $all_hosts        = inline_template ('<% @hosts.keys.sort.each do |key| -%>"<%= @hosts[key]["domain_internal"] %>", <% end -%>')
+  $domains_internal = $nagios_hiera['domains_internal']
+  $environments     = $nagios_hiera['environments']
 
   package { 'check-mk-server':
     ensure => installed,
@@ -35,10 +37,32 @@ class site_check_mk::server {
       content => template('site_check_mk/use_ssh.mk'),
       notify  => Exec['check_mk-refresh'],
       require => Package['check-mk-server'];
+    '/etc/check_mk/conf.d/hostgroups.mk':
+      content => template('site_check_mk/hostgroups.mk'),
+      notify  => Exec['check_mk-refresh'],
+      require => Package['check-mk-server'];
+    '/etc/check_mk/conf.d/host_contactgroups.mk':
+      content => template('site_check_mk/host_contactgroups.mk'),
+      notify  => Exec['check_mk-refresh'],
+      require => Package['check-mk-server'];
+    '/etc/check_mk/conf.d/ignored_services.mk':
+      source  => 'puppet:///modules/site_check_mk/ignored_services.mk',
+      notify  => Exec['check_mk-refresh'],
+      require => Package['check-mk-server'];
+    '/etc/check_mk/conf.d/extra_service_conf.mk':
+      source  => 'puppet:///modules/site_check_mk/extra_service_conf.mk',
+      notify  => Exec['check_mk-refresh'],
+      require => Package['check-mk-server'];
+    '/etc/check_mk/conf.d/extra_host_conf.mk':
+      source  => 'puppet:///modules/site_check_mk/extra_host_conf.mk',
+      notify  => Exec['check_mk-refresh'],
+      require => Package['check-mk-server'];
+
     '/etc/check_mk/all_hosts_static':
       content => $all_hosts,
       notify  => Exec['check_mk-refresh'],
       require => Package['check-mk-server'];
+
     '/etc/check_mk/.ssh':
       ensure  => directory,
       require => Package['check-mk-server'];
@@ -52,6 +76,7 @@ class site_check_mk::server {
       owner   => 'nagios',
       mode    => '0644',
       require => Package['check-mk-server'];
+
     # check_icmp must be suid root or called by sudo
     # see https://leap.se/code/issues/5171
     '/usr/lib/nagios/plugins/check_icmp':
@@ -59,6 +84,5 @@ class site_check_mk::server {
       require => Package['nagios-plugins-basic'];
   }
 
-
   include check_mk::agent::local_checks
 }
diff --git a/puppet/modules/site_check_mk/templates/host_contactgroups.mk b/puppet/modules/site_check_mk/templates/host_contactgroups.mk
new file mode 100644
index 00000000..6a534967
--- /dev/null
+++ b/puppet/modules/site_check_mk/templates/host_contactgroups.mk
@@ -0,0 +1,17 @@
+<%
+  contact_groups = []
+  @environments.keys.sort.each do |env_name|
+    hosts = ""
+    @nagios_hosts.keys.sort.each do |hostname|
+      hostdata = @nagios_hosts[hostname]
+      domain_internal = hostdata['domain_internal']
+      if hostdata['environment'] == env_name
+        hosts << '"' + domain_internal + '", '
+      end
+    end
+    contact_groups << '  ( "%s", [%s] )' % [env_name, hosts]
+  end
+%>
+host_contactgroups = [
+<%= contact_groups.join(",\n") %>
+]
diff --git a/puppet/modules/site_check_mk/templates/hostgroups.mk b/puppet/modules/site_check_mk/templates/hostgroups.mk
new file mode 100644
index 00000000..7158dcd1
--- /dev/null
+++ b/puppet/modules/site_check_mk/templates/hostgroups.mk
@@ -0,0 +1,17 @@
+<%
+  host_groups = []
+  @environments.keys.sort.each do |env_name|
+    hosts = ""
+    @nagios_hosts.keys.sort.each do |hostname|
+      hostdata = @nagios_hosts[hostname]
+      domain_internal = hostdata['domain_internal']
+      if hostdata['environment'] == env_name
+        hosts << '"' + domain_internal + '", '
+      end
+    end
+    host_groups << '  ( "%s", [%s] )' % [env_name, hosts]
+  end
+%>
+host_groups = [
+<%= host_groups.join(",\n") %>
+]
diff --git a/puppet/modules/site_config/manifests/default.pp b/puppet/modules/site_config/manifests/default.pp
index fc2179de..790b5a16 100644
--- a/puppet/modules/site_config/manifests/default.pp
+++ b/puppet/modules/site_config/manifests/default.pp
@@ -59,10 +59,10 @@ class site_config::default {
     include site_postfix::satellite
   }
 
-  # if class site_custom exists, include it.
+  # if class custom exists, include it.
   # possibility for users to define custom puppet recipes
-  if defined( '::site_custom') {
-    include ::site_custom
+  if defined( '::custom') {
+    include ::custom
   }
 
   include site_check_mk::agent
diff --git a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb
index 928a2b31..b0c2b7ad 100644
--- a/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb
+++ b/puppet/modules/site_config/templates/ipv4firewall_up.rules.erb
@@ -2,7 +2,7 @@
 *filter
 :INPUT DROP [0:0]
 :FORWARD DROP [0:0]
-:OUTPUT DROP [0:0]
+:OUTPUT ACCEPT [0:0]
 -A INPUT -i lo -j ACCEPT
 -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
 -A INPUT -p tcp -m state --state NEW,ESTABLISHED --dport 22 -j ACCEPT
@@ -11,14 +11,4 @@
 -A INPUT -p icmp -m icmp --icmp-type 8 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT
 -A INPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT
 -A INPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7
--A OUTPUT -o lo -j ACCEPT
--A OUTPUT -p icmp -m icmp --icmp-type 0 -m state --state RELATED,ESTABLISHED -j ACCEPT
--A OUTPUT -p icmp -m icmp --icmp-type 8 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT
--A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --sport 22 -j ACCEPT
--A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --sport <%= @ssh_port %> -j ACCEPT
--A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 80 -j ACCEPT
--A OUTPUT -p tcp -m state --state NEW,ESTABLISHED --dport 443 -j ACCEPT
--A OUTPUT -p udp -m udp --dport 53 -j ACCEPT
--A OUTPUT -p udp -m udp --dport 123 -j ACCEPT
--A OUTPUT -m limit --limit 5/min -j LOG --log-prefix "iptables denied: " --log-level 7
 COMMIT
diff --git a/puppet/modules/site_couchdb/files/runit_config b/puppet/modules/site_couchdb/files/runit_config
new file mode 100644
index 00000000..169b4832
--- /dev/null
+++ b/puppet/modules/site_couchdb/files/runit_config
@@ -0,0 +1,6 @@
+#!/bin/bash
+exec 2>&1
+export HOME=/home/bigcouch
+ulimit -H -n 32768
+ulimit -S -n 32768
+exec chpst -u bigcouch /opt/bigcouch/bin/bigcouch
diff --git a/puppet/modules/site_couchdb/manifests/add_users.pp b/puppet/modules/site_couchdb/manifests/add_users.pp
index f9ea7349..2f734ed4 100644
--- a/puppet/modules/site_couchdb/manifests/add_users.pp
+++ b/puppet/modules/site_couchdb/manifests/add_users.pp
@@ -1,5 +1,8 @@
 class site_couchdb::add_users {
 
+  Class['site_couchdb::create_dbs']
+    -> Class['site_couchdb::add_users']
+
   # Couchdb users
 
   ## leap_mx couchdb user
@@ -51,4 +54,13 @@ class site_couchdb::add_users {
     require => Couchdb::Query::Setup['localhost']
   }
 
+  ## replication couchdb user
+  ## read/write: all databases for replication
+  couchdb::add_user { $site_couchdb::couchdb_replication_user:
+    roles   => '["replication"]',
+    pw      => $site_couchdb::couchdb_replication_pw,
+    salt    => $site_couchdb::couchdb_replication_salt,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
 }
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch.pp b/puppet/modules/site_couchdb/manifests/bigcouch.pp
new file mode 100644
index 00000000..16593ec7
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/bigcouch.pp
@@ -0,0 +1,44 @@
+class site_couchdb::bigcouch {
+
+  $config         = $::site_couchdb::couchdb_config['bigcouch']
+  $cookie         = $config['cookie']
+  $ednp_port      = $config['ednp_port']
+
+  class { 'couchdb':
+    admin_pw            => $::site_couchdb::couchdb_admin_pw,
+    admin_salt          => $::site_couchdb::couchdb_admin_salt,
+    bigcouch            => true,
+    bigcouch_cookie     => $cookie,
+    ednp_port           => $ednp_port,
+    chttpd_bind_address => '127.0.0.1'
+  }
+
+  #
+  # stunnel must running correctly before bigcouch dbs can be set up.
+  #
+  Class['site_config::default']
+    -> Class['couchdb::bigcouch::package::cloudant']
+    -> Service['shorewall']
+    -> Exec['refresh_stunnel']
+    -> Class['site_couchdb::setup']
+    -> Class['site_couchdb::bigcouch::add_nodes']
+    -> Class['site_couchdb::bigcouch::settle_cluster']
+
+  include site_couchdb::bigcouch::add_nodes
+  include site_couchdb::bigcouch::settle_cluster
+  include site_couchdb::bigcouch::compaction
+
+  file { '/var/log/bigcouch':
+    ensure => directory
+  }
+
+  file { '/etc/sv/bigcouch/run':
+    ensure  => present,
+    source  => 'puppet:///modules/site_couchdb/runit_config',
+    owner   => root,
+    group   => root,
+    mode    => '0755',
+    require => Package['couchdb'],
+    notify  => Service['couchdb']
+  }
+}
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
index 97e85785..c8c43275 100644
--- a/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
+++ b/puppet/modules/site_couchdb/manifests/bigcouch/add_nodes.pp
@@ -1,6 +1,6 @@
 class site_couchdb::bigcouch::add_nodes {
   # loop through neighbors array and add nodes
-  $nodes = $::site_couchdb::bigcouch_config['neighbors']
+  $nodes = $::site_couchdb::bigcouch::config['neighbors']
 
   couchdb::bigcouch::add_node { $nodes:
     require => Couchdb::Query::Setup['localhost']
diff --git a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp
index aa843e2e..820b5be2 100644
--- a/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp
+++ b/puppet/modules/site_couchdb/manifests/bigcouch/settle_cluster.pp
@@ -1,11 +1,11 @@
 class site_couchdb::bigcouch::settle_cluster {
 
   exec { 'wait_for_couch_nodes':
-    command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 6 --wait 10'
+    command => '/srv/leap/bin/run_tests --test CouchDB/Are_configured_nodes_online? --retry 12 --wait 10'
   }
 
   exec { 'settle_cluster_membership':
-    command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 6 --wait 10',
+    command => '/srv/leap/bin/run_tests --test CouchDB/Is_cluster_membership_ok? --retry 12 --wait 10',
     require => Exec['wait_for_couch_nodes']
   }
 }
diff --git a/puppet/modules/site_couchdb/manifests/create_dbs.pp b/puppet/modules/site_couchdb/manifests/create_dbs.pp
index 41500d3a..4322f773 100644
--- a/puppet/modules/site_couchdb/manifests/create_dbs.pp
+++ b/puppet/modules/site_couchdb/manifests/create_dbs.pp
@@ -1,11 +1,14 @@
 class site_couchdb::create_dbs {
 
+  Class['site_couchdb::setup']
+    -> Class['site_couchdb::create_dbs']
+
   # Couchdb databases
 
   ### customer database
   ### r/w: webapp,
   couchdb::create_db { 'customers':
-    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
@@ -13,35 +16,35 @@ class site_couchdb::create_dbs {
   ## r: nickserver, leap_mx - needs to be restrict with design document
   ## r/w: webapp
   couchdb::create_db { 'identities':
-    members => "{ \"names\": [], \"roles\": [\"identities\"] }",
+    members => "{ \"names\": [], \"roles\": [\"replication\", \"identities\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
   ## keycache database
   ## r/w: nickserver
   couchdb::create_db { 'keycache':
-    members => "{ \"names\": [], \"roles\": [\"keycache\"] }",
+    members => "{ \"names\": [], \"roles\": [\"replication\", \"keycache\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
   ## sessions database
   ## r/w: webapp
   couchdb::create_db { 'sessions':
-    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
   ## shared database
   ## r/w: soledad
   couchdb::create_db { 'shared':
-    members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [] }",
+    members => "{ \"names\": [\"$site_couchdb::couchdb_soledad_user\"], \"roles\": [\"replication\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
   ## tickets database
   ## r/w: webapp
   couchdb::create_db { 'tickets':
-    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
@@ -49,14 +52,14 @@ class site_couchdb::create_dbs {
   ## r: soledad - needs to be restricted with a design document
   ## r/w: webapp
   couchdb::create_db { 'tokens':
-    members => "{ \"names\": [], \"roles\": [\"tokens\"] }",
+    members => "{ \"names\": [], \"roles\": [\"replication\", \"tokens\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
   ## users database
   ## r/w: webapp
   couchdb::create_db { 'users':
-    members => "{ \"names\": [], \"roles\": [\"users\"] }",
+    members => "{ \"names\": [], \"roles\": [\"replication\", \"users\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 
@@ -64,7 +67,7 @@ class site_couchdb::create_dbs {
   ## store messages to the clients such as payment reminders
   ## r/w: webapp
   couchdb::create_db { 'messages':
-    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [] }",
+    members => "{ \"names\": [\"$site_couchdb::couchdb_webapp_user\"], \"roles\": [\"replication\"] }",
     require => Couchdb::Query::Setup['localhost']
   }
 }
diff --git a/puppet/modules/site_couchdb/manifests/init.pp b/puppet/modules/site_couchdb/manifests/init.pp
index 3614661d..a11f6309 100644
--- a/puppet/modules/site_couchdb/manifests/init.pp
+++ b/puppet/modules/site_couchdb/manifests/init.pp
@@ -1,118 +1,68 @@
 class site_couchdb {
   tag 'leap_service'
 
-  $couchdb_config          = hiera('couch')
-  $couchdb_users           = $couchdb_config['users']
-
-  $couchdb_admin           = $couchdb_users['admin']
-  $couchdb_admin_user      = $couchdb_admin['username']
-  $couchdb_admin_pw        = $couchdb_admin['password']
-  $couchdb_admin_salt      = $couchdb_admin['salt']
-
-  $couchdb_leap_mx         = $couchdb_users['leap_mx']
-  $couchdb_leap_mx_user    = $couchdb_leap_mx['username']
-  $couchdb_leap_mx_pw      = $couchdb_leap_mx['password']
-  $couchdb_leap_mx_salt    = $couchdb_leap_mx['salt']
-
-  $couchdb_nickserver      = $couchdb_users['nickserver']
-  $couchdb_nickserver_user = $couchdb_nickserver['username']
-  $couchdb_nickserver_pw   = $couchdb_nickserver['password']
-  $couchdb_nickserver_salt = $couchdb_nickserver['salt']
-
-  $couchdb_soledad         = $couchdb_users['soledad']
-  $couchdb_soledad_user    = $couchdb_soledad['username']
-  $couchdb_soledad_pw      = $couchdb_soledad['password']
-  $couchdb_soledad_salt    = $couchdb_soledad['salt']
-
-  $couchdb_tapicero        = $couchdb_users['tapicero']
-  $couchdb_tapicero_user   = $couchdb_tapicero['username']
-  $couchdb_tapicero_pw     = $couchdb_tapicero['password']
-  $couchdb_tapicero_salt   = $couchdb_tapicero['salt']
-
-  $couchdb_webapp          = $couchdb_users['webapp']
-  $couchdb_webapp_user     = $couchdb_webapp['username']
-  $couchdb_webapp_pw       = $couchdb_webapp['password']
-  $couchdb_webapp_salt     = $couchdb_webapp['salt']
-
-  $couchdb_backup          = $couchdb_config['backup']
-
-  $bigcouch_config         = $couchdb_config['bigcouch']
-  $bigcouch_cookie         = $bigcouch_config['cookie']
-
-  $ednp_port               = $bigcouch_config['ednp_port']
-
-  class { 'couchdb':
-    bigcouch            => true,
-    admin_pw            => $couchdb_admin_pw,
-    admin_salt          => $couchdb_admin_salt,
-    bigcouch_cookie     => $bigcouch_cookie,
-    ednp_port           => $ednp_port,
-    chttpd_bind_address => '127.0.0.1'
-  }
-
-  # ensure that we don't have leftovers from previous installations
-  # where we installed the cloudant bigcouch package
-  # https://leap.se/code/issues/4971
-  class { 'couchdb::bigcouch::package::cloudant':
-    ensure => absent
-  }
+  $couchdb_config           = hiera('couch')
+  $couchdb_users            = $couchdb_config['users']
+
+  $couchdb_admin            = $couchdb_users['admin']
+  $couchdb_admin_user       = $couchdb_admin['username']
+  $couchdb_admin_pw         = $couchdb_admin['password']
+  $couchdb_admin_salt       = $couchdb_admin['salt']
+
+  $couchdb_leap_mx          = $couchdb_users['leap_mx']
+  $couchdb_leap_mx_user     = $couchdb_leap_mx['username']
+  $couchdb_leap_mx_pw       = $couchdb_leap_mx['password']
+  $couchdb_leap_mx_salt     = $couchdb_leap_mx['salt']
+
+  $couchdb_nickserver       = $couchdb_users['nickserver']
+  $couchdb_nickserver_user  = $couchdb_nickserver['username']
+  $couchdb_nickserver_pw    = $couchdb_nickserver['password']
+  $couchdb_nickserver_salt  = $couchdb_nickserver['salt']
+
+  $couchdb_soledad          = $couchdb_users['soledad']
+  $couchdb_soledad_user     = $couchdb_soledad['username']
+  $couchdb_soledad_pw       = $couchdb_soledad['password']
+  $couchdb_soledad_salt     = $couchdb_soledad['salt']
+
+  $couchdb_tapicero         = $couchdb_users['tapicero']
+  $couchdb_tapicero_user    = $couchdb_tapicero['username']
+  $couchdb_tapicero_pw      = $couchdb_tapicero['password']
+  $couchdb_tapicero_salt    = $couchdb_tapicero['salt']
+
+  $couchdb_webapp           = $couchdb_users['webapp']
+  $couchdb_webapp_user      = $couchdb_webapp['username']
+  $couchdb_webapp_pw        = $couchdb_webapp['password']
+  $couchdb_webapp_salt      = $couchdb_webapp['salt']
+
+  $couchdb_replication      = $couchdb_users['replication']
+  $couchdb_replication_user = $couchdb_replication['username']
+  $couchdb_replication_pw   = $couchdb_replication['password']
+  $couchdb_replication_salt = $couchdb_replication['salt']
+
+  $couchdb_backup           = $couchdb_config['backup']
+  $couchdb_mode             = $couchdb_config['mode']
+
+  if $couchdb_mode == 'multimaster' { include site_couchdb::bigcouch }
+  if $couchdb_mode == 'master'      { include site_couchdb::master }
+  if $couchdb_mode == 'mirror'      { include site_couchdb::mirror }
 
   Class['site_config::default']
-    -> Class['couchdb::bigcouch::package::cloudant']
     -> Service['shorewall']
-    -> Class['site_couchdb::stunnel']
-    -> Service['couchdb']
-    -> File['/root/.netrc']
-    -> Class['site_couchdb::bigcouch::add_nodes']
-    -> Class['site_couchdb::bigcouch::settle_cluster']
-    -> Class['site_couchdb::create_dbs']
-    -> Class['site_couchdb::add_users']
-
-  # /etc/couchdb/couchdb.netrc is deployed by couchdb::query::setup
-  # we symlink this to /root/.netrc for couchdb_scripts (eg. backup)
-  # and makes life easier for the admin (i.e. using curl/wget without
-  # passing credentials)
-  file {
-    '/root/.netrc':
-      ensure  => link,
-      target  => '/etc/couchdb/couchdb.netrc';
-
-    '/srv/leap/couchdb':
-      ensure => directory
-  }
-
-  couchdb::query::setup { 'localhost':
-    user  => $couchdb_admin_user,
-    pw    => $couchdb_admin_pw,
-  }
-
-  vcsrepo { '/srv/leap/couchdb/scripts':
-    ensure   => present,
-    provider => git,
-    source   => 'https://leap.se/git/couchdb_scripts',
-    revision => 'origin/master',
-    require  => File['/srv/leap/couchdb']
-  }
-
-  include site_couchdb::stunnel
-  include site_couchdb::bigcouch::add_nodes
-  include site_couchdb::bigcouch::settle_cluster
+    -> Exec['refresh_stunnel']
+    -> Class['couchdb']
+    -> Class['site_couchdb::setup']
+
+  include site_stunnel
+
+  include site_couchdb::setup
   include site_couchdb::create_dbs
   include site_couchdb::add_users
   include site_couchdb::designs
   include site_couchdb::logrotate
-  include site_couchdb::bigcouch::compaction
 
-  if $couchdb_backup { include site_couchdb::backup }
-
-  include site_shorewall::couchdb
-  include site_shorewall::couchdb::bigcouch
+  if $couchdb_backup   { include site_couchdb::backup }
 
   include site_check_mk::agent::couchdb
   include site_check_mk::agent::tapicero
 
-  file { '/var/log/bigcouch':
-    ensure => directory
-  }
-
 }
diff --git a/puppet/modules/site_couchdb/manifests/master.pp b/puppet/modules/site_couchdb/manifests/master.pp
new file mode 100644
index 00000000..a0a6633d
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/master.pp
@@ -0,0 +1,9 @@
+class site_couchdb::master {
+
+  class { 'couchdb':
+    admin_pw            => $site_couchdb::couchdb_admin_pw,
+    admin_salt          => $site_couchdb::couchdb_admin_salt,
+    chttpd_bind_address => '127.0.0.1'
+  }
+
+}
\ No newline at end of file
diff --git a/puppet/modules/site_couchdb/manifests/mirror.pp b/puppet/modules/site_couchdb/manifests/mirror.pp
new file mode 100644
index 00000000..abe35c4c
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/mirror.pp
@@ -0,0 +1,77 @@
+class site_couchdb::mirror {
+
+  Class['site_couchdb::add_users']
+    -> Class['site_couchdb::mirror']
+
+  class { 'couchdb':
+    admin_pw            => $site_couchdb::couchdb_admin_pw,
+    admin_salt          => $site_couchdb::couchdb_admin_salt,
+    chttpd_bind_address => '127.0.0.1'
+  }
+
+  $masters = $site_couchdb::couchdb_config['replication']['masters']
+  $master_node_names = keys($site_couchdb::couchdb_config['replication']['masters'])
+  $master_node = $masters[$master_node_names[0]]
+  $user = $site_couchdb::couchdb_replication_user
+  $password = $site_couchdb::couchdb_replication_pw
+  $from_host = $master_node['domain_internal']
+  $from_port = $master_node['couch_port']
+  $from = "http://${user}:${password}@${from_host}:${from_port}"
+
+  notice("mirror from: ${from}")
+
+  ### customer database
+  couchdb::mirror_db { 'customers':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## identities database
+  couchdb::mirror_db { 'identities':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## keycache database
+  couchdb::mirror_db { 'keycache':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## sessions database
+  couchdb::mirror_db { 'sessions':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## shared database
+  couchdb::mirror_db { 'shared':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## tickets database
+  couchdb::mirror_db { 'tickets':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## tokens database
+  couchdb::mirror_db { 'tokens':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## users database
+  couchdb::mirror_db { 'users':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+  ## messages db
+  couchdb::mirror_db { 'messages':
+    from => $from,
+    require => Couchdb::Query::Setup['localhost']
+  }
+
+}
diff --git a/puppet/modules/site_couchdb/manifests/setup.pp b/puppet/modules/site_couchdb/manifests/setup.pp
new file mode 100644
index 00000000..69bd1c6a
--- /dev/null
+++ b/puppet/modules/site_couchdb/manifests/setup.pp
@@ -0,0 +1,46 @@
+#
+# An initial setup class. All the other classes depend on this
+#
+class site_couchdb::setup {
+
+  # ensure that we don't have leftovers from previous installations
+  # where we installed the cloudant bigcouch package
+  # https://leap.se/code/issues/4971
+  class { 'couchdb::bigcouch::package::cloudant':
+    ensure => absent
+  }
+
+  $user = $site_couchdb::couchdb_admin_user
+
+  # /etc/couchdb/couchdb-admin.netrc is deployed by couchdb::query::setup
+  # we symlink to couchdb.netrc for puppet commands.
+  # we symlink this to /root/.netrc for couchdb_scripts (eg. backup)
+  # and makes life easier for the admin (i.e. using curl/wget without
+  # passing credentials)
+  file {
+    '/etc/couchdb/couchdb.netrc':
+      ensure  => link,
+      target  => "/etc/couchdb/couchdb-${user}.netrc";
+
+    '/root/.netrc':
+      ensure  => link,
+      target  => '/etc/couchdb/couchdb.netrc';
+
+    '/srv/leap/couchdb':
+      ensure => directory
+  }
+
+  couchdb::query::setup { 'localhost':
+    user  => $user,
+    pw    => $site_couchdb::couchdb_admin_pw,
+  }
+
+  vcsrepo { '/srv/leap/couchdb/scripts':
+    ensure   => present,
+    provider => git,
+    source   => 'https://leap.se/git/couchdb_scripts',
+    revision => 'origin/master',
+    require  => File['/srv/leap/couchdb']
+  }
+
+}
diff --git a/puppet/modules/site_couchdb/manifests/stunnel.pp b/puppet/modules/site_couchdb/manifests/stunnel.pp
deleted file mode 100644
index 91f1e3aa..00000000
--- a/puppet/modules/site_couchdb/manifests/stunnel.pp
+++ /dev/null
@@ -1,112 +0,0 @@
-class site_couchdb::stunnel {
-
-  $stunnel              = hiera('stunnel')
-
-  $couch_server         = $stunnel['couch_server']
-  $couch_server_accept  = $couch_server['accept']
-  $couch_server_connect = $couch_server['connect']
-
-  # Erlang Port Mapper Daemon (epmd) stunnel server/clients
-  $epmd_server          = $stunnel['epmd_server']
-  $epmd_server_accept   = $epmd_server['accept']
-  $epmd_server_connect  = $epmd_server['connect']
-  $epmd_clients         = $stunnel['epmd_clients']
-
-  # Erlang Distributed Node Protocol (ednp) stunnel server/clients
-  $ednp_server          = $stunnel['ednp_server']
-  $ednp_server_accept   = $ednp_server['accept']
-  $ednp_server_connect  = $ednp_server['connect']
-  $ednp_clients         = $stunnel['ednp_clients']
-
-
-
-  include site_config::x509::cert
-  include site_config::x509::key
-  include site_config::x509::ca
-
-  include x509::variables
-  $ca_path   = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
-  $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
-  $key_path  = "${x509::variables::keys}/${site_config::params::cert_name}.key"
-
-  # setup a stunnel server for the webapp to connect to couchdb
-  stunnel::service { 'couch_server':
-    accept     => $couch_server_accept,
-    connect    => $couch_server_connect,
-    client     => false,
-    cafile     => $ca_path,
-    key        => $key_path,
-    cert       => $cert_path,
-    verify     => '2',
-    pid        => '/var/run/stunnel4/couchserver.pid',
-    rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4',
-    require    => [
-      Class['Site_config::X509::Key'],
-      Class['Site_config::X509::Cert'],
-      Class['Site_config::X509::Ca'] ];
-  }
-
-
-  # setup stunnel server for Erlang Port Mapper Daemon (epmd), necessary for
-  # bigcouch clustering between each bigcouchdb node
-  stunnel::service { 'epmd_server':
-    accept     => $epmd_server_accept,
-    connect    => $epmd_server_connect,
-    client     => false,
-    cafile     => $ca_path,
-    key        => $key_path,
-    cert       => $cert_path,
-    verify     => '2',
-    pid        => '/var/run/stunnel4/epmd_server.pid',
-    rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4',
-    require    => [
-      Class['Site_config::X509::Key'],
-      Class['Site_config::X509::Cert'],
-      Class['Site_config::X509::Ca'] ];
-  }
-
-  # setup stunnel clients for Erlang Port Mapper Daemon (epmd) to connect
-  # to the above epmd stunnel server.
-  $epmd_client_defaults = {
-    'client'       => true,
-    'cafile'       => $ca_path,
-    'key'          => $key_path,
-    'cert'         => $cert_path,
-  }
-
-  create_resources(site_stunnel::clients, $epmd_clients, $epmd_client_defaults)
-
-  # setup stunnel server for Erlang Distributed Node Protocol (ednp), necessary
-  # for bigcouch clustering between each bigcouchdb node
-  stunnel::service { 'ednp_server':
-    accept     => $ednp_server_accept,
-    connect    => $ednp_server_connect,
-    client     => false,
-    cafile     => $ca_path,
-    key        => $key_path,
-    cert       => $cert_path,
-    verify     => '2',
-    pid        => '/var/run/stunnel4/ednp_server.pid',
-    rndfile    => '/var/lib/stunnel4/.rnd',
-    debuglevel => '4',
-    require    => [
-      Class['Site_config::X509::Key'],
-      Class['Site_config::X509::Cert'],
-      Class['Site_config::X509::Ca'] ];
-  }
-
-  # setup stunnel clients for Erlang Distributed Node Protocol (ednp) to connect
-  # to the above ednp stunnel server.
-  $ednp_client_defaults = {
-    'client'       => true,
-    'cafile'       => $ca_path,
-    'key'          => $key_path,
-    'cert'         => $cert_path,
-  }
-
-  create_resources(site_stunnel::clients, $ednp_clients, $ednp_client_defaults)
-
-  include site_check_mk::agent::stunnel
-}
diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp
index 6bcf3f5c..b28ce80e 100644
--- a/puppet/modules/site_haproxy/manifests/init.pp
+++ b/puppet/modules/site_haproxy/manifests/init.pp
@@ -2,25 +2,25 @@ class site_haproxy {
     $haproxy     = hiera('haproxy')
 
     class { 'haproxy':
-    enable           => true,
-    manage_service   => true,
-    global_options   => {
-      'log'     => '127.0.0.1 local0',
-      'maxconn' => '4096',
-      'stats'   => 'socket /var/run/haproxy.sock user haproxy group haproxy',
-      'chroot'  => '/usr/share/haproxy',
-      'user'    => 'haproxy',
-      'group'   => 'haproxy',
-      'daemon'  => ''
-    },
-    defaults_options => {
-      'log'             => 'global',
-      'retries'         => '3',
-      'option'          => 'redispatch',
-      'timeout connect' => '4000',
-      'timeout client'  => '20000',
-      'timeout server'  => '20000'
-    }
+      enable           => true,
+      manage_service   => true,
+      global_options   => {
+        'log'     => '127.0.0.1 local0',
+        'maxconn' => '4096',
+        'stats'   => 'socket /var/run/haproxy.sock user haproxy group haproxy',
+        'chroot'  => '/usr/share/haproxy',
+        'user'    => 'haproxy',
+        'group'   => 'haproxy',
+        'daemon'  => ''
+      },
+      defaults_options => {
+        'log'             => 'global',
+        'retries'         => '3',
+        'option'          => 'redispatch',
+        'timeout connect' => '4000',
+        'timeout client'  => '20000',
+        'timeout server'  => '20000'
+      }
   }
 
   # monitor haproxy
@@ -34,8 +34,8 @@ class site_haproxy {
   concat::fragment { 'leap_haproxy_webapp_couchdb':
     target  => '/etc/haproxy/haproxy.cfg',
     order   => '20',
-    content => template('site_haproxy/haproxy_couchdb.cfg.erb'),
+    content => template('site_haproxy/haproxy.cfg.erb'),
   }
-  
+
   include site_check_mk::agent::haproxy
 }
diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb
new file mode 100644
index 00000000..f42e8368
--- /dev/null
+++ b/puppet/modules/site_haproxy/templates/couch.erb
@@ -0,0 +1,32 @@
+frontend couch
+  bind localhost:<%= @listen_port %>
+  mode     http
+  option   httplog
+  option   dontlognull
+  option   http-server-close    # use client keep-alive, but close server connection.
+  use_backend couch_read if METH_GET
+  default_backend couch_write
+
+backend couch_write
+  mode http
+  balance  roundrobin
+  option   httpchk GET /        # health check using simple get to root
+  option   allbackups           # balance among all backups, not just one.
+  default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1
+<%- @servers.sort.each do |name,server| -%>
+<%-   next unless server['writable'] -%>
+    # <%=name%>
+    server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check
+<%- end -%>
+
+backend couch_read
+  mode http
+  balance  roundrobin
+  option   httpchk GET /        # health check using simple get to root
+  option   allbackups           # balance among all backups, not just one.
+  default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1
+<%- @servers.sort.each do |name,server| -%>
+    # <%=name%>
+    server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check
+<%- end -%>
+
diff --git a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb
new file mode 100644
index 00000000..8311b1a5
--- /dev/null
+++ b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb
@@ -0,0 +1,11 @@
+<%- @haproxy.each do |frontend, options| -%>
+<%-   if options['servers'] -%>
+
+##
+## <%= frontend %>
+##
+
+<%=     scope.function_templatewlv(["site_haproxy/#{frontend}.erb", options]) %>
+<%-   end -%>
+<%- end -%>
+
diff --git a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb
deleted file mode 100644
index 1fa01b96..00000000
--- a/puppet/modules/site_haproxy/templates/haproxy_couchdb.cfg.erb
+++ /dev/null
@@ -1,23 +0,0 @@
-
-listen bigcouch-in
-  mode     http
-  balance  roundrobin
-  option   httplog
-  option   dontlognull
-  option   httpchk GET /        # health check using simple get to root
-  option   http-server-close    # use client keep-alive, but close server connection.
-  option   allbackups           # balance among all backups, not just one.
-
-  bind localhost:4096
-
-  default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1
-
-<%- if @haproxy['servers'] -%>
-<%-   @haproxy['servers'].sort.each do |name,server| -%>
-<%-     backup = server['backup'] ? 'backup' : '' -%>
-  # <%=name%>
-  server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%=backup%> weight <%=server['weight']%> check
-
-<%-   end -%>
-<%- end -%>
-
diff --git a/puppet/modules/site_mx/manifests/couchdb.pp b/puppet/modules/site_mx/manifests/couchdb.pp
deleted file mode 100644
index b1f3bd02..00000000
--- a/puppet/modules/site_mx/manifests/couchdb.pp
+++ /dev/null
@@ -1,23 +0,0 @@
-class site_mx::couchdb {
-
-  $stunnel = hiera('stunnel')
-  $couch_client            = $stunnel['couch_client']
-  $couch_client_connect    = $couch_client['connect']
-
-  include x509::variables
-  $ca_path                 = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
-  $cert_path               = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
-  $key_path                = "${x509::variables::keys}/${site_config::params::cert_name}.key"
-
-  include site_stunnel
-
-  $couchdb_stunnel_client_defaults = {
-    'connect_port' => $couch_client_connect,
-    'client'     => true,
-    'cafile'     => $ca_path,
-    'key'        => $key_path,
-    'cert'       => $cert_path,
-  }
-
-  create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults)
-}
diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp
index c3d38a46..91014ed6 100644
--- a/puppet/modules/site_mx/manifests/init.pp
+++ b/puppet/modules/site_mx/manifests/init.pp
@@ -8,12 +8,12 @@ class site_mx {
   include site_config::x509::client_ca::ca
   include site_config::x509::client_ca::key
 
+  include site_stunnel
 
   include site_postfix::mx
   include site_haproxy
   include site_shorewall::mx
   include site_shorewall::service::smtp
-  include site_mx::couchdb
   include leap_mx
   include site_check_mk::agent::mx
 }
diff --git a/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
new file mode 100755
index 00000000..cf7c03e5
--- /dev/null
+++ b/puppet/modules/site_nagios/files/plugins/check_last_regex_in_log
@@ -0,0 +1,85 @@
+#!/bin/sh
+#
+# depends on nagios-plugins-common for /usr/lib/nagios/plugins/utils.sh
+# this package is installed using leap_platform by the Site_check_mk::Agent::Mrpe
+# class
+
+set -e
+
+usage()
+{
+cat << EOF
+usage: $0 -w <sec> -c <sec> -r <regexp> -f <filename>
+
+OPTIONS:
+  -h         Show this message
+  -r <regex> regex to grep for
+  -f <file>  logfile to search in
+  -w <sec>   warning  state after X seconds
+  -c <sec>   critical state after x seconds
+
+example: $0 -f /var/log/syslog -r 'tapicero' -w 300 -c 600
+EOF
+}
+
+
+. /usr/lib/nagios/plugins/utils.sh
+
+
+warn=0
+crit=0
+log=''
+regex=''
+
+set -- $(getopt hr:f:w:c: "$@")
+while [ $# -gt 0 ]
+do
+    case "$1" in
+    (-h) usage; exit 0 ;;
+    (-f) log="$2"; shift;;
+    (-r) regex="$2"; shift;;
+    (-w) warn="$2"; shift;;
+    (-c) crit="$2"; shift;;
+    (--) shift; break;;
+    (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
+    (*)  break;;
+    esac
+    shift
+done
+
+[ $warn -eq 0 -o $crit -eq 0 -o -z "$regex" -o -z "$log" ] && ( usage; exit $STATE_UNKNOWN)
+[ -f "$log" ] || (echo "$log doesn't exist"; exit $STATE_UNKNOWN)
+
+lastmsg=$(tac $log | grep -i $regex | head -1 | cut -d' ' -f 1-3)
+
+if [ -z "$lastmsg" ]
+then
+  summary="\"$regex\" in $log was not found"
+  state=$STATE_CRITICAL
+  state_text='CRITICAL'
+  diff_sec=0
+else
+  lastmsg_sec=$(date '+%s' -d "$lastmsg")
+  now_sec=$(date '+%s')
+
+  diff_sec=$(($now_sec - $lastmsg_sec))
+
+  if [ $diff_sec -lt $warn ]; then
+    state=$STATE_OK
+    state_text='OK'
+  elif [ $diff_sec -lt $crit ]; then
+    state=$STATE_WARNING
+    state_text='WARNING'
+  else
+    state=$STATE_CRITICAL
+    state_text='CRITICAL'
+  fi
+
+  summary="Last occurrence of \"$regex\" in $log was $diff_sec sec ago"
+fi
+
+# check_mk_agent output
+# echo "$state Tapicero_Heatbeat sec=$diff_sec;$warn;$crit;0; $state_text - $summary"
+
+echo "${state_text}: $summary | seconds=${diff_sec};$warn;$crit;0;"
+exit $state
diff --git a/puppet/modules/site_nagios/manifests/add_host_services.pp b/puppet/modules/site_nagios/manifests/add_host_services.pp
index 279809d1..bd968e6f 100644
--- a/puppet/modules/site_nagios/manifests/add_host_services.pp
+++ b/puppet/modules/site_nagios/manifests/add_host_services.pp
@@ -1,10 +1,13 @@
 define site_nagios::add_host_services (
   $domain_full_suffix,
   $domain_internal,
+  $domain_internal_suffix,
   $ip_address,
   $services,
   $ssh_port,
-  $openvpn_gateway_address='' ) {
+  $environment,
+  $openvpn_gateway_address='',
+  ) {
 
     $nagios_hostname = $domain_internal
 
@@ -16,6 +19,7 @@ define site_nagios::add_host_services (
       'hostname'    => $nagios_hostname,
       'ip_address'  => $ip_address,
       'openvpn_gw'  => $openvpn_gateway_address,
+      'environment' => $environment
     }
     $dynamic_parameters = {
       'service' => '%s'
diff --git a/puppet/modules/site_nagios/manifests/add_service.pp b/puppet/modules/site_nagios/manifests/add_service.pp
index 1b67d14e..72cd038a 100644
--- a/puppet/modules/site_nagios/manifests/add_service.pp
+++ b/puppet/modules/site_nagios/manifests/add_service.pp
@@ -1,5 +1,5 @@
 define site_nagios::add_service (
-  $hostname, $ip_address, $openvpn_gw = '', $service) {
+  $hostname, $ip_address, $service, $environment, $openvpn_gw = '') {
 
   $ssh      = hiera_hash('ssh')
   $ssh_port = $ssh['port']
@@ -9,19 +9,22 @@ define site_nagios::add_service (
       nagios_service {
         "${name}_ssh":
           use                 => 'generic-service',
-          check_command       => "check_ssh_port!$ssh_port",
+          check_command       => "check_ssh_port!${ssh_port}",
           service_description => 'SSH',
-          host_name           => $hostname;
+          host_name           => $hostname,
+          contact_groups      => $environment;
         "${name}_cert":
           use                 => 'generic-service',
           check_command       => 'check_https_cert',
           service_description => 'Website Certificate',
-          host_name           => $hostname;
+          host_name           => $hostname,
+          contact_groups      => $environment;
         "${name}_website":
           use                 => 'generic-service',
           check_command       => 'check_https',
           service_description => 'Website',
-          host_name           => $hostname
+          host_name           => $hostname,
+          contact_groups      => $environment;
       }
     }
     default:  {}
diff --git a/puppet/modules/site_nagios/manifests/plugins.pp b/puppet/modules/site_nagios/manifests/plugins.pp
new file mode 100644
index 00000000..90a01cfb
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/plugins.pp
@@ -0,0 +1,16 @@
+# Deploy generic plugins useful to all nodes
+# nagios::plugin won't work to deploy a plugin
+# because it complains with:
+# Could not find dependency Package[nagios-plugins] …
+# at /srv/leap/puppet/modules/nagios/manifests/plugin.pp:18
+class site_nagios::plugins {
+
+  file { [
+    '/usr/local/lib', '/usr/local/lib/nagios',
+    '/usr/local/lib/nagios/plugins' ]:
+      ensure => directory;
+    '/usr/local/lib/nagios/plugins/check_last_regex_in_log':
+      source => 'puppet:///modules/site_nagios/plugins/check_last_regex_in_log',
+      mode   => '0755';
+  }
+}
diff --git a/puppet/modules/site_nagios/manifests/server.pp b/puppet/modules/site_nagios/manifests/server.pp
index 85443917..092ca503 100644
--- a/puppet/modules/site_nagios/manifests/server.pp
+++ b/puppet/modules/site_nagios/manifests/server.pp
@@ -3,13 +3,19 @@ class site_nagios::server inherits nagios::base {
   # First, purge old nagios config (see #1467)
   class { 'site_nagios::server::purge': }
 
-  $nagios_hiera   = hiera('nagios')
-  $nagiosadmin_pw = htpasswd_sha1($nagios_hiera['nagiosadmin_pw'])
-  $nagios_hosts   = $nagios_hiera['hosts']
+  $nagios_hiera     = hiera('nagios')
+  $nagiosadmin_pw   = htpasswd_sha1($nagios_hiera['nagiosadmin_pw'])
+  $nagios_hosts     = $nagios_hiera['hosts']
+  $nagios_contacts  = hiera('contacts')
+  $environment      = $nagios_hiera['environments']
 
-  include nagios::defaults
   include nagios::base
-  class {'nagios':
+  include nagios::defaults::commands
+  include nagios::defaults::templates
+  include nagios::defaults::timeperiods
+  include nagios::defaults::plugins
+
+  class { 'nagios':
     # don't manage apache class from nagios, cause we already include
     # it in site_apache::common
     httpd              => 'absent',
@@ -46,6 +52,7 @@ class site_nagios::server inherits nagios::base {
   include site_nagios::server::apache
   include site_check_mk::server
   include site_shorewall::monitor
+  include site_nagios::server::icli
 
   augeas {
     'logrotate_nagios':
@@ -55,4 +62,8 @@ class site_nagios::server inherits nagios::base {
         'set missingok missingok', 'set ifempty notifempty',
         'set copytruncate copytruncate' ]
   }
+
+  create_resources ( site_nagios::server::hostgroup, $environment )
+  create_resources ( site_nagios::server::contactgroup, $environment )
+  create_resources ( site_nagios::server::add_contacts, $environment )
 }
diff --git a/puppet/modules/site_nagios/manifests/server/add_contacts.pp b/puppet/modules/site_nagios/manifests/server/add_contacts.pp
new file mode 100644
index 00000000..db507abf
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/server/add_contacts.pp
@@ -0,0 +1,16 @@
+define site_nagios::server::add_contacts ($contact_emails) {
+
+  $environment = $name
+
+  nagios_contact {
+    $environment:
+      alias                         => $environment,
+      service_notification_period   => '24x7',
+      host_notification_period      => '24x7',
+      service_notification_options  => 'w,u,c,r',
+      host_notification_options     => 'd,r',
+      service_notification_commands => 'notify-service-by-email',
+      host_notification_commands    => 'notify-host-by-email',
+      email                         => join($contact_emails, ', ')
+  }
+}
diff --git a/puppet/modules/site_nagios/manifests/server/contactgroup.pp b/puppet/modules/site_nagios/manifests/server/contactgroup.pp
new file mode 100644
index 00000000..188c54f1
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/server/contactgroup.pp
@@ -0,0 +1,6 @@
+define site_nagios::server::contactgroup ($contact_emails) {
+
+  nagios_contactgroup { $name:
+    members => $name
+  }
+}
diff --git a/puppet/modules/site_nagios/manifests/server/hostgroup.pp b/puppet/modules/site_nagios/manifests/server/hostgroup.pp
new file mode 100644
index 00000000..6f85ca6d
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/server/hostgroup.pp
@@ -0,0 +1,3 @@
+define site_nagios::server::hostgroup ($contact_emails) {
+  nagios_hostgroup { $name: }
+}
diff --git a/puppet/modules/site_nagios/manifests/server/icli.pp b/puppet/modules/site_nagios/manifests/server/icli.pp
new file mode 100644
index 00000000..26fba725
--- /dev/null
+++ b/puppet/modules/site_nagios/manifests/server/icli.pp
@@ -0,0 +1,26 @@
+# Install icli package and configure ncli aliases 
+class site_nagios::server::icli {
+  $nagios_hiera     = hiera('nagios')
+  $environments     = $nagios_hiera['environments']
+
+  package { 'icli':
+    ensure => installed;
+  }
+
+  file { '/root/.bashrc':
+    ensure => present;
+  }
+
+  file_line { 'icli aliases':
+    path => '/root/.bashrc',
+    line => 'source /root/.icli_aliases';
+  }
+
+  file { '/root/.icli_aliases':
+    content => template("${module_name}/icli_aliases.erb"),
+    mode    => '0644',
+    owner   => root,
+    group   => 0,
+    require => Package['icli'];
+  }
+}
\ No newline at end of file
diff --git a/puppet/modules/site_nagios/templates/icli_aliases.erb b/puppet/modules/site_nagios/templates/icli_aliases.erb
new file mode 100644
index 00000000..f1428f9e
--- /dev/null
+++ b/puppet/modules/site_nagios/templates/icli_aliases.erb
@@ -0,0 +1,7 @@
+alias ncli='icli -c /var/cache/nagios3/objects.cache -f /var/cache/nagios3/status.dat  -F /var/lib/nagios3/rw/nagios.cmd'
+alias ncli_problems='ncli -z '!o,!A''
+
+<% @environments.keys.sort.each do |env_name| %>
+alias ncli_<%= env_name %>='ncli -z '!o,!A' -g <%= env_name %>'
+alias ncli_<%= env_name %>_recheck='ncli -s Check_MK -g <%= env_name %> -r'
+<% end -%>
\ No newline at end of file
diff --git a/puppet/modules/site_obfsproxy/README b/puppet/modules/site_obfsproxy/README
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/puppet/modules/site_obfsproxy/README
diff --git a/puppet/modules/site_obfsproxy/manifests/init.pp b/puppet/modules/site_obfsproxy/manifests/init.pp
new file mode 100644
index 00000000..6275ebee
--- /dev/null
+++ b/puppet/modules/site_obfsproxy/manifests/init.pp
@@ -0,0 +1,39 @@
+class site_obfsproxy {
+  tag 'leap_service'
+  Class['site_config::default'] -> Class['site_obfsproxy']
+
+  $transport = 'scramblesuit'
+
+  $obfsproxy    = hiera('obfsproxy')
+  $scramblesuit = $obfsproxy['scramblesuit']
+  $scram_pass   = $scramblesuit['password']
+  $scram_port   = $scramblesuit['port']
+  $dest_ip      = $obfsproxy['gateway_address']
+  $dest_port    = '443'
+
+  if member($::services, 'openvpn') {
+    $openvpn      = hiera('openvpn')
+    $bind_address = $openvpn['gateway_address']
+  }
+  elsif member($::services, 'obfsproxy') {
+    $bind_address = hiera('ip_address')
+  }
+
+  include site_apt::preferences::twisted
+  include site_apt::preferences::obfsproxy
+
+  class { 'obfsproxy':
+    transport    => $transport,
+    bind_address => $bind_address,
+    port         => $scram_port,
+    param        => $scram_pass,
+    dest_ip      => $dest_ip,
+    dest_port    => $dest_port,
+  }
+
+  include site_shorewall::obfsproxy
+
+}
+
+
+
diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp
index b6331f12..d6f9150b 100644
--- a/puppet/modules/site_openvpn/manifests/init.pp
+++ b/puppet/modules/site_openvpn/manifests/init.pp
@@ -148,13 +148,17 @@ class site_openvpn {
   exec { 'restart_openvpn':
     command     => '/etc/init.d/openvpn restart',
     refreshonly => true,
-    subscribe   => File['/etc/openvpn'],
+    subscribe   => [
+                    File['/etc/openvpn'],
+                    Class['Site_config::X509::Key'],
+                    Class['Site_config::X509::Cert'],
+                    Class['Site_config::X509::Ca_bundle'] ],
     require     => [
-      Package['openvpn'],
-      File['/etc/openvpn'],
-      Class['Site_config::X509::Key'],
-      Class['Site_config::X509::Cert'],
-      Class['Site_config::X509::Ca_bundle'] ];
+                    Package['openvpn'],
+                    File['/etc/openvpn'],
+                    Class['Site_config::X509::Key'],
+                    Class['Site_config::X509::Cert'],
+                    Class['Site_config::X509::Ca_bundle'] ];
   }
 
   cron { 'add_gateway_ips.sh':
diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp
index 97cf2842..466f6d00 100644
--- a/puppet/modules/site_openvpn/manifests/server_config.pp
+++ b/puppet/modules/site_openvpn/manifests/server_config.pp
@@ -85,6 +85,18 @@ define site_openvpn::server_config(
         key     => 'tcp-nodelay',
         server  => $openvpn_configname;
     }
+  } elsif $proto == 'udp' {
+    if $config['fragment'] != 1500 {
+      openvpn::option {
+        "fragment ${openvpn_configname}":
+          key    => 'fragment',
+          value  => $config['fragment'],
+          server => $openvpn_configname;
+        "mssfix ${openvpn_configname}":
+          key    => 'mssfix',
+          server => $openvpn_configname;
+      }
+    }
   }
 
   openvpn::option {
diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp
index bdfee665..81f10b77 100644
--- a/puppet/modules/site_postfix/manifests/mx.pp
+++ b/puppet/modules/site_postfix/manifests/mx.pp
@@ -1,12 +1,12 @@
 class site_postfix::mx {
 
-  $domain_hash         = hiera ('domain')
+  $domain_hash         = hiera('domain')
   $domain              = $domain_hash['full_suffix']
   $host_domain         = $domain_hash['full']
   $cert_name           = hiera('name')
   $mynetworks          = join(hiera('mynetworks'), ' ')
 
-  $root_mail_recipient = hiera ('contacts')
+  $root_mail_recipient = hiera('contacts')
   $postfix_smtp_listen = 'all'
 
   include site_config::x509::cert
diff --git a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp
index d9b59f40..d56f6b54 100644
--- a/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp
+++ b/puppet/modules/site_postfix/manifests/mx/smtp_tls.pp
@@ -1,5 +1,6 @@
 class site_postfix::mx::smtp_tls {
 
+  include site_config::x509::ca
   include x509::variables
   $ca_path   = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
   $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
diff --git a/puppet/modules/site_shorewall/manifests/couchdb.pp b/puppet/modules/site_shorewall/manifests/couchdb.pp
deleted file mode 100644
index 73bed62b..00000000
--- a/puppet/modules/site_shorewall/manifests/couchdb.pp
+++ /dev/null
@@ -1,24 +0,0 @@
-class site_shorewall::couchdb {
-
-  include site_shorewall::defaults
-
-  $stunnel = hiera('stunnel')
-  $couch_server = $stunnel['couch_server']
-  $couch_stunnel_port = $couch_server['accept']
-
-  # define macro for incoming services
-  file { '/etc/shorewall/macro.leap_couchdb':
-    content => "PARAM   -       -       tcp    ${couch_stunnel_port}",
-    notify  => Service['shorewall'],
-    require => Package['shorewall']
-  }
-
-  shorewall::rule {
-      'net2fw-couchdb':
-        source      => 'net',
-        destination => '$FW',
-        action      => 'leap_couchdb(ACCEPT)',
-        order       => 200;
-  }
-
-}
diff --git a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp b/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp
deleted file mode 100644
index 20740650..00000000
--- a/puppet/modules/site_shorewall/manifests/couchdb/bigcouch.pp
+++ /dev/null
@@ -1,51 +0,0 @@
-class site_shorewall::couchdb::bigcouch {
-
-  include site_shorewall::defaults
-
-  $stunnel = hiera('stunnel')
-
-  # Erlang Port Mapper Daemon (epmd) stunnel server/clients
-  $epmd_clients         = $stunnel['epmd_clients']
-  $epmd_server          = $stunnel['epmd_server']
-  $epmd_server_port     = $epmd_server['accept']
-  $epmd_server_connect  = $epmd_server['connect']
-
-  # Erlang Distributed Node Protocol (ednp) stunnel server/clients
-  $ednp_clients         = $stunnel['ednp_clients']
-  $ednp_server          = $stunnel['ednp_server']
-  $ednp_server_port     = $ednp_server['accept']
-  $ednp_server_connect  = $ednp_server['connect']
-
-  # define macro for incoming services
-  file { '/etc/shorewall/macro.leap_bigcouch':
-    content => "PARAM   -       -       tcp    ${epmd_server_port},${ednp_server_port}",
-    notify  => Service['shorewall'],
-    require => Package['shorewall']
-  }
-
-  shorewall::rule {
-      'net2fw-bigcouch':
-        source      => 'net',
-        destination => '$FW',
-        action      => 'leap_bigcouch(ACCEPT)',
-        order       => 300;
-  }
-
-  # setup DNAT rules for each epmd
-  $epmd_shorewall_dnat_defaults = {
-    'source'          => '$FW',
-    'proto'           => 'tcp',
-    'destinationport' => regsubst($epmd_server_connect, '^([0-9.]+:)([0-9]+)$', '\2')
-  }
-  create_resources(site_shorewall::couchdb::dnat, $epmd_clients, $epmd_shorewall_dnat_defaults)
-
-  # setup DNAT rules for each ednp
-  $ednp_shorewall_dnat_defaults = {
-    'source'          => '$FW',
-    'proto'           => 'tcp',
-    'destinationport' => regsubst($ednp_server_connect, '^([0-9.]+:)([0-9]+)$', '\2')
-  }
-  create_resources(site_shorewall::couchdb::dnat, $ednp_clients, $ednp_shorewall_dnat_defaults)
-
-}
-
diff --git a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp b/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp
deleted file mode 100644
index f1bc9acf..00000000
--- a/puppet/modules/site_shorewall/manifests/couchdb/dnat.pp
+++ /dev/null
@@ -1,21 +0,0 @@
-define site_shorewall::couchdb::dnat (
-  $source,
-  $connect,
-  $connect_port,
-  $accept_port,
-  $proto,
-  $destinationport )
-{
-
-
-  shorewall::rule {
-    "dnat_${name}_${destinationport}":
-      action          => 'DNAT',
-      source          => $source,
-      destination     => "\$FW:127.0.0.1:${accept_port}",
-      proto           => $proto,
-      destinationport => $destinationport,
-      originaldest    => $connect,
-      order           => 200
-  }
-}
diff --git a/puppet/modules/site_shorewall/manifests/dnat_rule.pp b/puppet/modules/site_shorewall/manifests/dnat_rule.pp
index aa298408..f9fbe950 100644
--- a/puppet/modules/site_shorewall/manifests/dnat_rule.pp
+++ b/puppet/modules/site_shorewall/manifests/dnat_rule.pp
@@ -4,41 +4,45 @@ define site_shorewall::dnat_rule {
   if $port != 1194 {
     if $site_openvpn::openvpn_allow_unlimited {
       shorewall::rule {
-          "dnat_tcp_port_$port":
+          "dnat_tcp_port_${port}":
             action          => 'DNAT',
             source          => 'net',
             destination     => "\$FW:${site_openvpn::unlimited_gateway_address}:1194",
             proto           => 'tcp',
             destinationport => $port,
+            originaldest    => $site_openvpn::unlimited_gateway_address,
             order           => 100;
       }
       shorewall::rule {
-          "dnat_udp_port_$port":
+          "dnat_udp_port_${port}":
             action          => 'DNAT',
             source          => 'net',
             destination     => "\$FW:${site_openvpn::unlimited_gateway_address}:1194",
             proto           => 'udp',
             destinationport => $port,
+            originaldest    => $site_openvpn::unlimited_gateway_address,
             order           => 100;
       }
     }
     if $site_openvpn::openvpn_allow_limited {
       shorewall::rule {
-          "dnat_free_tcp_port_$port":
+          "dnat_free_tcp_port_${port}":
             action          => 'DNAT',
             source          => 'net',
             destination     => "\$FW:${site_openvpn::limited_gateway_address}:1194",
             proto           => 'tcp',
             destinationport => $port,
+            originaldest    => $site_openvpn::unlimited_gateway_address,
             order           => 100;
       }
       shorewall::rule {
-          "dnat_free_udp_port_$port":
+          "dnat_free_udp_port_${port}":
             action          => 'DNAT',
             source          => 'net',
             destination     => "\$FW:${site_openvpn::limited_gateway_address}:1194",
             proto           => 'udp',
             destinationport => $port,
+            originaldest    => $site_openvpn::unlimited_gateway_address,
             order           => 100;
       }
     }
diff --git a/puppet/modules/site_shorewall/manifests/obfsproxy.pp b/puppet/modules/site_shorewall/manifests/obfsproxy.pp
new file mode 100644
index 00000000..68fb9b9f
--- /dev/null
+++ b/puppet/modules/site_shorewall/manifests/obfsproxy.pp
@@ -0,0 +1,24 @@
+class site_shorewall::obfsproxy {
+
+  include site_shorewall::defaults
+
+  $obfsproxy    = hiera('obfsproxy')
+  $scramblesuit = $obfsproxy['scramblesuit']
+  $scram_port   = $scramblesuit['port']
+
+  # define macro for incoming services
+  file { '/etc/shorewall/macro.leap_obfsproxy':
+    content => "PARAM   -       -       tcp    $scram_port ",
+    notify  => Service['shorewall'],
+    require => Package['shorewall']
+  }
+
+  shorewall::rule {
+      'net2fw-obfs':
+        source      => 'net',
+        destination => '$FW',
+        action      => 'leap_obfsproxy(ACCEPT)',
+        order       => 200;
+  }
+
+}
diff --git a/puppet/modules/site_shorewall/manifests/stunnel/client.pp b/puppet/modules/site_shorewall/manifests/stunnel/client.pp
new file mode 100644
index 00000000..9a89a244
--- /dev/null
+++ b/puppet/modules/site_shorewall/manifests/stunnel/client.pp
@@ -0,0 +1,40 @@
+#
+# Adds some firewall magic to the stunnel.
+#
+# Using DNAT, this firewall rule allow a locally running program
+# to try to connect to the normal remote IP and remote port of the
+# service on another machine, but have this connection magically
+# routed through the locally running stunnel client.
+#
+# The network looks like this:
+#
+#   From the client's perspective:
+#
+#   |------- stunnel client --------------|    |---------- stunnel server -----------------------|
+#    consumer app -> localhost:accept_port  ->  connect:connect_port -> localhost:original_port
+#
+#   From the server's perspective:
+#
+#   |------- stunnel client --------------|    |---------- stunnel server -----------------------|
+#                                       ??  ->  *:accept_port -> localhost:connect_port -> service
+#
+
+define site_shorewall::stunnel::client(
+  $accept_port,
+  $connect,
+  $connect_port,
+  $original_port) {
+
+  include site_shorewall::defaults
+
+  shorewall::rule {
+    "stunnel_dnat_${name}":
+      action          => 'DNAT',
+      source          => '$FW',
+      destination     => "\$FW:127.0.0.1:${accept_port}",
+      proto           => 'tcp',
+      destinationport => $original_port,
+      originaldest    => $connect,
+      order           => 200
+  }
+}
diff --git a/puppet/modules/site_shorewall/manifests/stunnel/server.pp b/puppet/modules/site_shorewall/manifests/stunnel/server.pp
new file mode 100644
index 00000000..798cd631
--- /dev/null
+++ b/puppet/modules/site_shorewall/manifests/stunnel/server.pp
@@ -0,0 +1,22 @@
+#
+# Allow all incoming connections to stunnel server port
+#
+
+define site_shorewall::stunnel::server($port) {
+
+  include site_shorewall::defaults
+
+  file { "/etc/shorewall/macro.stunnel_server_${name}":
+    content => "PARAM   -       -       tcp    ${port}",
+    notify  => Service['shorewall'],
+    require => Package['shorewall']
+  }
+  shorewall::rule {
+    "net2fw-stunnel-server-${name}":
+      source      => 'net',
+      destination => '$FW',
+      action      => "stunnel_server_${name}(ACCEPT)",
+      order       => 200;
+  }
+
+}
\ No newline at end of file
diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp
index 9a05b6ed..1da2f1d5 100644
--- a/puppet/modules/site_sshd/manifests/init.pp
+++ b/puppet/modules/site_sshd/manifests/init.pp
@@ -53,7 +53,7 @@ class site_sshd {
   ##
   class { '::sshd':
     manage_nagios => false,
-    ports         => $ssh['port'],
+    ports         => [ $ssh['port'] ],
     use_pam       => 'yes',
     hardened_ssl  => 'yes',
     print_motd    => 'no',
diff --git a/puppet/modules/site_sshd/templates/ssh_config.erb b/puppet/modules/site_sshd/templates/ssh_config.erb
index 7e967413..36c0b6d5 100644
--- a/puppet/modules/site_sshd/templates/ssh_config.erb
+++ b/puppet/modules/site_sshd/templates/ssh_config.erb
@@ -21,3 +21,20 @@ Host *
     StrictHostKeyChecking no
 <% end -%>
 
+#
+# Tell SSH what host key algorithm we should use. I don't understand why this
+# is needed, since the man page says that "if hostkeys are known for the
+# destination host then [HostKeyAlgorithms default] is modified to prefer
+# their algorithms."
+#
+
+<% @hosts.sort.each do |name, host| -%>
+Host <%= name %> <%= host['domain_full'] %> <%= host['domain_internal'] %> <%= host['ip_address'] %>
+<% if host['host_pub_key'] -%>
+HostKeyAlgorithms <%= host['host_pub_key'].split(" ").first %>
+<% end -%>
+<% if host['port'] -%>
+Port <%= host['port'] %>
+<% end -%>
+
+<% end -%>
diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp
index 6e347d35..aed9775e 100644
--- a/puppet/modules/site_static/manifests/init.pp
+++ b/puppet/modules/site_static/manifests/init.pp
@@ -1,5 +1,10 @@
 class site_static {
   tag 'leap_service'
+
+  include site_config::x509::cert
+  include site_config::x509::key
+  include site_config::x509::ca_bundle
+
   $static        = hiera('static')
   $domains       = $static['domains']
   $formats       = $static['formats']
@@ -33,7 +38,7 @@ class site_static {
     include site_apt::preferences::passenger
     class { 'passenger':
       use_munin => false,
-      require => Class['site_apt::preferences::passenger']
+      require   => Class['site_apt::preferences::passenger']
     }
   }
 
diff --git a/puppet/modules/site_stunnel/manifests/client.pp b/puppet/modules/site_stunnel/manifests/client.pp
new file mode 100644
index 00000000..3b10ecb8
--- /dev/null
+++ b/puppet/modules/site_stunnel/manifests/client.pp
@@ -0,0 +1,49 @@
+#
+# Sets up stunnel and firewall configuration for
+# a single stunnel client
+#
+# As a client, we accept connections on localhost,
+# and connect to a remote $connect:$connect_port
+#
+
+define site_stunnel::client (
+  $accept_port,
+  $connect_port,
+  $connect,
+  $original_port,
+  $verify     = '2',
+  $pid        = $name,
+  $rndfile    = '/var/lib/stunnel4/.rnd',
+  $debuglevel = '4' ) {
+
+  include site_config::x509::cert
+  include site_config::x509::key
+  include site_config::x509::ca
+  include x509::variables
+  $ca_path   = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
+  $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
+  $key_path  = "${x509::variables::keys}/${site_config::params::cert_name}.key"
+
+  stunnel::service { $name:
+    accept     => "127.0.0.1:${accept_port}",
+    connect    => "${connect}:${connect_port}",
+    client     => true,
+    cafile     => $ca_path,
+    key        => $key_path,
+    cert       => $cert_path,
+    verify     => $verify,
+    pid        => "/var/run/stunnel4/${pid}.pid",
+    rndfile    => $rndfile,
+    debuglevel => $debuglevel,
+    sslversion => 'TLSv1';
+  }
+
+  site_shorewall::stunnel::client { $name:
+    accept_port   => $accept_port,
+    connect       => $connect,
+    connect_port  => $connect_port,
+    original_port => $original_port
+  }
+
+  include site_check_mk::agent::stunnel
+}
diff --git a/puppet/modules/site_stunnel/manifests/clients.pp b/puppet/modules/site_stunnel/manifests/clients.pp
index b75c9ac3..c0958b5f 100644
--- a/puppet/modules/site_stunnel/manifests/clients.pp
+++ b/puppet/modules/site_stunnel/manifests/clients.pp
@@ -1,33 +1,23 @@
-define site_stunnel::clients (
-  $accept_port,
-  $connect_port,
-  $connect,
-  $cafile,
-  $key,
-  $cert,
-  $client     = true,
-  $verify     = '2',
-  $pid        = $name,
-  $rndfile    = '/var/lib/stunnel4/.rnd',
-  $debuglevel = '4' ) {
+#
+# example hiera yaml:
+#
+#   stunnel:
+#     clients:
+#       ednp_clients:
+#         thrips_9002:
+#           accept_port: 4001
+#           connect: thrips.demo.bitmask.i
+#           connect_port: 19002
+#       epmd_clients:
+#         thrips_4369:
+#           accept_port: 4000
+#           connect: thrips.demo.bitmask.i
+#           connect_port: 14369
+#
+# In the above example, this resource definition is called twice, with $name
+# 'ednp_clients' and 'epmd_clients'
+#
 
-  stunnel::service { $name:
-    accept     => "127.0.0.1:${accept_port}",
-    connect    => "${connect}:${connect_port}",
-    client     => $client,
-    cafile     => $cafile,
-    key        => $key,
-    cert       => $cert,
-    verify     => $verify,
-    pid        => "/var/run/stunnel4/${pid}.pid",
-    rndfile    => $rndfile,
-    debuglevel => $debuglevel,
-    subscribe  => [
-      Class['Site_config::X509::Key'],
-      Class['Site_config::X509::Cert'],
-      Class['Site_config::X509::Ca'] ];
-
-  }
-
-  include site_check_mk::agent::stunnel
+define site_stunnel::clients {
+  create_resources(site_stunnel::client, $site_stunnel::clients[$name])
 }
diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp
index c7d6acc6..2e0cf5b8 100644
--- a/puppet/modules/site_stunnel/manifests/init.pp
+++ b/puppet/modules/site_stunnel/manifests/init.pp
@@ -1,3 +1,8 @@
+#
+# If you need something to happen after stunnel is started,
+# you can depend on Service['stunnel'] or Class['site_stunnel']
+#
+
 class site_stunnel {
 
   # include the generic stunnel module
@@ -13,5 +18,17 @@ class site_stunnel {
         ensure => absent;
     }
   }
+
+  $stunnel = hiera('stunnel')
+
+  # add server stunnels
+  create_resources(site_stunnel::servers, $stunnel['servers'])
+
+  # add client stunnels
+  $clients = $stunnel['clients']
+  $client_sections = keys($clients)
+  site_stunnel::clients { $client_sections: }
+
+  include site_stunnel::override_service
 }
 
diff --git a/puppet/modules/site_stunnel/manifests/override_service.pp b/puppet/modules/site_stunnel/manifests/override_service.pp
new file mode 100644
index 00000000..96187048
--- /dev/null
+++ b/puppet/modules/site_stunnel/manifests/override_service.pp
@@ -0,0 +1,13 @@
+class site_stunnel::override_service inherits stunnel::debian {
+
+  include site_config::x509::cert
+  include site_config::x509::key
+  include site_config::x509::ca
+
+  Service[stunnel] {
+    subscribe => [
+                  Class['Site_config::X509::Key'],
+                  Class['Site_config::X509::Cert'],
+                  Class['Site_config::X509::Ca'] ]
+  }
+}
diff --git a/puppet/modules/site_stunnel/manifests/servers.pp b/puppet/modules/site_stunnel/manifests/servers.pp
new file mode 100644
index 00000000..b6fac319
--- /dev/null
+++ b/puppet/modules/site_stunnel/manifests/servers.pp
@@ -0,0 +1,47 @@
+#
+# example hiera yaml:
+#
+#   stunnel:
+#     servers:
+#       couch_server:
+#         accept_port: 15984
+#         connect_port: 5984
+#
+
+define site_stunnel::servers (
+  $accept_port,
+  $connect_port,
+  $verify     = '2',
+  $pid        = $name,
+  $rndfile    = '/var/lib/stunnel4/.rnd',
+  $debuglevel = '4' ) {
+
+  include site_config::x509::cert
+  include site_config::x509::key
+  include site_config::x509::ca
+  include x509::variables
+  $ca_path   = "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"
+  $cert_path = "${x509::variables::certs}/${site_config::params::cert_name}.crt"
+  $key_path  = "${x509::variables::keys}/${site_config::params::cert_name}.key"
+
+  stunnel::service { $name:
+    accept     => $accept_port,
+    connect    => "127.0.0.1:${connect_port}",
+    client     => false,
+    cafile     => $ca_path,
+    key        => $key_path,
+    cert       => $cert_path,
+    verify     => $verify,
+    pid        => "/var/run/stunnel4/${pid}.pid",
+    rndfile    => '/var/lib/stunnel4/.rnd',
+    debuglevel => $debuglevel,
+    sslversion => 'TLSv1';
+  }
+
+  # allow incoming connections on $accept_port
+  site_shorewall::stunnel::server { $name:
+    port  => $accept_port
+  }
+
+  include site_check_mk::agent::stunnel
+}
diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp
index e62cb12d..80ccc5d3 100644
--- a/puppet/modules/site_tor/manifests/init.pp
+++ b/puppet/modules/site_tor/manifests/init.pp
@@ -11,23 +11,31 @@ class site_tor {
 
   $address        = hiera('ip_address')
 
-  class { 'tor::daemon': }
+  $openvpn        = hiera('openvpn', undef)
+  if $openvpn {
+    $openvpn_ports = $openvpn['ports']
+  }
+  else {
+    $openvpn_ports = []
+  }
+
+  include tor::daemon
   tor::daemon::relay { $nickname:
-    port             => 9001,
-    address          => $address,
-    contact_info     => obfuscate_email($contact_emails),
-    bandwidth_rate   => $bandwidth_rate,
-    my_family        => $family
+    port           => 9001,
+    address        => $address,
+    contact_info   => obfuscate_email($contact_emails),
+    bandwidth_rate => $bandwidth_rate,
+    my_family      => $family
   }
 
   if ( $tor_type == 'exit'){
-    tor::daemon::directory { $::hostname: port => 80 }
+    # Only enable the daemon directory if the node isn't also a webapp node
+    # or running openvpn on port 80
+    if ! member($::services, 'webapp') and ! member($openvpn_ports, '80') {
+      tor::daemon::directory { $::hostname: port => 80 }
+    }
   }
   else {
-    tor::daemon::directory { $::hostname:
-      port            => 80,
-      port_front_page => '';
-    }
     include site_tor::disable_exit
   }
 
diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp
index ff743fba..3ae4d266 100644
--- a/puppet/modules/site_webapp/manifests/couchdb.pp
+++ b/puppet/modules/site_webapp/manifests/couchdb.pp
@@ -7,10 +7,6 @@ class site_webapp::couchdb {
   $couchdb_webapp_user     = $webapp['couchdb_webapp_user']['username']
   $couchdb_webapp_password = $webapp['couchdb_webapp_user']['password']
 
-  $stunnel                 = hiera('stunnel')
-  $couch_client            = $stunnel['couch_client']
-  $couch_client_connect    = $couch_client['connect']
-
   include x509::variables
 
   file {
@@ -37,14 +33,4 @@ class site_webapp::couchdb {
   }
 
   include site_stunnel
-
-  $couchdb_stunnel_client_defaults = {
-    'connect_port' => $couch_client_connect,
-    'client'       => true,
-    'cafile'       => "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt",
-    'key'          => "${x509::variables::keys}/${site_config::params::cert_name}.key",
-    'cert'         => "${x509::variables::certs}/${site_config::params::cert_name}.crt",
-  }
-
-  create_resources(site_stunnel::clients, $couch_client, $couchdb_stunnel_client_defaults)
 }
diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp
new file mode 100644
index 00000000..16b6e2e7
--- /dev/null
+++ b/puppet/modules/site_webapp/manifests/hidden_service.pp
@@ -0,0 +1,43 @@
+class site_webapp::hidden_service {
+  $tor              = hiera('tor')
+  $hidden_service   = $tor['hidden_service']
+  $tor_domain       = "${hidden_service['address']}.onion"
+
+  include site_apache::common
+  include site_apache::module::headers
+  include site_apache::module::alias
+  include site_apache::module::expires
+  include site_apache::module::removeip
+
+  include tor::daemon
+  tor::daemon::hidden_service { 'webapp': ports => '80 127.0.0.1:80' }
+
+  file {
+    '/var/lib/tor/webapp/':
+      ensure  => directory,
+      owner   => 'debian-tor',
+      group   => 'debian-tor',
+      mode    => '2700';
+
+    '/var/lib/tor/webapp/private_key':
+      ensure  => present,
+      source  => "/srv/leap/files/nodes/${::hostname}/tor.key",
+      owner   => 'debian-tor',
+      group   => 'debian-tor',
+      mode    => '0600';
+
+    '/var/lib/tor/webapp/hostname':
+      ensure  => present,
+      content => $tor_domain,
+      owner   => 'debian-tor',
+      group   => 'debian-tor',
+      mode    => '0600';
+  }
+
+  apache::vhost::file {
+    'hidden_service':
+      content => template('site_apache/vhosts.d/hidden_service.conf.erb')
+  }
+
+  include site_shorewall::tor
+}
\ No newline at end of file
diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp
index 7fdd0c3f..9f97d2c5 100644
--- a/puppet/modules/site_webapp/manifests/init.pp
+++ b/puppet/modules/site_webapp/manifests/init.pp
@@ -10,6 +10,7 @@ class site_webapp {
   $webapp           = hiera('webapp')
   $api_version      = $webapp['api_version']
   $secret_token     = $webapp['secret_token']
+  $tor              = hiera('tor', false)
 
   Class['site_config::default'] -> Class['site_webapp']
 
@@ -53,8 +54,8 @@ class site_webapp {
 
   exec { 'bundler_update':
     cwd     => '/srv/leap/webapp',
-    command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle --without test development"',
-    unless  => '/usr/bin/bundle check',
+    command => '/bin/bash -c "/usr/bin/bundle check --path vendor/bundle || /usr/bin/bundle install --path vendor/bundle --without test development"',
+    unless  => '/usr/bin/bundle check --path vendor/bundle',
     user    => 'leap-webapp',
     timeout => 600,
     require => [
@@ -157,6 +158,20 @@ class site_webapp {
       notify  => Service['apache'];
   }
 
+  if $tor {
+    $hidden_service = $tor['hidden_service']
+    if $hidden_service['active'] {
+      include site_webapp::hidden_service
+    }
+  }
+
+
+  # needed for the soledad-sync check which is run on the
+  # webapp node (#6520)
+  package { 'python-u1db':
+    ensure => latest,
+  }
+
   include site_shorewall::webapp
   include site_check_mk::agent::webapp
 }
diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb
index 6461c5e8..0c75f3ca 100644
--- a/puppet/modules/site_webapp/templates/config.yml.erb
+++ b/puppet/modules/site_webapp/templates/config.yml.erb
@@ -18,3 +18,11 @@ production:
   minimum_client_version: "<%= @webapp['client_version']['min'] %>"
   default_service_level: "<%= @webapp['default_service_level'] %>"
   service_levels: <%= @webapp['service_levels'].to_json %>
+  allow_registration: <%= @webapp['allow_registration'].inspect %>
+  handle_blacklist: <%= @webapp['forbidden_usernames'].inspect %>
+<%- if @webapp['engines'] && @webapp['engines'].any? -%>
+  engines:
+<%-   @webapp['engines'].each do |engine| -%>
+    - <%= engine %>
+<%-   end -%>
+<%- end -%>
diff --git a/puppet/modules/sshd b/puppet/modules/sshd
-Subproject 5c23b33200fc6229ada7f4e13672b5da0d4bdd8
+Subproject 750a497758d94c2f5a6cad23cecc3dbde2d2f92
diff --git a/puppet/modules/stunnel b/puppet/modules/stunnel
-Subproject ec49fd93c2469bc5c13f7e6a7d25468613e1b84
+Subproject b0dc7c84b5f55aec12d7d65da812037913d9dbe
diff --git a/puppet/modules/tapicero/manifests/init.pp b/puppet/modules/tapicero/manifests/init.pp
index f2e723f5..28711b94 100644
--- a/puppet/modules/tapicero/manifests/init.pp
+++ b/puppet/modules/tapicero/manifests/init.pp
@@ -12,6 +12,8 @@ class tapicero {
   $couchdb_soledad_user    = $couchdb_users['soledad']['username']
   $couchdb_leap_mx_user    = $couchdb_users['leap_mx']['username']
 
+  $couchdb_mode            = $couchdb['mode']
+  $couchdb_replication     = $couchdb['replication']
 
   Class['site_config::default'] -> Class['tapicero']
 
@@ -93,7 +95,7 @@ class tapicero {
   vcsrepo { '/srv/leap/tapicero':
     ensure   => present,
     force    => true,
-    revision => 'origin/master',
+    revision => 'origin/version/0.6',
     provider => git,
     source   => 'https://leap.se/git/tapicero',
     owner    => 'tapicero',
diff --git a/puppet/modules/tapicero/templates/tapicero.yaml.erb b/puppet/modules/tapicero/templates/tapicero.yaml.erb
index 8e19b22f..510450ad 100644
--- a/puppet/modules/tapicero/templates/tapicero.yaml.erb
+++ b/puppet/modules/tapicero/templates/tapicero.yaml.erb
@@ -1,3 +1,5 @@
+<%- require 'json' -%>
+
 #
 # Default configuration options for Tapicero
 #
@@ -24,6 +26,10 @@ log_level: info
 options:
   # prefix for per user databases:
   db_prefix: "user-"
+  mode: <%= @couchdb_mode %>
+<%- if @couchdb_replication %>
+  replication: <%= @couchdb_replication.to_json %>
+<%- end -%>
 
   # security settings to be used for the per user databases
   security:
@@ -34,9 +40,11 @@ options:
         # explicit about this
         - <%= @couchdb_admin_user %>
       roles: []
-    readers:
+    members:
       names:
         - <%= @couchdb_soledad_user %>
         - <%= @couchdb_leap_mx_user %>
-      roles: []
+      roles:
+        - replication
+
 
diff --git a/tests/README.md b/tests/README.md
index debbf700..814c25b1 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,12 +1,25 @@
-This directory contains to kinds of tests:
+Tests
+---------------------------------
 
-White Box Tests
-================================
+tests/white-box/
 
-These tests are run on the server as superuser. They are for troubleshooting any problems with the internal setup of the server.
+    These tests are run on the server as superuser. They are for
+    troubleshooting any problems with the internal setup of the server.
 
-Black Box Tests
-================================
+tests/black-box/
+
+    These test are run the user's local machine. They are for troubleshooting
+    any external problems with the service exposed by the server.
+
+Additional Files
+---------------------------------
+
+tests/helpers/
+
+    Utility functions made available to all tests.
+
+tests/order.rb
+
+    Configuration file to specify which nodes should be tested in which order.
 
-These test are run the user's local machine. They are for troubleshooting any external problems with the service exposed by the server.
 
diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb
new file mode 100644
index 00000000..9b26eaaf
--- /dev/null
+++ b/tests/helpers/bonafide_helper.rb
@@ -0,0 +1,99 @@
+#
+# helper for the communication with the provider API for creating, authenticating, and deleting accounts.
+#
+
+class LeapTest
+
+  def assert_tmp_user
+    user = assert_create_user
+    assert_authenticate_user(user)
+    yield user if block_given?
+    assert_delete_user(user)
+  rescue StandardError, MiniTest::Assertion => exc
+    begin
+      assert_delete_user(user)
+    rescue
+    end
+    raise exc
+  end
+
+  def api_url(path)
+    api = property('api')
+    "https://%{domain}:%{port}#{path}" % {
+      :domain   => api['domain'],
+      :port     => api['port']
+    }
+  end
+
+  #
+  # attempts to create a user account via the API,
+  # returning the user object if successful.
+  #
+  def assert_create_user
+    user = SRP::User.new
+    url = api_url("/1/users.json")
+    assert_post(url, user.to_params) do |body|
+      assert response = JSON.parse(body), 'response should be JSON'
+      assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead."
+    end
+    user.ok = true
+    return user
+  end
+
+  #
+  # attempts to authenticate user. if successful,
+  # user object is updated with id and session token.
+  #
+  def assert_authenticate_user(user)
+    url = api_url("/1/sessions.json")
+    session = SRP::Session.new(user)
+    params = {'login' => user.username, 'A' => session.aa}
+    assert_post(url, params) do |response, body|
+      cookie = response['Set-Cookie'].split(';').first
+      assert(response = JSON.parse(body), 'response should be JSON')
+      assert(session.bb = response["B"], 'response should include "B"')
+      url = api_url("/1/sessions/login.json")
+      params = {'client_auth' => session.m, 'A' => session.aa}
+      options = {:headers => {'Cookie' => cookie}}
+      assert_put(url, params, options) do |body|
+        assert(response = JSON.parse(body), 'response should be JSON')
+        assert(response['M2'], 'response should include M2')
+        user.session_token = response['token']
+        user.id = response['id']
+        assert(user.session_token, 'response should include token')
+        assert(user.id, 'response should include user id')
+      end
+    end
+  end
+
+  #
+  # attempts to destroy a user account via the API.
+  #
+  def assert_delete_user(user)
+    if user && user.ok && user.id && user.session_token && !user.deleted
+      url = api_url("/1/users/#{user.id}.json")
+      options = {:headers => {
+        "Authorization" => "Token token=\"#{user.session_token}\""
+      }}
+      params = {
+        :identities => 'destroy'
+      }
+      user.deleted = true
+      delete(url, params, options) do |body, response, error|
+        assert error.nil?, "Error deleting user: #{error}"
+        assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+        assert(response = JSON.parse(body), 'Delete response should be JSON')
+        assert(response["success"], 'Deleting user should be a success')
+      end
+      domain = property('domain.full_suffix')
+      identities_url = couchdb_url("/identities/_design/Identity/_view/by_address?key=%22#{user.username}@#{domain}%22")
+      get(identities_url) do |body, response, error|
+        assert error.nil?, "Error checking identities db: #{error}"
+        assert response.code.to_i == 200, "Unable to check that user identity was deleted: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+        assert(response = JSON.parse(body), 'Couch response should be JSON')
+        assert response['rows'].empty?, "Identity should have been deleted for test user #{user.username} (id #{user.id}), but was not! Response was: #{body}."
+      end
+    end
+  end
+
+end
diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb
new file mode 100644
index 00000000..d4d3c0e0
--- /dev/null
+++ b/tests/helpers/couchdb_helper.rb
@@ -0,0 +1,103 @@
+class LeapTest
+
+  #
+  # generates a couchdb url for when couchdb is running
+  # remotely and is available via stunnel.
+  #
+  # example properties:
+  #
+  # stunnel:
+  #   clients:
+  #     couch_client:
+  #       couch1_5984:
+  #         accept_port: 4000
+  #         connect: couch1.bitmask.i
+  #         connect_port: 15984
+  #
+  def couchdb_urls_via_stunnel(path="", options=nil)
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf|
+      assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+      URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url|
+        remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
+        url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
+      }
+    end
+  end
+
+  #
+  # generates a couchdb url for accessing couchdb via haproxy
+  #
+  # example properties:
+  #
+  # haproxy:
+  #   couch:
+  #     listen_port: 4096
+  #     servers:
+  #       panda:
+  #         backup: false
+  #         host: localhost
+  #         port: 4000
+  #         weight: 100
+  #         writable: true
+  #
+  def couchdb_url_via_haproxy(path="", options=nil)
+    if options && options[:username] && options[:password]
+      userpart = "%{username}:%{password}@" % options
+    else
+      userpart = ""
+    end
+    port = assert_property('haproxy.couch.listen_port')
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via haproxy)'
+    }
+  end
+
+  #
+  # generates a couchdb url for when couchdb is running locally.
+  #
+  # example properties:
+  #
+  # couch:
+  #   port: 5984
+  #
+  def couchdb_url_via_localhost(path="", options=nil)
+    port = (options && options[:port]) || assert_property('couch.port')
+    if options && options[:username]
+      password = property("couch.users.%{username}.password" % options)
+      userpart = "%s:%s@" % [options[:username], password]
+    else
+      userpart = ""
+    end
+    return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+      url.memo = '(via direct localhost connection)'
+    }
+  end
+
+  #
+  # returns a single url for accessing couchdb
+  #
+  def couchdb_url(path="", options=nil)
+    if property('couch.port')
+      couchdb_url_via_localhost(path, options)
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options).first
+    end
+  end
+
+  #
+  # returns an array of urls for accessing couchdb
+  #
+  def couchdb_urls(path="", options=nil)
+    if property('couch.port')
+      [couchdb_url_via_localhost(path, options)]
+    elsif property('stunnel.clients.couch_client')
+      couchdb_urls_via_stunnel(path, options)
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/files_helper.rb b/tests/helpers/files_helper.rb
new file mode 100644
index 00000000..d6795889
--- /dev/null
+++ b/tests/helpers/files_helper.rb
@@ -0,0 +1,54 @@
+class LeapTest
+
+  #
+  # Matches the regexp in the file, and returns the first matched string (or fails if no match).
+  #
+  def file_match(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures.first
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # Matches the regexp in the file, and returns array of matched strings (or fails if no match).
+  #
+  def file_matches(filename, regexp)
+    if match = File.read(filename).match(regexp)
+      match.captures
+    else
+      fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+    end
+  end
+
+  #
+  # checks to make sure the given property path exists in $node (e.g. hiera.yaml)
+  # and returns the value
+  #
+  def assert_property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      fail "Required node property `#{property}` is missing." if latest.nil?
+    end
+    return latest
+  end
+
+  #
+  # a handy function to get the value of a long property path
+  # without needing to test the existance individually of each part
+  # in the tree.
+  #
+  # e.g. property("stunnel.clients.couch_client")
+  #
+  def property(property)
+    latest = $node
+    property.split('.').each do |segment|
+      latest = latest[segment]
+      return nil if latest.nil?
+    end
+    return latest
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb
new file mode 100644
index 00000000..0b13b754
--- /dev/null
+++ b/tests/helpers/http_helper.rb
@@ -0,0 +1,145 @@
+require 'net/http'
+
+class LeapTest
+
+  #
+  # In order to easily provide detailed error messages, it is useful
+  # to append a memo to a url string that details what this url is for
+  # (e.g. stunnel, haproxy, etc).
+  #
+  # So, the url happens to be a UrlString, the memo field is used
+  # if there is an error in assert_get.
+  #
+  class URLString < String
+    attr_accessor :memo
+  end
+
+  #
+  # aliases for http_send()
+  #
+  def get(url, params=nil, options=nil, &block)
+    http_send("GET", url, params, options, &block)
+  end
+  def delete(url, params=nil, options=nil, &block)
+    http_send("DELETE", url, params, options, &block)
+  end
+  def post(url, params=nil, options=nil, &block)
+    http_send("POST", url, params, options, &block)
+  end
+  def put(url, params=nil, options=nil, &block)
+    http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # send a GET, DELETE, POST, or PUT
+  # yields |body, response, error|
+  #
+  def http_send(method, url, params=nil, options=nil)
+    options ||= {}
+    response = nil
+
+    # build uri
+    uri = URI(url)
+    if params && (method == 'GET' || method == 'DELETE')
+      uri.query = URI.encode_www_form(params)
+    end
+
+    # build http
+    http = Net::HTTP.new uri.host, uri.port
+    if uri.scheme == 'https'
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http.use_ssl = true
+    end
+
+    # build request
+    request = build_request(method, uri, params, options)
+
+    # make http request
+    http.start do |agent|
+      response = agent.request(request)
+      yield response.body, response, nil
+    end
+  rescue => exc
+    yield nil, response, exc
+  end
+
+  #
+  # Aliases for assert_http_send()
+  #
+  def assert_get(url, params=nil, options=nil, &block)
+    assert_http_send("GET", url, params, options, &block)
+  end
+  def assert_delete(url, params=nil, options=nil, &block)
+    assert_http_send("DELETE", url, params, options, &block)
+  end
+  def assert_post(url, params=nil, options=nil, &block)
+    assert_http_send("POST", url, params, options, &block)
+  end
+  def assert_put(url, params=nil, options=nil, &block)
+    assert_http_send("PUT", url, params, options, &block)
+  end
+
+  #
+  # calls http_send, yielding results if successful or failing with
+  # descriptive infor otherwise.
+  #
+  def assert_http_send(method, url, params=nil, options=nil, &block)
+    options ||= {}
+    error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil)
+    http_send(method, url, params, options) do |body, response, error|
+      if body && response && response.code.to_i >= 200 && response.code.to_i < 300
+        if block
+          yield(body) if block.arity == 1
+          yield(response, body) if block.arity == 2
+        end
+      elsif response
+        fail ["Expected a 200 status code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n")
+      else
+        fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error
+      end
+    end
+  end
+
+  #
+  # only a warning for now, should be a failure in the future
+  #
+  def assert_auth_fail(url, params)
+    uri = URI(url)
+    get(url, params) do |body, response, error|
+      unless response.code.to_s == "401"
+        warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')."
+        return false
+      end
+    end
+    true
+  end
+
+  private
+
+  def build_request(method, uri, params, options)
+    request = case method
+      when "GET"    then Net::HTTP::Get.new(uri.request_uri)
+      when "DELETE" then Net::HTTP::Delete.new(uri.request_uri)
+      when "POST"   then Net::HTTP::Post.new(uri.request_uri)
+      when "PUT"    then Net::HTTP::Put.new(uri.request_uri)
+    end
+    if uri.user
+      request.basic_auth uri.user, uri.password
+    end
+    if params && (method == 'POST' || method == 'PUT')
+      if options[:format] == :json || options[:format] == 'json'
+        request["Content-Type"] = "application/json"
+        request.body = params.to_json
+      else
+        request.set_form_data(params) if params
+      end
+    end
+    if options[:headers]
+      options[:headers].each do |key, value|
+        request[key] = value
+      end
+    end
+    request
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/network_helper.rb b/tests/helpers/network_helper.rb
new file mode 100644
index 00000000..ff92d382
--- /dev/null
+++ b/tests/helpers/network_helper.rb
@@ -0,0 +1,79 @@
+class LeapTest
+
+  #
+  # tcp connection helper with timeout
+  #
+  def try_tcp_connect(host, port, timeout = 5)
+    addr     = Socket.getaddrinfo(host, nil)
+    sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+    Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
+      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      begin
+        socket.connect_nonblock(sockaddr)
+      rescue IO::WaitReadable
+        if IO.select([socket], nil, nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      rescue IO::WaitWritable
+        if IO.select(nil, [socket], nil, timeout) == nil
+          raise "Connection timeout"
+        else
+          socket.connect_nonblock(sockaddr)
+        end
+      end
+      return socket
+    end
+  end
+
+  def try_tcp_write(socket, timeout = 5)
+    begin
+      socket.write_nonblock("\0")
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Write timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def try_tcp_read(socket, timeout = 5)
+    begin
+      socket.read_nonblock(1)
+    rescue IO::WaitReadable
+      if IO.select([socket], nil, nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    rescue IO::WaitWritable
+      if IO.select(nil, [socket], nil, timeout) == nil
+        raise "Read timeout"
+      else
+        retry
+      end
+    end
+  end
+
+  def assert_tcp_socket(host, port, msg=nil)
+    begin
+      socket = try_tcp_connect(host, port, 1)
+      #try_tcp_write(socket,1)
+      #try_tcp_read(socket,1)
+    rescue StandardError => exc
+      fail ["Failed to open socket #{host}:#{port}", exc].join("\n")
+    ensure
+      socket.close if socket
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb
new file mode 100644
index 00000000..aad67dda
--- /dev/null
+++ b/tests/helpers/os_helper.rb
@@ -0,0 +1,38 @@
+class LeapTest
+
+  #
+  # works like pgrep command line
+  # return an array of hashes like so [{:pid => "1234", :process => "ls"}]
+  #
+  def pgrep(match)
+    output = `pgrep --full --list-name '#{match}'`
+    output.each_line.map{|line|
+      pid = line.split(' ')[0]
+      process = line.gsub(/(#{pid} |\n)/, '')
+      if process =~ /pgrep --full --list-name/
+        nil
+      else
+        {:pid => pid, :process => process}
+      end
+    }.compact
+  end
+
+  def assert_running(process, options={})
+    processes = pgrep(process)
+    assert processes.any?, "No running process for #{process}"
+    if options[:single]
+      assert processes.length == 1, "More than one process for #{process}"
+    end
+  end
+
+  #
+  # runs the specified command, failing on a non-zero exit status.
+  #
+  def assert_run(command)
+    output = `#{command}`
+    if $?.exitstatus != 0
+      fail "Error running `#{command}`:\n#{output}"
+    end
+  end
+
+end
\ No newline at end of file
diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py
new file mode 100755
index 00000000..2fb865fc
--- /dev/null
+++ b/tests/helpers/soledad_sync.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+#
+# Test Soledad sync
+#
+# This script performs a slightly modified U1DB sync to the Soledad server and
+# returns whether that sync was successful or not.
+#
+# It takes three arguments:
+#
+#   uuid   -- uuid of the user to sync
+#   token  -- a valid session token
+#   server -- the url of the soledad server we should connect to
+#
+# For example:
+#
+#   soledad_sync.py f6bef0586fcfdb8705e26a58f2d9e580 uYO-4ucEJFksJ6afjmcYwIyap2vW7bv6uLxk0w_RfCc https://199.119.112.9:2323/user-f6bef0586fcfdb8705e26a58f2d9e580
+#
+
+import os
+import sys
+import traceback
+import tempfile
+import shutil
+import u1db
+
+from u1db.remote.http_target import HTTPSyncTarget
+
+#
+# monkey patch U1DB's HTTPSyncTarget to perform token based auth
+#
+
+def set_token_credentials(self, uuid, token):
+    self._creds = {'token': (uuid, token)}
+
+def _sign_request(self, method, url_query, params):
+    uuid, token = self._creds['token']
+    auth = '%s:%s' % (uuid, token)
+    return [('Authorization', 'Token %s' % auth.encode('base64')[:-1])]
+
+HTTPSyncTarget.set_token_credentials = set_token_credentials
+HTTPSyncTarget._sign_request = _sign_request
+
+#
+# Create a temporary local u1db replica and attempt to sync to it.
+# Returns a failure message if something went wrong.
+#
+
+def soledad_sync(uuid, token, server):
+    tempdir = tempfile.mkdtemp()
+    try:
+        db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
+        creds = {'token': {'uuid': uuid, 'token': token}}
+        db.sync(server, creds=creds, autocreate=False)
+    finally:
+        shutil.rmtree(tempdir)
+
+#
+# exit codes:
+#
+# 0 - OK
+# 1 - WARNING
+# 2 - ERROR
+#
+
+if __name__ == '__main__':
+    try:
+        uuid, token, server = sys.argv[1:]
+        result = soledad_sync(uuid, token, server)
+        if result is None:
+            exit(0)
+        else:
+            print(result)
+            exit(1)
+    except Exception as exc:
+        print(exc.message or str(exc))
+        traceback.print_exc(file=sys.stdout)
+        exit(2)
diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb
new file mode 100644
index 00000000..5d30b459
--- /dev/null
+++ b/tests/helpers/srp_helper.rb
@@ -0,0 +1,171 @@
+#
+# Here are some very stripped down helper methods for SRP, useful only for
+# testing the client side.
+#
+
+require 'digest'
+require 'openssl'
+require 'securerandom'
+require 'base64'
+
+module SRP
+
+  ##
+  ## UTIL
+  ##
+
+  module Util
+    PRIME_N = <<-EOS.split.join.hex
+115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3
+    EOS
+    BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N)
+eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657
+6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da
+d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5
+7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb
+06e3
+    EOS
+    GENERATOR = 2 # g
+
+    def hn_xor_hg
+      byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR))
+    end
+
+    # a^n (mod m)
+    def modpow(a, n, m = BIG_PRIME_N)
+      r = 1
+      while true
+        r = r * a % m if n[0] == 1
+        n >>= 1
+        return r if n == 0
+        a = a * a % m
+      end
+    end
+
+    #  Hashes the (long) int args
+    def sha256_int(*args)
+      sha256_hex(*args.map{|a| "%02x" % a})
+    end
+
+    #  Hashes the hex args
+    def sha256_hex(*args)
+      h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('')
+      sha256_str([h].pack('H*'))
+    end
+
+    def sha256_str(s)
+      Digest::SHA2.hexdigest(s)
+    end
+
+    def bigrand(bytes)
+      OpenSSL::Random.random_bytes(bytes).unpack("H*")[0]
+    end
+
+    def multiplier
+      @muliplier ||= calculate_multiplier
+    end
+
+    protected
+
+    def calculate_multiplier
+      sha256_int(BIG_PRIME_N, GENERATOR).hex
+    end
+
+    def byte_xor_hex(a, b)
+      a = [a].pack('H*')
+      b = [b].pack('H*')
+      a.bytes.each_with_index.map do |a_byte, i|
+        (a_byte ^ (b[i].ord || 0)).chr
+      end.join
+    end
+  end
+
+  ##
+  ## SESSION
+  ##
+
+  class Session
+    include SRP::Util
+    attr_accessor :user
+    attr_accessor :bb
+
+    def initialize(user, aa=nil)
+      @user = user
+      @a = bigrand(32).hex
+    end
+
+    def m
+      @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k)
+    end
+
+    def aa
+      @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N)
+    end
+
+    protected
+
+    # client: K = H( (B - kg^x) ^ (a + ux) )
+    def client_secret
+      base = bb.hex
+      base -= modpow(GENERATOR, @user.private_key) * multiplier
+      base = base % BIG_PRIME_N
+      modpow(base, @user.private_key * u.hex + @a)
+    end
+
+    def k
+      @k ||= sha256_int(client_secret)
+    end
+
+    def n_xor_g_long
+      @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join
+    end
+
+    def login_hash
+      @login_hash ||= sha256_str(@user.username)
+    end
+
+    def u
+      @u ||= sha256_hex(aa, bb)
+    end
+  end
+
+  ##
+  ## Dummy USER
+  ##
+
+  class User
+    include SRP::Util
+
+    attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted
+
+    def initialize
+      @username = "test_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '')
+      @password = "password_" + SecureRandom.urlsafe_base64(10)
+      @salt     = bigrand(4).hex
+      @verifier = modpow(GENERATOR, private_key)
+      @ok       = false
+      @deleted  = false
+    end
+
+    def private_key
+      @private_key ||= calculate_private_key
+    end
+
+    def to_params
+      {
+        'user[login]' => @username,
+        'user[password_verifier]' => @verifier.to_s(16),
+        'user[password_salt]' => @salt.to_s(16)
+      }
+    end
+
+    private
+
+    def calculate_private_key
+      shex = '%x' % [@salt]
+      inner = sha256_str([@username, @password].join(':'))
+      sha256_hex(shex, inner).hex
+    end
+  end
+
+end
diff --git a/tests/order.rb b/tests/order.rb
index ffa6ae4e..4468686f 100644
--- a/tests/order.rb
+++ b/tests/order.rb
@@ -3,6 +3,10 @@ class LeapCli::Config::Node
   # returns a list of node names that should be tested before this node.
   # make sure to not return ourselves (please no dependency loops!).
   #
+  # NOTE: this method determines the order that nodes are tested in. To specify
+  # the order of tests on a particular node, each test can call class method
+  # LeapTest.depends_on().
+  #
   def test_dependencies
     dependents = LeapCli::Config::ObjectList.new
     unless services.include?('couchdb')
diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb
index 9d5da94f..450c4201 100644
--- a/tests/white-box/couchdb.rb
+++ b/tests/white-box/couchdb.rb
@@ -1,4 +1,4 @@
-raise SkipTest unless $node["services"].include?("couchdb")
+raise SkipTest unless service?(:couchdb)
 
 require 'json'
 
@@ -9,9 +9,11 @@ class CouchDB < LeapTest
   end
 
   def test_00_Are_daemons_running?
-    assert_running 'tapicero'
-    assert_running 'bin/beam'
-    assert_running 'bin/epmd'
+    assert_running '^tapicero', :single => true
+    if multimaster?
+      assert_running 'bin/beam'
+      assert_running 'bin/epmd'
+    end
     pass
   end
 
@@ -29,6 +31,7 @@ class CouchDB < LeapTest
   # compare the configured nodes to the nodes that are actually listed in bigcouch
   #
   def test_02_Is_cluster_membership_ok?
+    return unless multimaster?
     url = couchdb_backend_url("/nodes/_all_docs")
     neighbors = assert_property('couch.bigcouch.neighbors')
     neighbors << assert_property('domain.full')
@@ -48,7 +51,8 @@ class CouchDB < LeapTest
   # this seems backward to me, so it might be the other way around.
   #
   def test_03_Are_configured_nodes_online?
-    url = couchdb_url("/_membership")
+    return unless multimaster?
+    url = couchdb_url("/_membership", :username => 'admin')
     assert_get(url) do |body|
       response = JSON.parse(body)
       nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes']
@@ -66,11 +70,11 @@ class CouchDB < LeapTest
   end
 
   def test_04_Do_ACL_users_exist?
-    acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp']
-    url = couchdb_backend_url("/_users/_all_docs")
+    acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'tapicero', 'webapp', 'replication']
+    url = couchdb_backend_url("/_users/_all_docs", :username => 'admin')
     assert_get(url) do |body|
       response = JSON.parse(body)
-      assert_equal 6, response['total_rows']
+      assert_equal acl_users.count, response['total_rows']
       actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') }
       assert_equal acl_users.sort, actual_users.sort
     end
@@ -80,7 +84,8 @@ class CouchDB < LeapTest
   def test_05_Do_required_databases_exist?
     dbs_that_should_exist = ["customers","identities","keycache","sessions","shared","tickets","tokens","users"]
     dbs_that_should_exist.each do |db_name|
-      assert_get(couchdb_url("/"+db_name)) do |body|
+      url = couchdb_url("/"+db_name, :username => 'admin')
+      assert_get(url) do |body|
         assert response = JSON.parse(body)
         assert_equal db_name, response['db_name']
       end
@@ -88,22 +93,63 @@ class CouchDB < LeapTest
     pass
   end
 
-  private
+  # disable ACL enforcement, because it's a known issue with bigcouch
+  # and will only confuse the user
+  # see https://leap.se/code/issues/6030 for more details
+  #
+  ## for now, this just prints warnings, since we are failing these tests.
+  ##
 
-  def couchdb_url(path="", port=nil)
-    @port ||= begin
-      assert_property 'couch.port'
-      $node['couch']['port']
-    end
-    @password ||= begin
-      assert_property 'couch.users.admin.password'
-      $node['couch']['users']['admin']['password']
+  #def test_06_Is_ACL_enforced?
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  )
+  #  ok = assert_auth_fail(
+  #    couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+  #    {:limit => 1}
+  #  ) && ok
+  #  pass if ok
+  #end
+
+  def test_07_Can_records_be_created?
+    token = Token.new
+    url = couchdb_url("/tokens", :username => 'admin')
+    assert_post(url, token, :format => :json) do |body|
+      assert response = JSON.parse(body), "POST response should be JSON"
+      assert response["ok"], "POST response should be OK"
+      assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body|
+        assert response = JSON.parse(body), "DELETE response should be JSON"
+        assert response["ok"], "DELETE response should be OK"
+      end
     end
-    "http://admin:#{@password}@localhost:#{port || @port}#{path}"
+    pass
+  end
+
+  private
+
+  def multimaster?
+    mode == "multimaster"
+  end
+
+  def mode
+    assert_property('couch.mode')
+  end
+
+  # TODO: admin port is hardcoded for now but should be configurable.
+  def couchdb_backend_url(path="", options={})
+    options = {port: multimaster? && "5986"}.merge options
+    couchdb_url(path, options)
   end
 
-  def couchdb_backend_url(path="")
-    couchdb_url(path, "5986") # TODO: admin port is hardcoded for now but should be configurable.
+  require 'securerandom'
+  require 'digest/sha2'
+  class Token < Hash
+    def initialize
+      self['token'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '')
+      self['_id'] = Digest::SHA512.hexdigest(self['token'])
+      self['last_seen_at'] = Time.now
+    end
   end
 
 end
diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb
new file mode 100644
index 00000000..794a9a41
--- /dev/null
+++ b/tests/white-box/mx.rb
@@ -0,0 +1,50 @@
+raise SkipTest unless service?(:mx)
+
+require 'json'
+
+class Mx < LeapTest
+  depends_on "Network"
+
+  def setup
+  end
+
+  def test_01_Can_contact_couchdb?
+    dbs = ["identities"]
+    dbs.each do |db_name|
+      couchdb_urls("/"+db_name, url_options).each do |url|
+        assert_get(url) do |body|
+          assert response = JSON.parse(body)
+          assert_equal db_name, response['db_name']
+        end
+      end
+    end
+    pass
+  end
+
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
+    end
+  end
+
+  def test_03_Are_MX_daemons_running?
+    assert_running 'leap_mx'
+    assert_running '/usr/lib/postfix/master'
+    assert_running '/usr/sbin/unbound'
+    pass
+  end
+
+  private
+
+  def url_options
+    {
+      :username => property('couchdb_leap_mx_user.username'),
+      :password => property('couchdb_leap_mx_user.password')
+    }
+  end
+
+end
diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb
index e0b0339d..f2041710 100644
--- a/tests/white-box/network.rb
+++ b/tests/white-box/network.rb
@@ -26,38 +26,35 @@ class Network < LeapTest
   #     connect: "127.0.0.1:5984"
   #
   def test_02_Is_stunnel_running?
-    if $node['stunnel']
-      good_stunnel_pids = []
-      $node['stunnel'].each do |stunnel_type, stunnel_configs|
-        if stunnel_type =~ /_clients?$/
-          stunnel_configs.each do |stunnel_name, stunnel_conf|
-            config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
-            processes = pgrep(config_file_name)
-            assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
-            good_stunnel_pids += processes.map{|ps| ps[:pid]}
-            assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
-            assert_tcp_socket('localhost', port)
-          end
-        elsif stunnel_type =~ /_server$/
-          config_file_name = "/etc/stunnel/#{stunnel_type}.conf"
-          processes = pgrep(config_file_name)
-          assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
-          good_stunnel_pids += processes.map{|ps| ps[:pid]}
-          assert accept = stunnel_configs['accept'], "Field `accept` must be present in property `stunnel.#{stunnel_type}`"
-          assert_tcp_socket('localhost', accept)
-          assert connect = stunnel_configs['connect'], "Field `connect` must be present in property `stunnel.#{stunnel_type}`"
-          assert_tcp_socket(*connect.split(':'))
-        else
-          skip "Unknown stunnel type `#{stunnel_type}`"
-        end
+    ignore unless $node['stunnel']
+    good_stunnel_pids = []
+    $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs|
+      stunnel_configs.each do |stunnel_name, stunnel_conf|
+        config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+        processes = pgrep(config_file_name)
+        assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
+        good_stunnel_pids += processes.map{|ps| ps[:pid]}
+        assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+        assert_tcp_socket('localhost', port)
       end
-      all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
-      assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
-      pass
     end
+    $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf|
+      config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+      processes = pgrep(config_file_name)
+      assert_equal 6, processes.length, "There should be six stunnel processes running for `#{config_file_name}`"
+      good_stunnel_pids += processes.map{|ps| ps[:pid]}
+      assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', accept_port)
+      assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`"
+      assert_tcp_socket('localhost', connect_port)
+    end
+    all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
+    assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
+    pass
   end
 
   def test_03_Is_shorewall_running?
+    ignore unless File.exists?('/sbin/shorewall')
     assert_run('/sbin/shorewall status')
     pass
   end
diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb
index 5eb2bdb5..23a40426 100644
--- a/tests/white-box/openvpn.rb
+++ b/tests/white-box/openvpn.rb
@@ -1,6 +1,6 @@
-raise SkipTest unless $node["services"].include?("openvpn")
+raise SkipTest unless service?(:openvpn)
 
-class Openvpn < LeapTest
+class OpenVPN < LeapTest
   depends_on "Network"
 
   def setup
diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb
new file mode 100644
index 00000000..5a13e4a6
--- /dev/null
+++ b/tests/white-box/soledad.rb
@@ -0,0 +1,17 @@
+raise SkipTest unless service?(:soledad)
+
+require 'json'
+
+class Soledad < LeapTest
+  depends_on "Network"
+  depends_on "CouchDB" if service?(:couchdb)
+
+  def setup
+  end
+
+  def test_00_Is_Soledad_running?
+    assert_running 'soledad'
+    pass
+  end
+
+end
diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb
index 142ac2de..9f104899 100644
--- a/tests/white-box/webapp.rb
+++ b/tests/white-box/webapp.rb
@@ -1,57 +1,29 @@
-raise SkipTest unless $node["services"].include?("webapp")
+raise SkipTest unless service?(:webapp)
 
-require 'socket'
+require 'json'
 
 class Webapp < LeapTest
   depends_on "Network"
 
-  HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg'
-
   def setup
   end
 
-  #
-  # example properties:
-  #
-  # stunnel:
-  #   couch_client:
-  #     couch1_5984:
-  #       accept_port: 4000
-  #       connect: couch1.bitmask.i
-  #       connect_port: 15984
-  #
   def test_01_Can_contact_couchdb?
-    assert_property('stunnel.couch_client')
-    $node['stunnel']['couch_client'].values.each do |stunnel_conf|
-      assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
-      local_stunnel_url = "http://localhost:#{port}"
-      remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
-      msg = "(stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
-      assert_get(local_stunnel_url, nil, error_msg: msg) do |body|
-        assert_match /"couchdb":"Welcome"/, body, "Request to #{local_stunnel_url} should return couchdb welcome message."
-      end
+    url = couchdb_url("", url_options)
+    assert_get(url) do |body|
+      assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
     end
     pass
   end
 
-  #
-  # example properties:
-  #
-  # haproxy:
-  #   servers:
-  #     couch1:
-  #       backup: false
-  #       host: localhost
-  #       port: 4000
-  #       weight: 10
-  #
-  def test_02_Is_haproxy_working?
-    port = file_match(HAPROXY_CONFIG, /^  bind localhost:(\d+)$/)
-    url = "http://localhost:#{port}"
-    assert_get(url) do |body|
-      assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+  def test_02_Can_contact_couchdb_via_haproxy?
+    if property('haproxy.couch')
+      url = couchdb_url_via_haproxy("", url_options)
+      assert_get(url) do |body|
+        assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+      end
+      pass
     end
-    pass
   end
 
   def test_03_Are_daemons_running?
@@ -60,4 +32,94 @@ class Webapp < LeapTest
     pass
   end
 
+  #
+  # this is technically a black-box test. so, move this when we have support
+  # for black box tests.
+  #
+  def test_04_Can_access_webapp?
+    assert_get('https://' + $node['webapp']['domain'] + '/')
+    pass
+  end
+
+  def test_05_Can_create_and_authenticate_and_delete_user_via_API?
+    assert_tmp_user
+    pass
+  end
+
+  def test_06_Can_sync_Soledad?
+    soledad_config = property('definition_files.soledad_service')
+    if soledad_config && !soledad_config.empty?
+      soledad_server = pick_soledad_server(soledad_config)
+      if soledad_server
+        assert_tmp_user do |user|
+          assert_user_db_exists(user)
+          command = File.expand_path "../../helpers/soledad_sync.py", __FILE__
+          soledad_url = "https://#{soledad_server}/user-#{user.id}"
+          assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url}"
+          pass
+        end
+      end
+    else
+      skip 'No soledad service configuration'
+    end
+  end
+
+  private
+
+  def url_options
+    {
+      :username => property('couchdb_webapp_user.username'),
+      :password => property('couchdb_webapp_user.password')
+    }
+  end
+
+  #
+  # pick a random soledad server.
+  # I am not sure why, but using IP address directly does not work.
+  #
+  def pick_soledad_server(soledad_config_json_str)
+    soledad_config = JSON.parse(soledad_config_json_str)
+    host_name = soledad_config['hosts'].keys.shuffle.first
+    if host_name
+      hostname = soledad_config['hosts'][host_name]['hostname']
+      port = soledad_config['hosts'][host_name]['port']
+      return "#{hostname}:#{port}"
+    else
+      return nil
+    end
+  end
+
+  #
+  # returns true if the per-user db created by tapicero exists.
+  # we try three times, and give up after that.
+  #
+  def assert_user_db_exists(user)
+    last_body, last_response, last_error = nil
+    3.times do
+      sleep 0.1
+      get(couchdb_url("/user-#{user.id}/_design/docs")) do |body, response, error|
+        last_body, last_response, last_error = body, response, error
+        if response.code.to_i == 200
+          return
+        end
+      end
+      sleep 0.2
+    end
+    assert false, "Could not find user db for test user #{user.username}\nuuid=#{user.id}\nHTTP #{last_response.code} #{last_error} #{last_body}"
+  end
+
+  #
+  # I tried, but couldn't get this working:
+  # #
+  # # get an CSRF authenticity token
+  # #
+  # url = api_url("/")
+  # csrf_token = nil
+  # assert_get(url) do |body|
+  #   lines = body.split("\n").grep(/csrf-token/)
+  #   assert lines.any?, 'failed to find csrf-token'
+  #   csrf_token = lines.first.split('"')[1]
+  #   assert csrf_token, 'failed to find csrf-token'
+  # end
+
 end