summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/README.md25
-rw-r--r--tests/helpers/bonafide_helper.rb235
-rw-r--r--tests/helpers/client_side_db.py167
-rw-r--r--tests/helpers/couchdb_helper.rb142
-rw-r--r--tests/helpers/files_helper.rb54
-rw-r--r--tests/helpers/http_helper.rb157
-rw-r--r--tests/helpers/network_helper.rb79
-rw-r--r--tests/helpers/os_helper.rb41
-rw-r--r--tests/helpers/smtp_helper.rb45
-rwxr-xr-xtests/helpers/soledad_sync.py89
-rw-r--r--tests/helpers/srp_helper.rb171
-rw-r--r--tests/order.rb22
-rw-r--r--tests/white-box/couchdb.rb186
-rw-r--r--tests/white-box/dummy.rb71
-rw-r--r--tests/white-box/mx.rb267
-rw-r--r--tests/white-box/network.rb90
-rw-r--r--tests/white-box/openvpn.rb16
-rw-r--r--tests/white-box/soledad.rb17
-rw-r--r--tests/white-box/webapp.rb134
19 files changed, 2008 insertions, 0 deletions
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 00000000..814c25b1
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,25 @@
+Tests
+---------------------------------
+
+tests/white-box/
+
+ These tests are run on the server as superuser. They are for
+ troubleshooting any problems with the internal setup of the server.
+
+tests/black-box/
+
+ These test are run the user's local machine. They are for troubleshooting
+ any external problems with the service exposed by the server.
+
+Additional Files
+---------------------------------
+
+tests/helpers/
+
+ Utility functions made available to all tests.
+
+tests/order.rb
+
+ Configuration file to specify which nodes should be tested in which order.
+
+
diff --git a/tests/helpers/bonafide_helper.rb b/tests/helpers/bonafide_helper.rb
new file mode 100644
index 00000000..5b886228
--- /dev/null
+++ b/tests/helpers/bonafide_helper.rb
@@ -0,0 +1,235 @@
+#
+# helper for the communication with the provider API for creating, authenticating, and deleting accounts.
+#
+
+class LeapTest
+
+ def assert_tmp_user
+ user = assert_create_user
+ assert_authenticate_user(user)
+ yield user if block_given?
+ assert_delete_user(user)
+ rescue StandardError, MiniTest::Assertion => exc
+ begin
+ assert_delete_user(user)
+ rescue
+ end
+ raise exc
+ end
+
+ #
+ # attempts to create a user account via the API,
+ # returning the user object if successful.
+ #
+ def assert_create_user(username=nil, auth=nil)
+ user = SRP::User.new(username)
+ url = api_url("/users.json")
+ params = user.to_params
+ if auth
+ options = api_options(:auth => auth)
+ else
+ options = api_options
+ if property('webapp.invite_required')
+ @invite_code = generate_invite_code
+ params['user[invite_code]'] = @invite_code
+ end
+ end
+
+ assert_post(url, params, options) do |body|
+ assert response = JSON.parse(body), 'response should be JSON'
+ assert response['ok'], "Creating a user should be successful, got #{response.inspect} instead."
+ user.ok = true
+ user.id = response['id']
+ end
+ return user
+ end
+
+ # TODO: use the api for this instead.
+ def generate_invite_code
+ `cd /srv/leap/webapp/ && sudo -u leap-webapp RAILS_ENV=production bundle exec rake generate_invites[1]`.gsub(/\n/, "")
+ end
+
+ #
+ # attempts to authenticate user. if successful,
+ # user object is updated with id and session token.
+ #
+ def assert_authenticate_user(user)
+ url = api_url("/sessions.json")
+ session = SRP::Session.new(user)
+ params = {'login' => user.username, 'A' => session.aa}
+ assert_post(url, params, api_options) do |body, response|
+ cookie = response['Set-Cookie'].split(';').first
+ assert(response = JSON.parse(body), 'response should be JSON')
+ assert(session.bb = response["B"], 'response should include "B"')
+ url = api_url("/sessions/login.json")
+ params = {'client_auth' => session.m, 'A' => session.aa}
+ assert_put(url, params, api_options('Cookie' => cookie)) do |body|
+ assert(response = JSON.parse(body), 'response should be JSON')
+ assert(response['M2'], 'response should include M2')
+ user.session_token = response['token']
+ user.id = response['id']
+ assert(user.session_token, 'response should include token')
+ assert(user.id, 'response should include user id')
+ end
+ end
+ end
+
+ #
+ # attempts to destroy a user account via the API.
+ #
+ def assert_delete_user(user)
+ if user.is_a? String
+ assert_delete_user_by_login(user)
+ elsif user.is_a? SRP::User
+ assert_delete_srp_user(user)
+ end
+ end
+
+ #
+ # returns true if the identity exists, uses monitor token auth
+ #
+ def identity_exists?(address)
+ url = api_url("/identities/#{URI.encode(address)}.json")
+ options = {:ok_codes => [200, 404]}.merge(
+ api_options(:auth => :monitor)
+ )
+ assert_get(url, nil, options) do |body, response|
+ return response.code == "200"
+ end
+ end
+
+ def upload_public_key(user_id, public_key)
+ url = api_url("/users/#{user_id}.json")
+ params = {"user[public_key]" => public_key}
+ assert_put(url, params, api_options(:auth => :monitor))
+ end
+
+ #
+ # return user document as a Hash. uses monitor token auth
+ #
+ def find_user_by_id(user_id)
+ url = api_url("/users/#{user_id}.json")
+ assert_get(url, nil, api_options(:auth => :monitor)) do |body|
+ return JSON.parse(body)
+ end
+ end
+
+ #
+ # return user document as a Hash. uses monitor token auth
+ # NOTE: this relies on deprecated behavior of the API
+ # and will not work when multi-domain support is added.
+ #
+ def find_user_by_login(login)
+ url = api_url("/users/0.json?login=#{login}")
+ options = {:ok_codes => [200, 404]}.merge(
+ api_options(:auth => :monitor)
+ )
+ assert_get(url, nil, options) do |body, response|
+ if response.code == "200"
+ return JSON.parse(body)
+ else
+ return nil
+ end
+ end
+ end
+
+ private
+
+ def api_url(path)
+ unless path =~ /^\//
+ path = '/' + path
+ end
+ if property('testing.api_uri')
+ return property('testing.api_uri') + path
+ elsif property('api')
+ api = property('api')
+ return "https://%{domain}:%{port}/%{version}#{path}" % {
+ :domain => api['domain'],
+ :port => api['port'],
+ :version => api['version'] || 1
+ }
+ else
+ fail 'This node needs to have either testing.api_url or api.{domain,port} configured.'
+ end
+ end
+
+ #
+ # produces an options hash used for api http requests.
+ #
+ # argument options hash gets added to "headers"
+ # of the http request.
+ #
+ # special :auth key in argument will expand to
+ # add api_token_auth header.
+ #
+ # if you want to try manually:
+ #
+ # export API_URI=`grep api_uri /etc/leap/hiera.yaml | cut -d\" -f2`
+ # export TOKEN=`grep monitor_auth_token /etc/leap/hiera.yaml | awk '{print $2}'`
+ # curl -H "Accept: application/json" -H "Token: $TOKEN" $API_URI
+ #
+ def api_options(options={})
+ # note: must be :headers, not "headers"
+ hsh = {
+ :headers => {
+ "Accept" => "application/json"
+ }
+ }
+ if options[:auth]
+ hsh[:headers].merge!(api_token_auth(options.delete(:auth)))
+ end
+ hsh[:headers].merge!(options)
+ return hsh
+ end
+
+ #
+ # add token authentication to a http request.
+ #
+ # returns a hash suitable for adding to the 'headers' option
+ # of an http function.
+ #
+ def api_token_auth(token)
+ if token.is_a?(Symbol) && property('testing')
+ if token == :monitor
+ token_str = property('testing.monitor_auth_token')
+ else
+ raise ArgumentError.new 'no such token'
+ end
+ else
+ token_str = token
+ end
+ {"Authorization" => "Token token=\"#{token_str}\""}
+ end
+
+ #
+ # not actually used in any test, but useful when
+ # writing new tests.
+ #
+ def assert_delete_user_by_login(login_name)
+ user = find_user_by_login(login_name)
+ url = api_url("/users/#{user['id']}.json")
+ params = {:identities => 'destroy'}
+ delete(url, params, api_options(:auth => :monitor)) do |body, response, error|
+ assert error.nil?, "Error deleting user: #{error}"
+ assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+ assert(response = JSON.parse(body), 'Delete response should be JSON')
+ assert(response["success"], 'Deleting user should be a success')
+ end
+ end
+
+ def assert_delete_srp_user(user)
+ if user && user.ok && user.id && user.session_token && !user.deleted
+ url = api_url("users/#{user.id}.json")
+ params = {:identities => 'destroy'}
+ user.deleted = true
+ delete(url, params, api_options(:auth => user.session_token)) do |body, response, error|
+ assert error.nil?, "Error deleting user: #{error}"
+ assert response.code.to_i == 200, "Unable to delete user: HTTP response from API should have code 200, was #{response.code} #{error} #{body}"
+ assert(response = JSON.parse(body), 'Delete response should be JSON')
+ assert(response["success"], 'Deleting user should be a success')
+ end
+ end
+ end
+
+
+end
diff --git a/tests/helpers/client_side_db.py b/tests/helpers/client_side_db.py
new file mode 100644
index 00000000..2f8c220f
--- /dev/null
+++ b/tests/helpers/client_side_db.py
@@ -0,0 +1,167 @@
+import logging
+import os
+import tempfile
+import getpass
+import binascii
+import json
+
+try:
+ import requests
+ import srp._pysrp as srp
+except ImportError:
+ pass
+
+from twisted.internet.defer import inlineCallbacks
+
+from leap.soledad.client import Soledad
+
+
+"""
+Helper functions to give access to client-side Soledad database.
+Copied over from soledad/scripts folder.
+"""
+
+# create a logger
+logger = logging.getLogger(__name__)
+
+# DEBUG: enable debug logs
+# LOG_FORMAT = '%(asctime)s %(message)s'
+# logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG)
+
+
+safe_unhexlify = lambda x: binascii.unhexlify(x) if (
+ len(x) % 2 == 0) else binascii.unhexlify('0' + x)
+
+
+def _fail(reason):
+ logger.error('Fail: ' + reason)
+ exit(2)
+
+
+def get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file,
+ token):
+ # setup soledad info
+ logger.info('UUID is %s' % uuid)
+ logger.info('Server URL is %s' % server_url)
+ secrets_path = os.path.join(
+ basedir, '%s.secret' % uuid)
+ local_db_path = os.path.join(
+ basedir, '%s.db' % uuid)
+ # instantiate soledad
+ return Soledad(
+ uuid,
+ unicode(passphrase),
+ secrets_path=secrets_path,
+ local_db_path=local_db_path,
+ server_url=server_url,
+ cert_file=cert_file,
+ auth_token=token,
+ defer_encryption=True)
+
+
+def _get_api_info(provider):
+ info = requests.get(
+ 'https://'+provider+'/provider.json', verify=False).json()
+ return info['api_uri'], info['api_version']
+
+
+def _login(username, passphrase, provider, api_uri, api_version):
+ usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024)
+ auth = None
+ try:
+ auth = _authenticate(api_uri, api_version, usr).json()
+ except requests.exceptions.ConnectionError:
+ _fail('Could not connect to server.')
+ if 'errors' in auth:
+ _fail(str(auth['errors']))
+ return api_uri, api_version, auth
+
+
+def _authenticate(api_uri, api_version, usr):
+ api_url = "%s/%s" % (api_uri, api_version)
+ session = requests.session()
+ uname, A = usr.start_authentication()
+ params = {'login': uname, 'A': binascii.hexlify(A)}
+ init = session.post(
+ api_url + '/sessions', data=params, verify=False).json()
+ if 'errors' in init:
+ _fail('test user not found')
+ M = usr.process_challenge(
+ safe_unhexlify(init['salt']), safe_unhexlify(init['B']))
+ return session.put(api_url + '/sessions/' + uname, verify=False,
+ data={'client_auth': binascii.hexlify(M)})
+
+
+def _get_soledad_info(username, provider, passphrase, basedir):
+ api_uri, api_version = _get_api_info(provider)
+ auth = _login(username, passphrase, provider, api_uri, api_version)
+ # get soledad server url
+ service_url = '%s/%s/config/soledad-service.json' % \
+ (api_uri, api_version)
+ soledad_hosts = requests.get(service_url, verify=False).json()['hosts']
+ hostnames = soledad_hosts.keys()
+ # allow for choosing the host
+ host = hostnames[0]
+ if len(hostnames) > 1:
+ i = 1
+ print "There are many available hosts:"
+ for h in hostnames:
+ print " (%d) %s.%s" % (i, h, provider)
+ i += 1
+ choice = raw_input("Choose a host to use (default: 1): ")
+ if choice != '':
+ host = hostnames[int(choice) - 1]
+ server_url = 'https://%s:%d/user-%s' % \
+ (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'],
+ auth[2]['id'])
+ # get provider ca certificate
+ ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text
+ cert_file = os.path.join(basedir, 'ca.crt')
+ with open(cert_file, 'w') as f:
+ f.write(ca_cert)
+ return auth[2]['id'], server_url, cert_file, auth[2]['token']
+
+
+def _get_passphrase(args):
+ passphrase = args.passphrase
+ if passphrase is None:
+ passphrase = getpass.getpass(
+ 'Password for %s@%s: ' % (args.username, args.provider))
+ return passphrase
+
+
+def _get_basedir(args):
+ basedir = args.basedir
+ if basedir is None:
+ basedir = tempfile.mkdtemp()
+ elif not os.path.isdir(basedir):
+ os.mkdir(basedir)
+ logger.info('Using %s as base directory.' % basedir)
+ return basedir
+
+
+@inlineCallbacks
+def _export_key(args, km, fname, private=False):
+ address = args.username + "@" + args.provider
+ pkey = yield km.get_key(
+ address, OpenPGPKey, private=private, fetch_remote=False)
+ with open(args.export_private_key, "w") as f:
+ f.write(pkey.key_data)
+
+
+@inlineCallbacks
+def _export_incoming_messages(soledad, directory):
+ yield soledad.create_index("by-incoming", "bool(incoming)")
+ docs = yield soledad.get_from_index("by-incoming", '1')
+ i = 1
+ for doc in docs:
+ with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f:
+ f.write(doc.content["_enc_json"])
+ i += 1
+
+
+@inlineCallbacks
+def _get_all_docs(soledad):
+ _, docs = yield soledad.get_all_docs()
+ for doc in docs:
+ print json.dumps(doc.content, indent=4)
diff --git a/tests/helpers/couchdb_helper.rb b/tests/helpers/couchdb_helper.rb
new file mode 100644
index 00000000..b9085c1e
--- /dev/null
+++ b/tests/helpers/couchdb_helper.rb
@@ -0,0 +1,142 @@
+class LeapTest
+
+ #
+ # generates a couchdb url for when couchdb is running
+ # remotely and is available via stunnel.
+ #
+ # example properties:
+ #
+ # stunnel:
+ # clients:
+ # couch_client:
+ # couch1_5984:
+ # accept_port: 4000
+ # connect: couch1.bitmask.i
+ # connect_port: 15984
+ #
+ def couchdb_urls_via_stunnel(path="", options=nil)
+ path = path.gsub('"', '%22')
+ if options && options[:username] && options[:password]
+ userpart = "%{username}:%{password}@" % options
+ else
+ userpart = ""
+ end
+ assert_property('stunnel.clients.couch_client').values.collect do |stunnel_conf|
+ assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+ URLString.new("http://#{userpart}localhost:#{port}#{path}").tap {|url|
+ remote_ip_address = TCPSocket.gethostbyname(stunnel_conf['connect']).last
+ url.memo = "(via stunnel to %s:%s, aka %s)" % [stunnel_conf['connect'], stunnel_conf['connect_port'], remote_ip_address]
+ }
+ end
+ end
+
+ #
+ # generates a couchdb url for accessing couchdb via haproxy
+ #
+ # example properties:
+ #
+ # haproxy:
+ # couch:
+ # listen_port: 4096
+ # servers:
+ # panda:
+ # backup: false
+ # host: localhost
+ # port: 4000
+ # weight: 100
+ # writable: true
+ #
+ def couchdb_url_via_haproxy(path="", options=nil)
+ path = path.gsub('"', '%22')
+ if options && options[:username] && options[:password]
+ userpart = "%{username}:%{password}@" % options
+ else
+ userpart = ""
+ end
+ port = assert_property('haproxy.couch.listen_port')
+ return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+ url.memo = '(via haproxy)'
+ }
+ end
+
+ #
+ # generates a couchdb url for when couchdb is running locally.
+ #
+ # example properties:
+ #
+ # couch:
+ # port: 5984
+ #
+ def couchdb_url_via_localhost(path="", options=nil)
+ path = path.gsub('"', '%22')
+ port = (options && options[:port]) || assert_property('couch.port')
+ if options && options[:username]
+ password = property("couch.users.%{username}.password" % options)
+ userpart = "%s:%s@" % [options[:username], password]
+ else
+ userpart = ""
+ end
+ return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url|
+ url.memo = '(via direct localhost connection)'
+ }
+ end
+
+ #
+ # returns a single url for accessing couchdb
+ #
+ def couchdb_url(path="", options=nil)
+ if property('couch.port')
+ couchdb_url_via_localhost(path, options)
+ elsif property('stunnel.clients.couch_client')
+ couchdb_urls_via_stunnel(path, options).first
+ end
+ end
+
+ #
+ # returns an array of urls for accessing couchdb
+ #
+ def couchdb_urls(path="", options=nil)
+ if property('couch.port')
+ [couchdb_url_via_localhost(path, options)]
+ elsif property('stunnel.clients.couch_client')
+ couchdb_urls_via_stunnel(path, options)
+ end
+ end
+
+ def assert_destroy_user_db(user_id, options=nil)
+ db_name = "user-#{user_id}"
+ url = couchdb_url("/#{db_name}", options)
+ http_options = {:ok_codes => [200, 404]} # ignore missing dbs
+ assert_delete(url, nil, http_options)
+ end
+
+ def assert_create_user_db(user_id, options=nil)
+ db_name = "user-#{user_id}"
+ url = couchdb_url("/#{db_name}", options)
+ http_options = {:ok_codes => [200, 404]} # ignore missing dbs
+ assert_put(url, nil, :format => :json) do |body|
+ assert response = JSON.parse(body), "PUT response should be JSON"
+ assert response["ok"], "PUT response should be OK"
+ end
+ end
+
+ #
+ # returns true if the per-user db created by soledad-server exists.
+ #
+ def user_db_exists?(user_id, options=nil)
+ db_name = "user-#{user_id}"
+ url = couchdb_url("/#{db_name}", options)
+ get(url) do |body, response, error|
+ if response.nil?
+ fail "could not query couchdb #{url}: #{error}\n#{body}"
+ elsif response.code.to_i == 200
+ return true
+ elsif response.code.to_i == 404
+ return false
+ else
+ fail ["could not query couchdb #{url}: expected response code 200 or 404, but got #{response.code}.", error, body].compact.join("\n")
+ end
+ end
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/files_helper.rb b/tests/helpers/files_helper.rb
new file mode 100644
index 00000000..d6795889
--- /dev/null
+++ b/tests/helpers/files_helper.rb
@@ -0,0 +1,54 @@
+class LeapTest
+
+ #
+ # Matches the regexp in the file, and returns the first matched string (or fails if no match).
+ #
+ def file_match(filename, regexp)
+ if match = File.read(filename).match(regexp)
+ match.captures.first
+ else
+ fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+ end
+ end
+
+ #
+ # Matches the regexp in the file, and returns array of matched strings (or fails if no match).
+ #
+ def file_matches(filename, regexp)
+ if match = File.read(filename).match(regexp)
+ match.captures
+ else
+ fail "Regexp #{regexp.inspect} not found in file #{filename.inspect}."
+ end
+ end
+
+ #
+ # checks to make sure the given property path exists in $node (e.g. hiera.yaml)
+ # and returns the value
+ #
+ def assert_property(property)
+ latest = $node
+ property.split('.').each do |segment|
+ latest = latest[segment]
+ fail "Required node property `#{property}` is missing." if latest.nil?
+ end
+ return latest
+ end
+
+ #
+ # a handy function to get the value of a long property path
+ # without needing to test the existance individually of each part
+ # in the tree.
+ #
+ # e.g. property("stunnel.clients.couch_client")
+ #
+ def property(property)
+ latest = $node
+ property.split('.').each do |segment|
+ latest = latest[segment]
+ return nil if latest.nil?
+ end
+ return latest
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/http_helper.rb b/tests/helpers/http_helper.rb
new file mode 100644
index 00000000..0d0bb7d5
--- /dev/null
+++ b/tests/helpers/http_helper.rb
@@ -0,0 +1,157 @@
+require 'net/http'
+
+class LeapTest
+
+ #
+ # In order to easily provide detailed error messages, it is useful
+ # to append a memo to a url string that details what this url is for
+ # (e.g. stunnel, haproxy, etc).
+ #
+ # So, the url happens to be a UrlString, the memo field is used
+ # if there is an error in assert_get.
+ #
+ class URLString < String
+ attr_accessor :memo
+ end
+
+ #
+ # aliases for http_send()
+ #
+ def get(url, params=nil, options=nil, &block)
+ http_send("GET", url, params, options, &block)
+ end
+ def delete(url, params=nil, options=nil, &block)
+ http_send("DELETE", url, params, options, &block)
+ end
+ def post(url, params=nil, options=nil, &block)
+ http_send("POST", url, params, options, &block)
+ end
+ def put(url, params=nil, options=nil, &block)
+ http_send("PUT", url, params, options, &block)
+ end
+
+ #
+ # send a GET, DELETE, POST, or PUT
+ # yields |body, response, error|
+ #
+ def http_send(method, url, params=nil, options=nil)
+ options ||= {}
+ response = nil
+
+ # build uri
+ uri = URI(url)
+ if params && (method == 'GET' || method == 'DELETE')
+ uri.query = URI.encode_www_form(params)
+ end
+
+ # build http
+ http = Net::HTTP.new uri.host, uri.port
+ if uri.scheme == 'https'
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+ http.use_ssl = true
+ end
+
+ # build request
+ request = build_request(method, uri, params, options)
+
+ # make http request
+ http.start do |agent|
+ response = agent.request(request)
+ yield response.body, response, nil
+ end
+ rescue => exc
+ yield nil, response, exc
+ end
+
+ #
+ # Aliases for assert_http_send()
+ #
+ def assert_get(url, params=nil, options=nil, &block)
+ assert_http_send("GET", url, params, options, &block)
+ end
+ def assert_delete(url, params=nil, options=nil, &block)
+ assert_http_send("DELETE", url, params, options, &block)
+ end
+ def assert_post(url, params=nil, options=nil, &block)
+ assert_http_send("POST", url, params, options, &block)
+ end
+ def assert_put(url, params=nil, options=nil, &block)
+ assert_http_send("PUT", url, params, options, &block)
+ end
+
+ #
+ # calls http_send, yielding results if successful or failing with
+ # descriptive info otherwise.
+ #
+ # options:
+ # - error_msg: custom error message to display.
+ # - ok_codes: in addition to 2xx, codes in this array will not produce an error.
+ #
+ def assert_http_send(method, url, params=nil, options=nil, &block)
+ options ||= {}
+ error_msg = options[:error_msg] || (url.respond_to?(:memo) ? url.memo : nil)
+ http_send(method, url, params, options) do |body, response, error|
+ if response
+ code = response.code.to_i
+ ok = code >= 200 && code < 300
+ if options[:ok_codes]
+ ok ||= options[:ok_codes].include?(code)
+ end
+ if ok
+ if block
+ yield(body) if block.arity == 1
+ yield(body, response) if block.arity == 2
+ yield(body, response, error) if block.arity == 3
+ end
+ else
+ fail ["Expected success code from #{method} #{url}, but got #{response.code} instead.", error_msg, body].compact.join("\n")
+ end
+ else
+ fail ["Expected a response from #{method} #{url}, but got \"#{error}\" instead.", error_msg, body].compact.join("\n"), error
+ end
+ end
+ end
+
+ #
+ # only a warning for now, should be a failure in the future
+ #
+ def assert_auth_fail(url, params)
+ uri = URI(url)
+ get(url, params) do |body, response, error|
+ unless response.code.to_s == "401"
+ warn "Expected a '401 Unauthorized' response, but got #{response.code} instead (GET #{uri.request_uri} with username '#{uri.user}')."
+ return false
+ end
+ end
+ true
+ end
+
+ private
+
+ def build_request(method, uri, params, options)
+ request = case method
+ when "GET" then Net::HTTP::Get.new(uri.request_uri)
+ when "DELETE" then Net::HTTP::Delete.new(uri.request_uri)
+ when "POST" then Net::HTTP::Post.new(uri.request_uri)
+ when "PUT" then Net::HTTP::Put.new(uri.request_uri)
+ end
+ if uri.user
+ request.basic_auth uri.user, uri.password
+ end
+ if params && (method == 'POST' || method == 'PUT')
+ if options[:format] == :json || options[:format] == 'json'
+ request["Content-Type"] = "application/json"
+ request.body = params.to_json
+ else
+ request.set_form_data(params) if params
+ end
+ end
+ if options[:headers]
+ options[:headers].each do |key, value|
+ request[key] = value
+ end
+ end
+ request
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/network_helper.rb b/tests/helpers/network_helper.rb
new file mode 100644
index 00000000..713d57aa
--- /dev/null
+++ b/tests/helpers/network_helper.rb
@@ -0,0 +1,79 @@
+class LeapTest
+
+ #
+ # tcp connection helper with timeout
+ #
+ def try_tcp_connect(host, port, timeout = 5)
+ addr = Socket.getaddrinfo(host, nil)
+ sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+ Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
+ socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+ begin
+ socket.connect_nonblock(sockaddr)
+ rescue IO::WaitReadable
+ if IO.select([socket], nil, nil, timeout) == nil
+ raise "Connection timeout"
+ else
+ socket.connect_nonblock(sockaddr)
+ end
+ rescue IO::WaitWritable
+ if IO.select(nil, [socket], nil, timeout) == nil
+ raise "Connection timeout"
+ else
+ socket.connect_nonblock(sockaddr)
+ end
+ end
+ return socket
+ end
+ end
+
+ def try_tcp_write(socket, timeout = 5)
+ begin
+ socket.write_nonblock("\0")
+ rescue IO::WaitReadable
+ if IO.select([socket], nil, nil, timeout) == nil
+ raise "Write timeout"
+ else
+ retry
+ end
+ rescue IO::WaitWritable
+ if IO.select(nil, [socket], nil, timeout) == nil
+ raise "Write timeout"
+ else
+ retry
+ end
+ end
+ end
+
+ def try_tcp_read(socket, timeout = 5)
+ begin
+ socket.read_nonblock(1)
+ rescue IO::WaitReadable
+ if IO.select([socket], nil, nil, timeout) == nil
+ raise "Read timeout"
+ else
+ retry
+ end
+ rescue IO::WaitWritable
+ if IO.select(nil, [socket], nil, timeout) == nil
+ raise "Read timeout"
+ else
+ retry
+ end
+ end
+ end
+
+ def assert_tcp_socket(host, port, msg=nil)
+ begin
+ socket = try_tcp_connect(host, port, 1)
+ #try_tcp_write(socket,1)
+ #try_tcp_read(socket,1)
+ rescue StandardError => exc
+ fail ["Failed to open socket #{host}:#{port}", exc, msg].compact.join("\n")
+ ensure
+ socket.close if socket
+ end
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/os_helper.rb b/tests/helpers/os_helper.rb
new file mode 100644
index 00000000..9923d5b1
--- /dev/null
+++ b/tests/helpers/os_helper.rb
@@ -0,0 +1,41 @@
+class LeapTest
+
+ #
+ # works like pgrep command line
+ # return an array of hashes like so [{:pid => "1234", :process => "ls"}]
+ #
+ def pgrep(match)
+ output = `pgrep --full --list-name '#{match}'`
+ output.each_line.map{|line|
+ pid = line.split(' ')[0]
+ process = line.gsub(/(#{pid} |\n)/, '')
+ # filter out pgrep cmd itself
+ # on wheezy hosts, the "process" var contains the whole cmd including all parameters
+ # on jessie hosts, it only contains the first cmd (which is the default sheel invoked by 'sh')
+ if process =~ /^sh/
+ nil
+ else
+ {:pid => pid, :process => process}
+ end
+ }.compact
+ end
+
+ def assert_running(process, options={})
+ processes = pgrep(process)
+ assert processes.any?, "No running process for #{process}"
+ if options[:single]
+ assert processes.length == 1, "More than one process for #{process}"
+ end
+ end
+
+ #
+ # runs the specified command, failing on a non-zero exit status.
+ #
+ def assert_run(command)
+ output = `#{command} 2>&1`
+ if $?.exitstatus != 0
+ fail "Error running `#{command}`:\n#{output}"
+ end
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/smtp_helper.rb b/tests/helpers/smtp_helper.rb
new file mode 100644
index 00000000..ea7fb9fa
--- /dev/null
+++ b/tests/helpers/smtp_helper.rb
@@ -0,0 +1,45 @@
+require 'net/smtp'
+
+class LeapTest
+
+ TEST_EMAIL_USER = "test_user_email"
+ TEST_BAD_USER = "test_user_bad"
+
+ MSG_BODY = %(Since it seems that any heart which beats for freedom has the right only to a
+lump of lead, I too claim my share. If you let me live, I shall never stop
+crying for revenge and I shall avenge my brothers. I have finished. If you are
+not cowards, kill me!
+
+--Louise Michel)
+
+ def send_email(recipient, options={})
+ sender = options[:sender] || recipient
+ helo_domain = property('domain.full_suffix')
+ headers = {
+ "Date" => Time.now.utc,
+ "From" => sender,
+ "To" => recipient,
+ "Subject" => "Test Message",
+ "X-LEAP-TEST" => "true"
+ }.merge(options[:headers]||{})
+ message = []
+ headers.each do |key, value|
+ message << "#{key}: #{value}"
+ end
+ message << ""
+ message << MSG_BODY
+ Net::SMTP.start('localhost', 25, helo_domain) do |smtp|
+ smtp.send_message message.join("\n"), recipient, sender
+ end
+ end
+
+ def assert_send_email(recipient, options={})
+ begin
+ send_email(recipient, options)
+ rescue IOError, Net::OpenTimeout,
+ Net::ReadTimeout, Net::SMTPError => e
+ fail "Could not send mail to #{recipient} (#{e})"
+ end
+ end
+
+end \ No newline at end of file
diff --git a/tests/helpers/soledad_sync.py b/tests/helpers/soledad_sync.py
new file mode 100755
index 00000000..f4fc81ae
--- /dev/null
+++ b/tests/helpers/soledad_sync.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+"""
+soledad_sync.py
+
+This script exercises soledad synchronization.
+Its exit code is 0 if the sync took place correctly, 1 otherwise.
+
+It takes 5 arguments:
+
+ uuid: uuid of the user to sync
+ token: a valid session token
+ server: the url of the soledad server we should connect to
+ cert_file: the file containing the certificate for the CA that signed the
+ cert for the soledad server.
+ password: the password for the user to sync
+
+__author__: kali@leap.se
+"""
+import os
+import shutil
+import sys
+import tempfile
+
+# This is needed because the twisted shipped with wheezy is too old
+# to do proper ssl verification.
+os.environ['SKIP_TWISTED_SSL_CHECK'] = '1'
+
+from twisted.internet import defer, reactor
+from twisted.python import log
+
+from client_side_db import get_soledad_instance
+from leap.common.events import flags
+
+flags.set_events_enabled(False)
+
+NUMDOCS = 1
+USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0]
+
+
+def bail(msg, exitcode):
+ print "[!] %s" % msg
+ sys.exit(exitcode)
+
+
+def create_docs(soledad):
+ """
+ Populates the soledad database with dummy messages, so we can exercise
+ sending payloads during the sync.
+ """
+ deferreds = []
+ for index in xrange(NUMDOCS):
+ deferreds.append(soledad.create_doc({'payload': 'dummy'}))
+ return defer.gatherResults(deferreds)
+
+# main program
+
+if __name__ == '__main__':
+
+ tempdir = tempfile.mkdtemp()
+
+ def rm_tempdir():
+ shutil.rmtree(tempdir)
+
+ if len(sys.argv) < 6:
+ bail(USAGE, 2)
+
+ uuid, token, server, cert_file, passphrase = sys.argv[1:]
+ s = get_soledad_instance(
+ uuid, passphrase, tempdir, server, cert_file, token)
+
+ def onSyncDone(sync_result):
+ print "SYNC_RESULT:", sync_result
+ s.close()
+ rm_tempdir()
+ reactor.stop()
+
+ def log_and_exit(f):
+ log.err(f)
+ rm_tempdir()
+ reactor.stop()
+
+ def start_sync():
+ d = create_docs(s)
+ d.addCallback(lambda _: s.sync())
+ d.addCallback(onSyncDone)
+ d.addErrback(log_and_exit)
+
+ reactor.callWhenRunning(start_sync)
+ reactor.run()
diff --git a/tests/helpers/srp_helper.rb b/tests/helpers/srp_helper.rb
new file mode 100644
index 00000000..b30fa768
--- /dev/null
+++ b/tests/helpers/srp_helper.rb
@@ -0,0 +1,171 @@
+#
+# Here are some very stripped down helper methods for SRP, useful only for
+# testing the client side.
+#
+
+require 'digest'
+require 'openssl'
+require 'securerandom'
+require 'base64'
+
+module SRP
+
+ ##
+ ## UTIL
+ ##
+
+ module Util
+ PRIME_N = <<-EOS.split.join.hex
+115b8b692e0e045692cf280b436735c77a5a9e8a9e7ed56c965f87db5b2a2ece3
+ EOS
+ BIG_PRIME_N = <<-EOS.split.join.hex # 1024 bits modulus (N)
+eeaf0ab9adb38dd69c33f80afa8fc5e86072618775ff3c0b9ea2314c9c25657
+6d674df7496ea81d3383b4813d692c6e0e0d5d8e250b98be48e495c1d6089da
+d15dc7d7b46154d6b6ce8ef4ad69b15d4982559b297bcf1885c529f566660e5
+7ec68edbc3c05726cc02fd4cbf4976eaa9afd5138fe8376435b9fc61d2fc0eb
+06e3
+ EOS
+ GENERATOR = 2 # g
+
+ def hn_xor_hg
+ byte_xor_hex(sha256_int(BIG_PRIME_N), sha256_int(GENERATOR))
+ end
+
+ # a^n (mod m)
+ def modpow(a, n, m = BIG_PRIME_N)
+ r = 1
+ while true
+ r = r * a % m if n[0] == 1
+ n >>= 1
+ return r if n == 0
+ a = a * a % m
+ end
+ end
+
+ # Hashes the (long) int args
+ def sha256_int(*args)
+ sha256_hex(*args.map{|a| "%02x" % a})
+ end
+
+ # Hashes the hex args
+ def sha256_hex(*args)
+ h = args.map{|a| a.length.odd? ? "0#{a}" : a }.join('')
+ sha256_str([h].pack('H*'))
+ end
+
+ def sha256_str(s)
+ Digest::SHA2.hexdigest(s)
+ end
+
+ def bigrand(bytes)
+ OpenSSL::Random.random_bytes(bytes).unpack("H*")[0]
+ end
+
+ def multiplier
+ @muliplier ||= calculate_multiplier
+ end
+
+ protected
+
+ def calculate_multiplier
+ sha256_int(BIG_PRIME_N, GENERATOR).hex
+ end
+
+ def byte_xor_hex(a, b)
+ a = [a].pack('H*')
+ b = [b].pack('H*')
+ a.bytes.each_with_index.map do |a_byte, i|
+ (a_byte ^ (b[i].ord || 0)).chr
+ end.join
+ end
+ end
+
+ ##
+ ## SESSION
+ ##
+
+ class Session
+ include SRP::Util
+ attr_accessor :user
+ attr_accessor :bb
+
+ def initialize(user, aa=nil)
+ @user = user
+ @a = bigrand(32).hex
+ end
+
+ def m
+ @m ||= sha256_hex(n_xor_g_long, login_hash, @user.salt.to_s(16), aa, bb, k)
+ end
+
+ def aa
+ @aa ||= modpow(GENERATOR, @a).to_s(16) # A = g^a (mod N)
+ end
+
+ protected
+
+ # client: K = H( (B - kg^x) ^ (a + ux) )
+ def client_secret
+ base = bb.hex
+ base -= modpow(GENERATOR, @user.private_key) * multiplier
+ base = base % BIG_PRIME_N
+ modpow(base, @user.private_key * u.hex + @a)
+ end
+
+ def k
+ @k ||= sha256_int(client_secret)
+ end
+
+ def n_xor_g_long
+ @n_xor_g_long ||= hn_xor_hg.bytes.map{|b| "%02x" % b.ord}.join
+ end
+
+ def login_hash
+ @login_hash ||= sha256_str(@user.username)
+ end
+
+ def u
+ @u ||= sha256_hex(aa, bb)
+ end
+ end
+
+ ##
+ ## Dummy USER
+ ##
+
+ class User
+ include SRP::Util
+
+ attr_accessor :username, :password, :salt, :verifier, :id, :session_token, :ok, :deleted
+
+ def initialize(username=nil)
+ @username = username || "tmp_user_" + SecureRandom.urlsafe_base64(10).downcase.gsub(/[_-]/, '')
+ @password = "password_" + SecureRandom.urlsafe_base64(10)
+ @salt = bigrand(4).hex
+ @verifier = modpow(GENERATOR, private_key)
+ @ok = false
+ @deleted = false
+ end
+
+ def private_key
+ @private_key ||= calculate_private_key
+ end
+
+ def to_params
+ {
+ 'user[login]' => @username,
+ 'user[password_verifier]' => @verifier.to_s(16),
+ 'user[password_salt]' => @salt.to_s(16)
+ }
+ end
+
+ private
+
+ def calculate_private_key
+ shex = '%x' % [@salt]
+ inner = sha256_str([@username, @password].join(':'))
+ sha256_hex(shex, inner).hex
+ end
+ end
+
+end
diff --git a/tests/order.rb b/tests/order.rb
new file mode 100644
index 00000000..14aad9be
--- /dev/null
+++ b/tests/order.rb
@@ -0,0 +1,22 @@
+class LeapCli::Config::Node
+ #
+ # returns a list of node names that should be tested before this node.
+ # make sure to not return ourselves (please no dependency loops!).
+ #
+ # NOTE: this method determines the order that nodes are tested in. To specify
+ # the order of tests on a particular node, each test can call class method
+ # LeapTest.depends_on().
+ #
+ def test_dependencies
+ dependents = LeapCli::Config::ObjectList.new
+
+ # webapp, mx, and soledad depend on couchdb nodes
+ if services.include?('webapp') || services.include?('mx') || services.include?('soledad')
+ if !services.include?('couchdb')
+ dependents.merge! nodes_like_me[:services => 'couchdb']
+ end
+ end
+
+ dependents.keys.delete_if {|name| self.name == name}
+ end
+end \ No newline at end of file
diff --git a/tests/white-box/couchdb.rb b/tests/white-box/couchdb.rb
new file mode 100644
index 00000000..85dc6840
--- /dev/null
+++ b/tests/white-box/couchdb.rb
@@ -0,0 +1,186 @@
+raise SkipTest unless service?(:couchdb)
+
+require 'json'
+
+class CouchDB < LeapTest
+ depends_on "Network"
+
+ def setup
+ end
+
+ def test_00_Are_daemons_running?
+ assert_running 'bin/beam'
+ if multimaster?
+ assert_running 'bin/epmd'
+ end
+ pass
+ end
+
+ #
+ # check to make sure we can get welcome response from local couchdb
+ #
+ def test_01_Is_CouchDB_running?
+ assert_get(couchdb_url) do |body|
+ assert_match /"couchdb":"Welcome"/, body, "Could not get welcome message from #{couchdb_url}. Probably couchdb is not running."
+ end
+ pass
+ end
+
+ #
+ # compare the configured nodes to the nodes that are actually listed in bigcouch
+ #
+ def test_02_Is_cluster_membership_ok?
+ return unless multimaster?
+ url = couchdb_backend_url("/nodes/_all_docs")
+ neighbors = assert_property('couch.bigcouch.neighbors')
+ neighbors << assert_property('domain.full')
+ neighbors.sort!
+ assert_get(url) do |body|
+ response = JSON.parse(body)
+ nodes_in_db = response['rows'].collect{|row| row['id'].sub(/^bigcouch@/, '')}.sort
+ assert_equal neighbors, nodes_in_db, "The couchdb replication node list is wrong (/nodes/_all_docs)"
+ end
+ pass
+ end
+
+ #
+ # all configured nodes are in 'cluster_nodes'
+ # all nodes online and communicating are in 'all_nodes'
+ #
+ # this seems backward to me, so it might be the other way around.
+ #
+ def test_03_Are_configured_nodes_online?
+ return unless multimaster?
+ url = couchdb_url("/_membership", :username => 'admin')
+ assert_get(url) do |body|
+ response = JSON.parse(body)
+ nodes_configured_but_not_available = response['cluster_nodes'] - response['all_nodes']
+ nodes_available_but_not_configured = response['all_nodes'] - response['cluster_nodes']
+ if nodes_configured_but_not_available.any?
+ warn "These nodes are configured but not available:", nodes_configured_but_not_available
+ end
+ if nodes_available_but_not_configured.any?
+ warn "These nodes are available but not configured:", nodes_available_but_not_configured
+ end
+ if response['cluster_nodes'] == response['all_nodes']
+ pass
+ end
+ end
+ end
+
+ def test_04_Do_ACL_users_exist?
+ acl_users = ['_design/_auth', 'leap_mx', 'nickserver', 'soledad', 'webapp', 'replication']
+ url = couchdb_backend_url("/_users/_all_docs", :username => 'admin')
+ assert_get(url) do |body|
+ response = JSON.parse(body)
+ assert_equal acl_users.count, response['total_rows']
+ actual_users = response['rows'].map{|row| row['id'].sub(/^org.couchdb.user:/, '') }
+ assert_equal acl_users.sort, actual_users.sort
+ end
+ pass
+ end
+
+ def test_05_Do_required_databases_exist?
+ dbs_that_should_exist = ["customers","identities","keycache","shared","tickets","users", "tmp_users"]
+ dbs_that_should_exist << "tokens_#{rotation_suffix}"
+ dbs_that_should_exist << "sessions_#{rotation_suffix}"
+ dbs_that_should_exist.each do |db_name|
+ url = couchdb_url("/"+db_name, :username => 'admin')
+ assert_get(url) do |body|
+ assert response = JSON.parse(body)
+ assert_equal db_name, response['db_name']
+ end
+ end
+ pass
+ end
+
+ # disable ACL enforcement, because it's a known issue with bigcouch
+ # and will only confuse the user
+ # see https://leap.se/code/issues/6030 for more details
+ #
+ ## for now, this just prints warnings, since we are failing these tests.
+ ##
+
+ #def test_06_Is_ACL_enforced?
+ # ok = assert_auth_fail(
+ # couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+ # {:limit => 1}
+ # )
+ # ok = assert_auth_fail(
+ # couchdb_url('/users/_all_docs', :username => 'leap_mx'),
+ # {:limit => 1}
+ # ) && ok
+ # pass if ok
+ #end
+
+ def test_07_Can_records_be_created?
+ record = DummyRecord.new
+ url = couchdb_url("/tokens_#{rotation_suffix}", :username => 'admin')
+ assert_post(url, record, :format => :json) do |body|
+ assert response = JSON.parse(body), "POST response should be JSON"
+ assert response["ok"], "POST response should be OK"
+ assert_delete(File.join(url, response["id"]), :rev => response["rev"]) do |body|
+ assert response = JSON.parse(body), "DELETE response should be JSON"
+ assert response["ok"], "DELETE response should be OK"
+ end
+ end
+ pass
+ end
+
+ #
+ # This is not really a "test", just an attempt to make sure that
+ # the mx tests that fire off dummy emails don't fill up the
+ # storage db.
+ #
+ # mx tests can't run this because they don't have access to
+ # the storage db.
+ #
+ # This "test" is responsible for both creating the db if it does not
+ # exist, and destroying if it does.
+ #
+ # Yes, this is super hacky. Properly, we should add something to
+ # the soledad api to support create/delete of user storage dbs.
+ #
+ def test_99_Delete_mail_storage_used_in_mx_tests
+ user = find_user_by_login(TEST_EMAIL_USER)
+ if user
+ if user_db_exists?(user["id"])
+ # keep the test email db from filling up:
+ assert_destroy_user_db(user["id"], :username => 'admin')
+ end
+ # either way, make sure we leave a db for the mx tests:
+ assert_create_user_db(user["id"], :username => 'admin')
+ end
+ silent_pass
+ end
+
+ private
+
+ def multimaster?
+ mode == "multimaster"
+ end
+
+ def mode
+ assert_property('couch.mode')
+ end
+
+ # TODO: admin port is hardcoded for now but should be configurable.
+ def couchdb_backend_url(path="", options={})
+ options = {port: multimaster? && "5986"}.merge options
+ couchdb_url(path, options)
+ end
+
+ def rotation_suffix
+ rotation_suffix = Time.now.utc.to_i / 2592000 # monthly
+ end
+
+ require 'securerandom'
+ require 'digest/sha2'
+ class DummyRecord < Hash
+ def initialize
+ self['data'] = SecureRandom.urlsafe_base64(32).gsub(/^_*/, '')
+ self['_id'] = Digest::SHA512.hexdigest(self['data'])
+ end
+ end
+
+end
diff --git a/tests/white-box/dummy.rb b/tests/white-box/dummy.rb
new file mode 100644
index 00000000..a3e8ad68
--- /dev/null
+++ b/tests/white-box/dummy.rb
@@ -0,0 +1,71 @@
+# only run in the dummy case where there is no hiera.yaml file.
+raise SkipTest unless $node["dummy"]
+
+class Robot
+ def can_shoot_lasers?
+ "OHAI!"
+ end
+
+ def can_fly?
+ "YES!"
+ end
+end
+
+class TestDummy < LeapTest
+ def setup
+ @robot = Robot.new
+ end
+
+ def test_lasers
+ assert_equal "OHAI!", @robot.can_shoot_lasers?
+ pass
+ end
+
+ def test_fly
+ refute_match /^no/i, @robot.can_fly?
+ pass
+ end
+
+ def test_fail
+ fail "fail"
+ pass
+ end
+
+ def test_01_will_be_skipped
+ skip "test this later"
+ pass
+ end
+
+ def test_socket_failure
+ assert_tcp_socket('localhost', 900000)
+ pass
+ end
+
+ def test_warn
+ block_test do
+ warn "not everything", "is a success or failure"
+ end
+ end
+
+ # used to test extracting the proper caller even when in a block
+ def block_test
+ yield
+ end
+
+ def test_socket_success
+ fork {
+ Socket.tcp_server_loop('localhost', 12345) do |sock, client_addrinfo|
+ begin
+ sock.write('hi')
+ ensure
+ sock.close
+ exit
+ end
+ end
+ }
+ sleep 0.2
+ assert_tcp_socket('localhost', 12345)
+ pass
+ end
+
+end
diff --git a/tests/white-box/mx.rb b/tests/white-box/mx.rb
new file mode 100644
index 00000000..e0cb273a
--- /dev/null
+++ b/tests/white-box/mx.rb
@@ -0,0 +1,267 @@
+raise SkipTest unless service?(:mx)
+
+require 'date'
+require 'json'
+require 'net/smtp'
+
+class Mx < LeapTest
+ depends_on "Network"
+ depends_on "Webapp" if service?(:webapp)
+
+ def setup
+ end
+
+ def test_01_Can_contact_couchdb?
+ dbs = ["identities"]
+ dbs.each do |db_name|
+ couchdb_urls("/"+db_name, couch_url_options).each do |url|
+ assert_get(url) do |body|
+ assert response = JSON.parse(body)
+ assert_equal db_name, response['db_name']
+ end
+ end
+ end
+ pass
+ end
+
+ def test_02_Can_contact_couchdb_via_haproxy?
+ if property('haproxy.couch')
+ url = couchdb_url_via_haproxy("", couch_url_options)
+ assert_get(url) do |body|
+ assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+ end
+ pass
+ end
+ end
+
+ #
+ # this test picks a random identity document, then queries
+ # using the by_address view for that same document again.
+ #
+ def test_03_Can_query_identities_db?
+ ident = pick_random_identity
+ address = ident['address']
+ url_base = %(/identities/_design/Identity/_view/by_address)
+ params = %(?include_docs=true&reduce=false&startkey="#{address}"&endkey="#{address}")
+ assert_get(couchdb_url(url_base+params, couch_url_options)) do |body|
+ assert response = JSON.parse(body)
+ assert record = response['rows'].first
+ assert_equal address, record['doc']['address']
+ pass
+ end
+ end
+
+ def test_04_Are_MX_daemons_running?
+ assert_running '.*/usr/bin/twistd.*mx.tac'
+ assert_running '^/usr/lib/postfix/master$'
+ assert_running '^/usr/sbin/postfwd'
+ assert_running 'postfwd2::cache$'
+ assert_running 'postfwd2::policy$'
+ assert_running '^/usr/sbin/unbound$'
+ assert_running '^/usr/bin/freshclam'
+ assert_running '^/usr/sbin/opendkim'
+ if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0
+ assert_running '^/usr/sbin/clamd'
+ assert_running '^/usr/sbin/clamav-milter'
+ else
+ skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running.\nDon't worry, mail delivery will work without clamav. The download should finish soon."
+ end
+ pass
+ end
+
+ #
+ # TODO: test to make sure postmap returned the right result
+ #
+ def test_05_Can_postfix_query_leapmx?
+ ident = pick_random_identity(10, :with_public_key => true)
+ address = ident["address"]
+
+ #
+ # virtual alias map:
+ #
+ # user@domain => 41c29a80a44f4775513c64ac9cab91b9@deliver.local
+ #
+ assert_run("postmap -v -q \"#{address}\" tcp:localhost:4242")
+
+ #
+ # recipient access map:
+ #
+ # user@domain => [OK|REJECT|TEMP_FAIL]
+ #
+ # This map is queried by the mail server before delivery to the mail spool
+ # directory, and should check if the address is able to receive messages.
+ # Examples of reasons for denying delivery would be that the user is out of
+ # quota, is user, or have no pgp public key in the server.
+ #
+ # NOTE: in the future, when we support quota, we need to make sure that
+ # we don't randomly pick a user for this test that happens to be over quota.
+ #
+ assert_run("postmap -v -q \"#{address}\" tcp:localhost:2244")
+
+ #
+ # certificate validity map:
+ #
+ # fa:2a:70:1f:d8:16:4e:1a:3b:15:c1:67:00:f0 => [200|500]
+ #
+ # Determines whether a particular SMTP client cert is authorized
+ # to relay mail, based on the fingerprint.
+ #
+ if ident["cert_fingerprints"]
+ not_expired = ident["cert_fingerprints"].select {|key, value|
+ Time.now.utc < DateTime.strptime("2016-01-03", "%F").to_time.utc
+ }
+ if not_expired.any?
+ fingerprint = not_expired.first
+ assert_run("postmap -v -q #{fingerprint} tcp:localhost:2424")
+ end
+ end
+
+ pass
+ end
+
+ #
+ # The email sent by this test might get bounced back.
+ # In this case, the test will pass, but the bounce message will
+ # get sent to root, so the sysadmin will still figure out pretty
+ # quickly that something is wrong.
+ #
+ def test_06_Can_deliver_email?
+ addr = [TEST_EMAIL_USER, property('domain.full_suffix')].join('@')
+ bad_addr = [TEST_BAD_USER, property('domain.full_suffix')].join('@')
+
+ assert !identity_exists?(bad_addr), "the address #{bad_addr} must not exist."
+ if !identity_exists?(addr)
+ user = assert_create_user(TEST_EMAIL_USER, :monitor)
+ upload_public_key(user.id, TEST_EMAIL_PUBLIC_KEY)
+ end
+ assert identity_exists?(addr), "The identity #{addr} should have been created, but it doesn't exist yet."
+ assert_send_email(addr)
+ assert_raises(Net::SMTPError) do
+ send_email(bad_addr)
+ end
+ pass
+ end
+
+ private
+
+ def couch_url_options
+ {
+ :username => property('couchdb_leap_mx_user.username'),
+ :password => property('couchdb_leap_mx_user.password')
+ }
+ end
+
+ #
+ # returns a random identity record that also has valid address
+ # and destination fields.
+ #
+ # options:
+ #
+ # * :with_public_key -- searches only for identities with public keys
+ #
+ # note to self: for debugging, here is the curl you want:
+ # curl --netrc "127.0.0.1:5984/identities/_design/Identity/_view/by_address?startkey=\"xxxx@leap.se\"&endkey=\"xxxx@leap.se\"&reduce=false&include_docs=true"
+ #
+ def pick_random_identity(tries=5, options={})
+ assert_get(couchdb_url("/identities", couch_url_options)) do |body|
+ assert response = JSON.parse(body)
+ doc_count = response['doc_count'].to_i
+ if doc_count <= 1
+ # the design document counts as one document.
+ skip "There are no identity documents yet."
+ else
+ # try repeatedly to get a valid doc
+ for i in 1..tries
+ offset = rand(doc_count) # pick a random document
+ url = couchdb_url("/identities/_all_docs?include_docs=true&limit=1&skip=#{offset}", couch_url_options)
+ assert_get(url) do |body|
+ assert response = JSON.parse(body)
+ record = response['rows'].first
+ if record['id'] =~ /_design/
+ next
+ elsif record['doc'] && record['doc']['address']
+ next if record['doc']['destination'].nil? || record['doc']['destination'].empty?
+ next if options[:with_public_key] && !record_has_key?(record)
+ return record['doc']
+ else
+ fail "Identity document #{record['id']} is missing an address field. #{record['doc'].inspect}"
+ end
+ end
+ end
+ if options[:with_public_key]
+ skip "Could not find an Identity document with a public key for testing."
+ else
+ fail "Failed to find a valid Identity document (with address and destination)."
+ end
+ end
+ end
+ end
+
+ def record_has_key?(record)
+ !record['doc']['keys'].nil? &&
+ !record['doc']['keys'].empty? &&
+ !record['doc']['keys']['pgp'].nil? &&
+ !record['doc']['keys']['pgp'].empty?
+ end
+
+ TEST_EMAIL_PUBLIC_KEY=<<HERE
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+mI0EVvzIKQEEAN4f8FOGntJGTTD+fFUQS6y/ihn6tYLtyGZZbCOd0t/9kHt/raoR
+xEUks8rCOPMqHX+yeHsvDBtDyZYTvyhtfuWrBUbYGW+QZ4Pdvo+7NyLHPW0dKsCB
+Czrx7pxqpq1oq+LpUFqpSfjJTfYaGVDNXrPK144a7Rox2+MCbgq3twnFABEBAAG0
+EiA8dGVzdF91c2VyX2VtYWlsPoi4BBMBAgAiBQJW/MgpAhsvBgsJCAcDAgYVCAIJ
+CgsEFgIDAQIeAQIXgAAKCRAqYf65XmeSk0orBADUXjEiGnjzyBpXqaiVmJr4MyfP
+IfKTK4a+4qvR+2fseD7hteF98m26i1YRI5omLp4/MnxGSpgKFKIuWIdkEiLg7IJc
+pFZVdoDVufEtzbj9gmOHlnteksbCtuESyB0Hytsba4uS9afcTJdGiPNMHeniI/SY
+UKcCcIrQmpNIoOA5OLiNBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD
+7PK4xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6Ok
+NkhfGfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+c
+m3JuZy1k7QARAQABiQE9BBgBAgAJBQJW/MgpAhsuAKgJECph/rleZ5KTnSAEGQEC
+AAYFAlb8yCkACgkQsJSYitQUOv4w1wQAn3atI5EsmRyw6iC6UVWWJv/lKi1Priyt
+DsrdH5xUmHUgp6VU8Pw9Y6G+sv50KLfbVQ1l+8/3B71TjadsOxh+PBPsEyYpK6WX
+TVGy44IDvFWGyOod8tmfcFN9IpU5DmSk/vny9G7RK/nbnta2VnfZOzwm5i3cNkPr
+FGPL1z0K3qs0VwP+M7BXdqBRSFDDBpG1J0TrZioEjvKeOsT/Ul8mbVt7HQpcN93I
+wTO4uky0Woy2nb7SbTQw6wOpU54u7+5dSQ03ltUHg1owy6Y3CMOeFL+e9ALpAZAU
+aMwY7zMFhqlPVZZMfdMLRsdLin67RIM+OJ6A925AM52bEQT1YwkQlP4mvQY=
+=qclE
+-----END PGP PUBLIC KEY BLOCK-----
+HERE
+
+ TEST_EMAIL_PRIVATE_KEY = <<HERE
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+lQHYBFb8yCkBBADeH/BThp7SRk0w/nxVEEusv4oZ+rWC7chmWWwjndLf/ZB7f62q
+EcRFJLPKwjjzKh1/snh7LwwbQ8mWE78obX7lqwVG2BlvkGeD3b6Puzcixz1tHSrA
+gQs68e6caqataKvi6VBaqUn4yU32GhlQzV6zyteOGu0aMdvjAm4Kt7cJxQARAQAB
+AAP8DTFfcE6UG1AioJDU6KZ9oCaGONHLuxmNaArSofDrR/ODA9rLAUlp22N5LEdJ
+46NyOhXrEwHx2aK2k+vbVDbgrP4ZTH7GxIK/2KzmH4zX0fWUNsaRy94Q12lJegXH
+sH2Im8Jjxu16YwGgFNTX1fCPqLB6WdQpf1796s6+/3PnCDcCAOXTCul3N7V5Yl+9
+N2Anupn+qNDXKT/kiKIZLHsMbo7EriGWReG3lLj1cOJPC6Nf0uOEri4ErSjFEadR
+F2TNITsCAPdsZjc5RGppUXyBfxhQkAnZ0r+UT2meCH3g3EVh3W9SBrXNhwipNpW3
+bPzRjUCDtmA8EOvd93oPCZv4/tb50P8B/jC+QIZ3GncP1CFPSVDoIZ7OUU5M1330
+DP77vG1GxeQvYO/hlxL5/KdtTR6m5zlIuooDxUaNJz1w5/oVjlG3NZKpl7QSIDx0
+ZXN0X3VzZXJfZW1haWw+iLgEEwECACIFAlb8yCkCGy8GCwkIBwMCBhUIAgkKCwQW
+AgMBAh4BAheAAAoJECph/rleZ5KTSisEANReMSIaePPIGlepqJWYmvgzJ88h8pMr
+hr7iq9H7Z+x4PuG14X3ybbqLVhEjmiYunj8yfEZKmAoUoi5Yh2QSIuDsglykVlV2
+gNW58S3NuP2CY4eWe16SxsK24RLIHQfK2xtri5L1p9xMl0aI80wd6eIj9JhQpwJw
+itCak0ig4Dk4nQHYBFb8yCkBBAC+WMUQ+FC6GQ+pyaWlwTRsBAT4+Tp8w9jD7PK4
+xeEmVZDirP0VkW18UeQEueWJ63ia7wIGf1WyVH1tbvgVyRLsjT2cpKo8c6OkNkhf
+GfjTnUJPeBNy8734UDIdqZLXJl0z6Z1R0CfOjBqvV25kWUvMkz/NEgZBhE+cm3Ju
+Zy1k7QARAQABAAP9HrUaGvdpqTwVx3cHyXUhId6GzCuuKyaP4mZoGeBCcaQS2vQR
+YtiykwBwX/AlfwSFJmmHKB6EErWIA+QyaEFR/fO56cHD2TY3Ql0BGcuHIx3+9pkp
+biPBZdiiGz7oa6k6GWsbKSksqwV8poSXV7qbn+Bjm2xCM4VnjNZIrFtL7fkCAMOf
+e9yHBFoXfc175bkNXEUXrNS34kv2ODAlx6KyY+PS77D+nprpHpGCnLn77G+xH1Xi
+qvX1Dr/iSQU5Tzsd+tcCAPkYZulaC/9itwme7wIT3ur+mdqMHymsCzv9193iLgjJ
+9t7fARo18yB845hI9Xv7TwRcoyuSpfvuM05rCMRzydsCAOI1MZeKtZSogXVa9QTX
+sVGZeCkrujSVOgsA3w48OLc2OrwZskDfx5QHfeJnumjQLut5qsnZ+1onj9P2dGdn
+JaChe4kBPQQYAQIACQUCVvzIKQIbLgCoCRAqYf65XmeSk50gBBkBAgAGBQJW/Mgp
+AAoJELCUmIrUFDr+MNcEAJ92rSORLJkcsOogulFVlib/5SotT64srQ7K3R+cVJh1
+IKelVPD8PWOhvrL+dCi321UNZfvP9we9U42nbDsYfjwT7BMmKSull01RsuOCA7xV
+hsjqHfLZn3BTfSKVOQ5kpP758vRu0Sv5257WtlZ32Ts8JuYt3DZD6xRjy9c9Ct6r
+NFcD/jOwV3agUUhQwwaRtSdE62YqBI7ynjrE/1JfJm1bex0KXDfdyMEzuLpMtFqM
+tp2+0m00MOsDqVOeLu/uXUkNN5bVB4NaMMumNwjDnhS/nvQC6QGQFGjMGO8zBYap
+T1WWTH3TC0bHS4p+u0SDPjiegPduQDOdmxEE9WMJEJT+Jr0G
+=hvJM
+-----END PGP PRIVATE KEY BLOCK-----
+HERE
+
+end
diff --git a/tests/white-box/network.rb b/tests/white-box/network.rb
new file mode 100644
index 00000000..436fc8a8
--- /dev/null
+++ b/tests/white-box/network.rb
@@ -0,0 +1,90 @@
+require 'socket'
+require 'openssl'
+
+raise SkipTest if $node["dummy"]
+
+class Network < LeapTest
+
+ def setup
+ end
+
+ def test_01_Can_connect_to_internet?
+ assert_get('http://www.google.com/images/srpr/logo11w.png')
+ pass
+ end
+
+ #
+ # example properties:
+ #
+ # stunnel:
+ # ednp_clients:
+ # elk_9002:
+ # accept_port: 4003
+ # connect: elk.dev.bitmask.i
+ # connect_port: 19002
+ # couch_server:
+ # accept: 15984
+ # connect: "127.0.0.1:5984"
+ #
+ def test_02_Is_stunnel_running?
+ ignore unless $node['stunnel']
+ good_stunnel_pids = []
+ release = `facter lsbmajdistrelease`
+ if release.to_i > 7
+ # on jessie, there is only one stunnel proc running instead of 6
+ expected = 1
+ else
+ expected = 6
+ end
+ $node['stunnel']['clients'].each do |stunnel_type, stunnel_configs|
+ stunnel_configs.each do |stunnel_name, stunnel_conf|
+ config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+ processes = pgrep(config_file_name)
+ assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`"
+ good_stunnel_pids += processes.map{|ps| ps[:pid]}
+ assert port = stunnel_conf['accept_port'], 'Field `accept_port` must be present in `stunnel` property.'
+ assert_tcp_socket('localhost', port)
+ end
+ end
+ $node['stunnel']['servers'].each do |stunnel_name, stunnel_conf|
+ config_file_name = "/etc/stunnel/#{stunnel_name}.conf"
+ processes = pgrep(config_file_name)
+ assert_equal expected, processes.length, "There should be #{expected} stunnel processes running for `#{config_file_name}`"
+ good_stunnel_pids += processes.map{|ps| ps[:pid]}
+ assert accept_port = stunnel_conf['accept_port'], "Field `accept` must be present in property `stunnel.servers.#{stunnel_name}`"
+ assert_tcp_socket('localhost', accept_port)
+ assert connect_port = stunnel_conf['connect_port'], "Field `connect` must be present in property `stunnel.servers.#{stunnel_name}`"
+ assert_tcp_socket('localhost', connect_port,
+ "The local connect endpoint for stunnel `#{stunnel_name}` is unavailable.\n"+
+ "This is probably caused by a daemon that died or failed to start on\n"+
+ "port `#{connect_port}`, not stunnel itself.")
+ end
+ all_stunnel_pids = pgrep('/usr/bin/stunnel').collect{|process| process[:pid]}.uniq
+ assert_equal good_stunnel_pids.sort, all_stunnel_pids.sort, "There should not be any extra stunnel processes that are not configured in /etc/stunnel"
+ pass
+ end
+
+ def test_03_Is_shorewall_running?
+ ignore unless File.exists?('/sbin/shorewall')
+ assert_run('/sbin/shorewall status')
+ pass
+ end
+
+ THIRTY_DAYS = 60*60*24*30
+
+ def test_04_Are_server_certificates_valid?
+ cert_paths = ["/etc/x509/certs/leap_commercial.crt", "/etc/x509/certs/leap.crt"]
+ cert_paths.each do |cert_path|
+ if File.exists?(cert_path)
+ cert = OpenSSL::X509::Certificate.new(File.read(cert_path))
+ if Time.now > cert.not_after
+ fail "The certificate #{cert_path} expired on #{cert.not_after}"
+ elsif Time.now + THIRTY_DAYS > cert.not_after
+ fail "The certificate #{cert_path} will expire soon, on #{cert.not_after}"
+ end
+ end
+ end
+ pass
+ end
+
+end
diff --git a/tests/white-box/openvpn.rb b/tests/white-box/openvpn.rb
new file mode 100644
index 00000000..170d4503
--- /dev/null
+++ b/tests/white-box/openvpn.rb
@@ -0,0 +1,16 @@
+raise SkipTest unless service?(:openvpn)
+
+class OpenVPN < LeapTest
+ depends_on "Network"
+
+ def setup
+ end
+
+ def test_01_Are_daemons_running?
+ assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$'
+ assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$'
+ assert_running '^/usr/sbin/unbound$'
+ pass
+ end
+
+end
diff --git a/tests/white-box/soledad.rb b/tests/white-box/soledad.rb
new file mode 100644
index 00000000..d41bee58
--- /dev/null
+++ b/tests/white-box/soledad.rb
@@ -0,0 +1,17 @@
+raise SkipTest unless service?(:soledad)
+
+require 'json'
+
+class Soledad < LeapTest
+ depends_on "Network"
+ depends_on "CouchDB" if service?(:couchdb)
+
+ def setup
+ end
+
+ def test_00_Is_Soledad_running?
+ assert_running '.*/usr/bin/twistd.*--wsgi=leap.soledad.server.application'
+ pass
+ end
+
+end
diff --git a/tests/white-box/webapp.rb b/tests/white-box/webapp.rb
new file mode 100644
index 00000000..68f3dcd2
--- /dev/null
+++ b/tests/white-box/webapp.rb
@@ -0,0 +1,134 @@
+raise SkipTest unless service?(:webapp)
+
+require 'json'
+
+class Webapp < LeapTest
+ depends_on "Network"
+
+ def setup
+ end
+
+ def test_01_Can_contact_couchdb?
+ url = couchdb_url("", url_options)
+ assert_get(url) do |body|
+ assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+ end
+ pass
+ end
+
+ def test_02_Can_contact_couchdb_via_haproxy?
+ if property('haproxy.couch')
+ url = couchdb_url_via_haproxy("", url_options)
+ assert_get(url) do |body|
+ assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message."
+ end
+ pass
+ end
+ end
+
+ def test_03_Are_daemons_running?
+ assert_running '^/usr/sbin/apache2'
+ assert_running '^/usr/bin/ruby /usr/bin/nickserver'
+ pass
+ end
+
+ #
+ # this is technically a black-box test. so, move this when we have support
+ # for black box tests.
+ #
+ def test_04_Can_access_webapp?
+ assert_get('https://' + $node['webapp']['domain'] + '/')
+ pass
+ end
+
+ def test_05_Can_create_and_authenticate_and_delete_user_via_API?
+ if property('webapp.allow_registration')
+ assert_tmp_user
+ pass
+ else
+ skip "New user registrations are disabled."
+ end
+ end
+
+ def test_06_Can_sync_Soledad?
+ return unless property('webapp.allow_registration')
+ soledad_config = property('definition_files.soledad_service')
+ if soledad_config && !soledad_config.empty?
+ soledad_server = pick_soledad_server(soledad_config)
+ if soledad_server
+ assert_tmp_user do |user|
+ command = File.expand_path "../../helpers/soledad_sync.py", __FILE__
+ soledad_url = "https://#{soledad_server}/user-#{user.id}"
+ soledad_cert = "/usr/local/share/ca-certificates/leap_ca.crt"
+ assert_run "#{command} #{user.id} #{user.session_token} #{soledad_url} #{soledad_cert} #{user.password}"
+ assert_user_db_exists(user)
+ pass
+ end
+ end
+ else
+ skip 'No soledad service configuration'
+ end
+ end
+
+ private
+
+ def url_options
+ {
+ :username => property('webapp.couchdb_webapp_user.username'),
+ :password => property('webapp.couchdb_webapp_user.password')
+ }
+ end
+
+ #
+ # pick a random soledad server.
+ # I am not sure why, but using IP address directly does not work.
+ #
+ def pick_soledad_server(soledad_config_json_str)
+ soledad_config = JSON.parse(soledad_config_json_str)
+ host_name = soledad_config['hosts'].keys.shuffle.first
+ if host_name
+ hostname = soledad_config['hosts'][host_name]['hostname']
+ port = soledad_config['hosts'][host_name]['port']
+ return "#{hostname}:#{port}"
+ else
+ return nil
+ end
+ end
+
+ #
+ # returns true if the per-user db created by soledad-server exists.
+ # we try three times, and give up after that.
+ #
+ def assert_user_db_exists(user)
+ db_name = "user-#{user.id}"
+ repeatedly_try("/#{db_name}") do |body, response, error|
+ assert false, "Could not find user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}"
+ end
+ repeatedly_try("/#{db_name}/_design/docs") do |body, response, error|
+ assert false, "Could not find design docs for user db `#{db_name}` for test user `#{user.username}`\nuuid=#{user.id}\nHTTP #{response.code} #{error} #{body}"
+ end
+ end
+
+ #
+ # tries the URL repeatedly, giving up and yield the last response if
+ # no try returned a 200 http status code.
+ #
+ def repeatedly_try(url, &block)
+ last_body, last_response, last_error = nil
+ 3.times do
+ sleep 0.2
+ get(couchdb_url(url)) do |body, response, error|
+ last_body, last_response, last_error = body, response, error
+ # After moving to couchdb, webapp user is not allowed to Read user dbs,
+ # but the return code for non-existent databases is 404. See #7674
+ if response.code.to_i == 401
+ return
+ end
+ end
+ sleep 1
+ end
+ yield last_body, last_response, last_error
+ return
+ end
+
+end
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-25 08:07:18 +0000