diff options
Diffstat (limited to 'tests')
21 files changed, 338 insertions, 167 deletions
diff --git a/tests/example-provider/Vagrantfile b/tests/example-provider/Vagrantfile index 1e410f5e..e909e79b 100644 --- a/tests/example-provider/Vagrantfile +++ b/tests/example-provider/Vagrantfile @@ -42,6 +42,10 @@ Vagrant.configure("2") do |config| config.ssh.username = "vagrant" + # Enable private networking so the box can be accessed directly, + # not only via port forwaring + config.vm.network "private_network", type: "dhcp" + # forward leap_web ports config.vm.network "forwarded_port", guest: 443, host:4443 # forward pixelated ports diff --git a/tests/example-provider/vagrant/add-pixelated.sh b/tests/example-provider/vagrant/add-pixelated.sh index f9908947..38a7ea47 100755 --- a/tests/example-provider/vagrant/add-pixelated.sh +++ b/tests/example-provider/vagrant/add-pixelated.sh @@ -6,8 +6,8 @@ cd "$PROVIDERDIR" -if ! git submodule status files/puppet/modules/pixelated > /dev/null 2>&1; then - git submodule add https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated +if ! [ -d files/puppet/modules/pixelated ]; then + git clone https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated fi echo '{}' > services/pixelated.json @@ -26,7 +26,7 @@ $LEAP $OPTS -v 2 test --continue echo -e '\n===========================================================================================================\n\n' echo -e 'You are now ready to use your vagrant Pixelated provider.\n' -echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated Useragent.\n' -echo -e 'The Pixelated Useragent is available at https://localhost:8080\n' +echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated User Agent.\n' +echo -e 'The Pixelated User Agent is available at https://localhost:8080\n' echo -e 'Please add an exception for both sites in your browser dialog to allow the self-signed certificate.\n' diff --git a/tests/example-provider/vagrant/configure-leap.sh b/tests/example-provider/vagrant/configure-leap.sh index fd34d7ea..7a1efc71 100755 --- a/tests/example-provider/vagrant/configure-leap.sh +++ b/tests/example-provider/vagrant/configure-leap.sh @@ -1,45 +1,53 @@ -#!/bin/bash +#!/bin/sh +# Exit on failure +set -e +# shellcheck disable=SC1091 . /vagrant/vagrant/vagrant.config echo '===============================================' -echo 'configuring leap' +echo "Configuring LEAP in ${PROVIDERDIR}" echo '===============================================' # purge $PROVIDERDIR so this script can be run multiple times -[ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR +[ -e "$PROVIDERDIR" ] && rm -rf "$PROVIDERDIR" -mkdir -p $PROVIDERDIR -chown ${USER}:${USER} ${PROVIDERDIR} -cd $PROVIDERDIR +mkdir -p "$PROVIDERDIR" +chown "${USER}:${USER}" "${PROVIDERDIR}" +cd "$PROVIDERDIR" || exit -$LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform="$PLATFORMDIR" . -echo -e '\n@log = "./deploy.log"' >> Leapfile +$LEAP new --contacts "${contacts:?}" --domain "${provider_domain:?}" --name "${provider_name:?}" --platform="$PLATFORMDIR" . +printf '\n@log = "./deploy.log"' >> Leapfile -if [ ! -e /home/${USER}/.ssh/id_rsa ]; then - $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' +if [ ! -e "/home/${USER}/.ssh/id_rsa" ]; then + $SUDO ssh-keygen -f "/home/${USER}/.ssh/id_rsa" -P '' [ -d /root/.ssh ] || mkdir /root/.ssh - cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys + cat "/home/${USER}/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys fi -$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} +$SUDO mkdir -p "${PROVIDERDIR}/files/nodes/${NODE}" sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" -chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub +chown "${USER}:${USER}" "${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub" -$LEAP $OPTS add-user --self -$LEAP $OPTS cert ca -$LEAP $OPTS cert csr -$LEAP $OPTS node add $NODE ip_address:"$(facter ipaddress)" couch.mode:plain services:"$services" tags:production -echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json +$LEAP add-user --self +$LEAP cert ca +$LEAP cert csr + +# Try to see if there's a private IP for eth1 +# Otherwise take eth0 +# (virtualbox and libvirt backends behave differenently setting up +# direct accessible private networks. +# see https://www.vagrantup.com/docs/networking/private_network.html +IP="$(facter ipaddress_eth1)" +[ "$IP" = '' ] && IP="$(facter ipaddress_eth0)" +$LEAP node add "$NODE" ip_address:"${IP}" couch.mode:plain services:"${services:?}" tags:production -$LEAP $OPTS compile +echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json -$GIT init -$GIT add . -$GIT commit -m'configured provider' +$LEAP compile -$LEAP $OPTS node init $NODE +$LEAP node init "$NODE" if [ $? -eq 1 ]; then echo 'node init failed' exit 1 @@ -50,10 +58,7 @@ fi # workaround is to install rake as gem gem install rake -$LEAP $OPTS -v 2 deploy - -$GIT add . -$GIT commit -m'initialized and deployed provider' +$LEAP -v 2 deploy # Vagrant: leap_mx fails to start on jessie # https://leap.se/code/issues/7755 @@ -69,7 +74,7 @@ echo '===============================================' echo 'testing the platform' echo '===============================================' -$LEAP $OPTS -v 2 test --continue +$LEAP -v 2 test --continue echo '===============================================' echo 'setting node to demo-mode' @@ -80,13 +85,13 @@ postconf -e default_transport='error: in demo mode' curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" -echo -e '\n===========================================================================================================\n\n' -echo -e 'You are now ready to use your local LEAP provider.\n' +printf '\n===========================================================================================================\n\n' +printf 'You are now ready to use your local LEAP provider.\n' echo 'If you want to use the *Bitmask client* with your provider, please update your /etc/hosts with following dns overrides:' $LEAP list --print ip_address,domain.full,dns.aliases | sed 's/^.* //' | sed 's/, null//g' | tr -d '\]\[",' echo 'Please see https://leap.se/en/docs/platform/tutorials/vagrant#use-the-bitmask-client-to-do-an-initial-soledad-sync for more details how to use and test your LEAP provider.' -echo -e "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" -echo -e 'The LEAP webapp is now available at https://localhost:4443\n' -echo -e 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' +printf "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" +printf 'The LEAP webapp is now available at https://localhost:4443\n' +printf 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' diff --git a/tests/example-provider/vagrant/install-platform.pp b/tests/example-provider/vagrant/install-platform.pp index 223853c1..9cefcf7c 100755 --- a/tests/example-provider/vagrant/install-platform.pp +++ b/tests/example-provider/vagrant/install-platform.pp @@ -10,6 +10,3 @@ class { '::leap::cli::install': file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: ensure => directory } - -# install prerequisites for configuring the provider -include ::git diff --git a/tests/example-provider/vagrant/vagrant.config b/tests/example-provider/vagrant/vagrant.config index 60d2a52c..07222c3f 100644 --- a/tests/example-provider/vagrant/vagrant.config +++ b/tests/example-provider/vagrant/vagrant.config @@ -18,6 +18,4 @@ NODE='node1' SUDO="sudo -u ${USER}" PROVIDERDIR="/home/${USER}/leap/configuration" PLATFORMDIR="/srv/leap_platform" -LEAP="$SUDO /usr/local/bin/leap" -GIT="$SUDO git" - +LEAP="$SUDO /usr/local/bin/leap $OPTS" diff --git a/tests/platform-ci/Gemfile b/tests/platform-ci/Gemfile index 44a8637d..4cf14e43 100644 --- a/tests/platform-ci/Gemfile +++ b/tests/platform-ci/Gemfile @@ -16,4 +16,5 @@ group :test do # Install fog-aws because the gem dependency of leap_cli is now optional gem "fog-aws" gem "leap_cli", :git => 'https://leap.se/git/leap_cli.git' + gem "fog-aws" end diff --git a/tests/platform-ci/README.md b/tests/platform-ci/README.md index 60c17e41..2eae5ca0 100644 --- a/tests/platform-ci/README.md +++ b/tests/platform-ci/README.md @@ -1,15 +1,53 @@ -Continuous integration tests for the leap_platform code. +# Continuous integration tests for the leap_platform code -Usage: +# Setup + cd tests/platform-ci ./setup.sh + +# Run syntax checks and test if catalog compiles + bin/rake test:syntax - bin/rake test:catalog + bin/rake catalog For a list of all tasks: bin/rake -T -To create a virtual provider, run tests on it, then tear it down: +# Full integration test + +You can create a virtual provider using AWS, run tests on it, then tear it down +when the tests succeed. +In order to do so, you need to set your AWS credentials as environment variables: + + export AWS_ACCESS_KEY='...' + export AWS_SECRET_KEY='...' + +If you want to login to this machine during or after the deploy you need to + + export SSH_PRIVATE_KEY=$(cat ~/.ssh/id_rsa) + +then start the deply test with + + ./ci-build.sh + +# Running tests with docker and gitlab-runner + +Another possibility to run the platform tests is to use [gitlab-runner](https://docs.gitlab.com/runner/) +together with [Docker](https://www.docker.com/). + +Export `AWS_ACCESS_KEY`, `AWS_SECRET_KEY` and `SSH_PRIVATE_KEY` as shown above. +From the root dir of this repo run: + + gitlab-runner exec docker --env AWS_ACCESS_KEY="$AWS_ACCESS_KEY" --env AWS_SECRET_KEY="$AWS_SECRET_KEY" --env platform_PROVIDER_SSH_PRIVATE_KEY="$SSH_PRIVATE_KEY" deploy_test + +See `.gitlab-ci.yml` for all the different test jobs. + +To ssh into the VM you first need to enter the docker container: + + docker exec -u cirunner -it $(docker ps --latest -q) bash + +From there you can access the test provider config directory and ssh into the VM: - ./ci-build.sh + cd /builds/project-0/tests/platform-ci/provider/ + leap ssh citest0 diff --git a/tests/platform-ci/ci-build.sh b/tests/platform-ci/ci-build.sh index be492d32..9bdf75fb 100755 --- a/tests/platform-ci/ci-build.sh +++ b/tests/platform-ci/ci-build.sh @@ -12,9 +12,9 @@ # * AWS credentials as environment variables: # * `AWS_ACCESS_KEY` # * `AWS_SECRET_KEY` -# * ssh private key used to login to remove vm -# * `SSH_PRIVATE_KEY` -# +# * ssh private keys used to clone providers: +# * `bitmask_PROVIDER_SSH_PRIVATE_KEY` +# * `ibex_PROVIDER_SSH_PRIVATE_KEY` # exit if any commands returns non-zero status set -e @@ -39,6 +39,9 @@ ROOTDIR=$(readlink -f "$(dirname $0)") # leap_platform PLATFORMDIR=$(readlink -f "${ROOTDIR}/../..") +# deb repo component to configure +COMPONENT=${COMPONENT:-"master"} + # In the gitlab CI pipeline leap is installed in a different # stage by bundle. To debug you can run a single CI job locally # so we install leap_cli as gem here. @@ -63,22 +66,50 @@ deploy() { LEAP_CMD deploy "$TAG" } +leap_info() { + echo "Running leap info on $TAG" + LEAP_CMD info "${TAG}" +} + test() { LEAP_CMD test "$TAG" } +ssh_setup() { + # set the provider name from the first argument passed to the function + provider_name=$1 + # set CI_SSH_SECRET_PRIVATE_KEY to the variable name keyed off of the provider_name + CI_SSH_SECRET_PRIVATE_KEY=${provider_name}_PROVIDER_SSH_PRIVATE_KEY + # Set the SSH_PRIVATE_KEY to the value provided in the CI runner secret variable setting in gitlab + SSH_PRIVATE_KEY=${!CI_SSH_SECRET_PRIVATE_KEY} + echo "Working with provider: $provider_name" + [ -z "$SSH_PRIVATE_KEY" ] && fail "${provider_name}_PROVIDER_SSH_PRIVATE_KEY is not set - please provide it as env variable." + # Configure ssh keypair + [ -d ~/.ssh ] || /bin/mkdir ~/.ssh + /bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa + /bin/chmod 600 ~/.ssh/id_rsa + /bin/cp "${ROOTDIR}/provider/users/gitlab-runner-${provider_name}/gitlab-runner-${provider_name}_ssh.pub" ~/.ssh/id_rsa.pub +} + build_from_scratch() { - # when using gitlab-runner locally, CI_BUILD_ID is always 1 which + # setup ssh keys + ssh_setup platform + + # allow passing into the function the services, use a default set if empty + SERVICES=$1 + if [ -z "$SERVICES" ] + then + SERVICES='couchdb,soledad,mx,webapp,tor_relay,monitor' + fi + + # when using gitlab-runner locally, CI_JOB_ID is always 1 which # will conflict with running/terminating AWS instances in subsequent runs # therefore we pick a random number in this case [ "${CI_JOB_ID}" == "1" ] && CI_JOB_ID="000${RANDOM}" # create node(s) with unique id so we can run tests in parallel NAME="citest${CI_JOB_ID:-0}" - - TAG='single' - SERVICES='couchdb,soledad,mx,webapp,tor,monitor' # leap_platform/tests/platform-ci/provider PROVIDERDIR="${ROOTDIR}/provider" @@ -88,12 +119,11 @@ build_from_scratch() { # Create cloud.json needed for `leap vm` commands using AWS credentials which jq || ( apt-get update -y && apt-get install jq -y ) - # Dsiable xtrace + # Disable xtrace set +x [ -z "$AWS_ACCESS_KEY" ] && fail "\$AWS_ACCESS_KEY is not set - please provide it as env variable." [ -z "$AWS_SECRET_KEY" ] && fail "\$AWS_SECRET_KEY is not set - please provide it as env variable." - [ -z "$SSH_PRIVATE_KEY" ] && fail "\$SSH_PRIVATE_KEY is not set - please provide it as env variable." /usr/bin/jq ".platform_ci.auth |= .+ {\"aws_access_key_id\":\"$AWS_ACCESS_KEY\", \"aws_secret_access_key\":\"$AWS_SECRET_KEY\"}" < cloud.json.template > cloud.json # Enable xtrace again only if it was set at beginning of script @@ -102,7 +132,8 @@ build_from_scratch() { [ -d "./tags" ] || mkdir "./tags" /bin/echo "{\"environment\": \"$TAG\"}" | /usr/bin/json_pp > "${PROVIDERDIR}/tags/${TAG}.json" - pwd + # configure deb repo component + echo '{}' | jq ".sources.platform.apt |= { \"source\": \"http://deb.leap.se/platform\", \"component\": \"${COMPONENT}\" }" > common.json # remove old cached nodes echo "Removing old cached nodes..." @@ -123,53 +154,126 @@ build_from_scratch() { echo "Running leap node init on TAG: $TAG" LEAP_CMD node init "$TAG" - echo "Running leap info on $TAG" - LEAP_CMD info "${TAG}" } run() { - echo "Cloning $1 repo: $2" - git clone -q --depth 1 "$2" - cd "$1" - git rev-parse HEAD - echo -n "Operating in the $1 directory: " - pwd - echo "Listing current node information..." - LEAP_CMD list - echo "Attempting a deploy..." - deploy - echo "Attempting to run tests..." - test + provider_name=$1 + provider_URI=$2 + platform_branch=$3 + + # setup ssh keys + ssh_setup "$provider_name" + + # Setup the provider repository + echo "Setting up the provider repository: $provider_name by cloning $provider_URI" + git clone -q --depth 1 "$provider_URI" + cd "$provider_name" + echo -n "$provider_name repo at revision: " + git rev-parse HEAD + echo -n "Operating in the $provider_name directory: " + pwd + + + # If the third argument is set make sure we are on that platform branch + if [[ -n $platform_branch ]] + then + echo "Checking out $platform_branch branch of platform" + cd "$PLATFORMDIR" + git checkout -B "$platform_branch" + fi + + cd "${ROOTDIR}/${provider_name}" + echo "Listing current node information..." + LEAP_CMD list + + # Do the deployment + echo "Attempting a deploy..." + LEAP_CMD cert renew "$CI_JOB_NAME" + deploy + echo "Attempting to run tests..." + test +} + +soledad_migration() { + # check the version of soledad installed + # if the version is not greater than 0.9, we need to do the migration + if ! LEAP_CMD run "dpkg --compare-versions \$(dpkg -l |grep soledad-server|grep ^ii|awk '{ print \$3}') gt 0.8" vm |grep -q oops + then + echo "Need to migrate from soledad 0.8!" + if ! LEAP_CMD run 'systemctl stop leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl stop soledad-server' vm + then fail + fi + if ! LEAP_CMD run --stream '/usr/share/soledad-server/migration/0.9/migrate.py --log-file /dev/stdout --verbose --do-migrate | tee /var/log/leap/soledad_migration.log' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start soledad-server' vm + then fail + fi + fi } upgrade_test() { - # Checkout stable branch containing last release + # Checkout stable branch containing previous stable release # and deploy this cd "$PLATFORMDIR" - git remote add leap https://leap.se/git/leap_platform + # due to cache, this remote is sometimes already added + git remote add leap https://leap.se/git/leap_platform || true git fetch leap - git checkout -b leap_stable remotes/leap/stable + echo "Checking out leap/stable" + git checkout -b leap_stable remotes/leap/stable || true + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" - - build_from_scratch + LEAP_CMD --version + build_from_scratch 'couchdb,soledad,mx,webapp,tor,monitor' deploy + leap_info + # In 0.9 leap info did not output apt sources, so we do it manually + # but can remove it for next release + cat /etc/apt/sources.list.d/* test # Checkout HEAD of current branch and re-deploy cd "$PLATFORMDIR" - git checkout "$CI_COMMIT_REF" + echo "Checking out: $CI_COMMIT_SHA" + git checkout "$CI_COMMIT_SHA" + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" + LEAP_CMD --version + + # due to the 'tor' service no longer being valid in 0.10, we need to change + # that service to 'tor_relay'. This is done by changing the services array + # with jq to be set to the full correct list of services + jq '.services = ["couchdb","soledad","mx","webapp","tor_relay","monitor"]' < nodes/${NAME}.json deploy + + # pre-migration test + # allowed to fail because when a migration is needed, soledad-server refuses to start + test || /bin/true + + # check for soledad migration, and run it if necessary + soledad_migration + + leap_info + + # run the test again, this should succeed test cleanup @@ -192,12 +296,6 @@ cleanup() { # Ensure we don't output secret stuff to console even when running in verbose mode with -x set +x -# Configure ssh keypair -[ -d ~/.ssh ] || /bin/mkdir ~/.ssh -/bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa -/bin/chmod 600 ~/.ssh/id_rsa -/bin/cp "${ROOTDIR}/provider/users/gitlab-runner/gitlab-runner_ssh.pub" ~/.ssh/id_rsa.pub - # Enable xtrace again only if it was set at beginning of script [[ $xtrace == true ]] && set -x @@ -208,15 +306,16 @@ case "$CI_JOB_NAME" in ;; mail.bitmask.net) TAG='demomail' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; demo.bitmask.net) TAG='demovpn' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; deploy_test*) build_from_scratch deploy + leap_info test cleanup ;; diff --git a/tests/platform-ci/provider/nodes/catalogtest.json b/tests/platform-ci/provider/nodes/catalogtest.json index 05703666..bbf79d9e 100644 --- a/tests/platform-ci/provider/nodes/catalogtest.json +++ b/tests/platform-ci/provider/nodes/catalogtest.json @@ -10,7 +10,7 @@ "webapp", "monitor", "openvpn", - "tor", + "tor_relay", "obfsproxy", "static" ], diff --git a/tests/platform-ci/provider/provider.json b/tests/platform-ci/provider/provider.json index 218ff529..687f662a 100644 --- a/tests/platform-ci/provider/provider.json +++ b/tests/platform-ci/provider/provider.json @@ -10,7 +10,7 @@ "en": "You really should change this text" }, "contacts": { - "default": "root@example.org" + "default": "sysdevs@leap.se" }, "languages": ["en"], "default_language": "en", diff --git a/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub b/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub deleted file mode 100644 index 3e72b70f..00000000 --- a/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEtniDgIYEm4WtGgiQsZKBpY8x3tbzDBIoMLbZT496juCu4c3f+F5KkMPLmYRPcAupF8tVf+j7Fns7z69PuTjdGfe/cA9CTw/4sNAu3iLpunGR0d2Wtctez5mwz13bKRu9fck3H9p2F9Z47vMKtRTJJ6iIgaUVWU/eFd/MSMJeUVd2ns4Wr7SkHCBB3PV+QL1xl4+AZsUtnGVQ5cE4MZZFia/g6SlrKQYFtLRVIIpDuuaDSvULg1BFMhSCBDNygts8dKTJsCEQYeGVvHZaDwtKTnMqEIwBP4TkIoP+YWnZTPrGywFEJOlZ8b+4HdgdUAFLcFCycWMM9nVcWX7P2lIN gitlab-runner_ssh diff --git a/tests/platform-ci/setup.sh b/tests/platform-ci/setup.sh index e92dddc7..c10cb4c8 100755 --- a/tests/platform-ci/setup.sh +++ b/tests/platform-ci/setup.sh @@ -1,5 +1,5 @@ #!/bin/sh which bundle || /usr/bin/apt install bundle -/usr/local/bin/bundle install --binstubs --path=vendor --with=test --jobs "$(nproc)" -/usr/local/bin/bundle exec leap -v2 --yes help +bundle install --binstubs --path=vendor --with=test --jobs "$(nproc)" +bundle exec leap -v2 --yes help diff --git a/tests/server-tests/helpers/couchdb_helper.rb b/tests/server-tests/helpers/couchdb_helper.rb index efb2c2bf..0b6671ee 100644 --- a/tests/server-tests/helpers/couchdb_helper.rb +++ b/tests/server-tests/helpers/couchdb_helper.rb @@ -31,35 +31,6 @@ class LeapTest end # - # generates a couchdb url for accessing couchdb via haproxy - # - # example properties: - # - # haproxy: - # couch: - # listen_port: 4096 - # servers: - # panda: - # backup: false - # host: localhost - # port: 4000 - # weight: 100 - # writable: true - # - def couchdb_url_via_haproxy(path="", options=nil) - path = path.gsub('"', '%22') - if options && options[:username] && options[:password] - userpart = "%{username}:%{password}@" % options - else - userpart = "" - end - port = assert_property('haproxy.couch.listen_port') - return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| - url.memo = '(via haproxy)' - } - end - - # # generates a couchdb url for when couchdb is running locally. # # example properties: @@ -140,4 +111,4 @@ class LeapTest end end -end
\ No newline at end of file +end diff --git a/tests/server-tests/helpers/http_helper.rb b/tests/server-tests/helpers/http_helper.rb index 0d0bb7d5..3a1df9e7 100644 --- a/tests/server-tests/helpers/http_helper.rb +++ b/tests/server-tests/helpers/http_helper.rb @@ -5,7 +5,7 @@ class LeapTest # # In order to easily provide detailed error messages, it is useful # to append a memo to a url string that details what this url is for - # (e.g. stunnel, haproxy, etc). + # (e.g. stunnel, etc). # # So, the url happens to be a UrlString, the memo field is used # if there is an error in assert_get. @@ -154,4 +154,4 @@ class LeapTest request end -end
\ No newline at end of file +end diff --git a/tests/server-tests/helpers/os_helper.rb b/tests/server-tests/helpers/os_helper.rb index 9923d5b1..6a71388c 100644 --- a/tests/server-tests/helpers/os_helper.rb +++ b/tests/server-tests/helpers/os_helper.rb @@ -20,11 +20,29 @@ class LeapTest }.compact end - def assert_running(process, options={}) - processes = pgrep(process) - assert processes.any?, "No running process for #{process}" - if options[:single] - assert processes.length == 1, "More than one process for #{process}" + # + # passes if the specified process is runnin. + # + # arguments: + # + # match => VALUE -- scan process table for VALUE + # service => VALUE -- call systemctl is-active VALUE + # + # single => true|false -- if true, there must be one result + # + def assert_running(match:nil, service:nil, single:false) + if match + processes = pgrep(match) + assert processes.any?, "No running process for #{match}" + if single + assert processes.length == 1, "More than one process for #{match}" + end + elsif service + `systemctl is-active #{service} 2>&1` + if $?.exitstatus != 0 + output = `systemctl status #{service} 2>&1` + fail "Service '#{service}' is not running:\n#{output}" + end end end diff --git a/tests/server-tests/helpers/soledad_sync.py b/tests/server-tests/helpers/soledad_sync.py index f4fc81ae..a92ec68f 100755 --- a/tests/server-tests/helpers/soledad_sync.py +++ b/tests/server-tests/helpers/soledad_sync.py @@ -27,6 +27,7 @@ os.environ['SKIP_TWISTED_SSL_CHECK'] = '1' from twisted.internet import defer, reactor from twisted.python import log +from twisted.python.lockfile import FilesystemLock from client_side_db import get_soledad_instance from leap.common.events import flags @@ -35,6 +36,7 @@ flags.set_events_enabled(False) NUMDOCS = 1 USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0] +SYNC_TIMEOUT = 60 def bail(msg, exitcode): @@ -42,6 +44,13 @@ def bail(msg, exitcode): sys.exit(exitcode) +def obtain_lock(): + scriptname = os.path.basename(__file__) + lockfile = os.path.join(tempfile.gettempdir(), scriptname + '.lock') + lock = FilesystemLock(lockfile) + return lock.lock() + + def create_docs(soledad): """ Populates the soledad database with dummy messages, so we can exercise @@ -64,16 +73,30 @@ if __name__ == '__main__': if len(sys.argv) < 6: bail(USAGE, 2) + if not obtain_lock(): + bail("another instance is already running", 1) + uuid, token, server, cert_file, passphrase = sys.argv[1:] s = get_soledad_instance( uuid, passphrase, tempdir, server, cert_file, token) + def syncWithTimeout(_): + d = s.sync() + reactor.callLater(SYNC_TIMEOUT, d.cancel) + return d + def onSyncDone(sync_result): print "SYNC_RESULT:", sync_result s.close() rm_tempdir() reactor.stop() + def trap_cancel(f): + f.trap(defer.CancelledError) + log.err("sync timed out after %s seconds" % SYNC_TIMEOUT) + rm_tempdir() + reactor.stop() + def log_and_exit(f): log.err(f) rm_tempdir() @@ -81,8 +104,9 @@ if __name__ == '__main__': def start_sync(): d = create_docs(s) - d.addCallback(lambda _: s.sync()) + d.addCallback(syncWithTimeout) d.addCallback(onSyncDone) + d.addErrback(trap_cancel) d.addErrback(log_and_exit) reactor.callWhenRunning(start_sync) diff --git a/tests/server-tests/white-box/couchdb.rb b/tests/server-tests/white-box/couchdb.rb index 44a2769b..dcf71bc7 100644 --- a/tests/server-tests/white-box/couchdb.rb +++ b/tests/server-tests/white-box/couchdb.rb @@ -9,9 +9,9 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running 'bin/beam' + assert_running match: 'bin/beam' if multimaster? - assert_running 'bin/epmd' + assert_running match: 'bin/epmd' end pass end diff --git a/tests/server-tests/white-box/mx.rb b/tests/server-tests/white-box/mx.rb index ecc8686c..dfad0eed 100644 --- a/tests/server-tests/white-box/mx.rb +++ b/tests/server-tests/white-box/mx.rb @@ -24,16 +24,6 @@ class Mx < LeapTest pass end - def test_02_Can_contact_couchdb_via_haproxy? - if property('haproxy.couch') - url = couchdb_url_via_haproxy("", couch_url_options) - assert_get(url) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." - end - pass - end - end - # # this test picks a random identity document, then queries # using the by_address view for that same document again. @@ -52,17 +42,17 @@ class Mx < LeapTest end def test_04_Are_MX_daemons_running? - assert_running '.*/usr/bin/twistd.*mx.tac' - assert_running '^/usr/lib/postfix/master$' - assert_running '^/usr/sbin/postfwd' - assert_running 'postfwd2::cache$' - assert_running 'postfwd2::policy$' - assert_running '^/usr/sbin/unbound' - assert_running '^/usr/bin/freshclam' - assert_running '^/usr/sbin/opendkim' + assert_running match: '.*/usr/bin/twistd.*mx.tac' + assert_running match: '^/usr/lib/postfix/master$' + assert_running match: '^/usr/sbin/postfwd' + assert_running match: 'postfwd2::cache$' + assert_running match: 'postfwd2::policy$' + assert_running match: '^/usr/sbin/unbound' + assert_running match: '^/usr/bin/freshclam' + assert_running match: '^/usr/sbin/opendkim' if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0 - assert_running '^/usr/sbin/clamd' - assert_running '^/usr/sbin/clamav-milter' + assert_running match: '^/usr/sbin/clamd' + assert_running match: '^/usr/sbin/clamav-milter' pass else skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running." diff --git a/tests/server-tests/white-box/openvpn.rb b/tests/server-tests/white-box/openvpn.rb index d5cc2265..adda34a9 100644 --- a/tests/server-tests/white-box/openvpn.rb +++ b/tests/server-tests/white-box/openvpn.rb @@ -7,10 +7,46 @@ class OpenVPN < LeapTest end def test_01_Are_daemons_running? - assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' - assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' - assert_running '^/usr/sbin/unbound' + assert_running match: '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' + assert_running match: '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' + assert_running match: '^/usr/sbin/unbound' pass end + def test_02_Can_connect_to_openvpn? + # because of the way the firewall rules are currently set up, you can only + # connect to the standard 1194 openvpn port when you are connecting + # from the same host as openvpn is running on. + # + # so, this is disabled for now: + # $node['openvpn']['ports'].each {|port| ...} + # + + $node['openvpn']['protocols'].each do |protocol| + assert_openvpn_is_bound_to_port($node['openvpn']['gateway_address'], protocol, 1194) + end + pass + end + + private + + # + # asserting succeeds if openvpn appears to be correctly bound and we can + # connect to it. we don't actually try to establish a vpn connection in this + # test, we just check to see that it sort of looks like it is openvpn running + # on the port. + # + def assert_openvpn_is_bound_to_port(ip_address, protocol, port) + protocol = protocol.downcase + if protocol == 'udp' + # this sends a magic string to openvpn to attempt to start the protocol. + nc_output = `/bin/echo -e "\\x38\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00" | timeout 0.5 nc -u #{ip_address} #{port}`.strip + assert !nc_output.empty?, "Could not connect to OpenVPN daemon at #{ip_address} on port #{port} (#{protocol})." + elsif protocol == 'tcp' + assert system("openssl s_client -connect #{ip_address}:#{port} 2>&1 | grep -q CONNECTED"), + "Could not connect to OpenVPN daemon at #{ip_address} on port #{port} (#{protocol})." + else + assert false, "invalid openvpn protocol #{protocol}" + end + end end diff --git a/tests/server-tests/white-box/soledad.rb b/tests/server-tests/white-box/soledad.rb index 7c6918f9..112d6b9b 100644 --- a/tests/server-tests/white-box/soledad.rb +++ b/tests/server-tests/white-box/soledad.rb @@ -10,7 +10,7 @@ class Soledad < LeapTest end def test_00_Is_Soledad_running? - assert_running '/usr/bin/python /usr/bin/twistd --uid=soledad --gid=soledad --pidfile=/var/run/soledad.pid.*' + assert_running service: 'soledad-server' pass end diff --git a/tests/server-tests/white-box/webapp.rb b/tests/server-tests/white-box/webapp.rb index da1ec8c5..42adf219 100644 --- a/tests/server-tests/white-box/webapp.rb +++ b/tests/server-tests/white-box/webapp.rb @@ -4,6 +4,7 @@ require 'json' class Webapp < LeapTest depends_on "Network" + depends_on "Soledad" if service?(:soledad) def setup end @@ -16,19 +17,9 @@ class Webapp < LeapTest pass end - def test_02_Can_contact_couchdb_via_haproxy? - if property('haproxy.couch') - url = couchdb_url_via_haproxy("", url_options) - assert_get(url) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." - end - pass - end - end - def test_03_Are_daemons_running? - assert_running '^/usr/sbin/apache2' - assert_running '^ruby /usr/bin/nickserver' + assert_running match: '^/usr/sbin/apache2' + assert_running match: 'ruby /usr/bin/nickserver' pass end |