#!/bin/bash usage() { cat << EOF usage: $0 [options] command [arguments...] This script runs the leap platform deploy tests OPTIONS -c|--config file specify config file -a|--all run command on all nodes -V|--versions show versions/git revision of leap_cli and leap_platoform in provider dir -h|--help show help COMMANDS bootstrap bootstrap node(s): - leap local start - leap node int - sets up hostname and runs apt-get dist-upgrade - leap local save create_provider creates a provider instance deploy deploy node(s) reset_deploy reset and deploy node(s) EOF } add_nodes() { suffix=$IP_SUFFIX_START for i in $@ do node=${i%:*} services=${i#*:} let suffix++ ip="${IP_PREFIX}.$suffix" case $services in openvpn) config="openvpn.gateway_address:${IP_PREFIX}.98 openvpn.second_gateway_address:${IP_PREFIX}.99" ;; *) config='' ;; esac $LEAP_CMD node add --local $node ip_address:$ip $config services:$services done } bootstrap_nodes() { for vm in $@ do $LEAP_CMD $OPTS local start $vm wait_for_node $vm $LEAP_CMD $OPTS node init $vm # set hostname + do dist-upgrade $LEAP_CMD $OPTS deploy $vm --tags site_apt::dist_upgrade,site_config::hosts $LEAP_CMD $OPTS local save $vm done } create_provider() { if [ -e $PROVIDERDIR ] then echo $PROVIDERDIR exists - exiting exit 1 fi mkdir -p $PROVIDERDIR cd $PROVIDERDIR $LEAP_CMD $OPTS new --contacts $CONTACTS --domain $DOMAIN --name $PROVIDER --platform=$PLATFORMDIR . cd $PLATFORMDIR git checkout $PLATFORM_BRANCH git submodule sync git submodule update --init cd $PROVIDERDIR # for now, we use the vagrant pubkey until https://leap.se/code/issues/2039 is solved $LEAP_CMD $OPTS add-user --self --ssh-pub-key=$SSHKEY $LEAP_CMD $OPTS cert ca && leap cert csr # copy for faster testing #cp $ROOTDIR/dh.pem.test $PROVIDERDIR/files/ca/dh.pem $LEAP_CMD $OPTS cert dh add_nodes $NODES } deploy() { # remove colors until #1751 is fixed RMCOLORS='s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' # we need to deploy with verbose level 2, and filter out unwanted stuff # until puppet errors show up in verbose level 0 +1 (#1750) FILTER_CLI='= read|= loading|= no change| - executing| = executing| = applying| = ran git| = checking| = synching| = skipping file_path| - cd .*; rsync -| - hiera| = created hiera/| = updated hiera/| - cd /root/| - rolling backexecuting| - files/ca/client_ca.crt|\[bin,puppet\] ->|] Hostname updated.| = Updating submodule puppet/modules|Warning: Permanently added.*to the list of known hosts.| = leap command v| = leap platform v| - \[.*\] ok' FILTER_PUPPET='] notice: |] No change to hostname|] Puppet apply complete|] warning: Dynamic lookup|] warning: Scope\(Class|Skipping because of failed dependencies' FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET|$FILTER_COMMON)" for vm in $@ do [ -e $LOGDIR ] || mkdir -p $LOGDIR date=`date +"%F-%H%M%S"` LOG1="$LOGDIR/deploy-$vm.log" LOG2="$LOGDIR/deploy-$vm-$date.log" ERRLOG1="$LOGDIR/deploy-$vm-$date-error.log" ERRLOG2="$LOGDIR/deploy-error.log" echo "Deploying \"$vm\" on `date`"|tee -a $LOG1 $LOG2 $LEAP_CMD $OPTS -v 2 deploy $vm 2>&1 | sed -r "$RMCOLORS" | tee -a $LOG1 $LOG2 | egrep -v "$FILTER_ALL" | tee -a $ERRLOG1 $ERRLOG2 > /dev/null if [ -s $ERRLOG1 ] then versions=`versions` msg="From: $MAIL_FROM\nTo: $MAIL_TO\nSubject: leap_platform $PLATFORM_BRANCH branch on \"$vm\" had errors !\n\n" echo "`date`: $msg" | tee -a $LOG1 $LOG2 $ERRLOG2 echo msg="${msg} Output of error log below:\n\n`cat $ERRLOG1` \n\n" msg="${msg}-------------------------------------------------------------------\n\n" msg="${msg}error log: ${ERRLOG1}\n" msg="${msg}comlete log: ${LOG2}\n\n" msg="${msg}Tested on `date` on \"$vm\" with following versions/git commit IDs: \n\n$versions" cat $ERRLOG1 if [ "$MAIL_TO" != '' ]; then echo "Sending this mail to $MAIL_TO:" printf "$msg" | /usr/sbin/sendmail -t fi else echo "Deploy to $vm on `date` went fine."| tee -a $LOG1 $LOG2 rm $ERRLOG1 fi done } get_ip () { grep ip_address $PROVIDERDIR/nodes/$1.json |cut -f 2 -d:|sed 's/[",]//g' } reset_deploy() { update_platform update_leap_cli cd $PROVIDERDIR log_start echo "Starting deploy_nodes for nodes $@ as background tasks on `date`" for i in $@ do $LEAP_CMD $OPTS local reset $i wait_for_node $i deploy $i & done # needed in a detached screen session, otherwise it would terminate before deploy jobs # have finished echo "Waiting for last deploy process has finished..." wait } log_start() { echo echo "Starting $0 on `date`" } ip_pingable () { ping -q -W10 -c1 $1 >/dev/null 2>&1 return $? } ssh_up () { nc -w 4 $1 22 > /dev/null return $? } wait_for_node() { vm=$1 ip=`get_ip $vm` online=0 echo "Waiting for ssh on VM $vm (IP: $ip) to come up..." while [ $online -eq 0 ] do ssh_up $ip && online=1 sleep 1 done } update_leap_cli () { cd $LEAP_SRC git pull } update_platform () { cd $PLATFORMDIR git pull } versions () { cd $PLATFORMDIR platform_head=`git rev-parse HEAD` platform_commit=`git show | head -5` cd $LEAP_SRC cli_head=`git rev-parse HEAD` cli_commit=`git show $cli_head | head -5` cd $PROVIDERDIR [ -d .git ] && provider_head=`git rev-parse HEAD` if [ -n $provider_head ] then provider_head='not under version control' fi echo "Provider ($PROVIDERDIR): $provider_head" echo #echo leap cli: $cli_head $LEAP_CMD -v 2 list | grep ' = leap command v' echo "$cli_commit" echo echo #echo "leap_platform:" $LEAP_CMD -v 2 list | grep ' = leap platform v' echo "$platform_commit" echo echo } config="" all=false print_versions=false # default in lib/leap_cli/leapfile.rb IP_PREFIX='10.5.5' if ! options=$(getopt -o avVc:h -l all,verbose,versions,config:,help -- "$@") then # something went wrong, getopt will put out an error message for us usage exit 1 fi eval set -- "$options" while [ $# -gt 0 ] do case $1 in -h|--help) usage; exit 1;; -c|--config) config=$2; shift ;; -a|--all) all=true;; -v|--verbose) VERBOSE=1 ;; -V|--versions) print_versions=true;; (--) shift; break;; (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; (*) break;; esac shift done cmd=$1 shift nodelist=$@ #echo "config: $config" #echo "cmd: $cmd" #echo "nodelist: $nodelist" #echo "all: $all" if [ -z $config ] then usage echo "Please provide a config file" exit 1 fi # provider specific config source $config # common config for all providers source /etc/leap/platform-test-common.cfg [ -e $PROVIDERDIR ] && cd $PROVIDERDIR if $print_versions then versions=`versions` echo "$versions" echo exit 0 fi if [ -z $LEAP_CMD -o -z $LEAP_SRC ] then echo "please provide a path to the leap_cli binary and the source in the config file, using the LEAP_CMD and LEAP_SRC var." exit 1 fi if $all ; then # use NODES variable from the config file nodes=$NODES # strip services from nodes_services # i.e. nodes_services='redevcouchdb1:couchdb redevcouchdb2:couchdb' # -> nodes='redevcouchdb1 redevcouchdb2' nodes=`echo "$NODES" | sed 's/:[[:alnum:]]*//g'` else # use nodelist provided via cmdline parameters nodes=$nodelist fi # check if another process is already running with same config file procs=`pgrep -fc "leap-platform-test.*-c.*$config"` if [ $procs -gt 1 ]; then echo "Other process(es) found running for config $config - exiting." exit 1 fi case $cmd in add_nodes) add_nodes "$nodes";; bootstrap) bootstrap_nodes "$nodes";; create_provider) create_provider;; deploy) deploy "$nodes";; reset_deploy) reset_deploy "$nodes";; (*) usage; echo "Please specify a command."; exit 1;; esac