From dda2836ca6c588f990dab94861bb6e4fae5eb476 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 5 Jul 2016 13:56:34 +0200 Subject: Refactor for local builds It's now possible to deploy locally. Large parts still need to get refactored. This is mostly working for these commands: - reset_deploy - simple_deploy --- leap-platform-test | 257 +++++++++++++++++++++++++++-------------------------- 1 file changed, 130 insertions(+), 127 deletions(-) diff --git a/leap-platform-test b/leap-platform-test index bc36369..0d614dd 100755 --- a/leap-platform-test +++ b/leap-platform-test @@ -23,7 +23,7 @@ OPTIONS -c|--config file specify config file -h|--help show help -l|--lock refuse to deploy if lockfile from previous failures exists - -v|--verbose show also successful puppet resource changes, not only errors + -v|--verbose show also successful puppet resource changes, not only errors -V|--versions show versions/git revision of leap_cli and leap_platoform in provider dir COMMANDS @@ -65,13 +65,13 @@ add_nodes() { destroy_vms() { for vm in "$@" do - $LEAP_CMD local destroy $vm + $LEAP_CMD local destroy "$vm" done } bootstrap_nodes() { - for vm in $@ + for vm in "$@" do $LEAP_CMD $OPTS local start "$vm" wait_for_node "$vm" @@ -84,7 +84,7 @@ bootstrap_nodes() { # make sure machines are rebooted in order to be able to load kernel modules after a kernel update # https://leap.se/code/issues/6494 - cd ${PROVIDERDIR}/test + cd "${PROVIDERDIR}/test" vagrant reload $vm $LEAP_CMD $OPTS local save "$vm" @@ -93,7 +93,7 @@ bootstrap_nodes() { cleanup () { echo "cleaning up..." - [ -e $LOCKFILE ] && rm $LOCKFILE + [ -e "$LOCKFILE" ] && rm "$LOCKFILE" } @@ -117,7 +117,7 @@ check_for_running_instances() { fi exit 1 else - rm $FAILURE_LOCKFILE + rm "$FAILURE_LOCKFILE" fi fi } @@ -152,72 +152,65 @@ create_provider() { git commit -m"finished create_provider" } +# deploys to all nodes in a given env deploy() { returncode=0 # we need to deploy with verbose level 2, and filter out unwanted stuff # until puppet errors show up in verbose level 0 +1 (#1750) - FILTER_CLI=' - \[.*\] Changed /etc/hostname to| - \[.*\] Changed hostname to|= read|= loading|= no change| - executing| = executing| = applying| = ran git| = checking| = synching| = skipping file_path| - cd .*; rsync -| - hiera| = created hiera/| = updated hiera/| = updated secrets.json| - cd /root/| - rolling backexecuting| - files/|\[bin,tests,puppet\] ->|] Hostname updated.| = Updating submodule puppet/modules|Warning: Permanently added.*to the list of known hosts.| = leap command v| = leap platform v| - \[.*\] ok| - \[.*\] STARTING APPLY| - \[.*\] APPLY COMPLETE|net.ssh.authentication.agent.*could not connect to ssh-agent|Deploying | - mx/dkim.pub, mx/dkim.key -> .*/srv/leap/files' + FILTER_CLI=' - \[.*\] Changed /etc/hostname to| - \[.*\] Changed hostname to|= read|= loading|= no change| - executing| = executing| = applying| = ran git| = checking| = synching| = skipping file_path| - cd .*; rsync -| - hiera| = created hiera/| = updated hiera/| = updated secrets.json| - cd /root/| - rolling backexecuting| - files/|\[bin,tests,puppet\] ->|] Hostname updated.| = Updating submodule puppet/modules|Warning: Permanently added.*to the list of known hosts.| = leap command v| = leap platform v| - \[.*\] ok| - \[.*\] STARTING APPLY| - \[.*\] APPLY COMPLETE|net.ssh.service.forward.*: could not establish forwarding of authentication agent|Deploying | - mx/dkim.pub, mx/dkim.key -> .*/srv/leap/files' FILTER_PUPPET="] notice: |^Notice: |] No change to hostname|] Puppet apply complete \(changes made\).|] completed in |] warning: Dynamic lookup|] warning: Scope\(Class|Skipping because of failed dependencies|warning: You cannot collect without storeconfigs being set|warning: Not collecting exported resources without storeconfigs|warning: default \`to_a' will be obsolete|Warning: Found multiple default providers for vcsrepo" - #FILTER_PUPPET_DEPREC="templatewrapper.rb:76:in \`method_missing" - if [ -n "$FILTER_COMMON" ] then - #FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET|$FILTER_PUPPET_DEPREC|$FILTER_COMMON)" FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET|$FILTER_COMMON)" else - #FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET|$FILTER_PUPPET_DEPREC)" FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET)" fi + [ -e "$LOGDIR" ] || mkdir -p "$LOGDIR" + date=$( date +"%F-%H%M%S" ) + LOG1="$LOGDIR/$TAG.log" + LOG2="$LOGDIR/deploy-$TAG-$date.log" + ERRLOG1="$LOGDIR/deploy-$TAG-$date-error.log" + ERRLOG2="$LOGDIR/deploy-error.log" + echo "Deploying tag \"$TAG\" on $( date )"|tee -a "$LOG1" "$LOG2" - for vm in "$@" - do - [ -e "$LOGDIR" ] || mkdir -p "$LOGDIR" - date=$( date +"%F-%H%M%S" ) - LOG1="$LOGDIR/deploy-$vm.log" - LOG2="$LOGDIR/deploy-$vm-$date.log" - ERRLOG1="$LOGDIR/deploy-$vm-$date-error.log" - ERRLOG2="$LOGDIR/deploy-error.log" - echo "Deploying \"$vm\" on $( date )"|tee -a "$LOG1" "$LOG2" - - if $verbose - then - $LEAP_CMD $OPTS -v 2 deploy "$vm" 2>&1 | tee -a "$LOG1" "$LOG2" "$LOG_GLOBAL" - cat $LOG2 | strip_colors | egrep -iv "$FILTER_ALL" | tee -a "$ERRLOG1" "$ERRLOG2" "$ERRLOG_GLOBAL" > /dev/null - else - $LEAP_CMD $OPTS -v 2 deploy "$vm" 2>&1 | tee -a "$LOG1" "$LOG2" "$LOG_GLOBAL" | strip_colors | egrep -iv "$FILTER_ALL" | tee -a "$ERRLOG1" "$ERRLOG2" "$ERRLOG_GLOBAL" > /dev/null - fi - - # send an host-specific error mail on deploy failures - if [ -s "$ERRLOG1" ] - then - touch $FAILURE_LOCKFILE - returncode=1 - versions=$( versions ) - subj="WARNING - \"leap deploy\" of platform $PLATFORM_BRANCH branch on \"$vm\" had errors !" - echo "$( date ): $msg" | tee -a "$LOG1" "$LOG2" "$ERRLOG2" - echo - msg="Output of error log below:\n\n$( cat "$ERRLOG1" ) \n\n" - msg="${msg}-------------------------------------------------------------------\n\n" - msg="${msg}error log: ${ERRLOG1}\n" - msg="${msg}comlete log: ${LOG2}\n\n" - - msg="${msg}Tested on $( date ) on \"$vm\" with following versions/git commit IDs: \n\n$versions" - - cat "$ERRLOG1" - - if [ "$MAIL_TO" != '' ]; then - echo "Sending this mail to $MAIL_TO:" - sendemail -f "$MAIL_FROM" -t "$MAIL_TO" -o tls=no -m "${msg}" -u "${subj}" - fi - else - echo "Deploy to $vm on $( date ) went fine."| tee -a "$LOG1" "$LOG2" - rm "$ERRLOG1" - fi - done + if $verbose + then + $LEAP_CMD "$OPTS" -v 2 deploy "$TAG" 2>&1 | tee -a "$LOG1" "$LOG2" "$LOG_GLOBAL" + strip_colors < "$LOG2" | egrep -iv "$FILTER_ALL" | tee -a "$ERRLOG1" "$ERRLOG2" "$ERRLOG_GLOBAL" > /dev/null + else + $LEAP_CMD "$OPTS" -v 2 deploy "$TAG" 2>&1 | tee -a "$LOG1" "$LOG2" "$LOG_GLOBAL" | strip_colors | egrep -iv "$FILTER_ALL" | tee -a "$ERRLOG1" "$ERRLOG2" "$ERRLOG_GLOBAL" > /dev/null + fi + + # send an tag-specific error mail on deploy failures + if [ -s "$ERRLOG1" ] + then + touch "$FAILURE_LOCKFILE" + returncode=1 + versions=$( versions ) + subj="WARNING - \"leap deploy\" of platform $PLATFORM_BRANCH branch on tag \"$TAG\" had errors !" + echo "$( date ): $msg" | tee -a "$LOG1" "$LOG2" "$ERRLOG2" + echo + msg="Output of error log below:\n\n$( cat "$ERRLOG1" ) \n\n" + msg="${msg}-------------------------------------------------------------------\n\n" + msg="${msg}error log: ${ERRLOG1}\n" + msg="${msg}comlete log: ${LOG2}\n\n" + + msg="${msg}Tested on $( date ) on tag \"$TAG\" with following versions/git commit IDs: \n\n$versions" + + cat "$ERRLOG1" + + if [ "$MAIL_TO" != '' ]; then + echo "Sending this mail to $MAIL_TO:" + sendemail -f "$MAIL_FROM" -t "$MAIL_TO" -o tls=no -m "${msg}" -u "${subj}" + fi + else + echo "Deploy to tag $TAG on $( date ) went fine."| tee -a "$LOG1" "$LOG2" + rm "$ERRLOG1" + fi return $returncode } @@ -291,7 +284,7 @@ init_deploy() { log_start - echo "Starting init_deploy for nodes $@ as background tasks on $( date )" + echo "Starting init_deploy for nodes $* as background tasks on $( date )" for i in $nodes do @@ -331,41 +324,61 @@ reset_deploy() { # 'leap node init' is not run again, this is usually sufficient. The # following steps happen: # - #. make sure the platform and leap cli are up-to-date #. reset the nodes to their saved state (saved is after bootstrap_nodes has # been run) #. deploy the nodes #. send email if MAIL_TO is set + cd "$PROVIDERDIR" - local nodes="$*" - update_platform - update_leap_cli + log_start + + echo "Starting reset_deploy on $(date)" + + $LEAP_CMD $OPTS local start + $LEAP_CMD $OPTS local reset + wait_for_nodes + + # run cutom provisioning script, i.e. specified in ~/.leaprc + cd test + vagrant provision + cd .. + + deploy + + if [ -e "$FAILURE_LOCKFILE" ] + then + print_global_errorlog + deploy_failure_email + returncode=1 + fi + summary + return $returncode +} + +simple_deploy() { + returncode=0 + # Simple deploy to an existing,initialized and optional already deployed node. + # Following steps happen: + # + #. deploy the nodes + #. send email if MAIL_TO is set cd "$PROVIDERDIR" log_start - echo "Starting reset_deploy for nodes $@ as background tasks on $( date )" + echo "Starting simple_deploy on $(date)" - for i in $nodes - do - $LEAP_CMD $OPTS local start "$i" - $LEAP_CMD $OPTS local reset "$i" - wait_for_node "$i" + $LEAP_CMD $OPTS local start + wait_for_nodes - # run cutom provisioning script, i.e. specified in ~/.leaprc - cd test - vagrant provision "$i" - cd .. + # run cutom provisioning script, i.e. specified in ~/.leaprc + cd test + vagrant provision + cd .. - deploy "$i" & - done - - # needed in a detached screen session, otherwise it would terminate before deploy jobs - # have finished - echo "Waiting until last deploy process has finished..." - wait + deploy if [ -e "$FAILURE_LOCKFILE" ] then @@ -396,20 +409,20 @@ destroy_deploy() { log_start - echo "Starting destroy_deploy for nodes $@ as background tasks on $( date )" + echo "Starting destroy_deploy for nodes $* as background tasks on $(date)" destroy_vms "$@" for i in $nodes do - bootstrap_nodes "$i" + bootstrap_nodes "$i" - # run cutom provisioning script, i.e. specified in ~/.leaprc - cd test - vagrant provision "$i" - cd .. + # run cutom provisioning script, i.e. specified in ~/.leaprc + cd test + vagrant provision "$i" + cd .. - deploy "$i" & + deploy "$i" & done # needed in a detached screen session, otherwise it would terminate before deploy jobs @@ -434,7 +447,7 @@ print_global_errorlog () { echo -e "\n==================================================================\n" echo "Errors during deploy:" - cat $ERRLOG_GLOBAL | strip_colors + strip_colors < "$ERRLOG_GLOBAL" echo -e "\n==================================================================\n\n" @@ -444,7 +457,7 @@ print_global_errorlog () { summary () { echo "Complete deploy log: $LOG_GLOBAL" - [ -f $ERRLOG_GLOBAL ] && echo "Error deploy log: $ERRLOG_GLOBAL" + [ -f "$ERRLOG_GLOBAL" ] && echo "Error deploy log: $ERRLOG_GLOBAL" versions } @@ -475,11 +488,11 @@ run_tests () { if [ "$test_failure" -ne 0 ] then echo 'WARNING - "leap test" failed !' | tee -a "$TEST_LOG1" "$TEST_LOG2" - touch $FAILURE_LOCKFILE + touch "$FAILURE_LOCKFILE" else echo 'OK - "leap test" is all green !' | tee -a "$TEST_LOG1" "$TEST_LOG2" fi - return $test_failure + return "$test_failure" } update_leap_cli () { @@ -502,8 +515,8 @@ update_platform () { git fetch git checkout master - git branch -D $PLATFORM_BRANCH - git checkout -f -b $PLATFORM_BRANCH origin/$PLATFORM_BRANCH + git branch -D "$PLATFORM_BRANCH" + git checkout -f -b "$PLATFORM_BRANCH" "origin/$PLATFORM_BRANCH" git clean -fd git submodule sync git submodule update --init @@ -538,20 +551,18 @@ wait_for_node() { done } - -# http://docs.vagrantup.com/v2/providers/default.html -export VAGRANT_DEFAULT_PROVIDER="libvirt" - +wait_for_nodes() { + sleep 10 +} config="" -all=false verbose=false print_versions=false # default in lib/leap_cli/leapfile.rb IP_PREFIX='10.5.5' -if ! options=$(getopt -o avVlc:h -l all,lock,verbose,versions,config:,help -- "$@") +if ! options=$(getopt -o vVlc:h -l lock,verbose,versions,config:,help -- "$@") then # something went wrong, getopt will put out an error message for us usage @@ -563,7 +574,6 @@ eval set -- "$options" while [ $# -gt 0 ] do case $1 in - -a|--all) all=true;; -c|--config) config=$2; shift ;; -h|--help) usage; exit 1;; -l|--lock) lock=true;; @@ -579,19 +589,25 @@ done cmd=$1 shift -nodelist="$*" -if [ -z "$config" ] + +# common config for all providers +source ".platform-test.conf" + +# custom user config +source ~/.config/leap/platform_ci/.platform-test.conf &> /dev/null || /bin/true + +if [ ! -z "$config" ] then - usage - echo "Please provide a config file" - exit 1 + # source provider specific config + source "$config" fi -# provider specific config -source "$config" -# common config for all providers -source /etc/leap/platform-test-common.cfg +if [ -z "$TAG" ] +then + echo "Please specify a TAG to deploy." + exit 1 +fi date=$( date +"%F-%H%M%S" ) LOG_GLOBAL="$LOGDIR/deploy-$date.log" @@ -600,8 +616,8 @@ ERRLOG_GLOBAL="$LOGDIR/deploy-$date-error.log" TEST_LOG1="$LOGDIR/test.log" TEST_LOG2="$LOGDIR/test-$date.log" -LOCKFILE_DIR="/var/run/lock/leap_ci/${PLATFORM_BRANCH}" -[ -e $LOCKFILE_DIR ] || mkdir -p $LOCKFILE_DIR +LOCKFILE_DIR="$ROOTDIR/builds/lock" +[ -e "$LOCKFILE_DIR" ] || mkdir -p "$LOCKFILE_DIR" LOCKFILE="${LOCKFILE_DIR}/$(basename $0).lock" FAILURE_LOCKFILE="${LOCKFILE_DIR}/failure.lock" exitcode=0 @@ -618,30 +634,16 @@ then exit 0 fi -if [ -z "$LEAP_CMD" -o -z "$LEAP_SRC" ] +if [ -z "$LEAP_CMD" ] then - echo "please provide a path to the leap_cli binary and the source in the config file, using the LEAP_CMD and LEAP_SRC var." + echo "please provide a path to the leap_cli binary using the LEAP_CMD var." exit 1 fi check_for_running_instances || exit $? -# set branch specifc lockfile so a deploy test cannot be run twice in parallel -touch $LOCKFILE - -if $all ; then - # use NODES variable from the config file - nodes=$NODES - - # strip services from nodes_services - # i.e. nodes_services='redevcouchdb1:couchdb redevcouchdb2:couchdb' - # -> nodes='redevcouchdb1 redevcouchdb2' - nodes=$( echo "$NODES" | sed 's/:[[:alnum:],]*//g' ) -else - # use nodelist provided via cmdline parameters - nodes=$nodelist -fi - +# set global lockfile +touch "$LOCKFILE" case $cmd in add_nodes) add_nodes "$nodes";; @@ -651,6 +653,7 @@ case $cmd in destroy_deploy) destroy_deploy "$nodes";; init_deploy) init_deploy "$nodes";; reset_deploy) reset_deploy "$nodes";; + simple_deploy) simple_deploy "$nodes";; test) run_tests "$nodes";; (*) usage; echo "Please specify a command."; exit 1;; esac -- cgit v1.2.3