Use two lockfiles now, one for indication a running process of leap-platform-test, and a different one that indicates the last deploy failed

1 files changed, 34 insertions, 14 deletions

diff --git a/leap-platform-test b/leap-platform-test
index 8d6741f..a48eff4 100755
--- a/leap-platform-test
+++ b/leap-platform-test
@@ -2,6 +2,12 @@
 #
 
 
+# in case the script gets canceled, use a trap to finally remove the 
+# lockfile that indicates a running process
+
+trap "cleanup" INT TERM
+
+
 usage()
 {
     cat << EOF
@@ -63,8 +69,6 @@ destroy_vms() {
 
 bootstrap_nodes() {
 
-    check_for_running_instances || exit $?
-
     for vm in $@
     do
         $LEAP_CMD $OPTS local start "$vm"
@@ -82,18 +86,25 @@ bootstrap_nodes() {
     done
 }
 
+cleanup () {
+    echo "cleaning up..."
+    [ -e $LOCKFILE ] && rm $LOCKFILE
+    exit
+}
+
+
 check_for_running_instances() {
 
-    if [[ $( pidof -x "$( basename "$0" )" -o %PPID ) ]]
+    if [[ -f $LOCKFILE ]]
     then 
-        echo "Other process(es) found running for $(basename "$0") - exiting."
+        echo "Lockfile found at $LOCKFILE - maybe other process(es) found running for $(basename "$0") - exiting. Please investigate and then remove lockfile."
         exit 1
     fi
 
-    if [[ $lock && -f $LOCKFILE ]]
+    if [[ $lock && -f $FAILURE_LOCKFILE ]]
     then
-        subj="WARNING: CI lock found - previous deploy tests failed !"
-        msg="CI lock found, and --lock in use. This means that leap test failed on the previous run.\n Please investigate and then remove $LOCKFILE\n\n"
+        subj="WARNING: CI failure lockfile found for branch ${PLATFORM_BRANCH} - previous deploy tests failed !"
+        msg="CI lock found, and --lock in use. This means that leap test failed on the previous run.\n Please investigate and then remove $FAILURE_LOCKFILE\n\n"
         if [ "$MAIL_TO" != '' ]; then
             echo "$subj Sending mail to $MAIL_TO:"
             sendemail -f "$MAIL_FROM" -t "$MAIL_TO" -o tls=no -m "${msg}" -u "${subj}"
@@ -105,7 +116,6 @@ check_for_running_instances() {
 
 create_provider() {
     
-    check_for_running_instances || exit $?
 
     if [ -e "$PROVIDERDIR" ]
     then
@@ -183,6 +193,7 @@ deploy() {
                 echo "Sending this mail to $MAIL_TO:"
                 sendemail -f "$MAIL_FROM" -t "$MAIL_TO" -o tls=no -m "${msg}" -u "${subj}" 
             fi
+            touch $FAILURE_LOCKFILE
         else
             echo "Deploy to $vm on $( date ) went fine."| tee -a "$LOG1" "$LOG2"
             rm "$ERRLOG1"
@@ -252,7 +263,6 @@ init_deploy() {
     #. run tests
     #. send email
 
-    check_for_running_instances || exit $?
 
     local nodes="$*"
     update_platform
@@ -300,7 +310,6 @@ reset_deploy() {
     #. run tests
     #. send email
 
-    check_for_running_instances || exit $?
 
     local nodes="$*"
     update_platform
@@ -345,7 +354,6 @@ destroy_deploy() {
     #. run tests
     #. send email
 
-    check_for_running_instances || exit $?
 
     local nodes="$*"
     update_platform
@@ -400,11 +408,11 @@ run_tests () {
     then
         echo 'WARNING - "leap test" failed !' | tee -a "$TEST_LOG1" "$TEST_LOG2"
         if $lock ; then
-            touch $LOCKFILE
+            touch $FAILURE_LOCKFILE
         fi
     else
         echo 'OK - "leap test" is all green !' | tee -a "$TEST_LOG1" "$TEST_LOG2"
-        rm -f $LOCKFILE
+        rm -f $FAILURE_LOCKFILE
     fi
     return "$test_failure"
 
@@ -521,7 +529,17 @@ date=$( date +"%F-%H%M%S" )
 
 TEST_LOG1="$LOGDIR/test.log"
 TEST_LOG2="$LOGDIR/test-$date.log"
-LOCKFILE="/var/run/lock/leap_ci_deploy_${PLATFORM_BRANCH}.lock"
+
+LOCKFILE_DIR="/var/run/lock/leap_ci/${PLATFORM_BRANCH}"
+[ -e $LOCKFILE_DIR ] || mkdir -p $LOCKFILE_DIR
+LOCKFILE="${LOCKFILE_DIR}/$(basename $0).lock"
+FAILURE_LOCKFILE="${LOCKFILE_DIR}/failure.lock"
+
+check_for_running_instances || exit $?
+
+# set branch specifc lockfile so a deploy test cannot be run twice in parallel
+touch $LOCKFILE
+
 
 [ -e "$PROVIDERDIR" ] && cd  "$PROVIDERDIR"
 
@@ -564,3 +582,5 @@ case $cmd in
     test)             run_tests "$nodes";;
     (*)               usage; echo "Please specify a command."; exit 1;;
 esac
+
+cleanup