1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
|
#!/bin/bash
usage()
{
cat << EOF
usage: $0 [options] command [arguments...]
This script runs the leap platform deploy tests
OPTIONS
-c|--config file specify config file
-a|--all run command on all nodes
-V|--versions show versions/git revision of leap_cli and leap_platoform in provider dir
-h|--help show help
COMMANDS
bootstrap <node(s)> bootstrap node(s):
- leap local start
- leap node int
- sets up hostname and runs apt-get dist-upgrade
- leap local save
create_provider creates a provider instance
deploy <node(s)> deploy node(s)
reset_deploy <node(s)> reset and deploy node(s)
EOF
}
add_nodes() {
suffix=$IP_SUFFIX_START
for i in $@
do
node=${i%:*}
services=${i#*:}
let suffix++
ip="${IP_PREFIX}.$suffix"
case $services in
openvpn)
config="openvpn.gateway_address:${IP_PREFIX}.98 openvpn.second_gateway_address:${IP_PREFIX}.99"
;;
*)
config=''
;;
esac
$LEAP_CMD node add --local $node ip_address:$ip $config services:$services
done
}
bootstrap_nodes() {
for vm in $@
do
$LEAP_CMD $OPTS local start $vm
wait_for_node $vm
$LEAP_CMD $OPTS node init $vm
# set hostname + do dist-upgrade
$LEAP_CMD $OPTS deploy $vm --tags site_apt::dist_upgrade,site_config::hosts,site_squid_deb_proxy::client
$LEAP_CMD $OPTS local save $vm
done
}
create_provider() {
if [ -e $PROVIDERDIR ]
then
echo $PROVIDERDIR exists - exiting
exit 1
fi
mkdir -p $PROVIDERDIR
cd $PROVIDERDIR
$LEAP_CMD $OPTS new --contacts $CONTACTS --domain $DOMAIN --name $PROVIDER --platform=$PLATFORMDIR .
cd $PLATFORMDIR
git checkout $PLATFORM_BRANCH
git submodule sync
git submodule update --init
cd $PROVIDERDIR
# for now, we use the vagrant pubkey until https://leap.se/code/issues/2039 is solved
$LEAP_CMD $OPTS add-user --self --ssh-pub-key=$SSHKEY
$LEAP_CMD $OPTS cert ca && leap cert csr
# copy for faster testing
#cp $ROOTDIR/dh.pem.test $PROVIDERDIR/files/ca/dh.pem
$LEAP_CMD $OPTS cert dh
add_nodes $NODES
}
deploy() {
# we need to deploy with verbose level 2, and filter out unwanted stuff
# until puppet errors show up in verbose level 0 +1 (#1750)
FILTER_CLI='= read|= loading|= no change| - executing| = executing| = applying| = ran git| = checking| = synching| = skipping file_path| - cd .*; rsync -| - hiera| = created hiera/| = updated hiera/| = updated secrets.json| - cd /root/| - rolling backexecuting| - files/|\[bin,tests,puppet\] ->|] Hostname updated.| = Updating submodule puppet/modules|Warning: Permanently added.*to the list of known hosts.| = leap command v| = leap platform v| - \[.*\] ok'
FILTER_PUPPET='] notice: |] No change to hostname|] Puppet apply complete \(changes made\).|] warning: Dynamic lookup|] warning: Scope\(Class|Skipping because of failed dependencies'
if [ -n "$FILTER_COMMON" ]
then
FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET|$FILTER_COMMON)"
else
FILTER_ALL="($FILTER_CLI|$FILTER_PUPPET)"
fi
for vm in $@
do
[ -e $LOGDIR ] || mkdir -p $LOGDIR
date=`date +"%F-%H%M%S"`
LOG1="$LOGDIR/deploy-$vm.log"
LOG2="$LOGDIR/deploy-$vm-$date.log"
ERRLOG1="$LOGDIR/deploy-$vm-$date-error.log"
ERRLOG2="$LOGDIR/deploy-error.log"
echo "Deploying \"$vm\" on `date`"|tee -a $LOG1 $LOG2
$LEAP_CMD $OPTS -v 2 deploy $vm 2>&1 | tee -a $LOG1 $LOG2 | egrep -v "$FILTER_ALL" | tee -a $ERRLOG1 $ERRLOG2 > /dev/null
if [ -s $ERRLOG1 ]
then
touch $deploy_failure
versions=`versions`
msg="From: $MAIL_FROM\nTo: $MAIL_TO\nSubject: leap_platform $PLATFORM_BRANCH branch on \"$vm\" had errors !\n\n"
echo "`date`: $msg" | tee -a $LOG1 $LOG2 $ERRLOG2
echo
msg="${msg} Output of error log below:\n\n`cat $ERRLOG1` \n\n"
msg="${msg}-------------------------------------------------------------------\n\n"
msg="${msg}error log: ${ERRLOG1}\n"
msg="${msg}comlete log: ${LOG2}\n\n"
msg="${msg}Tested on `date` on \"$vm\" with following versions/git commit IDs: \n\n$versions"
cat $ERRLOG1
if [ "$MAIL_TO" != '' ]; then
echo "Sending this mail to $MAIL_TO:"
printf "$msg" | /usr/sbin/sendmail -t
fi
else
echo "Deploy to $vm on `date` went fine."| tee -a $LOG1 $LOG2
rm $ERRLOG1
fi
done
}
get_ip () {
grep ip_address $PROVIDERDIR/nodes/$1.json |cut -f 2 -d:|sed 's/[",]//g'
}
reset_deploy() {
local nodes=$@
update_platform
update_leap_cli
cd $PROVIDERDIR
log_start
echo "Starting deploy_nodes for nodes $@ as background tasks on `date`"
for i in $nodes
do
$LEAP_CMD $OPTS local reset $i
wait_for_node $i
deploy $i &
done
# needed in a detached screen session, otherwise it would terminate before deploy jobs
# have finished
echo "Waiting for last deploy process has finished..."
wait
if [ ! -e $deploy_failure -a -n $MAIL_TO ]
then
versions=`versions`
msg="From: $MAIL_FROM\nTo: $MAIL_TO\nSubject: Success ! leap_platform $PLATFORM_BRANCH branch deployed fine\n\n"
msg="${msg}Tested on `date` on these nodes: \"$nodes\"\nwith following versions/git commit IDs: \n\n$versions"
echo "Sending success mail to $MAIL_TO"
printf "$msg" | /usr/sbin/sendmail -t
fi
}
log_start() {
echo
echo "Starting $0 on `date`"
}
ip_pingable () {
ping -q -W10 -c1 $1 >/dev/null 2>&1
return $?
}
ssh_up () {
nc -w 4 $1 22 > /dev/null
return $?
}
wait_for_node() {
vm=$1
ip=`get_ip $vm`
online=0
echo "Waiting for ssh on VM $vm (IP: $ip) to come up..."
while [ $online -eq 0 ]
do
ssh_up $ip && online=1
sleep 1
done
}
update_leap_cli () {
cd $LEAP_SRC
git pull
}
update_platform () {
cd $PLATFORMDIR
git pull
}
versions () {
cd $PLATFORMDIR
platform_head=`git rev-parse HEAD`
platform_commit=`git show | head -6`
cd $LEAP_SRC
cli_head=`git rev-parse HEAD`
cli_commit=`git show $cli_head | head -6`
cd $PROVIDERDIR
[ -d .git ] && provider_head=`git rev-parse HEAD`
if [ -n $provider_head ]
then
provider_head='not under version control'
fi
echo "Provider ($PROVIDERDIR): $provider_head"
echo
#echo leap cli: $cli_head
$LEAP_CMD -v 2 list | grep ' = leap command v'
echo "$cli_commit"
echo
echo
#echo "leap_platform:"
$LEAP_CMD -v 2 list | grep ' = leap platform v'
echo "$platform_commit"
echo
echo
}
# http://docs.vagrantup.com/v2/providers/default.html
export VAGRANT_DEFAULT_PROVIDER="libvirt"
config=""
all=false
print_versions=false
# default in lib/leap_cli/leapfile.rb
IP_PREFIX='10.5.5'
deploy_failure="/tmp/deploy_failure"
[ -e $deploy_failure ] && rm $deploy_failure
if ! options=$(getopt -o avVc:h -l all,verbose,versions,config:,help -- "$@")
then
# something went wrong, getopt will put out an error message for us
usage
exit 1
fi
eval set -- "$options"
while [ $# -gt 0 ]
do
case $1 in
-h|--help) usage; exit 1;;
-c|--config) config=$2; shift ;;
-a|--all) all=true;;
-v|--verbose) VERBOSE=1 ;;
-V|--versions) print_versions=true;;
(--) shift; break;;
(-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
(*) break;;
esac
shift
done
cmd=$1
shift
nodelist=$@
#echo "config: $config"
#echo "cmd: $cmd"
#echo "nodelist: $nodelist"
#echo "all: $all"
if [ -z $config ]
then
usage
echo "Please provide a config file"
exit 1
fi
# provider specific config
source $config
# common config for all providers
source /etc/leap/platform-test-common.cfg
[ -e $PROVIDERDIR ] && cd $PROVIDERDIR
if $print_versions
then
versions=`versions`
echo "$versions"
echo
exit 0
fi
if [ -z $LEAP_CMD -o -z $LEAP_SRC ]
then
echo "please provide a path to the leap_cli binary and the source in the config file, using the LEAP_CMD and LEAP_SRC var."
exit 1
fi
if $all ; then
# use NODES variable from the config file
nodes=$NODES
# strip services from nodes_services
# i.e. nodes_services='redevcouchdb1:couchdb redevcouchdb2:couchdb'
# -> nodes='redevcouchdb1 redevcouchdb2'
nodes=`echo "$NODES" | sed 's/:[[:alnum:],]*//g'`
else
# use nodelist provided via cmdline parameters
nodes=$nodelist
fi
if [[ "`pidof -x $(basename $0) -o %PPID`" ]]
then
echo "Other process(es) found running for $(basename $0) - exiting."
exit 1
fi
case $cmd in
add_nodes) add_nodes "$nodes";;
bootstrap) bootstrap_nodes "$nodes";;
create_provider) create_provider;;
deploy) deploy "$nodes";;
reset_deploy) reset_deploy "$nodes";;
(*) usage; echo "Please specify a command."; exit 1;;
esac
|