From 0d5407168827cb293abd9f0b5ecd406b16783f01 Mon Sep 17 00:00:00 2001 From: mh Date: Wed, 11 Aug 2010 09:59:09 +0200 Subject: introduce caching for xen plugins calls to the xen binaries such as xm are quite heavy and we therefore cache now a lot of the data that is generated for these plugins and use them as well for the other ones. --- files/plugins/xen | 11 +++++---- files/plugins/xen-cpu | 20 +++++++--------- files/plugins/xen_mem | 9 ++++--- files/plugins/xen_traffic_all | 55 ++++++++++++++++++++++++++----------------- files/plugins/xen_vbd | 34 +++++++++++++++++--------- files/plugins/xen_vm | 7 +++++- 6 files changed, 84 insertions(+), 52 deletions(-) (limited to 'files') diff --git a/files/plugins/xen b/files/plugins/xen index 575cd3b..a9f1a1e 100755 --- a/files/plugins/xen +++ b/files/plugins/xen @@ -8,8 +8,6 @@ # autoconf (optinal - used by munin-config) # -MAXDOMAINS=16 - if [ "$1" = "autoconf" ]; then if which xm > /dev/null ; then echo yes @@ -19,6 +17,11 @@ if [ "$1" = "autoconf" ]; then exit 1 fi +# we cache xm list for 5 min for perfomance reasons +((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE '^\/var\/lib\/munin\/plugin-state\/xm_list\.state$') && \ + [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || \ + /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state + if [ "$1" = "config" ]; then echo 'graph_title Xen Domain Utilerisation' @@ -27,7 +30,7 @@ if [ "$1" = "config" ]; then echo 'graph_vlabel mS' echo 'graph_category xen' echo 'graph_info This graph shows of many mS wall time where used by a domain' - /usr/sbin/xm list | grep -v "^Name .* Console$" | \ + cat /var/lib/munin/plugin-state/xm_list.state | \ while read name domid mem cpu state time console; do name=`echo $name | sed -e"s/-/_/"` echo "$name.label $name" @@ -43,7 +46,7 @@ if [ "$1" = "config" ]; then exit 0 fi -/usr/sbin/xm list | grep -v "^Name" | grep -v "^Name .* Console$" | \ +cat /var/lib/munin/plugin-state/xm_list.state | \ while read name domid mem cpu state time console; do name=`echo $name | sed -e"s/-/_/"` time=`echo $time | sed -e "s/\.//"` diff --git a/files/plugins/xen-cpu b/files/plugins/xen-cpu index 0ef301e..b456a14 100755 --- a/files/plugins/xen-cpu +++ b/files/plugins/xen-cpu @@ -28,6 +28,10 @@ use strict; $ENV{PATH} = '/bin:/usr/bin:/usr/sbin'; +# we cache xm list for 5 min for perfomance reasons +system('((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE \'^\/var\/lib\/munin\/plugin-state\/xm_list\.state$\') && [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state'); +system('((find /var/lib/munin/plugin-state/xm_top.state -mmin -5 2>&1 | grep -qE \'^\/var\/lib\/munin\/plugin-state\/xm_top\.state$\') && [ `cat /var/lib/munin/plugin-state/xm_top.state | wc -l` -gt 1 ]) || /usr/sbin/xentop -b -i1 > /var/lib/munin/plugin-state/xm_top.state'); + my $arg; undef($arg); if (defined($ARGV[0])) { $arg = 'config' if ($ARGV[0] eq 'config'); @@ -53,8 +57,9 @@ if (defined($ARGV[0])) { 'graph_info' => 'Display the % of CPU Usage for each domain', ); - my @domains = `$XM list`; - shift(@domains); # we dont need the header line + my @domains = `cat /var/lib/munin/plugin-state/xm_list.state`; + # the header line is not in the cached file + #shift(@domains); # we dont need the header line my $cnt = "0"; foreach my $domain ( @domains ) { my ($dom,undef) = split(/\s/, $domain, 2); @@ -81,16 +86,7 @@ if (defined($ARGV[0])) { # Nothing was passed as an argument, so let's just return the proper values -my @chunks; undef(@chunks); - -{ - # run the xentop command a few times because the first reading is not always accurate - local $/ = undef; - @chunks = split(/^xentop - .*$/m, `$XMTOP -b -i2 -d2`); -} - -# Take only the last run of xentop -my @stats = split (/\n/,pop(@chunks)); +my @stats = `cat /var/lib/munin/plugin-state/xm_top.state`; # remove the first 4 items that are junk that we don't need. shift(@stats); diff --git a/files/plugins/xen_mem b/files/plugins/xen_mem index 5e985a5..16d91cf 100644 --- a/files/plugins/xen_mem +++ b/files/plugins/xen_mem @@ -43,7 +43,10 @@ #%# family=auto #%# capabilities=autoconf -XM="/usr/sbin/xm" +# we cache xm list for 5 min for perfomance reasons +((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE '^\/var\/lib\/munin\/plugin-state\/xm_list\.state$') && \ + [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || \ + /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state if [ "$1" = "autoconf" ]; then echo yes @@ -60,7 +63,7 @@ if [ "$1" = "config" ]; then echo 'graph_info This graph shows affected memory for each domain.' echo 'Domain_0.label Domain-0' echo 'Domain_0.draw AREA' -$XM list | grep -v 'Mem' | grep -v 'Domain-0' | while read i; do +cat /var/lib/munin/plugin-state/xm_list.state | grep -v 'Mem' | grep -v 'Domain-0' | while read i; do name=`echo $i | awk '{ print $1 }' | sed 's/[\/.-]/_/g'` echo -n "$name.label " echo $i | awk '{ print $1 }' @@ -70,7 +73,7 @@ done exit 0 fi -$XM list | grep -v 'Mem' | while read i; do +cat /var/lib/munin/plugin-state/xm_list.state | grep -v 'Mem' | while read i; do name=`echo $i | awk '{ print $1 }' | sed 's/[\/.-]/_/g'` echo -n "$name.value " echo $i | awk '{ print $3 * 1024 * 1024 }' diff --git a/files/plugins/xen_traffic_all b/files/plugins/xen_traffic_all index 76d4ded..c5bbfbb 100644 --- a/files/plugins/xen_traffic_all +++ b/files/plugins/xen_traffic_all @@ -11,6 +11,11 @@ #%# family=auto #%# capabilities=autoconf +# we cache xm list for 5 min for perfomance reasons +((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE '^\/var\/lib\/munin\/plugin-state\/xm_list\.state$') && \ + [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || \ + /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state + if [ "$1" = "autoconf" ]; then if which xm > /dev/null ; then echo yes @@ -32,34 +37,42 @@ if [ "$1" = "config" ]; then echo 'graph_vlabel bits received (-) / sent (+) per ${graph_period}' echo 'graph_args --base 1024 -l 0' echo 'graph_category xen' - DOMAINS=$(xm list | awk '{print $1}' | egrep -v "^(Name|Domain-0)") + DOMAINS=$(cat /var/lib/munin/plugin-state/xm_list.state | awk '{print $1}' | egrep -v "^(Name|Domain-0)") for dom in $DOMAINS; do - devs=$( xm network-list $dom |\ - egrep "^[0-9]+" | sed 's@^.*vif/\([0-9]*\)/\([0-9]*\).*$@vif\1.\2@') - real_name=$( echo $dom | sed -e's/-/_/g' ) - name=$real_name - for dev in $devs; do - if [ ${#devs} -gt 1 ]; then - name=$real_name"_"`echo $dev | sed 's/\./\_/'` - fi + # we update network devices only twice an hour + ((find /var/lib/munin/plugin-state/xm_net_$dom.state -mmin -30 > /dev/null 2>&1) && \ + [ `cat /var/lib/munin/plugin-state/xm_net_$dom.state | wc -l` -gt 0 ]) || \ + (/usr/sbin/xm network-list $dom |\ + egrep "^[0-9]+" | sed 's@^.*vif/\([0-9]*\)/\([0-9]*\).*$@vif\1.\2@' > /var/lib/munin/plugin-state/xm_net_$dom.state) + devs=$(cat /var/lib/munin/plugin-state/xm_net_$dom.state) + real_name=$( echo $dom | sed -e's/-/_/g' ) + name=$real_name + for dev in $devs; do + if [ ${#devs} -gt 1 ]; then + name=$real_name"_"`echo $dev | sed 's/\./\_/'` + fi - echo $name'Down.label received' - echo $name'Down.type COUNTER' - echo $name'Down.graph no' - echo "${name}Down.cdef ${name}Down,8,*" - echo "${name}Up.label ${name}" - echo $name'Up.type COUNTER' - echo "${name}Up.negative ${name}Down" - echo "${name}Up.cdef ${name}Up,8,*" - done + echo $name'Down.label received' + echo $name'Down.type COUNTER' + echo $name'Down.graph no' + echo "${name}Down.cdef ${name}Down,8,*" + echo "${name}Up.label ${name}" + echo $name'Up.type COUNTER' + echo "${name}Up.negative ${name}Down" + echo "${name}Up.cdef ${name}Up,8,*" + done done exit 0 fi -DOMAINS=$(xm list | awk '{print $1}' | egrep -v "^(Name|Domain-0)") +DOMAINS=$(cat /var/lib/munin/plugin-state/xm_list.state | awk '{print $1}' | egrep -v "^(Name|Domain-0)") for dom in $DOMAINS; do - devs=$( xm network-list $dom |\ - egrep "^[0-9]+" | sed 's@^.*vif/\([0-9]*\)/\([0-9]*\).*$@vif\1.\2@') + # we update network devices only twice an hour + ((find /var/lib/munin/plugin-state/xm_net_$dom.state -mmin -30 > /dev/null 2>&1) && \ + [ `cat /var/lib/munin/plugin-state/xm_net_$dom.state | wc -l` -gt 0 ]) || \ + (/usr/sbin/xm network-list $dom |\ + egrep "^[0-9]+" | sed 's@^.*vif/\([0-9]*\)/\([0-9]*\).*$@vif\1.\2@' > /var/lib/munin/plugin-state/xm_net_$dom.state) + devs=$(cat /var/lib/munin/plugin-state/xm_net_$dom.state) real_name=$( echo $dom | sed -e's/-/_/g' ) name=$real_name for dev in $devs; do diff --git a/files/plugins/xen_vbd b/files/plugins/xen_vbd index 238bf08..4eca5a6 100755 --- a/files/plugins/xen_vbd +++ b/files/plugins/xen_vbd @@ -16,6 +16,10 @@ $XM = '/usr/sbin/xm'; $XMTOP = '/usr/sbin/xentop'; +# we cache xm list for 5 min for perfomance reasons +system('((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE \'^\/var\/lib\/munin\/plugin-state\/xm_list\.state$\') && [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state'); +system('((find /var/lib/munin/plugin-state/xm_top.state -mmin -5 2>&1 | grep -qE \'^\/var\/lib\/munin\/plugin-state\/xm_top\.state$\') && [ `cat /var/lib/munin/plugin-state/xm_top.state | wc -l` -gt 1 ]) || /usr/sbin/xentop -b -i1 > /var/lib/munin/plugin-state/xm_top.state'); + # ah, parameters coming in if ( defined($ARGV[0])) { @@ -46,7 +50,7 @@ if ( defined($ARGV[0])) 'graph_info' => 'Display the I/O operations for each domain', ); - @domains = `$XM list`; + @domains = `cat /var/lib/munin/plugin-state/xm_list.state`; shift(@domains); # we don't need the header line foreach $domain ( @domains ) @@ -82,16 +86,24 @@ if ( defined($ARGV[0])) # No args, get rolling -# NAME STATE CPU(sec) CPU(%) MEM(k) MEM(%) MAXMEM(k) MAXMEM(%) VCPUS NETS NETTX(k) NETRX(k) VBDS VBD_OO VBD_RD VBD_WR SSID -$tmpfile = "/tmp/munin_xen_vbd_tmp"; - -system("$XMTOP -b -i1 > $tmpfile"); -open(I,"$tmpfile"); -while(){ - chomp; - s/^\s*//g; - @tmp=split(/\s+/, $_); - next if $tmp[0] eq "NAME"; +my @stats = `cat /var/lib/munin/plugin-state/xm_top.state`; + +# remove the first 4 items that are junk that we don't need. +shift(@stats); +shift(@stats); +shift(@stats); +shift(@stats); + +my %vals; undef(%vals); + +foreach my $domain (@stats) { + # trim the leading whitespace + $domain =~ s/^\s+//; + my @tmp = split(/\s+/, $domain); + + # we need to change - and . to _ or things get weird with the graphs + # some decent quoting would probably fix this, but this works for now + $tmp[0] =~ s/[-.]/_/g; $domname = $tmp[0]; $domname =~ s/[-.]/_/g; diff --git a/files/plugins/xen_vm b/files/plugins/xen_vm index a0dc0b7..a69b5fe 100644 --- a/files/plugins/xen_vm +++ b/files/plugins/xen_vm @@ -45,6 +45,11 @@ XM="/usr/sbin/xm" +# we cache xm list for 5 min for perfomance reasons +((find /var/lib/munin/plugin-state/xm_list.state -mmin -5 2>&1 | grep -qE '^\/var\/lib\/munin\/plugin-state\/xm_list\.state$') && \ + [ `cat /var/lib/munin/plugin-state/xm_list.state | wc -l` -gt 1 ]) || \ + /usr/sbin/xm list | grep -v "^Name .* Console$" > /var/lib/munin/plugin-state/xm_list.state + if [ "$1" = "autoconf" ]; then echo yes exit 0 @@ -61,7 +66,7 @@ if [ "$1" = "config" ]; then exit 0 fi -domains=`$XM list | wc -l` +domains=`cat /var/lib/munin/plugin-state/xm_list.state | wc -l` echo -n "domains.value " echo $(($domains-2)) -- cgit v1.2.3