Merge ~daniel-nichter/percona-toolkit/bash-tool-libs r119.

lib/bash/alt_cmds.sh

@@ -0,0 +1,34 @@
+# This program is copyright 2011 Percona Inc.
+# Feedback and improvements are welcome.
+#
+# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+# systems, you can issue `man perlgpl' or `man perlartistic' to read these
+# licenses.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA  02111-1307  USA.
+# ###########################################################################
+# alt_cmds package
+# ###########################################################################
+
+# Package: alt_cmds
+# alt_cmds provides alternatives to commands that aren't on all systems.
+
+set -u
+
+# seq N, return 1, ..., 5
+_seq() {
+   local i=$1
+   awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
+}
+
+# ###########################################################################
+# End alt_cmds package
+# ###########################################################################

lib/bash/collect.sh

@@ -0,0 +1,238 @@
+# This program is copyright 2011 Percona Inc.
+# Feedback and improvements are welcome.
+#
+# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+# systems, you can issue `man perlgpl' or `man perlartistic' to read these
+# licenses.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA  02111-1307  USA.
+# ###########################################################################
+# collect package
+# ###########################################################################
+
+# Package: collect
+# collect collects system information.
+
+set -u
+
+# Global variables.
+CMD_GDB=${CMD_GDB:-"gdb"}
+CMD_IOSTAT=${CMD_IOSTAT:-"iostat"}
+CMD_MPSTAT=${CMD_MPSTAT:-"mpstat"}
+CMD_MYSQL=${CMD_MSSQL:-"mysql"}
+CMD_MYSQLADMIN=${CMD_MYSQL_ADMIN:-"mysqladmin"}
+CMD_OPCONTROL=${CMD_OPCONTROL:-"opcontrol"}
+CMD_OPREPORT=${CMD_OPREPORT:-"opreport"}
+CMD_PMAP=${CMD_PMAP:-"pmap"}
+CMD_STRACE=${CMD_STRACE:-"strace"}
+CMD_TCPDUMP=${CMD_TCPDUMP:-"tcpdump"}
+CMD_VMSTAT=${CMD_VMSTAT:-"vmstat"}
+
+collect() {
+   local d=$1  # directory to save results in
+   local p=$2  # prefix for each result file
+
+   # Get pidof mysqld; pidof doesn't exist on some systems.  We try our best...
+   local mysqld_pid=$(pidof -s mysqld);
+   if [ -z "$mysqld_pid" ]; then
+      mysqld_pid=$(pgrep -o -x mysqld);
+   fi
+   if [ -z "$mysqld_pid" ]; then
+      mysqld_pid=$(ps -eaf | grep 'mysql[d]' | grep -v mysqld_safe | awk '{print $2}' | head -n1);
+   fi
+
+   # Get memory allocation info before anything else.
+   if [ -x "$CMD_PMAP" -a "$mysqld_pid" ]; then
+      if $CMD_PMAP --help 2>&1 | grep -- -x >/dev/null 2>&1 ; then
+         $CMD_PMAP -x $mysqld_pid > "$d/$p-pmap"
+      else
+         # Some pmap's apparently don't support -x (issue 116).
+         $CMD_PMAP $mysqld_pid > "$d/$p-pmap"
+      fi
+   fi
+
+   # Getting a GDB stacktrace can be an intensive operation,
+   # so do this only if necessary.
+   if [ "$OPT_COLLECT_GDB" = "yes" -a "$mysqld_pid" ]; then
+      $CMD_GDB                     \
+         -ex "set pagination 0"    \
+         -ex "thread apply all bt" \
+         --batch -p $mysqld_pid    \
+         >> "$d/$p-stacktrace"
+   else
+      echo "GDB (--collect-gdb) was not enabled" >> "$d/$p-stacktrace"
+   fi
+
+   # Get MySQL's variables if possible.  Then sleep long enough that we probably
+   # complete SHOW VARIABLES if all's well.  (We don't want to run mysql in the
+   # foreground, because it could hang.)
+   $CMD_MYSQL "$EXT_ARGV" -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 &
+   sleep .2
+
+   # Get the major.minor version number.  Version 3.23 doesn't matter for our
+   # purposes, and other releases have x.x.x* version conventions so far.
+   local mysql_version="$(awk '/^version[^_]/{print substr($2,1,3)}' "$d/$p-variables")"
+
+   # Is MySQL logging its errors to a file?  If so, tail that file.
+   local mysql_error_log="$(awk '/log_error/{print $2}' "$d/$p-variables")"
+   if [ -z "$mysql_error_log" -a "$mysqld_pid" ]; then
+      # Try getting it from the open filehandle...
+      mysql_error_log="$(ls -l /proc/$mysqld_pid/fd | awk '/ 2 ->/{print $NF}')"
+   fi
+
+   local tail_error_log_pid=""
+   if [ "$mysql_error_log" ]; then
+      echo "The MySQL error log seems to be ${mysql_error_log}"
+      tail -f "$mysql_error_log" >"$d/$p-log_error" 2>&1 &
+      tail_error_log_pid=$!
+      # Send a mysqladmin debug to the server so we can potentially learn about
+      # locking etc.
+      $CMD_MYSQLADMIN "$EXT_ARGV" debug
+   else
+      echo "Could not find the MySQL error log"
+   fi
+
+   # Get a sample of these right away, so we can get these without interaction
+   # with the other commands we're about to run.
+   local innostat="SHOW /*!40100 ENGINE*/ INNODB STATUS\G"
+   local proclist="SHOW FULL PROCESSLIST\G"
+   if [ "${mysql_version}" '>' "5.1" ]; then
+      local mutex="SHOW ENGINE INNODB MUTEX"
+   else
+      local mutex="SHOW MUTEX STATUS"
+   fi
+   $CMD_MYSQL "$EXT_ARGV" -e "$innostat"        >> "$d/$p-innodbstatus1" 2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e "$proclist"        >> "$d/$p-processlist1"  2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables1"   2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e "$mutex"           >> "$d/$p-mutex-status1" 2>&1 &
+
+   # If TCP dumping is specified, start that on the server's port.
+   local tcpdump_pid=""
+   if [ "$OPT_COLLECT_TCPDUMP" = "yes" ]; then
+      local port=$(awk '/^port/{print $2}' "$d/$p-variables")
+      if [ "$port" ]; then
+         $CMD_TCPDUMP -i any -s 4096 -w "$d/$p-tcpdump" port ${port} &
+         tcpdump_pid=$!
+      fi
+   fi
+
+   # Next, start oprofile gathering data during the whole rest of this process.
+   # The --init should be a no-op if it has already been init-ed.
+   local have_oprofile="no"
+   if [ "$OPT_COLLECT_OPROFILE" = "yes" ]; then
+      if $CMD_OPCONTROL --init; then
+         $CMD_OPCONTROL --start --no-vmlinux
+         have_oprofile="yes"
+      fi
+   elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then
+      # Don't run oprofile and strace at the same time.
+      $CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 &
+      local strace_pid=$!
+   fi
+
+   # Grab a few general things first.  Background all of these so we can start
+   # them all up as quickly as possible.  We use mysqladmin -c even though it is
+   # buggy and won't stop on its own in 5.1 and newer, because there is a chance
+   # that we will get and keep a connection to the database; in troubled times
+   # the database tends to exceed max_connections, so reconnecting in the loop
+   # tends not to work very well.
+   ps -eaf                                      >> "$d/$p-ps"             2>&1 &
+   sysctl -a                                    >> "$d/$p-sysctl"         2>&1 &
+   top -bn1                                     >> "$d/$p-top"            2>&1 &
+   $CMD_VMSTAT 1 $OPT_INTERVAL                  >> "$d/$p-vmstat"         2>&1 &
+   $CMD_VMSTAT $OPT_INTERVAL 2                  >> "$d/$p-vmstat-overall" 2>&1 &
+   $CMD_IOSTAT -dx  1 $OPT_INTERVAL             >> "$d/$p-iostat"         2>&1 &
+   $CMD_IOSTAT -dx  $OPT_INTERVAL 2             >> "$d/$p-iostat-overall" 2>&1 &
+   $CMD_MPSTAT -P ALL 1 $OPT_INTERVAL           >> "$d/$p-mpstat"         2>&1 &
+   $CMD_MPSTAT -P ALL $OPT_INTERVAL 1           >> "$d/$p-mpstat-overall" 2>&1 &
+   lsof -nP -p $mysqld_pid -bw                  >> "$d/$p-lsof"           2>&1 &
+   $CMD_MYSQLADMIN "$EXT_ARGV" ext -i1 -c$OPT_INTERVAL >> "$d/$p-mysqladmin"     2>&1 &
+   local mysqladmin_pid=$!
+
+   # This loop gathers data for the rest of the duration, and defines the time
+   # of the whole job.
+   echo "Loop start: $(date +'TS %s.%N %F %T')"
+   for a in $(_seq $OPT_RUN_TIME); do
+      # We check the disk, but don't exit, because we need to stop jobs if we
+      # need to exit.
+      disk_space $d > $d/$p-disk-space
+      check_disk_space          \
+         $d/$p-disk-space       \
+         "$OPT_DISK_BYTE_LIMIT" \
+         "$OPT_DISK_PCT_LIMIT"  \
+         || break
+
+      # Synchronize ourselves onto the clock tick, so the sleeps are 1-second
+      sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}')
+      local ts="$(date +"TS %s.%N %F %T")"
+
+      # Collect the stuff for this cycle
+      (cat /proc/diskstats  2>&1; echo $ts) >> "$d/$p-diskstats"   &
+      (cat /proc/stat       2>&1; echo $ts) >> "$d/$p-procstat"    &
+      (cat /proc/vmstat     2>&1; echo $ts) >> "$d/$p-procvmstat"  &
+      (cat /proc/meminfo    2>&1; echo $ts) >> "$d/$p-meminfo"     &
+      (cat /proc/slabinfo   2>&1; echo $ts) >> "$d/$p-slabinfo"    &
+      (cat /proc/interrupts 2>&1; echo $ts) >> "$d/$p-interrupts"  &
+      (df -h                2>&1; echo $ts) >> "$d/$p-df"          &
+      (netstat -antp        2>&1; echo $ts) >> "$d/$p-netstat"     &
+      (netstat -s           2>&1; echo $ts) >> "$d/$p-netstat_s"   &
+   done
+   echo "Loop end: $(date +'TS %s.%N %F %T')"
+
+   if [ "$have_oprofile" = "yes" ]; then
+      $CMD_OPCONTROL --stop
+      $CMD_OPCONTROL --dump
+      kill $(pidof oprofiled);  # TODO: what if system doesn't have pidof?
+      $CMD_OPCONTROL --save=pt_collect_$p
+
+      # Attempt to generate a report; if this fails, then just tell the user
+      # how to generate the report.
+      local mysqld_path=$(which mysqld);
+      if [ "$mysqld_path" -a -f "$mysqld_path" ]; then
+         $CMD_OPREPORT            \
+            --demangle=smart      \
+            --symbols             \
+            --merge tgid          \
+            session:pt_collect_$p \
+            "$mysqld_path"        \
+            > "$d/$p-opreport"
+      else
+         echo "oprofile data saved to pt_collect_$p; you should be able"      \
+              "to get a report by running something like 'opreport"           \
+              "--demangle=smart --symbols --merge tgid session:pt_collect_$p" \
+              "/path/to/mysqld'"                                              \
+            > "$d/$p-opreport"
+      fi
+   elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then
+      kill -s 2 $strace_pid
+      sleep 1
+      kill -s 15 $strace_pid
+      # Sometimes strace leaves threads/processes in T status.
+      kill -s 18 $mysqld_pid
+   fi
+
+   $CMD_MYSQL "$EXT_ARGV" -e "$innostat"        >> "$d/$p-innodbstatus2" 2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e "$proclist"        >> "$d/$p-processlist2"  2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables2"   2>&1 &
+   $CMD_MYSQL "$EXT_ARGV" -e "$mutex"           >> "$d/$p-mutex-status2" 2>&1 &
+
+   # Kill backgrounded tasks.
+   kill $mysqladmin_pid
+   [ "$tail_error_log_pid" ] && kill $tail_error_log_pid
+   [ "$tcpdump_pid" ] && kill $tcpdump_pid
+
+   # Finally, record what system we collected this data from.
+   hostname > "$d/$p-hostname"
+}
+
+# ###########################################################################
+# End tmpdir package
+# ###########################################################################

lib/bash/parse_options.sh

@@ -28,7 +28,7 @@ set -u
 # sub will be scoped locally.
 declare -a ARGV   # non-option args (probably input files)
 declare EXT_ARGV  # everything after -- (args for an external command)
-OPT_ERR=${OPT_ERR:""}
+OPT_ERR=${OPT_ERR:-""}
 
 # Sub: usage
 #   Print usage (--help) and list the program's options.

lib/bash/safeguards.sh

@@ -0,0 +1,51 @@
+# This program is copyright 2011 Percona Inc.
+# Feedback and improvements are welcome.
+#
+# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+# systems, you can issue `man perlgpl' or `man perlartistic' to read these
+# licenses.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA  02111-1307  USA.
+# ###########################################################################
+# safeguards package
+# ###########################################################################
+
+# Package: safeguards
+# safeguards is a collection of function to help avoid blowing things up.
+
+set -u
+
+disk_space() {
+   local filesystem=${1:-"$PWD"}
+   # Filesystem    512-blocks       Used  Available  Capacity  Mounted on
+   # /dev/disk0s2   236306352  190223184   45571168       81%  /
+   df -m -P $filesystem
+}
+
+check_disk_space() {
+   local file=$1
+   local mb=${2:-"0"}
+   local pct=${3:-"0"}
+
+   local avail=$(cat $file | awk '/^\//{print $4}');
+   local full=$(cat $file  | awk '/^\//{print $5}' | sed -e 's/%//g');
+   if [ "${avail}" -le "$mb" -o "$full" -le "$pct" ]; then
+      echo "Not enough free space (${full}% full, ${avail}MB free)"
+      echo "Wanted less than ${pct}% full and more than ${mb}MB"
+      return 1
+   fi
+   return 0
+}
+
+
+# ###########################################################################
+# End safeguards package
+# ###########################################################################

t/lib/bash/collect.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+TESTS=19
+
+TMPFILE="$TEST_TMPDIR/parse-opts-output"
+TMPDIR="$TEST_TMPDIR"
+PATH="$PATH:$PERCONA_TOOLKIT_SANDBOX/bin"
+
+mkdir "$TMPDIR/collect" 2>/dev/null
+
+source "$LIB_DIR/log_warn_die.sh"
+source "$LIB_DIR/parse_options.sh"
+source "$LIB_DIR/safeguards.sh"
+source "$LIB_DIR/alt_cmds.sh"
+source "$LIB_DIR/collect.sh"
+
+parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf
+
+# Prefix (with path) for the collect files.
+local p="$TMPDIR/collect/2011_12_05"
+
+# Default collect, no extras like gdb, tcpdump, etc.
+collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1
+
+# Even if this system doesn't have all the cmds, collect should still
+# create all the default files.
+ls -1 $TMPDIR/collect | sort > $TMPDIR/collect-files
+no_diff \
+   $TMPDIR/collect-files \
+   $T_LIB_DIR/samples/bash/collect001.txt \
+   "Default collect files"
+
+cmd_ok \
+   "grep -q 'Avail' $p-df" \
+   "df"
+
+# hostname is the last thing collected, so if it's ok,
+# then the sub reached its end.
+is \
+   "`cat $p-hostname`" \
+   "`hostname`" \
+   "hostname"
+
+cmd_ok \
+   "grep -q -i 'buffer pool' $p-innodbstatus1" \
+   "innodbstatus1"
+
+cmd_ok \
+   "grep -q -i 'buffer pool' $p-innodbstatus2" \
+   "innodbstatus2"
+
+cmd_ok \
+   "grep -q 'error log seems to be /tmp/12345/data/mysqld.log' $p-output" \
+   "Finds MySQL error log"
+
+cmd_ok \
+   "grep -q 'Status information:' $p-log_error" \
+   "debug"
+
+cmd_ok \
+   "grep -q 'COMMAND[ ]\+PID[ ]\+USER' $p-lsof" \
+   "lsof"
+
+cmd_ok \
+   "grep -q 'buf/buf0buf.c' $p-mutex-status1" \
+   "mutex-status1"
+
+cmd_ok \
+   "grep -q 'buf/buf0buf.c' $p-mutex-status2" \
+   "mutex-status2"
+
+cmd_ok \
+   "grep -q '^| Uptime' $p-mysqladmin" \
+   "mysqladmin ext"
+
+cmd_ok \
+   "grep -qP 'Database\tTable\tIn_use' $p-opentables1" \
+   "opentables1"
+
+cmd_ok \
+   "grep -qP 'Database\tTable\t\In_use' $p-opentables2" \
+   "opentables2"
+
+cmd_ok \
+   "grep -q '1. row' $p-processlist1" \
+   "processlist1"
+
+cmd_ok \
+   "grep -q '1. row' $p-processlist2" \
+   "processlist2"
+
+cmd_ok \
+   "grep -q 'mysqld' $p-ps" \
+   "ps"
+
+cmd_ok \
+   "grep -qP '^warning_count\t\d' $p-variables" \
+   "variables"
+
+local iters=$(cat $p-df | grep -c '^TS ')
+is "$iters" "1" "1 iteration/1s run time"
+
+# ###########################################################################
+# Try longer run time.
+# ###########################################################################
+
+parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 2 -- --defaults-file=/tmp/12345/my.sandbox.cnf
+
+rm $TMPDIR/collect/*
+
+collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1
+
+local iters=$(cat $p-df | grep -c '^TS ')
+is "$iters" "2" "2 iteration/2s run time"
+
+# ############################################################################
+# Done
+# ############################################################################

t/lib/bash/parse_options.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-TESTS=24
+TESTS=25
 
 TMPFILE="$TEST_TMPDIR/parse-opts-output"
 
@@ -14,17 +14,15 @@ source "$LIB_DIR/parse_options.sh"
 TMPDIR="$TEST_TMPDIR"
 parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE
 
-TEST_NAME="No warnings or errors"
+is "`cat $TMPFILE`" "" "No warnings or errors"
-is "`cat $TMPFILE`" ""
 
-TEST_NAME="Default opts"
+is "$OPT_STRING_OPT" "" "Default string option"
-is "$OPT_STRING_OPT" ""
+is "$OPT_STRING_OPT2" "foo" "Default string option with default"
-is "$OPT_STRING_OPT2" "foo"
+is "$OPT_TYPELESS_OPTION" "" "Default typeless option"
-is "$OPT_TYPELESS_OPTION" ""
+is "$OPT_NOPTION" "yes" "Defailt neg option"
-is "$OPT_NOPTION" "yes"
+is "$OPT_INT_OPT" "" "Default int option"
-is "$OPT_INT_OPT" ""
+is "$OPT_INT_OPT2" "42" "Default int option with default"
-is "$OPT_INT_OPT2" "42"
+is "$OPT_VERSION" "" "--version"
-is "$OPT_VERSION" ""
 
 # ############################################################################
 # Specify some opts, but use default values for the rest.
@@ -32,14 +30,13 @@ is "$OPT_VERSION" ""
 
 parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt 50 --typeless-option --string-opt bar
 
-TEST_NAME="User-specified opts with defaults"
+is "$OPT_STRING_OPT" "bar" "Specified string option (spec)"
-is "$OPT_STRING_OPT" "bar" # specified
+is "$OPT_STRING_OPT2" "foo" "Default string option with default (spec)"
-is "$OPT_STRING_OPT2" "foo"
+is "$OPT_TYPELESS_OPTION" "yes" "Specified typeless option (spec)"
-is "$OPT_TYPELESS_OPTION" "yes" # specified
+is "$OPT_NOPTION" "yes" "Default neg option (spec)"
-is "$OPT_NOPTION" "yes"
+is "$OPT_INT_OPT" "50" "Specified int option (spec)"
-is "$OPT_INT_OPT" "50" # specified
+is "$OPT_INT_OPT2" "42" "Default int option with default (spec)"
-is "$OPT_INT_OPT2" "42"
+is "$OPT_VERSION" "" "--version (spec)"
-is "$OPT_VERSION" ""
 
 # ############################################################################
 # Negate an option like --no-option.
@@ -47,23 +44,20 @@ is "$OPT_VERSION" ""
 
 parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption
 
-TEST_NAME="Negated option"
+is "$OPT_STRING_OPT" "" "Default string option (neg)"
-is "$OPT_STRING_OPT" ""
+is "$OPT_STRING_OPT2" "foo" "Default string option with default (net)"
-is "$OPT_STRING_OPT2" "foo"
+is "$OPT_TYPELESS_OPTION" "" "Default typeless option (neg)"
-is "$OPT_TYPELESS_OPTION" ""
+is "$OPT_NOPTION" "no" "Negated option (neg)"
-is "$OPT_NOPTION" "no" # negated
+is "$OPT_INT_OPT" "" "Default int option (neg)"
-is "$OPT_INT_OPT" ""
+is "$OPT_INT_OPT2" "42" "Default int option with default (neg)"
-is "$OPT_INT_OPT2" "42"
+is "$OPT_VERSION" "" "--version (neg)"
-is "$OPT_VERSION" ""
 
 # ############################################################################
 # Short form.
 # ############################################################################
 
 parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v
-
+is "$OPT_VERSION" "yes" "Short form"
-TEST_NAME="Short form"
-is "$OPT_VERSION" "yes"
 
 # ############################################################################
 # An unknown option should produce an error.
@@ -74,13 +68,9 @@ is "$OPT_VERSION" "yes"
    parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1
 )
 local err=$?
-TEST_NAME="Non-zero exit on unknown option"
+is "$err" "1" "Non-zero exit on unknown option"
-is "$err" "1"
+cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" "Error on unknown option"
-
-TEST_NAME="Error on unknown option"
-cmd_ok "grep -q 'Unknown option: foo' $TMPFILE"
 
 # ############################################################################
 # Done
 # ############################################################################
-exit

t/lib/bash/tmpdir.sh

@@ -5,40 +5,31 @@ TESTS=9
 source "$LIB_DIR/log_warn_die.sh"
 source "$LIB_DIR/tmpdir.sh"
 
-TEST_NAME="TMPDIR not defined"
+is "$TMPDIR" "" "TMPDIR not defined"
-is "$TMPDIR" ""
 
-TEST_NAME="mk_tmpdir makes secure tmpdir"
 mk_tmpdir
-cmd_ok "test -d $TMPDIR"
+cmd_ok "test -d $TMPDIR" "mk_tmpdir makes secure tmpdir"
 
 tmpdir=$TMPDIR;
 
-TEST_NAME="rm_tmpdir"
 rm_tmpdir
-cmd_ok "test ! -d $tmpdir"
+cmd_ok "test ! -d $tmpdir" "rm_tmpdir"
 
-TEST_NAME="rm_tmpdir resets TMPDIR"
+is "$TMPDIR" "" "rm_tmpdir resets TMPDIR"
-is "$TMPDIR" "" 
 
 # --tmpdir
 OPT_TMPDIR="/tmp/use--tmpdir"
 
-TEST_NAME="TMPDIR not defined"
+is "$TMPDIR" "" "TMPDIR not defined"
-is "$TMPDIR" ""
 
-TEST_NAME="--tmpdir does not exist yet"
+cmd_ok "test ! -d $OPT_TMPDIR" "--tmpdir does not exist yet"
-cmd_ok "test ! -d $OPT_TMPDIR"
 
 mk_tmpdir
-TEST_NAME="mk_tmpdir uses --tmpdir"
+is "$TMPDIR" "/tmp/use--tmpdir" "mk_tmpdir uses --tmpdir"
-is "$TMPDIR" "/tmp/use--tmpdir"
 
-TEST_NAME="mk_tmpdir creates --tmpdir"
+cmd_ok "test -d $TMPDIR" "mk_tmpdir creates --tmpdir"
-cmd_ok "test -d $TMPDIR"
 
 tmpdir=$TMPDIR;
 
-TEST_NAME="rm_tmpdir removes --tmpdir"
 rm_tmpdir
-cmd_ok "test ! -d $tmpdir"
+cmd_ok "test ! -d $tmpdir" "rm_tmpdir removes --tmpdir"

t/lib/samples/bash/collect001.txt

@@ -0,0 +1,34 @@
+2011_12_05-df
+2011_12_05-disk-space
+2011_12_05-diskstats
+2011_12_05-hostname
+2011_12_05-innodbstatus1
+2011_12_05-innodbstatus2
+2011_12_05-interrupts
+2011_12_05-iostat
+2011_12_05-iostat-overall
+2011_12_05-log_error
+2011_12_05-lsof
+2011_12_05-meminfo
+2011_12_05-mpstat
+2011_12_05-mpstat-overall
+2011_12_05-mutex-status1
+2011_12_05-mutex-status2
+2011_12_05-mysqladmin
+2011_12_05-netstat
+2011_12_05-netstat_s
+2011_12_05-opentables1
+2011_12_05-opentables2
+2011_12_05-output
+2011_12_05-processlist1
+2011_12_05-processlist2
+2011_12_05-procstat
+2011_12_05-procvmstat
+2011_12_05-ps
+2011_12_05-slabinfo
+2011_12_05-stacktrace
+2011_12_05-sysctl
+2011_12_05-top
+2011_12_05-variables
+2011_12_05-vmstat
+2011_12_05-vmstat-overall

t/lib/samples/bash/po002.sh

@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+
+:
+
+# ############################################################################
+# Documentation
+# ############################################################################
+:<<'DOCUMENTATION'
+=pod
+
+=head1 NAME
+
+pt-stalk - Wait for a condition to occur then begin collecting data.
+
+=head1 OPTIONS
+
+=over
+
+=item --collect
+
+default: yes; negatable: yes
+
+Collect system information.
+
+=item --collect-gdb
+
+Collect GDB stacktraces.
+
+=item --collect-oprofile
+
+Collect oprofile data.
+
+=item --collect-strace
+
+Collect strace data.
+
+=item --collect-tcpdump
+
+Collect tcpdump data.
+
+=item --cycles
+
+type: int; default: 5
+
+Number of times condition must be met before triggering collection.
+
+=item --daemonize
+
+default: yes; negatable: yes
+
+Daemonize the tool.
+
+=item --dest
+
+type: string
+
+Where to store collected data.
+
+=item --disk-byte-limit
+
+type: int; default: 100
+
+Exit if the disk has less than this many MB free.
+
+=item --disk-pct-limit
+
+type: int; default: 5
+
+Exit if the disk is less than this %full.
+
+=item --execute-command
+
+type: string; default: pt-collect
+
+Location of the C<pt-collect> tool.
+
+=item --function
+
+type: string; default: status
+
+Built-in function name or plugin file name which returns the value of C<VARIABLE>.
+
+Possible values are:
+
+=over
+
+=item * status
+
+Grep the value of C<VARIABLE> from C<mysqladmin extended-status>.
+
+=item * processlist
+
+Count the number of processes in C<mysqladmin processlist> whose
+C<VARIABLE> column matches C<MATCH>.  For example:
+
+   TRIGGER_FUNCTION="processlist" \
+   VARIABLE="State"               \
+   MATCH="statistics"             \
+   THRESHOLD="10"
+
+The above triggers when more than 10 processes are in the "statistics" state.
+C<MATCH> must be specified for this trigger function.
+
+=item * magic
+
+TODO
+
+=item * plugin file name
+
+A plugin file allows you to specify a custom trigger function.  The plugin
+file must contain a function called C<trg_plugin>.  For example:
+
+   trg_plugin() {
+      # Do some stuff.
+      echo "$value"
+   }
+
+The last output if the function (its "return value") must be a number.
+This number is compared to C<THRESHOLD>.  All L<"ENVIRONMENT"> variables
+are available to the function.
+
+Do not alter the tool's existing global variables.  Prefix any plugin-specific
+global variables with "PLUGIN_".
+
+=back
+
+=item --help
+
+Print help and exit.
+
+=item --interval
+
+type: int; default: 1
+
+Interval between checks.
+
+=item --iterations
+
+type: int
+
+Exit after triggering C<pt-collect> this many times.  By default, the tool
+will collect as many times as it's triggered.
+
+=item --log
+
+type: string; default: /var/log/pt-stalk.log
+
+Print all output to this file when daemonized.
+
+=item --match
+
+type: string
+
+Match pattern for C<processles> L<"--function">.
+
+=item --notify-by-email
+
+type: string
+
+Send mail to this list of addresses when C<pt-collect> triggers.
+
+=item --pid FILE
+
+type: string; default: /var/run/pt-stalk.pid
+
+Create a PID file when daemonized.
+
+=item --retention-time
+
+type: int; default: 30
+
+Remove samples after this many days.
+
+=item --run-time
+
+type: int; default: 30
+
+How long to collect statistics data for?
+
+Make sure that this isn't longer than SLEEP.
+
+=item --sleep
+
+type: int; default: 300
+
+How long to sleep after collecting?
+
+=item --threshold N
+
+type: int; default: 25
+
+Max number of C<N> to tolerate.
+
+=item --variable NAME
+
+type: string; default: Threads_running
+
+This is the thing to check for.
+
+=item --version
+
+Print tool's version and exit.
+
+=back
+
+=head1 ENVIRONMENT
+
+No env vars used.
+
+=cut
+
+DOCUMENTATION

util/test-bash-functions

@@ -48,6 +48,7 @@ fi
 load_tests() {
    local test_files="$@"
    local i=0
+   local n_tests=0
    for t in $test_files; do
       # Return unless the test file is bash.  There may be other types of
       # files in the tool's test dir.
@@ -57,14 +58,16 @@ load_tests() {
       head -n 1 $t | grep -q bash || continue
 
       tests[$i]=$t
+      i=$((i + 1))
+
       number_of_tests=$(grep --max-count 1 '^TESTS=[0-9]' $t | cut -d'=' -f2)
       if [ -z "$number_of_tests" ]; then
-         i=$(( i + 1 ))
+         n_tests=$(( $n_tests + 1 ))
       else
-         i=$(( i + $number_of_tests ))
+         n_tests=$(( $n_tests + $number_of_tests ))
       fi
    done
-   echo "1..$i"
+   echo "1..$n_tests"
 }
 
 # Source a test file to run whatever it contains (hopefully tests!).
@@ -72,8 +75,6 @@ run_test() {
    local t=$1  # test file name, e.g. "group-by-all-01" for pt-diskstats
    rm -rf $TEST_TMPDIR/* >/dev/null 2>&1
 
-   TEST_NUMBER=1  # test number in this test file
-
    # Tests assume that they're being ran from their own dir, so they access
    # sample files like "samples/foo.txt".  So cd to the dir of the test file
    # and run it.  But the test file may have been given as a relative path,
@@ -92,7 +93,7 @@ run_test() {
 # Print a TAP-style test result.
 result() {
    local result=$1
-   local test_name=${TEST_NAME:-"$TEST_NUMBER"}
+   local test_name=$2
    if [ $result -eq 0 ]; then
       echo "ok $testno - $TEST_FILE $test_name"
    else
@@ -104,7 +105,6 @@ result() {
       fi
    fi
    testno=$((testno + 1))
-   TEST_NUMBER=$((TEST_NUMBER + 1))
    return $result
 }
 
@@ -115,23 +115,26 @@ result() {
 no_diff() {
    local got=$1
    local expected=$2
+   local test_name=$3
    test_command="diff $got $expected"
    eval $test_command > $TEST_TMPDIR/failed_result 2>&1
-   result $?
+   result $? "$test_name"
 }
 
 is() {
    local got=$1
    local expected=$2
+   local test_name=$3
    test_command="\"$got\" == \"$expected\""
    test "$got" = "$expected"
-   result $?
+   result $? "$test_name"
 }
 
 cmd_ok() {
    local test_command=$1
+   local test_name=$2
    eval $test_command
-   result $?
+   result $? "$test_name"
 }
 
 # ############################################################################