percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-17 17:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge lp:~daniel-nichter/percona-toolkit/bash-tool-libs.

Browse Source
This commit is contained in:
Daniel Nichter
2011-12-27 15:37:09 -07:00
parent eab12a1c81 0056014339
commit 09cd535d3a
21 changed files with 1918 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/bash/alt_cmds.sh Normal file
View File

@@ -0,0 +1,34 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# alt_cmds package
# ###########################################################################
# Package: alt_cmds
# alt_cmds provides alternatives to commands that aren't on all systems.
set -u
# seq N, return 1, ..., 5
_seq() {
local i=$1
awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
}
# ###########################################################################
# End alt_cmds package
# ###########################################################################

294
lib/bash/collect.sh Normal file
View File

@@ -0,0 +1,294 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# collect package
# ###########################################################################
# Package: collect
# collect collects system information.
set -u
# Global variables.
CMD_GDB=${CMD_GDB:-"gdb"}
CMD_IOSTAT=${CMD_IOSTAT:-"iostat"}
CMD_MPSTAT=${CMD_MPSTAT:-"mpstat"}
CMD_MYSQL=${CMD_MSSQL:-"mysql"}
CMD_MYSQLADMIN=${CMD_MYSQL_ADMIN:-"mysqladmin"}
CMD_OPCONTROL=${CMD_OPCONTROL:-"opcontrol"}
CMD_OPREPORT=${CMD_OPREPORT:-"opreport"}
CMD_PMAP=${CMD_PMAP:-"pmap"}
CMD_STRACE=${CMD_STRACE:-"strace"}
CMD_TCPDUMP=${CMD_TCPDUMP:-"tcpdump"}
CMD_VMSTAT=${CMD_VMSTAT:-"vmstat"}
collect() {
local d=$1 # directory to save results in
local p=$2 # prefix for each result file
# Get pidof mysqld; pidof doesn't exist on some systems. We try our best...
local mysqld_pid=$(pidof -s mysqld);
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(pgrep -o -x mysqld);
fi
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(ps -eaf | grep 'mysql[d]' | grep -v mysqld_safe | awk '{print $2}' | head -n1);
fi
# Get memory allocation info before anything else.
if [ -x "$CMD_PMAP" -a "$mysqld_pid" ]; then
if $CMD_PMAP --help 2>&1 | grep -- -x >/dev/null 2>&1 ; then
$CMD_PMAP -x $mysqld_pid > "$d/$p-pmap"
else
# Some pmap's apparently don't support -x (issue 116).
$CMD_PMAP $mysqld_pid > "$d/$p-pmap"
fi
fi
# Getting a GDB stacktrace can be an intensive operation,
# so do this only if necessary.
if [ "$OPT_COLLECT_GDB" = "yes" -a "$mysqld_pid" ]; then
$CMD_GDB \
-ex "set pagination 0" \
-ex "thread apply all bt" \
--batch -p $mysqld_pid \
>> "$d/$p-stacktrace"
else
echo "GDB (--collect-gdb) was not enabled" >> "$d/$p-stacktrace"
fi
# Get MySQL's variables if possible. Then sleep long enough that we probably
# complete SHOW VARIABLES if all's well. (We don't want to run mysql in the
# foreground, because it could hang.)
$CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 &
sleep .2
# Get the major.minor version number. Version 3.23 doesn't matter for our
# purposes, and other releases have x.x.x* version conventions so far.
local mysql_version="$(awk '/^version[^_]/{print substr($2,1,3)}' "$d/$p-variables")"
# Is MySQL logging its errors to a file? If so, tail that file.
local mysql_error_log="$(awk '/log_error/{print $2}' "$d/$p-variables")"
if [ -z "$mysql_error_log" -a "$mysqld_pid" ]; then
# Try getting it from the open filehandle...
mysql_error_log="$(ls -l /proc/$mysqld_pid/fd | awk '/ 2 ->/{print $NF}')"
fi
local tail_error_log_pid=""
if [ "$mysql_error_log" ]; then
echo "The MySQL error log seems to be ${mysql_error_log}"
tail -f "$mysql_error_log" >"$d/$p-log_error" 2>&1 &
tail_error_log_pid=$!
# Send a mysqladmin debug to the server so we can potentially learn about
# locking etc.
$CMD_MYSQLADMIN $EXT_ARGV debug
else
echo "Could not find the MySQL error log"
fi
# Get a sample of these right away, so we can get these without interaction
# with the other commands we're about to run.
local innostat="SHOW /*!40100 ENGINE*/ INNODB STATUS\G"
if [ "${mysql_version}" '>' "5.1" ]; then
local mutex="SHOW ENGINE INNODB MUTEX"
else
local mutex="SHOW MUTEX STATUS"
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 &
open_tables >> "$d/$p-opentables1" 2>&1 &
# If TCP dumping is specified, start that on the server's port.
local tcpdump_pid=""
if [ "$OPT_COLLECT_TCPDUMP" = "yes" ]; then
local port=$(awk '/^port/{print $2}' "$d/$p-variables")
if [ "$port" ]; then
$CMD_TCPDUMP -i any -s 4096 -w "$d/$p-tcpdump" port ${port} &
tcpdump_pid=$!
fi
fi
# Next, start oprofile gathering data during the whole rest of this process.
# The --init should be a no-op if it has already been init-ed.
local have_oprofile="no"
if [ "$OPT_COLLECT_OPROFILE" = "yes" ]; then
if $CMD_OPCONTROL --init; then
$CMD_OPCONTROL --start --no-vmlinux
have_oprofile="yes"
fi
elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then
# Don't run oprofile and strace at the same time.
$CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 &
local strace_pid=$!
fi
# Grab a few general things first. Background all of these so we can start
# them all up as quickly as possible.
ps -eaf >> "$d/$p-ps" 2>&1 &
sysctl -a >> "$d/$p-sysctl" 2>&1 &
top -bn1 >> "$d/$p-top" 2>&1 &
$CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 &
$CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 &
$CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 &
$CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 &
$CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 &
$CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 &
lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 &
# Collect multiple snapshots of the status variables. We use
# mysqladmin -c even though it is buggy and won't stop on its
# own in 5.1 and newer, because there is a chance that we will
# get and keep a connection to the database; in troubled times
# the database tends to exceed max_connections, so reconnecting
# in the loop tends not to work very well.
$CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" 2>&1 &
local mysqladmin_pid=$!
local have_lock_waits_table=0
$CMD_MYSQL $EXT_ARGV -e "SHOW TABLES FROM INFORMATION_SCHEMA" \
| grep -qi "INNODB_LOCK_WAITS"
if [ $? -eq 0 ]; then
have_lock_waits_table=1
fi
# This loop gathers data for the rest of the duration, and defines the time
# of the whole job.
echo "Loop start: $(date +'TS %s.%N %F %T')"
for loopno in $(_seq $OPT_RUN_TIME); do
# We check the disk, but don't exit, because we need to stop jobs if we
# need to exit.
disk_space $d > $d/$p-disk-space
check_disk_space \
$d/$p-disk-space \
"$OPT_DISK_BYTE_LIMIT" \
"$OPT_DISK_PCT_LIMIT" \
|| break
# Synchronize ourselves onto the clock tick, so the sleeps are 1-second
sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}')
local ts="$(date +"TS %s.%N %F %T")"
# Collect the stuff for this cycle
(cat /proc/diskstats 2>&1; echo $ts) >> "$d/$p-diskstats" &
(cat /proc/stat 2>&1; echo $ts) >> "$d/$p-procstat" &
(cat /proc/vmstat 2>&1; echo $ts) >> "$d/$p-procvmstat" &
(cat /proc/meminfo 2>&1; echo $ts) >> "$d/$p-meminfo" &
(cat /proc/slabinfo 2>&1; echo $ts) >> "$d/$p-slabinfo" &
(cat /proc/interrupts 2>&1; echo $ts) >> "$d/$p-interrupts" &
(df -h 2>&1; echo $ts) >> "$d/$p-df" &
(netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" &
(netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" &
($CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \
>> "$d/$p-processlist"
if [ $have_lock_waits_table -eq 1 ]; then
(lock_waits 2>&1; echo $ts) >>"$d/$p-lock-waits"
fi
done
echo "Loop end: $(date +'TS %s.%N %F %T')"
if [ "$have_oprofile" = "yes" ]; then
$CMD_OPCONTROL --stop
$CMD_OPCONTROL --dump
kill $(pidof oprofiled); # TODO: what if system doesn't have pidof?
$CMD_OPCONTROL --save=pt_collect_$p
# Attempt to generate a report; if this fails, then just tell the user
# how to generate the report.
local mysqld_path=$(which mysqld);
if [ "$mysqld_path" -a -f "$mysqld_path" ]; then
$CMD_OPREPORT \
--demangle=smart \
--symbols \
--merge tgid \
session:pt_collect_$p \
"$mysqld_path" \
> "$d/$p-opreport"
else
echo "oprofile data saved to pt_collect_$p; you should be able" \
"to get a report by running something like 'opreport" \
"--demangle=smart --symbols --merge tgid session:pt_collect_$p" \
"/path/to/mysqld'" \
> "$d/$p-opreport"
fi
elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then
kill -s 2 $strace_pid
sleep 1
kill -s 15 $strace_pid
# Sometimes strace leaves threads/processes in T status.
kill -s 18 $mysqld_pid
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 &
open_tables >> "$d/$p-opentables2" 2>&1 &
# Kill backgrounded tasks.
kill $mysqladmin_pid
[ "$tail_error_log_pid" ] && kill $tail_error_log_pid
[ "$tcpdump_pid" ] && kill $tcpdump_pid
# Finally, record what system we collected this data from.
hostname > "$d/$p-hostname"
}
open_tables() {
local open_tables=$($CMD_MYSQLADMIN $EXT_ARGV ext | grep "Open_tables" | awk '{print $4}')
if [ -n "$open_tables" -a $open_tables -le 1000 ]; then
$CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' 2>&1 &
else
echo "Too many open tables: $open_tables"
fi
}
lock_waits() {
local sql1="SELECT
CONCAT('thread ', b.trx_mysql_thread_id, ' from ', p.host) AS who_blocks,
IF(p.command = \"Sleep\", p.time, 0) AS idle_in_trx,
MAX(TIMESTAMPDIFF(SECOND, r.trx_wait_started, CURRENT_TIMESTAMP)) AS max_wait_time,
COUNT(*) AS num_waiters
FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS AS w
INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS b ON b.trx_id = w.blocking_trx_id
INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS r ON r.trx_id = w.requesting_trx_id
LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id
GROUP BY who_blocks ORDER BY num_waiters DESC\G"
$CMD_MYSQL $EXT_ARGV -e "$sql1"
local sql2="SELECT
r.trx_id AS waiting_trx_id,
r.trx_mysql_thread_id AS waiting_thread,
TIMESTAMPDIFF(SECOND, r.trx_wait_started, CURRENT_TIMESTAMP) AS wait_time,
r.trx_query AS waiting_query,
l.lock_table AS waiting_table_lock,
b.trx_id AS blocking_trx_id, b.trx_mysql_thread_id AS blocking_thread,
SUBSTRING(p.host, 1, INSTR(p.host, ':') - 1) AS blocking_host,
SUBSTRING(p.host, INSTR(p.host, ':') +1) AS blocking_port,
IF(p.command = \"Sleep\", p.time, 0) AS idle_in_trx,
b.trx_query AS blocking_query
FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS AS w
INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS b ON b.trx_id = w.blocking_trx_id
INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS r ON r.trx_id = w.requesting_trx_id
INNER JOIN INFORMATION_SCHEMA.INNODB_LOCKS AS l ON w.requested_lock_id = l.lock_id
LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id
ORDER BY wait_time DESC\G"
$CMD_MYSQL $EXT_ARGV -e "$sql2"
}
# ###########################################################################
# End collect package
# ###########################################################################

71
lib/bash/daemon.sh Normal file
View File

@@ -0,0 +1,71 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# daemon package
# ###########################################################################
# Package: daemon
# daemon handles daemon related tasks like checking a PID file.
set -u
# Sub: make_pid_file
# Check and make a PID file.
#
# Arguments:
# file - File to write PID to.
# pid - PID to write into file.
make_pid_file() {
local file=$1
local pid=$2
# Yes there's a race condition here, between checking if the file exists
# and creating it, but it's not important enough to handle.
if [ -f "$file" ]; then
# PID file already exists. See if the pid it contains is still running.
# If yes, then die. Else, the pid file is stale and we can reclaim it.
local old_pid=$(cat $file)
if [ -z "$old_pid" ]; then
# PID file is empty, so be safe and die since we can't check a
# non-existent pid.
die "PID file $file already exists but it is empty"
else
kill -0 $old_pid 2>/dev/null
if [ $? -eq 0 ]; then
die "PID file $file already exists and its PID ($old_pid) is running"
else
echo "Overwriting PID file $file because its PID ($old_pid)" \
"is not running"
fi
fi
fi
# PID file doesn't exist, or it does but its pid is stale.
echo "$pid" > $file
}
remove_pid_file() {
local file=$1
if [ -f "$file" ]; then
rm $file
fi
}
# ###########################################################################
# End daemon package
# ###########################################################################

46
lib/bash/log_warn_die.sh Normal file
View File

@@ -0,0 +1,46 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# log_warn_die package
# ###########################################################################
# Package: log_warn_die
# log_warn_die provides standard log(), warn(), and die() subs.
set -u
# Global variables.
EXIT_STATUS=0
log() {
TS=$(date +%F-%T | tr :- _);
echo "$TS $1"
}
warn() {
log "$1" >&2
EXIT_STATUS=$((EXIT_STATUS | 1))
}
die() {
warn "$1"
exit 1
}
# ###########################################################################
# End log_warn_die package
# ###########################################################################

286
lib/bash/parse_options.sh Normal file
View File

@@ -0,0 +1,286 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# parse_options package
# ###########################################################################
# Package: parse_options
# parse_options parses Perl POD options from Bash tools and creates
# global variables for each option.
set -u
# Global variables. These must be global because declare inside a
# sub will be scoped locally.
ARGV="" # Non-option args (probably input files)
EXT_ARGV="" # Everything after -- (args for an external command)
OPT_ERRS=0 # How many command line option errors
OPT_VERSION="no" # If --version was specified
OPT_HELP="no" # If --help was specified
# Sub: usage
# Print usage (--help) and list the program's options.
#
# Arguments:
# file - Program file with Perl POD which has usage and options.
#
# Required Global Variables:
# TIMDIR - Temp directory set by <set_TMPDIR()>.
# TOOL - Tool's name.
#
# Optional Global Variables:
# OPT_ERR - Command line option error message.
usage() {
local file=$1
local usage=$(grep '^Usage: ' $file)
echo $usage >&2
echo >&2
echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2
}
usage_or_errors() {
local file=$1
if [ "$OPT_VERSION" = "yes" ]; then
local version=$(grep '^pt-[^ ]\+ [0-9]' $file)
echo "$version"
return 1
fi
if [ "$OPT_HELP" = "yes" ]; then
usage "$file"
echo >&2
echo "Command line options:" >&2
echo >&2
for opt in $(ls $TMPDIR/po/); do
local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://')
echo "--$opt" >&2
echo " $desc" >&2
echo >&2
done
return 1
fi
if [ $OPT_ERRS -gt 0 ]; then
echo >&2
usage $file
return 1
fi
# No --help, --version, or errors.
return 0
}
# Sub: parse_options
# Parse Perl POD options from a program file.
#
# Arguments:
# file - Program file with Perl POD options.
#
# Required Global Variables:
# TIMDIR - Temp directory set by <set_TMPDIR()>.
#
# Set Global Variables:
# This sub decalres a global var for each option by uppercasing the
# option, removing the option's leading --, changing all - to _, and
# prefixing with "OPT_". E.g. --foo-bar becomes OPT_FOO_BAR.
parse_options() {
local file=$1
shift
# Parse the program options (po) from the POD. Each option has
# a spec file like:
# $ cat po/string-opt2
# long=string-opt2
# type=string
# default=foo
# That's the spec for --string-opt2. Each line is a key:value pair
# from the option's POD line like "type: string; default: foo".
mkdir $TMPDIR/po/ 2>/dev/null
rm -rf $TMPDIR/po/*
(
export PO_DIR="$TMPDIR/po"
cat $file | perl -ne '
BEGIN { $/ = ""; }
next unless $_ =~ m/^=head1 OPTIONS/;
while ( defined(my $para = <>) ) {
last if $para =~ m/^=head1/;
chomp;
if ( $para =~ m/^=item --(\S+)/ ) {
my $opt = $1;
my $file = "$ENV{PO_DIR}/$opt";
open my $opt_fh, ">", $file or die "Cannot open $file: $!";
printf $opt_fh "long:$opt\n";
$para = <>;
chomp;
if ( $para =~ m/^[a-z ]+:/ ) {
map {
chomp;
my ($attrib, $val) = split(/: /, $_);
printf $opt_fh "$attrib:$val\n";
} split(/; /, $para);
$para = <>;
chomp;
}
my ($desc) = $para =~ m/^([^?.]+)/;
printf $opt_fh "desc:$desc.\n";
close $opt_fh;
}
}
last;
'
)
# Evaluate the program options into existence as global variables
# transformed like --my-op == $OPT_MY_OP. If an option has a default
# value, it's assigned that value. Else, it's value is an empty string.
for opt_spec in $(ls $TMPDIR/po/); do
local opt=""
local default_val=""
local neg=0
while read line; do
local key=`echo $line | cut -d ':' -f 1`
local val=`echo $line | cut -d ':' -f 2`
case "$key" in
long)
opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:])
;;
default)
default_val="$val"
;;
"short form")
;;
type)
;;
desc)
;;
negatable)
if [ "$val" = "yes" ]; then
neg=1
fi
;;
*)
echo "Invalid attribute in $TMPDIR/po/$opt_spec: $line" >&2
exit 1
esac
done < $TMPDIR/po/$opt_spec
if [ -z "$opt" ]; then
echo "No long attribute in option spec $TMPDIR/po/$opt_spec" >&2
exit 1
fi
if [ $neg -eq 1 ]; then
if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
echo "Option $opt_spec is negatable but not default: yes" >&2
exit 1
fi
fi
eval "OPT_${opt}"="$default_val"
done
# Parse the command line options. Anything after -- is put into
# EXT_ARGV. Options must begin with one or two hyphens (--help or -h),
# else the item is put into ARGV (it's probably a filename, directory,
# etc.) The program option specs parsed above are used to valid the
# command line options. All options have already been eval'd into
# existence, but we re-eval opts specified on the command line to update
# the corresponding global variable's value. For example, if --foo has
# a default value 100, then $OPT_FOO=100 already, but if --foo=500 is
# specified on the command line, then we re-eval $OPT_FOO=500 to update
# $OPT_FOO.
for opt; do
if [ $# -eq 0 ]; then
break # no more opts
fi
opt=$1
if [ "$opt" = "--" ]; then
shift
EXT_ARGV="$@"
break
fi
shift
if [ $(expr "$opt" : "-") -eq 0 ]; then
# Option does not begin with a hyphen (-), so treat it as
# a filename, directory, etc.
if [ -z "$ARGV" ]; then
ARGV="$opt"
else
ARGV="$ARGV $opt"
fi
continue
fi
# Save real opt from cmd line for error messages.
local real_opt="$opt"
# Strip leading -- or --no- from option.
if $(echo $opt | grep -q '^--no-'); then
neg=1
opt=$(echo $opt | sed 's/^--no-//')
else
neg=0
opt=$(echo $opt | sed 's/^-*//')
fi
# Find the option's spec file.
if [ -f "$TMPDIR/po/$opt" ]; then
spec="$TMPDIR/po/$opt"
else
spec=$(grep "^short form:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "Unknown option: $real_opt" >&2
continue
fi
fi
# Get the value specified for the option, if any. If the opt's spec
# says it has a type, then it requires a value and that value should
# be the next item ($1). Else, typeless options (like --version) are
# either "yes" if specified, else "no" if negatable and --no-opt.
required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2)
if [ -n "$required_arg" ]; then
if [ $# -eq 0 ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "$real_opt requires a $required_arg argument" >&2
continue
else
val="$1"
shift
fi
else
if [ $neg -eq 0 ]; then
val="yes"
else
val="no"
fi
fi
# Get and transform the opt's long form. E.g.: -q == --quiet == QUIET.
opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:])
# Re-eval the option to update its global variable value.
eval "OPT_$opt"="$val"
done
}
# ###########################################################################
# End parse_options package
# ###########################################################################

80
lib/bash/safeguards.sh Normal file
View File

@@ -0,0 +1,80 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# safeguards package
# ###########################################################################
# Package: safeguards
# safeguards is a collection of function to help avoid blowing things up.
set -u
disk_space() {
local filesystem=${1:-"$PWD"}
# Filesystem 1024-blocks Used Available Capacity Mounted on
# /dev/disk0s2 118153176 94409664 23487512 81% /
df -P -k $filesystem
}
# Sub: check_disk_space
# Check if there is or will be enough disk space. Input is a file
# with output from <disk_space()>, i.e. `df -P -k`. The df output
# must use 1k blocks, but the mb arg from the user is in MB.
#
# Arguments:
# file - File with output from <disk_space()>.
# mb - Minimum MB free.
# pc - Minimum percent free.
# mb_margin - Add this many MB to the real MB used.
#
# Returns:
# 0 if there is/will be enough disk space, else 1.
check_disk_space() {
local file=$1
local mb=${2:-"0"}
local pc=${3:-"0"}
local mb_margin=${4:-"0"}
# Convert MB to KB because the df output should be in 1k blocks.
local kb=$(($mb * 1024))
local kb_margin=$(($mb_margin * 1024))
local kb_used=$(cat $file | awk '/^\//{print $3}');
local kb_free=$(cat $file | awk '/^\//{print $4}');
local pc_used=$(cat $file | awk '/^\//{print $5}' | sed -e 's/%//g');
if [ "$kb_margin" -gt "0" ]; then
local kb_total=$(($kb_used + $kb_free))
kb_used=$(($kb_used + $kb_margin))
kb_free=$(($kb_free - $kb_margin))
pc_used=$(awk "BEGIN { printf(\"%d\", $kb_used/$kb_total * 100) }")
fi
local pc_free=$((100 - $pc_used))
if [ "$kb_free" -le "$kb" -o "$pc_free" -le "$pc" ]; then
warn "Not enough free disk space: ${pc_free}% free, ${kb_free} KB free; wanted more than ${pc}% free or ${kb} KB free"
return 1
fi
return 0
}
# ###########################################################################
# End safeguards package
# ###########################################################################

70
lib/bash/tmpdir.sh Normal file
View File

@@ -0,0 +1,70 @@
# This program is copyright 2011 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# tmpdir package
# ###########################################################################
# Package: tmpdir
# tmpdir make a secure temporary directory using mktemp.
set -u
# Global variables.
TMPDIR=""
# Sub: mk_tmpdir
# Create a secure tmpdir and set TMPDIR.
#
# Optional Arguments:
# dir - User-specified tmpdir (default none).
#
# Set Global Variables:
# TMPDIR - Absolute path of secure temp directory.
mk_tmpdir() {
local dir=${1:-""}
if [ -n "$dir" ]; then
if [ ! -d "$dir" ]; then
mkdir $dir || die "Cannot make tmpdir $dir"
fi
TMPDIR="$dir"
else
local tool=`basename $0`
local pid="$$"
TMPDIR=`mktemp -d /tmp/${tool}.${pid}.XXXXX` \
|| die "Cannot make secure tmpdir"
fi
}
# Sub: rm_tmpdir
# Remove the tmpdir and unset TMPDIR.
#
# Optional Global Variables:
# TMPDIR - TMPDIR set by <mk_tmpdir()>.
#
# Set Global Variables:
# TMPDIR - Set to "".
rm_tmpdir() {
if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then
rm -rf $TMPDIR
fi
TMPDIR=""
}
# ###########################################################################
# End tmpdir package
# ###########################################################################

23
t/lib/bash.t Normal file
View File

@@ -0,0 +1,23 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use PerconaTest;
my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/;
push @ARGV, "$trunk/t/lib/bash/*.sh" unless @ARGV;
$ENV{LIB_DIR} = "$trunk/lib/bash";
$ENV{T_LIB_DIR} = "$trunk/t/lib";
system("$trunk/util/test-bash-functions $trunk/t/lib/samples/bash/dummy.sh @ARGV");
exit;

114
t/lib/bash/collect.sh Normal file
View File

@@ -0,0 +1,114 @@
#!/usr/bin/env bash
TESTS=18
TMPFILE="$TEST_TMPDIR/parse-opts-output"
TMPDIR="$TEST_TMPDIR"
PATH="$PATH:$PERCONA_TOOLKIT_SANDBOX/bin"
mkdir "$TMPDIR/collect" 2>/dev/null
source "$LIB_DIR/log_warn_die.sh"
source "$LIB_DIR/parse_options.sh"
source "$LIB_DIR/safeguards.sh"
source "$LIB_DIR/alt_cmds.sh"
source "$LIB_DIR/collect.sh"
parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf
# Prefix (with path) for the collect files.
local p="$TMPDIR/collect/2011_12_05"
# Default collect, no extras like gdb, tcpdump, etc.
collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1
# Even if this system doesn't have all the cmds, collect should still
# create all the default files.
ls -1 $TMPDIR/collect | sort > $TMPDIR/collect-files
no_diff \
$TMPDIR/collect-files \
$T_LIB_DIR/samples/bash/collect001.txt \
"Default collect files"
cmd_ok \
"grep -q 'Avail' $p-df" \
"df"
# hostname is the last thing collected, so if it's ok,
# then the sub reached its end.
is \
"`cat $p-hostname`" \
"`hostname`" \
"hostname"
cmd_ok \
"grep -q -i 'buffer pool' $p-innodbstatus1" \
"innodbstatus1"
cmd_ok \
"grep -q -i 'buffer pool' $p-innodbstatus2" \
"innodbstatus2"
cmd_ok \
"grep -q 'error log seems to be /tmp/12345/data/mysqld.log' $p-output" \
"Finds MySQL error log"
cmd_ok \
"grep -q 'Status information:' $p-log_error" \
"debug"
cmd_ok \
"grep -q 'COMMAND[ ]\+PID[ ]\+USER' $p-lsof" \
"lsof"
cmd_ok \
"grep -q 'buf0buf.c' $p-mutex-status1" \
"mutex-status1"
cmd_ok \
"grep -q 'buf0buf.c' $p-mutex-status2" \
"mutex-status2"
cmd_ok \
"grep -q '^| Uptime' $p-mysqladmin" \
"mysqladmin ext"
cmd_ok \
"grep -qP 'Database\tTable\tIn_use' $p-opentables1" \
"opentables1"
cmd_ok \
"grep -qP 'Database\tTable\t\In_use' $p-opentables2" \
"opentables2"
cmd_ok \
"grep -q '1. row' $p-processlist" \
"processlist"
cmd_ok \
"grep -q 'mysqld' $p-ps" \
"ps"
cmd_ok \
"grep -qP '^wait_timeout\t\d' $p-variables" \
"variables"
local iters=$(cat $p-df | grep -c '^TS ')
is "$iters" "1" "1 iteration/1s run time"
# ###########################################################################
# Try longer run time.
# ###########################################################################
parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 2 -- --defaults-file=/tmp/12345/my.sandbox.cnf
rm $TMPDIR/collect/*
collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1
local iters=$(cat $p-df | grep -c '^TS ')
is "$iters" "2" "2 iteration/2s run time"
# ############################################################################
# Done
# ############################################################################

65
t/lib/bash/daemon.sh Normal file
View File

@@ -0,0 +1,65 @@
#!/usr/bin/env bash
TESTS=7
TMPDIR="$TEST_TMPDIR"
local file="$TMPDIR/pid-file"
source "$LIB_DIR/log_warn_die.sh"
source "$LIB_DIR/daemon.sh"
cmd_ok \
"test ! -f $file" \
"PID file doesn't exist"
make_pid_file $file $$
cmd_ok \
"test -f $file" \
"PID file created"
local pid=`cat $file`
is \
"$pid" \
"$$" \
"Correct PID"
remove_pid_file $file
cmd_ok \
"test ! -f $file" \
"PID file removed"
# ###########################################################################
# PID file already exists and proc is running.
# ###########################################################################
echo $$ > $file
(
make_pid_file $file $$ >$TMPDIR/output 2>&1
)
cmd_ok \
"grep -q \"PID file /tmp/percona-toolkit.test/pid-file already exists and its PID ($$) is running\" $TMPDIR/output" \
"Does not overwrite PID file is PID is running"
echo 999999 > $file
make_pid_file $file $$ >$TMPDIR/output 2>&1
cmd_ok \
"grep -q 'Overwriting PID file /tmp/percona-toolkit.test/pid-file because its PID (999999) is not running' $TMPDIR/output" \
"Overwrites PID file if PID is not running"
pid=`cat $file`
is \
"$pid" \
"$$" \
"Correct PID"
rm $file
rm $TMPDIR/output
# ###########################################################################
# Done.
# ###########################################################################

87
t/lib/bash/parse_options.sh Normal file
View File

@@ -0,0 +1,87 @@
#!/usr/bin/env bash
TESTS=26
TMPFILE="$TEST_TMPDIR/parse-opts-output"
source "$LIB_DIR/log_warn_die.sh"
source "$LIB_DIR/parse_options.sh"
# ############################################################################
# Parse options from POD using all default values.
# ############################################################################
TOOL="pt-stalk"
TMPDIR="$TEST_TMPDIR"
parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE
is "`cat $TMPFILE`" "" "No warnings or errors"
is "$OPT_STRING_OPT" "" "Default string option"
is "$OPT_STRING_OPT2" "foo" "Default string option with default"
is "$OPT_TYPELESS_OPTION" "" "Default typeless option"
is "$OPT_NOPTION" "yes" "Defailt neg option"
is "$OPT_INT_OPT" "" "Default int option"
is "$OPT_INT_OPT2" "42" "Default int option with default"
is "$OPT_VERSION" "" "--version"
# ############################################################################
# Specify some opts, but use default values for the rest.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt 50 --typeless-option --string-opt bar
is "$OPT_STRING_OPT" "bar" "Specified string option (spec)"
is "$OPT_STRING_OPT2" "foo" "Default string option with default (spec)"
is "$OPT_TYPELESS_OPTION" "yes" "Specified typeless option (spec)"
is "$OPT_NOPTION" "yes" "Default neg option (spec)"
is "$OPT_INT_OPT" "50" "Specified int option (spec)"
is "$OPT_INT_OPT2" "42" "Default int option with default (spec)"
is "$OPT_VERSION" "" "--version (spec)"
# ############################################################################
# Negate an option like --no-option.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption
is "$OPT_STRING_OPT" "" "Default string option (neg)"
is "$OPT_STRING_OPT2" "foo" "Default string option with default (net)"
is "$OPT_TYPELESS_OPTION" "" "Default typeless option (neg)"
is "$OPT_NOPTION" "no" "Negated option (neg)"
is "$OPT_INT_OPT" "" "Default int option (neg)"
is "$OPT_INT_OPT2" "42" "Default int option with default (neg)"
is "$OPT_VERSION" "" "--version (neg)"
# ############################################################################
# Short form.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v
is "$OPT_VERSION" "yes" "Short form"
# ############################################################################
# An unknown option should produce an error.
# ############################################################################
# Have to call this in a subshell because the error will cause an exit.
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1
cmd_ok "grep -q 'Unknown option: --foo' $TMPFILE" "Error on unknown option"
usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1
local err=$?
is "$err" "1" "Non-zero exit on unknown option"
# ###########################################################################
# --help
# ###########################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --help
usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1
no_diff \
"$TMPFILE" \
"$T_LIB_DIR/samples/bash/help001.txt" \
"--help"
# ############################################################################
# Done
# ############################################################################

55
t/lib/bash/safeguards.sh Normal file
View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
TESTS=11
source "$LIB_DIR/log_warn_die.sh"
source "$LIB_DIR/safeguards.sh"
TMPDIR="$TEST_TMPDIR"
SAMPLE="$T_LIB_DIR/samples/bash"
disk_space "/" > $TMPDIR/df-out
cmd_ok \
"grep -q Avail $TMPDIR/df-out" \
"disk_space()"
is \
"`wc -l $TMPDIR/df-out | awk '{print $1}'`" \
"2" \
"2-line df output"
check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 >$TMPDIR/out 2>&1
is "$?" "0" "Enough disk space"
is \
"`cat $TMPDIR/out`" \
"" \
"No output if enough disk space"
check_disk_space "$SAMPLE/diskspace001.txt" 24000 18 >$TMPDIR/out 2>&1
is "$?" "1" "Not enough MB free"
cmd_ok \
"grep -q '19% free, 23487512 KB free; wanted more than 18% free or 24576000 KB free' $TMPDIR/out" \
"Warning if not enough disk space"
check_disk_space "$SAMPLE/diskspace001.txt" 22000 19 >$TMPDIR/out 2>&1
is "$?" "1" "Not enough % free"
# ###########################################################################
# Check with a margin (amount we plan to use in the future).
# ###########################################################################
check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 100
is "$?" "0" "Enough disk space with margin"
check_disk_space "$SAMPLE/diskspace001.txt" 23000 18 100 >$TMPDIR/out 2>&1
is "$?" "1" "Not enough MB free with margin"
check_disk_space "$SAMPLE/diskspace001.txt" 100 5 20000 >$TMPDIR/out 2>&1
is "$?" "1" "Not enough % free with margin"
cmd_ok \
"grep -q '3% free,' $TMPDIR/out" \
"Calculates % free with margin"
# ###########################################################################
# Done
# ###########################################################################

41
t/lib/bash/tmpdir.sh Normal file
View File

@@ -0,0 +1,41 @@
#!/usr/bin/env bash
TESTS=9
source "$LIB_DIR/log_warn_die.sh"
source "$LIB_DIR/tmpdir.sh"
is "$TMPDIR" "" "TMPDIR not defined"
mk_tmpdir
cmd_ok "test -d $TMPDIR" "mk_tmpdir makes secure tmpdir"
tmpdir=$TMPDIR;
rm_tmpdir
cmd_ok "test ! -d $tmpdir" "rm_tmpdir"
is "$TMPDIR" "" "rm_tmpdir resets TMPDIR"
# ###########################################################################
# User-specified tmpdir.
# ###########################################################################
local dir="/tmp/use--tmpdir"
is "$TMPDIR" "" "TMPDIR not defined"
cmd_ok "test ! -d $dir" "--tmpdir does not exist yet"
mk_tmpdir $dir
is "$TMPDIR" "$dir" "mk_tmpdir uses --tmpdir"
cmd_ok "test -d $dir" "mk_tmpdir creates --tmpdir"
rm_tmpdir
cmd_ok "test ! -d $tmpdir" "rm_tmpdir removes --tmpdir"
# ###########################################################################
# Done
# ###########################################################################

33
t/lib/samples/bash/collect001.txt Normal file
View File

@@ -0,0 +1,33 @@
2011_12_05-df
2011_12_05-disk-space
2011_12_05-diskstats
2011_12_05-hostname
2011_12_05-innodbstatus1
2011_12_05-innodbstatus2
2011_12_05-interrupts
2011_12_05-iostat
2011_12_05-iostat-overall
2011_12_05-log_error
2011_12_05-lsof
2011_12_05-meminfo
2011_12_05-mpstat
2011_12_05-mpstat-overall
2011_12_05-mutex-status1
2011_12_05-mutex-status2
2011_12_05-mysqladmin
2011_12_05-netstat
2011_12_05-netstat_s
2011_12_05-opentables1
2011_12_05-opentables2
2011_12_05-output
2011_12_05-processlist
2011_12_05-procstat
2011_12_05-procvmstat
2011_12_05-ps
2011_12_05-slabinfo
2011_12_05-stacktrace
2011_12_05-sysctl
2011_12_05-top
2011_12_05-variables
2011_12_05-vmstat
2011_12_05-vmstat-overall

2
t/lib/samples/bash/diskspace001.txt Normal file
View File

@@ -0,0 +1,2 @@
Filesystem 1024-blocks Used Available Capacity Mounted on
/dev/disk0s2 118153176 94409664 23487512 81% /

4
t/lib/samples/bash/dummy.sh Normal file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
# This is a dummy script for testing the Bash libs. t/lib/bashLibs.t
# calls "util/test-bash-functions dummy.sh <lib-test-files.sh>".

30
t/lib/samples/bash/help001.txt Normal file
View File

@@ -0,0 +1,30 @@
Usage: pt-stalk [OPTIONS] [-- MYSQL_OPTIONS]
For more information, 'man pt-stalk' or 'perldoc /Users/daniel/p/bash-tool-libs/t/lib/samples/bash/po001.sh'.
Command line options:
--help
Print help and exit.
--int-opt
Int option without a default.
--int-opt2
Int option with a default.
--noption
Negatable option.
--string-opt
String option without a default.
--string-opt2
String option with a default.
--typeless-option
Just an option.
--version
Print tool's version and exit.

308
t/lib/samples/bash/po001.sh Normal file
View File

@@ -0,0 +1,308 @@
#!/usr/bin/env bash
# This is a fake script for testing the parse_options.sh lib.
exit 0;
# ############################################################################
# Documentation
# ############################################################################
:<<'DOCUMENTATION'
=pod
=head1 NAME
pt-stalk - Wait for a condition to occur then begin collecting data.
=head1 SYNOPSIS
Usage: pt-stalk [OPTIONS] [-- MYSQL_OPTIONS]
pt-stalk watches for a condition to become true, and when it does, executes
a script. By default it executes L<pt-collect>, but that can be customized.
This tool is useful for gathering diagnostic data when an infrequent event
occurs, so an expert person can review the data later.
=head1 RISKS
The following section is included to inform users about the potential risks,
whether known or unknown, of using this tool. The two main categories of risks
are those created by the nature of the tool (e.g. read-only tools vs. read-write
tools) and those created by bugs.
pt-stalk is a read-only tool. It should be very low-risk.
At the time of this release, we know of no bugs that could cause serious harm
to users.
The authoritative source for updated information is always the online issue
tracking system. Issues that affect this tool will be marked as such. You can
see a list of such issues at the following URL:
L<http://www.percona.com/bugs/pt-stalk>.
See also L<"BUGS"> for more information on filing bugs and getting help.
=head1 DESCRIPTION
Although pt-stalk comes pre-configured to do a specific thing, in general
this tool is just a skeleton script for the following flow of actions:
=over
=item 1.
Loop infinitely, sleeping between iterations.
=item 2.
In each iteration, run some command and get the output.
=item 3.
If the command fails or the output is larger than the threshold,
execute the collection script; but do not execute if the destination disk
is too full.
=back
By default, the tool is configured to execute mysqladmin extended-status and
extract the value of the Threads_connected variable; if this is greater than
100, it runs the collection script. This is really just placeholder code,
and almost certainly needs to be customized!
If the tool does execute the collection script, it will wait for a while
before checking and executing again. This is to prevent a continuous
condition from causing a huge number of executions to fire off.
The name 'stalk' is because 'watch' is already taken, and 'stalk' is fun.
=head1 CONFIGURING
If the file F<pt-stalk.conf> exists in the current working directory, then
L<"ENVIRONMENT"> variables are imported from it. For example, the config
file has the format:
INTERVAL=10
GDB=yes
See L<"ENVIRONMENT">.
=head1 OPTIONS
=over
=item --string-opt
type: string
String option without a default.
=item --string-opt2
type: string; default: foo
String option with a default.
=item --typeless-option
Just an option.
=item --noption
default: yes; negatable: yes
Negatable option.
=item --int-opt
type: int
Int option without a default.
=item --int-opt2
type: int; default: 42
Int option with a default.
=item --version
short form: -v
Print tool's version and exit.
=item --help
Print help and exit.
=back
=head1 ENVIRONMENT
The following environment variables configure how, what, and when the tool
runs. They are all optional and can be specified either on the command line
or in the F<pt-stalk.conf> config file (see L<"CONFIGURING">).
=over
=item THRESHOLD (default 100)
This is the max number of <whatever> we want to tolerate.
=item VARIABLE (default Threads_connected}
This is the thing to check for.
=item CYCLES (default 1)
How many times must the condition be met before the script will fire?
=item GDB (default no)
Collect GDB stacktraces?
=item OPROFILE (default yes)
Collect oprofile data?
=item STRACE (default no)
Collect strace data?
=item TCPDUMP (default yes)
Collect tcpdump data?
=item EMAIL
Send mail to this list of addresses when the script triggers.
=item MYSQLOPTIONS
Any options to pass to mysql/mysqladmin, such as -u, -p, etc
=item INTERVAL (default 30)
This is the interval between checks.
=item MAYBE_EMPTY (default no)
If the command you're running to detect the condition is allowed to return
nothing (e.g. a grep line that might not even exist if there's no problem),
then set this to "yes".
=item COLLECT (default ${HOME}/bin/pt-collect)
This is the location of the 'collect' script.
=item DEST (default ${HOME}/collected/)
This is where to store the collected data.
=item DURATION (default 30)
How long to collect statistics data for? Make sure that this isn't longer
than SLEEP.
=item SLEEP (default DURATION * 10)
How long to sleep after collecting?
=item PCT_THRESHOLD (default 95)
Bail out if the disk is more than this %full.
=item MB_THRESHOLD (default 100)
Bail out if the disk has less than this many MB free.
=item PURGE (default 30)
Remove samples after this many days.
=back
=head1 SYSTEM REQUIREMENTS
This tool requires Bash v3 or newer.
=head1 BUGS
For a list of known bugs, see L<http://www.percona.com/bugs/pt-stalk>.
Please report bugs at L<https://bugs.launchpad.net/percona-toolkit>.
Include the following information in your bug report:
=over
=item * Complete command-line used to run the tool
=item * Tool L<"--version">
=item * MySQL version of all servers involved
=item * Output from the tool including STDERR
=item * Input files (log/dump/config files, etc.)
=back
If possible, include debugging output by running the tool with C<PTDEBUG>;
see L<"ENVIRONMENT">.
=head1 DOWNLOADING
Visit L<http://www.percona.com/software/percona-toolkit/> to download the
latest release of Percona Toolkit. Or, get the latest release from the
command line:
wget percona.com/get/percona-toolkit.tar.gz
wget percona.com/get/percona-toolkit.rpm
wget percona.com/get/percona-toolkit.deb
You can also get individual tools from the latest release:
wget percona.com/get/TOOL
Replace C<TOOL> with the name of any tool.
=head1 AUTHORS
Baron Schwartz, Justin Swanhart, and Fernando Ipar
=head1 ABOUT PERCONA TOOLKIT
This tool is part of Percona Toolkit, a collection of advanced command-line
tools developed by Percona for MySQL support and consulting. Percona Toolkit
was forked from two projects in June, 2011: Maatkit and Aspersa. Those
projects were created by Baron Schwartz and developed primarily by him and
Daniel Nichter, both of whom are employed by Percona. Visit
L<http://www.percona.com/software/> for more software developed by Percona.
=head1 COPYRIGHT, LICENSE, AND WARRANTY
This program is copyright 2010-2011 Baron Schwartz, 2011 Percona Inc.
Feedback and improvements are welcome.
THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
systems, you can issue `man perlgpl' or `man perlartistic' to read these
licenses.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-stalk 1.0.1
=cut
DOCUMENTATION

212
t/lib/samples/bash/po002.sh Normal file
View File

@@ -0,0 +1,212 @@
#!/usr/bin/env bash
:
# ############################################################################
# Documentation
# ############################################################################
:<<'DOCUMENTATION'
=pod
=head1 NAME
pt-stalk - Wait for a condition to occur then begin collecting data.
=head1 OPTIONS
=over
=item --collect
default: yes; negatable: yes
Collect system information.
=item --collect-gdb
Collect GDB stacktraces.
=item --collect-oprofile
Collect oprofile data.
=item --collect-strace
Collect strace data.
=item --collect-tcpdump
Collect tcpdump data.
=item --cycles
type: int; default: 5
Number of times condition must be met before triggering collection.
=item --daemonize
default: yes; negatable: yes
Daemonize the tool.
=item --dest
type: string
Where to store collected data.
=item --disk-byte-limit
type: int; default: 100
Exit if the disk has less than this many MB free.
=item --disk-pct-limit
type: int; default: 5
Exit if the disk is less than this %full.
=item --execute-command
type: string; default: pt-collect
Location of the C<pt-collect> tool.
=item --function
type: string; default: status
Built-in function name or plugin file name which returns the value of C<VARIABLE>.
Possible values are:
=over
=item * status
Grep the value of C<VARIABLE> from C<mysqladmin extended-status>.
=item * processlist
Count the number of processes in C<mysqladmin processlist> whose
C<VARIABLE> column matches C<MATCH>. For example:
TRIGGER_FUNCTION="processlist" \
VARIABLE="State" \
MATCH="statistics" \
THRESHOLD="10"
The above triggers when more than 10 processes are in the "statistics" state.
C<MATCH> must be specified for this trigger function.
=item * magic
TODO
=item * plugin file name
A plugin file allows you to specify a custom trigger function. The plugin
file must contain a function called C<trg_plugin>. For example:
trg_plugin() {
# Do some stuff.
echo "$value"
}
The last output if the function (its "return value") must be a number.
This number is compared to C<THRESHOLD>. All L<"ENVIRONMENT"> variables
are available to the function.
Do not alter the tool's existing global variables. Prefix any plugin-specific
global variables with "PLUGIN_".
=back
=item --help
Print help and exit.
=item --interval
type: int; default: 1
Interval between checks.
=item --iterations
type: int
Exit after triggering C<pt-collect> this many times. By default, the tool
will collect as many times as it's triggered.
=item --log
type: string; default: /var/log/pt-stalk.log
Print all output to this file when daemonized.
=item --match
type: string
Match pattern for C<processles> L<"--function">.
=item --notify-by-email
type: string
Send mail to this list of addresses when C<pt-collect> triggers.
=item --pid FILE
type: string; default: /var/run/pt-stalk.pid
Create a PID file when daemonized.
=item --retention-time
type: int; default: 30
Remove samples after this many days.
=item --run-time
type: int; default: 30
How long to collect statistics data for?
Make sure that this isn't longer than SLEEP.
=item --sleep
type: int; default: 300
How long to sleep after collecting?
=item --threshold N
type: int; default: 25
Max number of C<N> to tolerate.
=item --variable NAME
type: string; default: Threads_running
This is the thing to check for.
=item --version
Print tool's version and exit.
=back
=head1 ENVIRONMENT
No env vars used.
=cut
DOCUMENTATION

44
util/test-bash-functions
View File

@@ -34,9 +34,10 @@ die() {
# Paths
# ############################################################################
TMPDIR="/tmp/percona-toolkit.test"
if [ ! -d $TMPDIR ]; then
mkdir $TMPDIR
# Do not use TMPDIR because the tools use it for their own secure tmpdir.
TEST_TMPDIR="/tmp/percona-toolkit.test"
if [ ! -d $TEST_TMPDIR ]; then
mkdir $TEST_TMPDIR
fi
# ############################################################################
@@ -47,6 +48,7 @@ fi
load_tests() {
local test_files="$@"
local i=0
local n_tests=0
for t in $test_files; do
# Return unless the test file is bash. There may be other types of
# files in the tool's test dir.
@@ -56,22 +58,22 @@ load_tests() {
head -n 1 $t | grep -q bash || continue
tests[$i]=$t
i=$((i + 1))
number_of_tests=$(grep --max-count 1 '^TESTS=[0-9]' $t | cut -d'=' -f2)
if [ -z "$number_of_tests" ]; then
i=$(( i + 1 ))
n_tests=$(( $n_tests + 1 ))
else
i=$(( i + $number_of_tests ))
n_tests=$(( $n_tests + $number_of_tests ))
fi
done
echo "1..$i"
echo "1..$n_tests"
}
# Source a test file to run whatever it contains (hopefully tests!).
run_test() {
local t=$1 # test file name, e.g. "group-by-all-01" for pt-diskstats
rm -rf $TMPDIR/* >/dev/null 2>&1
TEST_NUMBER=1 # test number in this test file
rm -rf $TEST_TMPDIR/* >/dev/null 2>&1
# Tests assume that they're being ran from their own dir, so they access
# sample files like "samples/foo.txt". So cd to the dir of the test file
@@ -91,19 +93,18 @@ run_test() {
# Print a TAP-style test result.
result() {
local result=$1
local test_name=${TEST_NAME:-"$TEST_NUMBER"}
local test_name=$2
if [ $result -eq 0 ]; then
echo "ok $testno - $TEST_FILE $test_name"
else
echo "not ok $testno - $TEST_FILE $test_name"
failed_tests=$(( failed_tests + 1))
echo "# Failed '$test_command'" >&2
if [ -f $TMPDIR/failed_result ]; then
cat $TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2
if [ -f $TEST_TMPDIR/failed_result ]; then
cat $TEST_TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2
fi
fi
testno=$((testno + 1))
TEST_NUMBER=$((TEST_NUMBER + 1))
return $result
}
@@ -114,17 +115,26 @@ result() {
no_diff() {
local got=$1
local expected=$2
local test_name=$3
test_command="diff $got $expected"
eval $test_command > $TMPDIR/failed_result 2>&1
result $?
eval $test_command > $TEST_TMPDIR/failed_result 2>&1
result $? "$test_name"
}
is() {
local got=$1
local expected=$2
local test_name=$3
test_command="\"$got\" == \"$expected\""
test "$got" = "$expected"
result $?
result $? "$test_name"
}
cmd_ok() {
local test_command=$1
local test_name=$2
eval $test_command
result $? "$test_name"
}
# ############################################################################
@@ -157,6 +167,6 @@ for t in "${tests[@]}"; do
run_test $t
done
rm -rf $TMPDIR
rm -rf $TEST_TMPDIR
exit $failed_tests

43
util/update-modules
View File

@@ -49,7 +49,11 @@ file_is_modified() {
pkgs_in_tool() {
local tool=$1
pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1)
if [ "$tool_lang" = "perl" ]; then
pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1)
else
pkgs=$(grep '^# [a-z_]* package' $tool | awk '{print $2}')
fi
}
replace_pkg_in_tool() {
@@ -72,7 +76,8 @@ replace_pkg_in_tool() {
head -n $pkg_start_line $tool_file > $tmp_file
echo "# $pkg package
if [ "$tool_lang" = "perl" ]; then
echo "# $pkg package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/$pkg.pm
@@ -80,11 +85,24 @@ replace_pkg_in_tool() {
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{" >> $tmp_file
else
echo "# $pkg package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/bash/$pkg.sh
# t/lib/bash/$pkg.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
" >> $tmp_file
fi
$BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file
echo "}
# ###########################################################################
if [ "$tool_lang" = "perl" ]; then
echo "}"
fi
echo "# ###########################################################################
# End $pkg package" >> $tmp_file
tail -n +$pkg_end_line $tool_file >> $tmp_file
@@ -97,14 +115,19 @@ replace_pkg_in_tool() {
# ############################################################################
tool_file=$1
if [ -z "$tool_file" ]; then
die "Usage: $0 TOOL [MODULE...]"
fi
if [ ! -f $tool_file ]; then
die "$tool_file does not exist"
fi
if [ -z "$(head -n 1 $tool_file | grep perl)" ]; then
die "$tool_file is not a Perl tool"
if [ -n "$(head -n 1 $tool_file | grep perl)" ]; then
tool_lang="perl"
else
tool_lang="bash"
fi
tool=$(basename $tool_file)
@@ -124,11 +147,17 @@ for pkg in $pkgs; do
continue
fi
pkg_file="$BRANCH/lib/$pkg.pm"
if [ "$tool_lang" = "perl" ]; then
pkg_file="$BRANCH/lib/$pkg.pm"
else
pkg_file="$BRANCH/lib/bash/$pkg.sh"
fi
if [ ! -f $pkg_file ]; then
warn "$pkg_file does not exist"
continue
fi
if file_is_modified $pkg_file; then
warn "$pkg_file has uncommitted changes"
continue