percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-25 05:44:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge daniel changes

Browse Source
This commit is contained in:
baron@percona.com
2012-01-31 11:49:10 -05:00
parent 950a386a72 afd0bf8982
commit f56fbcba8d
8 changed files with 276 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
bin/pt-stalk
View File

@@ -60,7 +60,7 @@ HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV
OPT_ERRS=0 # How many command line option errors
OPT_VERSION="" # If --version was specified
OPT_HELP="" # If --help was specified
PO_DIR="$TMPDIR/po" # Directory with program option spec files
PO_DIR="" # Directory with program option spec files
usage() {
local file="$1"
@@ -133,6 +133,12 @@ usage_or_errors() {
return 0
}
option_error() {
local err="$1"
OPT_ERRS=$(($OPT_ERRS + 1))
echo "$err" >&2
}
parse_options() {
local file="$1"
shift
@@ -273,7 +279,7 @@ _parse_config_files() {
echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue
config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"
config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"
[ "$config_opt" = "" ] && continue
@@ -317,8 +323,7 @@ _parse_command_line() {
if [ "$next_opt_is_val" ]; then
next_opt_is_val=""
if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "$real_opt requires a $required_arg argument" >&2
option_error "$real_opt requires a $required_arg argument"
continue
fi
val="$opt"
@@ -353,8 +358,7 @@ _parse_command_line() {
else
spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "Unknown option: $real_opt" >&2
option_error "Unknown option: $real_opt"
continue
fi
fi
@@ -368,8 +372,7 @@ _parse_command_line() {
fi
else
if [ "$val" ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "Option $real_opt does not take a value" >&2
option_error "Option $real_opt does not take a value"
continue
fi
if [ "$opt_is_negated" ]; then
@@ -464,11 +467,28 @@ rm_tmpdir() {
set -u
CMD_PIDOF="$(which pidof)"
CMD_PGREP="$(which pgrep)"
_seq() {
local i="$1"
awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
}
_pidof() {
local cmd="$1"
if ! pidof "$cmd" 2>/dev/null; then
ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }'
fi
}
_lsof() {
local pid="$1"
if ! lsof -p $pid 2>/dev/null; then
/bin/ls -l /proc/$pid/fd 2>/dev/null
fi
}
# ###########################################################################
# End alt_cmds package
# ###########################################################################
@@ -599,20 +619,17 @@ CMD_OPCONTROL="$(which opcontrol)"
CMD_OPREPORT="$(which opreport)"
CMD_PMAP="$(which pmap)"
CMD_STRACE="$(which strace)"
CMD_SYSCTL="$(which sysctl)"
CMD_TCPDUMP="$(which tcpdump)"
CMD_VMSTAT="$(which vmstat)"
[ -z "$CMD_SYSCTL" -a -x "/sbin/sysctl" ] && CMD_SYSCTL="/sbin/sysctl"
collect() {
local d="$1" # directory to save results in
local p="$2" # prefix for each result file
local mysqld_pid=$(pidof -s mysqld);
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(pgrep -o -x mysqld);
fi
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(ps -eaf | grep 'mysql[d]' | grep -v mysqld_safe | awk '{print $2}' | head -n1);
fi
local mysqld_pid=$(_pidof mysqld | head -n1)
if [ "$CMD_PMAP" -a "$mysqld_pid" ]; then
if $CMD_PMAP --help 2>&1 | grep -- -x >/dev/null 2>&1 ; then
@@ -630,7 +647,7 @@ collect() {
>> "$d/$p-stacktrace"
fi
$CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" &
sleep .2
local mysql_version="$(awk '/^version[^_]/{print substr($2,1,3)}' "$d/$p-variables")"
@@ -642,12 +659,13 @@ collect() {
local tail_error_log_pid=""
if [ "$mysql_error_log" ]; then
echo "The MySQL error log seems to be ${mysql_error_log}"
tail -f "$mysql_error_log" >"$d/$p-log_error" 2>&1 &
log "The MySQL error log seems to be $mysql_error_log"
tail -f "$mysql_error_log" >"$d/$p-log_error" &
tail_error_log_pid=$!
$CMD_MYSQLADMIN $EXT_ARGV debug
else
echo "Could not find the MySQL error log"
log "Could not find the MySQL error log"
fi
local innostat="SHOW /*!40100 ENGINE*/ INNODB STATUS\G"
@@ -656,9 +674,9 @@ collect() {
else
local mutex="SHOW MUTEX STATUS"
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 &
open_tables >> "$d/$p-opentables1" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" &
open_tables >> "$d/$p-opentables1" &
local tcpdump_pid=""
if [ "$CMD_TCPDUMP" -a "$OPT_COLLECT_TCPDUMP" ]; then
@@ -669,45 +687,49 @@ collect() {
fi
fi
local have_oprofile="no"
local have_oprofile=""
if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then
if $CMD_OPCONTROL --init; then
$CMD_OPCONTROL --start --no-vmlinux
have_oprofile="yes"
fi
elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then
$CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 &
elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then
$CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" &
local strace_pid=$!
fi
ps -eaf >> "$d/$p-ps" 2>&1 &
sysctl -a >> "$d/$p-sysctl" 2>&1 &
top -bn1 >> "$d/$p-top" 2>&1 &
lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 &
ps -eaf >> "$d/$p-ps" &
top -bn1 >> "$d/$p-top" &
[ "$mysqld_pid" ] && _lsof $mysqld_pid >> "$d/$p-lsof" &
if [ "$CMD_SYSCTL" ]; then
$CMD_SYSCTL -a >> "$d/$p-sysctl" &
fi
if [ "$CMD_VMSTAT" ]; then
$CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 &
$CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 &
$CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" &
$CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" &
fi
if [ "$CMD_IOSTAT" ]; then
$CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 &
$CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 &
$CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" &
$CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" &
fi
if [ "$CMD_MPSTAT" ]; then
$CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 &
$CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 &
$CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" &
$CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" &
fi
$CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" 2>&1 &
$CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" &
local mysqladmin_pid=$!
local have_lock_waits_table=0
local have_lock_waits_table=""
$CMD_MYSQL $EXT_ARGV -e "SHOW TABLES FROM INFORMATION_SCHEMA" \
| grep -i "INNODB_LOCK_WAITS" >/dev/null 2>&1
if [ $? -eq 0 ]; then
have_lock_waits_table=1
have_lock_waits_table="yes"
fi
echo "Loop start: $(date +'TS %s.%N %F %T')"
log "Loop start: $(date +'TS %s.%N %F %T')"
for loopno in $(_seq $OPT_RUN_TIME); do
disk_space $d > $d/$p-disk-space
check_disk_space \
@@ -719,43 +741,53 @@ collect() {
sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}')
local ts="$(date +"TS %s.%N %F %T")"
if [ -d "/proc" ]; then
if [ -f "/proc/diskstats" ]; then
(cat /proc/diskstats 2>&1; echo $ts) >> "$d/$p-diskstats" &
(echo $ts; cat /proc/diskstats) >> "$d/$p-diskstats" &
fi
if [ -f "/proc/stat" ]; then
(cat /proc/stat 2>&1; echo $ts) >> "$d/$p-procstat" &
(echo $ts; cat /proc/stat) >> "$d/$p-procstat" &
fi
if [ -f "/proc/vmstat" ]; then
(cat /proc/vmstat 2>&1; echo $ts) >> "$d/$p-procvmstat" &
(echo $ts; cat /proc/vmstat) >> "$d/$p-procvmstat" &
fi
if [ -f "/proc/meminfo" ]; then
(cat /proc/meminfo 2>&1; echo $ts) >> "$d/$p-meminfo" &
(echo $ts; cat /proc/meminfo) >> "$d/$p-meminfo" &
fi
if [ -f "/proc/slabinfo" ]; then
(cat /proc/slabinfo 2>&1; echo $ts) >> "$d/$p-slabinfo" &
(echo $ts; cat /proc/slabinfo) >> "$d/$p-slabinfo" &
fi
if [ -f "/proc/interrupts" ]; then
(cat /proc/interrupts 2>&1; echo $ts) >> "$d/$p-interrupts" &
(echo $ts; cat /proc/interrupts) >> "$d/$p-interrupts" &
fi
fi
(df -h 2>&1; echo $ts) >> "$d/$p-df" &
(netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" &
(netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" &
($CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \
>> "$d/$p-processlist"
(echo $ts; df -h) >> "$d/$p-df" &
if [ $have_lock_waits_table -eq 1 ]; then
(lock_waits 2>&1; echo $ts) >>"$d/$p-lock-waits"
(echo $ts; netstat -antp) >> "$d/$p-netstat" &
(echo $ts; netstat -s) >> "$d/$p-netstat_s" &
(echo $ts; $CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G") \
>> "$d/$p-processlist" &
if [ "$have_lock_waits_table" ]; then
(echo $ts; lock_waits) >>"$d/$p-lock-waits" &
fi
done
echo "Loop end: $(date +'TS %s.%N %F %T')"
log "Loop end: $(date +'TS %s.%N %F %T')"
if [ "$have_oprofile" = "yes" ]; then
if [ "$have_oprofile" ]; then
$CMD_OPCONTROL --stop
$CMD_OPCONTROL --dump
kill $(pidof oprofiled); # TODO: what if system doesn't have pidof?
local oprofiled_pid=$(_pidof oprofiled)
if [ "$oprofiled_pid" ]; then
kill $oprofiled_pid
else
warn "Cannot kill oprofiled because its PID cannot be determined"
fi
$CMD_OPCONTROL --save=pt_collect_$p
local mysqld_path=$(which mysqld);
@@ -768,7 +800,7 @@ collect() {
"$mysqld_path" \
> "$d/$p-opreport"
else
echo "oprofile data saved to pt_collect_$p; you should be able" \
log "oprofile data saved to pt_collect_$p; you should be able" \
"to get a report by running something like 'opreport" \
"--demangle=smart --symbols --merge tgid session:pt_collect_$p" \
"/path/to/mysqld'" \
@@ -778,26 +810,33 @@ collect() {
kill -s 2 $strace_pid
sleep 1
kill -s 15 $strace_pid
kill -s 18 $mysqld_pid
[ "$mysqld_pid" ] && kill -s 18 $mysqld_pid
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 &
open_tables >> "$d/$p-opentables2" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" &
open_tables >> "$d/$p-opentables2" &
kill $mysqladmin_pid
[ "$tail_error_log_pid" ] && kill $tail_error_log_pid
[ "$tcpdump_pid" ] && kill $tcpdump_pid
[ "$tcpdump_pid" ] && kill $tcpdump_pid
hostname > "$d/$p-hostname"
for file in "$d/$p-"*; do
if [ -z "$(grep -v '^TS ' --max-count 1 "$file")" ]; then
log "Removing empty file $file";
rm "$file"
fi
done
}
open_tables() {
local open_tables=$($CMD_MYSQLADMIN $EXT_ARGV ext | grep "Open_tables" | awk '{print $4}')
if [ -n "$open_tables" -a $open_tables -le 1000 ]; then
$CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' 2>&1 &
$CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' &
else
echo "Too many open tables: $open_tables"
log "Too many open tables: $open_tables"
fi
}
@@ -841,9 +880,10 @@ lock_waits() {
# ###########################################################################
# Global variables
# ###########################################################################
TRIGGER_FUNCTION=""
RAN_WITH=""
EXIT_REASON=""
TOOL=$(basename $0)
TOOL="pt-stalk"
OKTORUN=1
ITER=1
@@ -896,12 +936,21 @@ grep_processlist() {
}
set_trg_func() {
if [ -f "$OPT_FUNCTION" ]; then
source $OPT_FUNCTION
local func="$1"
if [ -f "$func" ]; then
# Trigger function is a file with Bash code; source it.
. "$func"
TRIGGER_FUNCTION="trg_plugin"
return 0 # success
else
TRIGGER_FUNCTION="trg_$OPT_FUNCTION"
# Trigger function is name of a built-in function.
func=$(echo "$func" | tr [:upper:] [:lower:])
if [ "$func" = "status" -o "$func" = "processlist" ]; then
TRIGGER_FUNCTION="trg_$func"
return 0 # success
fi
fi
return 1 # error
}
trg_status() {
@@ -958,7 +1007,7 @@ purge_samples() {
# "pt_collect_" here needs to match $CMD_OPCONTROL --save=pt_collect_$p
# in collect(). TODO: fix this
find "$oprofile_dir" -type d -name 'pt_collect_*' \
-depth -mtime +$retention_time -exec rm -f '{}' \;
-depth -mtime +$retention_time -exec rm -rf '{}' \;
fi
}
@@ -1042,6 +1091,7 @@ stalk() {
(
collect "$OPT_DEST" "$prefix"
) >> "$OPT_DEST/$prefix-output" 2>&1 &
log "Collector PID $!"
else
# There will not be enough disk space, so do not collect.
warn "Collect canceled because there will not be enough disk space after collecting another $margin MB"
@@ -1087,9 +1137,6 @@ main() {
# Make a secure tmpdir.
mk_tmpdir
# Set TRIGGER_FUNCTION based on --function.
set_trg_func
# Stalk while oktorun.
stalk
@@ -1104,8 +1151,8 @@ main() {
# Execute the program if it was not included from another file.
# This makes it possible to include without executing, and thus test.
if [ "$(basename "$0")" = "pt-stalk" ] \
|| [ "$(basename "$0")" = "bash" -a "$_" = "$0" ]; then
if [ "${0##*/}" = "$TOOL" ] \
|| [ "${0##*/}" = "bash" -a "$_" = "$0" ]; then
# Check that mysql and mysqladmin are in PATH. If not, we're
# already dead in the water, so don't bother with cmd line opts,
@@ -1115,10 +1162,17 @@ if [ "$(basename "$0")" = "pt-stalk" ] \
[ -n "$(mysqladmin --help)" ] \
|| die "Cannot execute mysqladmin. Check that it is in PATH."
# Parse command line options. We must do this first so we can
# see if --daemonize was specified.
mk_tmpdir
parse_options "$0" "$@"
# Verify and set TRIGGER_FUNCTION based on --function.
if ! set_trg_func "$OPT_FUNCTION"; then
option_error "Invalid --function value: $OPT_FUNCTION"
fi
usage_or_errors "$0"
po_status=$?
rm_tmpdir

18
lib/bash/alt_cmds.sh
View File

@@ -23,12 +23,30 @@
set -u
# Global variables.
CMD_PIDOF="$(which pidof)"
CMD_PGREP="$(which pgrep)"
# seq N, return 1, ..., 5
_seq() {
local i="$1"
awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
}
_pidof() {
local cmd="$1"
if ! pidof "$cmd" 2>/dev/null; then
ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }'
fi
}
_lsof() {
local pid="$1"
if ! lsof -p $pid 2>/dev/null; then
/bin/ls -l /proc/$pid/fd 2>/dev/null
fi
}
# ###########################################################################
# End alt_cmds package
# ###########################################################################

141
lib/bash/collect.sh
View File

@@ -33,21 +33,19 @@ CMD_OPCONTROL="$(which opcontrol)"
CMD_OPREPORT="$(which opreport)"
CMD_PMAP="$(which pmap)"
CMD_STRACE="$(which strace)"
CMD_SYSCTL="$(which sysctl)"
CMD_TCPDUMP="$(which tcpdump)"
CMD_VMSTAT="$(which vmstat)"
# Try to find command manually.
[ -z "$CMD_SYSCTL" -a -x "/sbin/sysctl" ] && CMD_SYSCTL="/sbin/sysctl"
collect() {
local d="$1" # directory to save results in
local p="$2" # prefix for each result file
# Get pidof mysqld; pidof doesn't exist on some systems. We try our best...
local mysqld_pid=$(pidof -s mysqld);
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(pgrep -o -x mysqld);
fi
if [ -z "$mysqld_pid" ]; then
mysqld_pid=$(ps -eaf | grep 'mysql[d]' | grep -v mysqld_safe | awk '{print $2}' | head -n1);
fi
# Get pidof mysqld.
local mysqld_pid=$(_pidof mysqld | head -n1)
# Get memory allocation info before anything else.
if [ "$CMD_PMAP" -a "$mysqld_pid" ]; then
@@ -72,7 +70,7 @@ collect() {
# Get MySQL's variables if possible. Then sleep long enough that we probably
# complete SHOW VARIABLES if all's well. (We don't want to run mysql in the
# foreground, because it could hang.)
$CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" &
sleep .2
# Get the major.minor version number. Version 3.23 doesn't matter for our
@@ -88,14 +86,15 @@ collect() {
local tail_error_log_pid=""
if [ "$mysql_error_log" ]; then
echo "The MySQL error log seems to be ${mysql_error_log}"
tail -f "$mysql_error_log" >"$d/$p-log_error" 2>&1 &
log "The MySQL error log seems to be $mysql_error_log"
tail -f "$mysql_error_log" >"$d/$p-log_error" &
tail_error_log_pid=$!
# Send a mysqladmin debug to the server so we can potentially learn about
# locking etc.
$CMD_MYSQLADMIN $EXT_ARGV debug
else
echo "Could not find the MySQL error log"
log "Could not find the MySQL error log"
fi
# Get a sample of these right away, so we can get these without interaction
@@ -106,9 +105,9 @@ collect() {
else
local mutex="SHOW MUTEX STATUS"
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 &
open_tables >> "$d/$p-opentables1" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" &
open_tables >> "$d/$p-opentables1" &
# If TCP dumping is specified, start that on the server's port.
local tcpdump_pid=""
@@ -122,35 +121,39 @@ collect() {
# Next, start oprofile gathering data during the whole rest of this process.
# The --init should be a no-op if it has already been init-ed.
local have_oprofile="no"
local have_oprofile=""
if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then
if $CMD_OPCONTROL --init; then
$CMD_OPCONTROL --start --no-vmlinux
have_oprofile="yes"
fi
elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then
elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then
# Don't run oprofile and strace at the same time.
$CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 &
$CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" &
local strace_pid=$!
fi
# Grab a few general things first. Background all of these so we can start
# them all up as quickly as possible.
ps -eaf >> "$d/$p-ps" 2>&1 &
sysctl -a >> "$d/$p-sysctl" 2>&1 &
top -bn1 >> "$d/$p-top" 2>&1 &
lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 &
ps -eaf >> "$d/$p-ps" &
top -bn1 >> "$d/$p-top" &
[ "$mysqld_pid" ] && _lsof $mysqld_pid >> "$d/$p-lsof" &
if [ "$CMD_SYSCTL" ]; then
$CMD_SYSCTL -a >> "$d/$p-sysctl" &
fi
if [ "$CMD_VMSTAT" ]; then
$CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 &
$CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 &
$CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" &
$CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" &
fi
if [ "$CMD_IOSTAT" ]; then
$CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 &
$CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 &
$CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" &
$CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" &
fi
if [ "$CMD_MPSTAT" ]; then
$CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 &
$CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 &
$CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" &
$CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" &
fi
# Collect multiple snapshots of the status variables. We use
@@ -159,19 +162,19 @@ collect() {
# get and keep a connection to the database; in troubled times
# the database tends to exceed max_connections, so reconnecting
# in the loop tends not to work very well.
$CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" 2>&1 &
$CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" &
local mysqladmin_pid=$!
local have_lock_waits_table=0
local have_lock_waits_table=""
$CMD_MYSQL $EXT_ARGV -e "SHOW TABLES FROM INFORMATION_SCHEMA" \
| grep -i "INNODB_LOCK_WAITS" >/dev/null 2>&1
if [ $? -eq 0 ]; then
have_lock_waits_table=1
have_lock_waits_table="yes"
fi
# This loop gathers data for the rest of the duration, and defines the time
# of the whole job.
echo "Loop start: $(date +'TS %s.%N %F %T')"
log "Loop start: $(date +'TS %s.%N %F %T')"
for loopno in $(_seq $OPT_RUN_TIME); do
# We check the disk, but don't exit, because we need to stop jobs if we
# need to exit.
@@ -186,44 +189,56 @@ collect() {
sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}')
local ts="$(date +"TS %s.%N %F %T")"
# Collect the stuff for this cycle
# #####################################################################
# Collect data for this cycle.
# #####################################################################
if [ -d "/proc" ]; then
if [ -f "/proc/diskstats" ]; then
(cat /proc/diskstats 2>&1; echo $ts) >> "$d/$p-diskstats" &
(echo $ts; cat /proc/diskstats) >> "$d/$p-diskstats" &
fi
if [ -f "/proc/stat" ]; then
(cat /proc/stat 2>&1; echo $ts) >> "$d/$p-procstat" &
(echo $ts; cat /proc/stat) >> "$d/$p-procstat" &
fi
if [ -f "/proc/vmstat" ]; then
(cat /proc/vmstat 2>&1; echo $ts) >> "$d/$p-procvmstat" &
(echo $ts; cat /proc/vmstat) >> "$d/$p-procvmstat" &
fi
if [ -f "/proc/meminfo" ]; then
(cat /proc/meminfo 2>&1; echo $ts) >> "$d/$p-meminfo" &
(echo $ts; cat /proc/meminfo) >> "$d/$p-meminfo" &
fi
if [ -f "/proc/slabinfo" ]; then
(cat /proc/slabinfo 2>&1; echo $ts) >> "$d/$p-slabinfo" &
(echo $ts; cat /proc/slabinfo) >> "$d/$p-slabinfo" &
fi
if [ -f "/proc/interrupts" ]; then
(cat /proc/interrupts 2>&1; echo $ts) >> "$d/$p-interrupts" &
(echo $ts; cat /proc/interrupts) >> "$d/$p-interrupts" &
fi
fi
(df -h 2>&1; echo $ts) >> "$d/$p-df" &
(netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" &
(netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" &
($CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \
>> "$d/$p-processlist"
(echo $ts; df -h) >> "$d/$p-df" &
if [ $have_lock_waits_table -eq 1 ]; then
(lock_waits 2>&1; echo $ts) >>"$d/$p-lock-waits"
(echo $ts; netstat -antp) >> "$d/$p-netstat" &
(echo $ts; netstat -s) >> "$d/$p-netstat_s" &
(echo $ts; $CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G") \
>> "$d/$p-processlist" &
if [ "$have_lock_waits_table" ]; then
(echo $ts; lock_waits) >>"$d/$p-lock-waits" &
fi
done
echo "Loop end: $(date +'TS %s.%N %F %T')"
log "Loop end: $(date +'TS %s.%N %F %T')"
if [ "$have_oprofile" = "yes" ]; then
if [ "$have_oprofile" ]; then
$CMD_OPCONTROL --stop
$CMD_OPCONTROL --dump
kill $(pidof oprofiled); # TODO: what if system doesn't have pidof?
local oprofiled_pid=$(_pidof oprofiled)
if [ "$oprofiled_pid" ]; then
kill $oprofiled_pid
else
warn "Cannot kill oprofiled because its PID cannot be determined"
fi
$CMD_OPCONTROL --save=pt_collect_$p
# Attempt to generate a report; if this fails, then just tell the user
@@ -238,7 +253,7 @@ collect() {
"$mysqld_path" \
> "$d/$p-opreport"
else
echo "oprofile data saved to pt_collect_$p; you should be able" \
log "oprofile data saved to pt_collect_$p; you should be able" \
"to get a report by running something like 'opreport" \
"--demangle=smart --symbols --merge tgid session:pt_collect_$p" \
"/path/to/mysqld'" \
@@ -249,28 +264,40 @@ collect() {
sleep 1
kill -s 15 $strace_pid
# Sometimes strace leaves threads/processes in T status.
kill -s 18 $mysqld_pid
[ "$mysqld_pid" ] && kill -s 18 $mysqld_pid
fi
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 &
open_tables >> "$d/$p-opentables2" 2>&1 &
$CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" &
$CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" &
open_tables >> "$d/$p-opentables2" &
# Kill backgrounded tasks.
kill $mysqladmin_pid
[ "$tail_error_log_pid" ] && kill $tail_error_log_pid
[ "$tcpdump_pid" ] && kill $tcpdump_pid
[ "$tcpdump_pid" ] && kill $tcpdump_pid
# Finally, record what system we collected this data from.
hostname > "$d/$p-hostname"
# Remove "empty" files, i.e. ones that are truly empty or
# just contain timestamp lines. When a command above fails,
# it may leave an empty file.
for file in "$d/$p-"*; do
# If there's not at least 1 line that's not a TS,
# then the file is empty.
if [ -z "$(grep -v '^TS ' --max-count 1 "$file")" ]; then
log "Removing empty file $file";
rm "$file"
fi
done
}
open_tables() {
local open_tables=$($CMD_MYSQLADMIN $EXT_ARGV ext | grep "Open_tables" | awk '{print $4}')
if [ -n "$open_tables" -a $open_tables -le 1000 ]; then
$CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' 2>&1 &
$CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' &
else
echo "Too many open tables: $open_tables"
log "Too many open tables: $open_tables"
fi
}

22
lib/bash/parse_options.sh
View File

@@ -54,7 +54,7 @@ HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV
OPT_ERRS=0 # How many command line option errors
OPT_VERSION="" # If --version was specified
OPT_HELP="" # If --help was specified
PO_DIR="$TMPDIR/po" # Directory with program option spec files
PO_DIR="" # Directory with program option spec files
# Sub: usage
# Print usage (--help) and list the program's options.
@@ -65,9 +65,6 @@ PO_DIR="$TMPDIR/po" # Directory with program option spec files
# Required Global Variables:
# TIMDIR - Temp directory set by <set_TMPDIR()>.
# TOOL - Tool's name.
#
# Optional Global Variables:
# OPT_ERR - Command line option error message.
usage() {
local file="$1"
@@ -140,6 +137,12 @@ usage_or_errors() {
return 0
}
option_error() {
local err="$1"
OPT_ERRS=$(($OPT_ERRS + 1))
echo "$err" >&2
}
# Sub: parse_options
# Parse Perl POD options from a program file.
#
@@ -322,7 +325,7 @@ _parse_config_files() {
# Strip leading and trailing spaces, and spaces around the first =,
# and end-of-line # comments.
config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"
config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"
# Skip blank lines.
[ "$config_opt" = "" ] && continue
@@ -384,8 +387,7 @@ _parse_command_line() {
if [ "$next_opt_is_val" ]; then
next_opt_is_val=""
if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "$real_opt requires a $required_arg argument" >&2
option_error "$real_opt requires a $required_arg argument"
continue
fi
val="$opt"
@@ -425,8 +427,7 @@ _parse_command_line() {
else
spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "Unknown option: $real_opt" >&2
option_error "Unknown option: $real_opt"
continue
fi
fi
@@ -446,8 +447,7 @@ _parse_command_line() {
else
# Option does not take a value.
if [ "$val" ]; then
OPT_ERRS=$(($OPT_ERRS + 1))
echo "Option $real_opt does not take a value" >&2
option_error "Option $real_opt does not take a value"
continue
fi
if [ "$opt_is_negated" ]; then

16
t/lib/bash/collect.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
TESTS=19
TESTS=20
TMPFILE="$TEST_TMPDIR/parse-opts-output"
TMPDIR="$TEST_TMPDIR"
@@ -108,6 +108,20 @@ cmd_ok \
local iters=$(cat $p-df | grep -c '^TS ')
is "$iters" "1" "1 iteration/1s run time"
empty_files=0
for file in $p-*; do
if ! [ -s $file ]; then
empty_files=1
break
fi
if [ -z "$(grep -v '^TS ' --max-count 1 $file)" ]; then
empty_files=1
break
fi
done
is "$empty_files" "0" "No empty files"
# ###########################################################################
# Try longer run time.
# ###########################################################################

11
t/lib/bash/parse_options.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
TESTS=73
TESTS=78
TMPFILE="$TEST_TMPDIR/parse-opts-output"
TOOL="pt-stalk"
@@ -166,6 +166,15 @@ is "$OPT_INT_OPT" "100" "Two --config int option"
is "$ARGV" "" "Two --config ARGV"
is "$EXT_ARGV" "--host=127.1 --user=daniel" "Two--config External ARGV"
# Spaces before and after the option[=value] lines.
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --config $T_LIB_DIR/samples/bash/config004.conf
is "$OPT_STRING_OPT" "foo" "Default string option (spacey)"
is "$OPT_TYPELESS_OPTION" "yes" "Default typeless option (spacey)"
is "$OPT_INT_OPT" "123" "Default int option (spacey)"
is "$ARGV" "" "ARGV (spacey)"
is "$EXT_ARGV" "" "External ARGV (spacey)"
# ############################################################################
# Option values with spaces.
# ############################################################################

3
t/lib/samples/bash/config004.conf Normal file
View File

@@ -0,0 +1,3 @@
typeless-option
int-opt=123
string-opt=foo

9
t/pt-stalk/pt-stalk.t
View File

@@ -24,7 +24,7 @@ if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 20;
plan tests => 21;
}
my $cnf = "/tmp/12345/my.sandbox.cnf";
@@ -169,6 +169,13 @@ like(
"Trigger file logs how pt-stalk was ran"
);
chomp($output = `cat $log_file | grep 'Collector PID'`);
like(
$output,
qr/Collector PID \d+/,
"Collector PID logged"
);
# ###########################################################################
# Triggered but --no-collect.
# ###########################################################################