percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-09 07:30:02 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
6f84e4bfd98c4808435df4b652288e2cfdcb35df
percona-toolkit/bin/pt-stalk

846 lines
21 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# This program is part of Percona Toolkit: http://www.percona.com/software/
# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
# notices and disclaimers.
set -u
# ###########################################################################
# log_warn_die package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/bash/log_warn_die.sh
# t/lib/bash/log_warn_die.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
EXIT_STATUS=0
log() {
TS=$(date +%F-%T | tr :- _);
echo "$TS $1"
}
warn() {
log "$1" >&2
EXIT_STATUS=$((EXIT_STATUS | 1))
}
die() {
warn "$1"
exit 1
}
# ###########################################################################
# End log_warn_die package
# ###########################################################################
# ###########################################################################
# parse_options package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/bash/parse_options.sh
# t/lib/bash/parse_options.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
declare -a ARGV # non-option args (probably input files)
declare EXT_ARGV # everything after -- (args for an external command)
OPT_ERR=${OPT_ERR:""}
usage() {
local file=$1
local usage=$(grep '^Usage: ' $file)
if [ "$OPT_ERR" ]; then
echo "Error: ${OPT_ERR}" >&2
fi
echo $usage >&2
echo >&2
echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2
}
parse_options() {
local file=$1
shift
mkdir $TMPDIR/po/ 2>/dev/null
rm -rf $TMPDIR/po/*
awk -v "po_dir"="$TMPDIR/po" '
/^=head1 OPTIONS/ {
getline
while ($0 !~ /^=head1/) {
if ($0 ~ /^=item --.*/) {
long_opt = substr($2, 3, length($2) - 2)
spec_file = po_dir "/" long_opt
trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file
getline # blank line
getline # specs or description
if ($0 ~ /^[a-z]/ ) {
print "long:" long_opt "; " $0 | trf
close(trf)
}
else {
print "long:" long_opt > spec_file
close(spec_file)
}
}
getline
}
exit
}' $file
for opt_spec in $(ls $TMPDIR/po/); do
local opt=""
local default_val=""
local neg=0
while read line; do
local key=`echo $line | cut -d ':' -f 1`
local val=`echo $line | cut -d ':' -f 2`
case "$key" in
long)
opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:])
;;
default)
default_val="$val"
;;
shortform)
;;
type)
;;
negatable)
if [ "$val" = "yes" ]; then
neg=1
fi
;;
*)
die "Invalid attribute in $TMPDIR/po/$opt_spec: $line"
esac
done < $TMPDIR/po/$opt_spec
if [ -z "$opt" ]; then
die "No long attribute in option spec $TMPDIR/po/$opt_spec"
fi
if [ $neg -eq 1 ]; then
if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
die "Option $opt_spec is negatable but not default: yes"
fi
fi
eval "OPT_${opt}"="$default_val"
done
local i=0 # ARGV index
for opt; do
if [ $# -eq 0 ]; then
break # no more opts
fi
opt=$1
if [ "$opt" = "--" ]; then
shift
EXT_ARGV="$@"
break
fi
if [ "$opt" = "--version" ]; then
version=$(grep '^pt-[^ ]\+ [0-9]' $0)
echo "$version"
exit 0
fi
if [ "$opt" = "--help" ]; then
usage $file
exit 0
fi
shift
if [ $(expr "$opt" : "-") -eq 0 ]; then
ARGV[i]="$opt"
i=$((i+1))
continue
fi
local real_opt="$opt"
if $(echo $opt | grep -q '^--no-'); then
neg=1
opt=$(echo $opt | sed 's/^--no-//')
else
neg=0
opt=$(echo $opt | sed 's/^-*//')
fi
if [ -f "$TMPDIR/po/$opt" ]; then
spec="$TMPDIR/po/$opt"
else
spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
die "Unknown option: $opt"
fi
fi
required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2)
if [ -n "$required_arg" ]; then
if [ $# -eq 0 ]; then
die "$real_opt requires a $required_arg argument"
else
val="$1"
shift
fi
else
if [ $neg -eq 0 ]; then
val="yes"
else
val="no"
fi
fi
opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:])
eval "OPT_$opt"="$val"
done
}
# ###########################################################################
# End parse_options package
# ###########################################################################
# ###########################################################################
# tmpdir package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/bash/tmpdir.sh
# t/lib/bash/tmpdir.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
TMPDIR=""
OPT_TMPDIR=${OPT_TMPDIR:""}
mk_tmpdir() {
if [ -n "$OPT_TMPDIR" ]; then
TMPDIR="$OPT_TMPDIR"
if [ ! -d "$TMPDIR" ]; then
mkdir $TMPDIR || die "Cannot make $TMPDIR"
fi
else
local tool=`basename $0`
local pid="$$"
TMPDIR=`mktemp -d /tmp/${tool}.${pid}.XXXXX` \
|| die "Cannot make secure tmpdir"
fi
}
rm_tmpdir() {
if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then
rm -rf $TMPDIR
fi
TMPDIR=""
}
# ###########################################################################
# End tmpdir package
# ###########################################################################
# ###########################################################################
# Global variables
# ###########################################################################
OKTORUN=1
ITER=0
# ###########################################################################
# Subroutines
# ###########################################################################
grep_processlist() {
local file=$1
local col=$2
local pat=${3:-""}
local gt=${4:-0}
local quiet=${5:-0}
awk "
BEGIN {
FS=\"|\"
OFS=\" | \"
n_cols=0
found=0
}
/^\|/ {
if ( n_cols ) {
val=colno_for_name[\"$col\"]
if ((\"$pat\" && match(\$val, \"$pat\")) || ($gt && \$val > $gt) ) {
found++
if (!$quiet) print \$0
}
}
else {
for (i = 1; i <= NF; i++) {
gsub(/^[ ]*/, \"\", \$i)
gsub(/[ ]*$/, \"\", \$i)
if ( \$i != \"\" ) {
name_for_colno[i]=\$i
colno_for_name[\$i]=i
n_cols++
}
}
}
}
END {
if ( found )
exit 0
exit 1
}
" $file
}
set_trg_func() {
if [ -f "$OPT_FUNCTION" ]; then
source $OPT_FUNCTION
TRIGGER_FUNCTION="trg_plugin"
else
TRIGGER_FUNCTION="trg_$OPT_FUNCTION"
fi
}
trg_status() {
local var=$1
mysqladmin "$EXT_ARGV" extended-status | grep "$OPT_VARIABLE" | awk '{print $4}'
}
trg_processlist() {
local var=$1
local tmpfile="$TMPDIR/processlist"
mysqladmin "$EXT_ARGV" processlist > $tmpfile-1
grep_processlist $tmpfile-1 $var $OPT_MATCH 0 0 > $tmpfile-2
wc -l $tmpfile-2 | awk '{print $1}'
rm -rf $tmpfile*
return
}
trg_magic() {
echo "TODO"
return
}
oktorun() {
if [ $OKTORUN -eq 0 ]; then
return 1 # stop running
fi
if [ -n "$OPT_ITERATIONS" ] && [ $ITER -ge $OPT_ITERATIONS ]; then
return 1 # stop running
fi
return 0 # continue running
}
sleep_ok() {
local seconds=$1
local msg=${2:""}
if oktorun; then
if [ -n "$msg" ]; then
log $msg
fi
sleep $seconds
fi
}
purge_samples() {
# Delete things more than $PURGE days old
#find "$OPT_DEST" -type f -mtime +$OPT_PURGE -exec rm -f '{}' \;
#find "/var/lib/oprofile/samples" -type d -name 'pt_collect_*' \
# -depth -mtime +$OPT_PURGE -exec rm -f '{}' \;
:
}
sigtrap() {
if [ $OKTORUN -eq 1 ]; then
warn "Caught signal, exiting"
OKTORUN=0
else
warn "Caught signal again, forcing exit"
exit $EXIT_STATUS
fi
}
collect() {
log "$OPT_COLLECT triggered"
ITER=$((ITER + 1))
# PREFIX="$(date +%F-%T | tr :- _)"
# echo "${NOTE}" > "${DEST}/${PREFIX}-trigger"
# Run pt-collect.
$OPT_EXECUTE_COMMAND \
-i "$OPT_RUN_TIME" \
-g "$OPT_COLLECT_GDB" \
-o "$OPT_COLLECT_OPROFILE" \
-s "$OPT_COLLECT_STRACE" \
-t "$OPT_COLLECT_TCPDUMP" \
-f "$OPT_DISK_PCT_LIMIT" \
-m "$OPT_DISK_BYTE_LIMIT" \
-- "$EXT_ARGV"
}
stalk() {
# We increment this variable every time that the check is true,
# and set it to 0 if it's false.
local cycles_true=0
local matched="no"
while oktorun; do
# This is where we decide whether to execute 'collect'.
# The idea is to generate a number and store into $detected,
# and if $detected > $OPT_THRESHOLD, then we'll execute pt-collect.
local value=$($TRIGGER_FUNCTION $OPT_VARIABLE)
local trg_exit_status=$?
if [ -z "$value" ]; then
# No value. Maybe we failed to connect to MySQL?
warn "Detected value is empty; something failed? Trigger exit status: $trg_exit_status"
matched="no"
cycles_true=0
elif [ $value -gt $OPT_THRESHOLD ]; then
matched="yes"
cycles_true=$(($cycles_true + 1))
else
matched="no"
cycles_true=0
fi
log "Check results: $OPT_VARIABLE=$value, matched=$matched, cycles_true=$cycles_true"
if [ "$matched" = "yes" -a $cycles_true -ge $OPT_CYCLES ]; then
collect
sleep_ok "$OPT_SLEEP" "Sleeping $OPT_SLEEP seconds to avoid DOS attack"
else
sleep_ok "$OPT_INTERVAL"
fi
purge_samples
done
}
# ###########################################################################
# Main program loop, called below if tool is ran from the command line.
# ###########################################################################
main() {
trap sigtrap SIGHUP SIGINT SIGTERM
# Note: $$ is the parent's PID, but we're a child proc.
# Bash 4 has $BASHPID but we can't rely on that. Consequently,
# we don't know our own PID. See the usage of $! below.
log "$0 started"
# Make a secure tmpdir.
mk_tmpdir
# Make the collection location
# mkdir -p "$OPT_DEST" || die "Can't make the destination directory"
# test -d "$OPT_DEST" || die "$OPT_DEST isn't a directory"
# test -w "$OPT_DEST" || die "$OPT_DEST isn't writable"
# Test if we have root; warn if not, but it isn't critical.
if [ "$(id -u)" != "0" ]; then
log 'Not running with root privileges!';
fi
# Set TRIGGER_FUNCTION based on --function.
set_trg_func
# Stalk while oktorun.
stalk
# Remove the secure tmpdir.
rm_tmpdir
log "$0 exit status $EXIT_STATUS"
exit $EXIT_STATUS
}
# Execute the program if it was not included from another file.
# This makes it possible to include without executing, and thus test.
if [ "$(basename "$0")" = "pt-stalk" ] \
|| [ "$(basename "$0")" = "bash" -a "$_" = "$0" ]; then
# Parse command line options. We must do this first so we can
# see if --daemonize was specified.
mk_tmpdir
parse_options $0 "$@"
rm_tmpdir
if [ "$OPT_DAEMONIZE" = "yes" ]; then
main "$@" </dev/null 1>>$OPT_LOG 2>&1 &
# The child PID is $BASHPID but that special var is only
# in Bash 4+, so we can't rely on it. Consequently, we
# use $! to get the PID of the child we just forked.
echo "$!" > $OPT_PID
else
main "$@"
fi
fi
# ############################################################################
# Documentation
# ############################################################################
:<<'DOCUMENTATION'
=pod
=head1 NAME
pt-stalk - Wait for a condition to occur then begin collecting data.
=head1 SYNOPSIS
Usage: pt-stalk [OPTIONS] [-- MYSQL OPTIONS]
pt-stalk watches for a condition to become true, and when it does, executes
a script. By default it executes L<pt-collect>, but that can be customized.
This tool is useful for gathering diagnostic data when an infrequent event
occurs, so an expert person can review the data later.
=head1 RISKS
The following section is included to inform users about the potential risks,
whether known or unknown, of using this tool. The two main categories of risks
are those created by the nature of the tool (e.g. read-only tools vs. read-write
tools) and those created by bugs.
pt-stalk is a read-only tool. It should be very low-risk.
At the time of this release, we know of no bugs that could cause serious harm
to users.
The authoritative source for updated information is always the online issue
tracking system. Issues that affect this tool will be marked as such. You can
see a list of such issues at the following URL:
L<http://www.percona.com/bugs/pt-stalk>.
See also L<"BUGS"> for more information on filing bugs and getting help.
=head1 DESCRIPTION
Although pt-stalk comes pre-configured to do a specific thing, in general
this tool is just a skeleton script for the following flow of actions:
=over
=item 1.
Loop infinitely, sleeping between iterations.
=item 2.
In each iteration, run some command and get the output.
=item 3.
If the command fails or the output is larger than the threshold,
execute the collection script; but do not execute if the destination disk
is too full.
=back
By default, the tool is configured to execute mysqladmin extended-status and
extract the value of the Threads_running variable; if this is greater than
25, it runs the collection script. This is really just placeholder code,
and almost certainly needs to be customized!
If the tool does execute the collection script, it will wait for a while
before checking and executing again. This is to prevent a continuous
condition from causing a huge number of executions to fire off.
The name 'stalk' is because 'watch' is already taken, and 'stalk' is fun.
=head1 CONFIGURING
TODO
=head1 OPTIONS
=over
=item --collect
default: yes; negatable: yes
Collect system information.
=item --collect-gdb
Collect GDB stacktraces.
=item --collect-oprofile
Collect oprofile data.
=item --collect-strace
Collect strace data.
=item --collect-tcpdump
Collect tcpdump data.
=item --cycles
type: int; default: 5
Number of times condition must be met before triggering collection.
=item --daemonize
default: yes; negatable: yes
Daemonize the tool.
=item --dest
type: string
Where to store collected data.
=item --disk-byte-limit
type: int; default: 100
Exit if the disk has less than this many MB free.
=item --disk-pct-limit
type: int; default: 5
Exit if the disk is less than this %full.
=item --execute-command
type: string; default: pt-collect
Location of the C<pt-collect> tool.
=item --function
type: string; default: status
Built-in function name or plugin file name which returns the value of C<VARIABLE>.
Possible values are:
=over
=item * status
Grep the value of C<VARIABLE> from C<mysqladmin extended-status>.
=item * processlist
Count the number of processes in C<mysqladmin processlist> whose
C<VARIABLE> column matches C<MATCH>. For example:
TRIGGER_FUNCTION="processlist" \
VARIABLE="State" \
MATCH="statistics" \
THRESHOLD="10"
The above triggers when more than 10 processes are in the "statistics" state.
C<MATCH> must be specified for this trigger function.
=item * magic
TODO
=item * plugin file name
A plugin file allows you to specify a custom trigger function. The plugin
file must contain a function called C<trg_plugin>. For example:
trg_plugin() {
# Do some stuff.
echo "$value"
}
The last output if the function (its "return value") must be a number.
This number is compared to C<THRESHOLD>. All L<"ENVIRONMENT"> variables
are available to the function.
Do not alter the tool's existing global variables. Prefix any plugin-specific
global variables with "PLUGIN_".
=back
=item --help
Print help and exit.
=item --interval
type: int; default: 1
Interval between checks.
=item --iterations
type: int
Exit after triggering C<pt-collect> this many times. By default, the tool
will collect as many times as it's triggered.
=item --log
type: string; default: /var/log/pt-stalk.log
Print all output to this file when daemonized.
=item --match
type: string
Match pattern for C<processles> L<"--function">.
=item --notify-by-email
type: string
Send mail to this list of addresses when C<pt-collect> triggers.
=item --pid FILE
type: string; default: /var/run/pt-stalk.pid
Create a PID file when daemonized.
=item --retention-time
type: int; default: 30
Remove samples after this many days.
=item --run-time
type: int; default: 30
How long to collect statistics data for?
Make sure that this isn't longer than SLEEP.
=item --sleep
type: int; default: 300
How long to sleep after collecting?
=item --threshold N
type: int; default: 25
Max number of C<N> to tolerate.
=item --variable NAME
type: string; default: Threads_running
This is the thing to check for.
=item --version
Print tool's version and exit.
=back
=head1 ENVIRONMENT
No env vars used.
=head1 SYSTEM REQUIREMENTS
This tool requires Bash v3 or newer.
=head1 BUGS
For a list of known bugs, see L<http://www.percona.com/bugs/pt-stalk>.
Please report bugs at L<https://bugs.launchpad.net/percona-toolkit>.
Include the following information in your bug report:
=over
=item * Complete command-line used to run the tool
=item * Tool L<"--version">
=item * MySQL version of all servers involved
=item * Output from the tool including STDERR
=item * Input files (log/dump/config files, etc.)
=back
If possible, include debugging output by running the tool with C<PTDEBUG>;
see L<"ENVIRONMENT">.
=head1 DOWNLOADING
Visit L<http://www.percona.com/software/percona-toolkit/> to download the
latest release of Percona Toolkit. Or, get the latest release from the
command line:
wget percona.com/get/percona-toolkit.tar.gz
wget percona.com/get/percona-toolkit.rpm
wget percona.com/get/percona-toolkit.deb
You can also get individual tools from the latest release:
wget percona.com/get/TOOL
Replace C<TOOL> with the name of any tool.
=head1 AUTHORS
Baron Schwartz, Justin Swanhart, Fernando Ipar, and Daniel Nichter
=head1 ABOUT PERCONA TOOLKIT
This tool is part of Percona Toolkit, a collection of advanced command-line
tools developed by Percona for MySQL support and consulting. Percona Toolkit
was forked from two projects in June, 2011: Maatkit and Aspersa. Those
projects were created by Baron Schwartz and developed primarily by him and
Daniel Nichter, both of whom are employed by Percona. Visit
L<http://www.percona.com/software/> for more software developed by Percona.
=head1 COPYRIGHT, LICENSE, AND WARRANTY
This program is copyright 2010-2011 Baron Schwartz, 2011 Percona Inc.
Feedback and improvements are welcome.
THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
systems, you can issue `man perlgpl' or `man perlartistic' to read these
licenses.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-stalk 2.0.0
=cut
DOCUMENTATION
Reference in New Issue View Git Blame Copy Permalink