Implement --daemonize, --log, and --pid. Catch signals. Merge lp:~daniel-nichter/percona-toolkit/bash-tool-libs r117. Set -u globally. Fix --interval specs. Refactor internals more.

This commit is contained in:
Daniel Nichter
2011-12-02 16:19:16 -07:00
4 changed files with 346 additions and 234 deletions

View File

@@ -4,6 +4,8 @@
# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
# notices and disclaimers.
set -u
# ###########################################################################
# log_warn_die package
# This package is a copy without comments from the original. The original
@@ -13,8 +15,6 @@
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
set -u
EXIT_STATUS=0
log() {
@@ -45,8 +45,6 @@ die() {
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
set -u
declare -a ARGV # non-option args (probably input files)
declare EXT_ARGV # everything after -- (args for an external command)
OPT_ERR=${OPT_ERR:""}
@@ -55,17 +53,12 @@ usage() {
local file=$1
local usage=$(grep '^Usage: ' $file)
local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help)
if [ "$OPT_ERR" ]; then
echo "Error: ${OPT_ERR}" >&2
fi
echo $usage >&2
echo >&2
echo "Options:" >&2
echo >&2
cat $TMPDIR/help >&2
echo >&2
echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2
}
@@ -73,57 +66,79 @@ parse_options() {
local file=$1
shift
local opt=""
local val=""
local default=""
local version=""
local i=0
awk '
mkdir $TMPDIR/po/ 2>/dev/null
rm -rf $TMPDIR/po/*
awk -v "po_dir"="$TMPDIR/po" '
/^=head1 OPTIONS/ {
getline
while ($0 !~ /^=head1/) {
if ($0 ~ /^=item --.*/) {
long_opt=substr($2, 3, length($2) - 2)
short_opt=""
required_arg=""
if ($3) {
if ($3 ~ /-[a-z]/)
short_opt=substr($3, 3, length($3) - 3)
else
required_arg=$3
}
if ($4 ~ /[A-Z]/)
required_arg=$4
long_opt = substr($2, 3, length($2) - 2)
spec_file = po_dir "/" long_opt
trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file
getline # blank line
getline # short description line
getline # specs or description
if ($0 ~ /default: /) {
i=index($0, "default: ")
default=substr($0, i + 9, length($0) - (i + 9))
if ($0 ~ /^[a-z]/ ) {
print "long:" long_opt "; " $0 | trf
close(trf)
}
else {
print "long:" long_opt > spec_file
close(spec_file)
}
else
default=""
print long_opt "," short_opt "," required_arg "," default
}
getline
}
exit
}' $file > $TMPDIR/options
}' $file
while read spec; do
opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:])
default=$(echo $spec | cut -d',' -f4)
eval "OPT_${opt}"="$default"
done < <(cat $TMPDIR/options)
for opt_spec in $(ls $TMPDIR/po/); do
local opt=""
local default_val=""
local neg=0
while read line; do
local key=`echo $line | cut -d ':' -f 1`
local val=`echo $line | cut -d ':' -f 2`
case "$key" in
long)
opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:])
;;
default)
default_val="$val"
;;
shortform)
;;
type)
;;
negatable)
if [ "$val" = "yes" ]; then
neg=1
fi
;;
*)
die "Invalid attribute in $TMPDIR/po/$opt_spec: $line"
esac
done < $TMPDIR/po/$opt_spec
if [ -z "$opt" ]; then
die "No long attribute in option spec $TMPDIR/po/$opt_spec"
fi
if [ $neg -eq 1 ]; then
if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
die "Option $opt_spec is negatable but not default: yes"
fi
fi
eval "OPT_${opt}"="$default_val"
done
local i=0 # ARGV index
for opt; do
if [ $# -eq 0 ]; then
break
break # no more opts
fi
opt=$1
if [ "$opt" = "--" ]; then
@@ -140,30 +155,51 @@ parse_options() {
usage $file
exit 0
fi
shift
shift
if [ $(expr "$opt" : "-") -eq 0 ]; then
ARGV[i]="$opt"
i=$((i+1))
continue
fi
opt=$(echo $opt | sed 's/^-*//')
spec=$(grep -E "^$opt,|,$opt," "$TMPDIR/options")
if [ -z "$spec" ]; then
die "Unknown option: $opt"
local real_opt="$opt"
if $(echo $opt | grep -q '^--no-'); then
neg=1
opt=$(echo $opt | sed 's/^--no-//')
else
neg=0
opt=$(echo $opt | sed 's/^-*//')
fi
opt=$(echo $spec | cut -d',' -f1)
required_arg=$(echo $spec | cut -d',' -f3)
val="yes"
if [ -f "$TMPDIR/po/$opt" ]; then
spec="$TMPDIR/po/$opt"
else
spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
die "Unknown option: $opt"
fi
fi
required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2)
if [ -n "$required_arg" ]; then
if [ $# -eq 0 ]; then
die "--$opt requires a $required_arg argument"
die "$real_opt requires a $required_arg argument"
else
val="$1"
shift
fi
else
if [ $neg -eq 0 ]; then
val="yes"
else
val="no"
fi
fi
opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:])
eval "OPT_${opt}"="$val"
opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:])
eval "OPT_$opt"="$val"
done
}
@@ -180,8 +216,6 @@ parse_options() {
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
set -u
TMPDIR=""
OPT_TMPDIR=${OPT_TMPDIR:""}
@@ -219,7 +253,6 @@ ITER=0
# ###########################################################################
# Subroutines
# ###########################################################################
set +u
grep_processlist() {
local file=$1
@@ -308,7 +341,7 @@ oktorun() {
sleep_ok() {
local seconds=$1
local msg=$2
local msg=${2:""}
if oktorun; then
if [ -n "$msg" ]; then
log $msg
@@ -325,6 +358,16 @@ purge_samples() {
:
}
sigtrap() {
if [ $OKTORUN -eq 1 ]; then
warn "Caught signal, exiting"
OKTORUN=0
else
warn "Caught signal again, forcing exit"
exit $EXIT_STATUS
fi
}
collect() {
log "$OPT_COLLECT triggered"
ITER=$((ITER + 1))
@@ -344,34 +387,13 @@ collect() {
-- "$EXT_ARGV"
}
# ###########################################################################
# Main program loop, called below if tool is ran from the command line.
# ###########################################################################
main() {
mk_tmpdir
parse_options $0 "$@"
# Make the collection location
# mkdir -p "$OPT_DEST" || die "Can't make the destination directory"
# test -d "$OPT_DEST" || die "$OPT_DEST isn't a directory"
# test -w "$OPT_DEST" || die "$OPT_DEST isn't writable"
# Test if we have root; warn if not, but it isn't critical.
if [ "$(id -u)" != "0" ]; then
log 'Not running with root privileges!';
fi
stalk() {
# We increment this variable every time that the check is true,
# and set it to 0 if it's false.
local cycles_true=0
local matched="no"
# Set TRIGGER_FUNCTION based on --function.
set_trg_func
while oktorun; do
# This is where we decide whether to execute 'collect'.
# The idea is to generate a number and store into $detected,
# and if $detected > $OPT_THRESHOLD, then we'll execute pt-collect.
@@ -395,26 +417,75 @@ main() {
if [ "$matched" = "yes" -a $cycles_true -ge $OPT_CYCLES ]; then
collect
sleep_ok $OPT_SLEEP "Sleeping $OPT_SLEEP seconds to avoid DOS attack"
sleep_ok "$OPT_SLEEP" "Sleeping $OPT_SLEEP seconds to avoid DOS attack"
else
sleep_ok $OPT_INTERVAL
sleep_ok "$OPT_INTERVAL"
fi
purge_samples
done
}
# Remove the secure tmpdir. This is not actually called because
# this tool runs forever.
# ###########################################################################
# Main program loop, called below if tool is ran from the command line.
# ###########################################################################
main() {
trap sigtrap SIGHUP SIGINT SIGTERM
# Note: $$ is the parent's PID, but we're a child proc.
# Bash 4 has $BASHPID but we can't rely on that. Consequently,
# we don't know our own PID. See the usage of $! below.
log "$0 started"
# Make a secure tmpdir.
mk_tmpdir
# Make the collection location
# mkdir -p "$OPT_DEST" || die "Can't make the destination directory"
# test -d "$OPT_DEST" || die "$OPT_DEST isn't a directory"
# test -w "$OPT_DEST" || die "$OPT_DEST isn't writable"
# Test if we have root; warn if not, but it isn't critical.
if [ "$(id -u)" != "0" ]; then
log 'Not running with root privileges!';
fi
# Set TRIGGER_FUNCTION based on --function.
set_trg_func
# Stalk while oktorun.
stalk
# Remove the secure tmpdir.
rm_tmpdir
log "$0 exit status $EXIT_STATUS"
exit $EXIT_STATUS
}
# Execute the program if it was not included from another file.
# This makes it possible to include without executing, and thus test.
if [ "$(basename "$0")" = "pt-stalk" ] || [ "$(basename "$0")" = "bash" -a "$_" = "$0" ]; then
main "$@"
fi
if [ "$(basename "$0")" = "pt-stalk" ] \
|| [ "$(basename "$0")" = "bash" -a "$_" = "$0" ]; then
exit $EXIT_STATUS
# Parse command line options. We must do this first so we can
# see if --daemonize was specified.
mk_tmpdir
parse_options $0 "$@"
rm_tmpdir
if [ "$OPT_DAEMONIZE" = "yes" ]; then
main "$@" </dev/null 1>>$OPT_LOG 2>&1 &
# The child PID is $BASHPID but that special var is only
# in Bash 4+, so we can't rely on it. Consequently, we
# use $! to get the PID of the child we just forked.
echo "$!" > $OPT_PID
else
main "$@"
fi
fi
# ############################################################################
# Documentation
@@ -608,9 +679,11 @@ global variables with "PLUGIN_".
Print help and exit.
=item --interval SECONDS
=item --interval
Interval between checks. (default: 1)
type: int; default: 1
Interval between checks.
=item --iterations

View File

@@ -46,17 +46,12 @@ usage() {
local file=$1
local usage=$(grep '^Usage: ' $file)
local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help)
if [ "$OPT_ERR" ]; then
echo "Error: ${OPT_ERR}" >&2
fi
echo $usage >&2
echo >&2
echo "Options:" >&2
echo >&2
cat $TMPDIR/help >&2
echo >&2
echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2
}
@@ -77,57 +72,102 @@ parse_options() {
local file=$1
shift
local opt=""
local val=""
local default=""
local version=""
local i=0
awk '
# Parse the program options (po) from the POD. Each option has
# a spec file like:
# $ cat po/string-opt2
# long=string-opt2
# type=string
# default=foo
# That's the spec for --string-opt2. Each line is a key:value pair
# from the option's POD line like "type: string; default: foo".
mkdir $TMPDIR/po/ 2>/dev/null
rm -rf $TMPDIR/po/*
awk -v "po_dir"="$TMPDIR/po" '
/^=head1 OPTIONS/ {
getline
while ($0 !~ /^=head1/) {
if ($0 ~ /^=item --.*/) {
long_opt=substr($2, 3, length($2) - 2)
short_opt=""
required_arg=""
if ($3) {
if ($3 ~ /-[a-z]/)
short_opt=substr($3, 3, length($3) - 3)
else
required_arg=$3
}
if ($4 ~ /[A-Z]/)
required_arg=$4
long_opt = substr($2, 3, length($2) - 2)
spec_file = po_dir "/" long_opt
trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file
getline # blank line
getline # short description line
getline # specs or description
if ($0 ~ /default: /) {
i=index($0, "default: ")
default=substr($0, i + 9, length($0) - (i + 9))
if ($0 ~ /^[a-z]/ ) {
# spec line like "type: int; default: 100"
print "long:" long_opt "; " $0 | trf
close(trf)
}
else {
# no specs, should be description of option
print "long:" long_opt > spec_file
close(spec_file)
}
else
default=""
print long_opt "," short_opt "," required_arg "," default
}
getline
}
exit
}' $file > $TMPDIR/options
}' $file
while read spec; do
opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:])
default=$(echo $spec | cut -d',' -f4)
eval "OPT_${opt}"="$default"
done < <(cat $TMPDIR/options)
# Evaluate the program options into existence as global variables
# transformed like --my-op == $OPT_MY_OP. If an option has a default
# value, it's assigned that value. Else, it's value is an empty string.
for opt_spec in $(ls $TMPDIR/po/); do
local opt=""
local default_val=""
local neg=0
while read line; do
local key=`echo $line | cut -d ':' -f 1`
local val=`echo $line | cut -d ':' -f 2`
case "$key" in
long)
opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:])
;;
default)
default_val="$val"
;;
shortform)
;;
type)
;;
negatable)
if [ "$val" = "yes" ]; then
neg=1
fi
;;
*)
die "Invalid attribute in $TMPDIR/po/$opt_spec: $line"
esac
done < $TMPDIR/po/$opt_spec
if [ -z "$opt" ]; then
die "No long attribute in option spec $TMPDIR/po/$opt_spec"
fi
if [ $neg -eq 1 ]; then
if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
die "Option $opt_spec is negatable but not default: yes"
fi
fi
eval "OPT_${opt}"="$default_val"
done
# Parse the command line options. Anything after -- is put into
# EXT_ARGV. Options must begin with one or two hyphens (--help or -h),
# else the item is put into ARGV (it's probably a filename, directory,
# etc.) The program option specs parsed above are used to valid the
# command line options. All options have already been eval'd into
# existence, but we re-eval opts specified on the command line to update
# the corresponding global variable's value. For example, if --foo has
# a default value 100, then $OPT_FOO=100 already, but if --foo=500 is
# specified on the command line, then we re-eval $OPT_FOO=500 to update
# $OPT_FOO.
local i=0 # ARGV index
for opt; do
if [ $# -eq 0 ]; then
break
break # no more opts
fi
opt=$1
if [ "$opt" = "--" ]; then
@@ -144,30 +184,62 @@ parse_options() {
usage $file
exit 0
fi
shift
shift
if [ $(expr "$opt" : "-") -eq 0 ]; then
# Option does not begin with a hyphen (-), so treat it as
# a filename, directory, etc.
ARGV[i]="$opt"
i=$((i+1))
continue
fi
opt=$(echo $opt | sed 's/^-*//')
spec=$(grep -E "^$opt,|,$opt," "$TMPDIR/options")
if [ -z "$spec" ]; then
die "Unknown option: $opt"
# Save real opt from cmd line for error messages.
local real_opt="$opt"
# Strip leading -- or --no- from option.
if $(echo $opt | grep -q '^--no-'); then
neg=1
opt=$(echo $opt | sed 's/^--no-//')
else
neg=0
opt=$(echo $opt | sed 's/^-*//')
fi
opt=$(echo $spec | cut -d',' -f1)
required_arg=$(echo $spec | cut -d',' -f3)
val="yes"
# Find the option's spec file.
if [ -f "$TMPDIR/po/$opt" ]; then
spec="$TMPDIR/po/$opt"
else
spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
die "Unknown option: $opt"
fi
fi
# Get the value specified for the option, if any. If the opt's spec
# says it has a type, then it requires a value and that value should
# be the next item ($1). Else, typeless options (like --version) are
# either "yes" if specified, else "no" if negatable and --no-opt.
required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2)
if [ -n "$required_arg" ]; then
if [ $# -eq 0 ]; then
die "--$opt requires a $required_arg argument"
die "$real_opt requires a $required_arg argument"
else
val="$1"
shift
fi
else
if [ $neg -eq 0 ]; then
val="yes"
else
val="no"
fi
fi
opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:])
eval "OPT_${opt}"="$val"
# Get and transform the opt's long form. E.g.: -q == --quiet == QUIET.
opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:])
# Re-eval the option to update its global variable value.
eval "OPT_$opt"="$val"
done
}

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
TESTS=37
TESTS=24
TMPFILE="$TEST_TMPDIR/parse-opts-output"
@@ -11,54 +11,59 @@ source "$LIB_DIR/parse_options.sh"
# Parse options from POD using all default values.
# ############################################################################
TMPDIR="$TEST_TMPDIR"
parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE
TEST_NAME="No warnings or errors"
is "`cat $TMPFILE`" ""
TEST_NAME="Default opts"
is "$OPT_THRESHOLD" "100"
is "$OPT_VARIABLE" "Threads_connected"
is "$OPT_CYCLES" "1"
is "$OPT_GDB" "no"
is "$OPT_OPROFILE" "yes"
is "$OPT_STRACE" "no"
is "$OPT_TCPDUMP" "yes"
is "$OPT_EMAIL" ""
is "$OPT_INTERVAL" "30"
is "$OPT_MAYBE_EMPTY" "no"
is "$OPT_COLLECT" "${HOME}/bin/pt-collect"
is "$OPT_DEST" "${HOME}/collected/"
is "$OPT_DURATION" "30"
is "$OPT_SLEEP" "300"
is "$OPT_PCT_THRESHOLD" "95"
is "$OPT_MB_THRESHOLD" "100"
is "$OPT_PURGE" "30"
is "$OPT_STRING_OPT" ""
is "$OPT_STRING_OPT2" "foo"
is "$OPT_TYPELESS_OPTION" ""
is "$OPT_NOPTION" "yes"
is "$OPT_INT_OPT" ""
is "$OPT_INT_OPT2" "42"
is "$OPT_VERSION" ""
# ############################################################################
# Specify some opts, but use default values for the rest.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --threshold 50 --gdb yes --email user@example.com
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt 50 --typeless-option --string-opt bar
TEST_NAME="User-specified opts with defaults"
is "$OPT_THRESHOLD" "50" # specified
is "$OPT_VARIABLE" "Threads_connected"
is "$OPT_CYCLES" "1"
is "$OPT_GDB" "yes" # specified
is "$OPT_OPROFILE" "yes"
is "$OPT_STRACE" "no"
is "$OPT_TCPDUMP" "yes"
is "$OPT_EMAIL" "user@example.com" # specified
is "$OPT_INTERVAL" "30"
is "$OPT_MAYBE_EMPTY" "no"
is "$OPT_COLLECT" "${HOME}/bin/pt-collect"
is "$OPT_DEST" "${HOME}/collected/"
is "$OPT_DURATION" "30"
is "$OPT_SLEEP" "300"
is "$OPT_PCT_THRESHOLD" "95"
is "$OPT_MB_THRESHOLD" "100"
is "$OPT_PURGE" "30"
is "$OPT_STRING_OPT" "bar" # specified
is "$OPT_STRING_OPT2" "foo"
is "$OPT_TYPELESS_OPTION" "yes" # specified
is "$OPT_NOPTION" "yes"
is "$OPT_INT_OPT" "50" # specified
is "$OPT_INT_OPT2" "42"
is "$OPT_VERSION" ""
# ############################################################################
# Negate an option like --no-option.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption
TEST_NAME="Negated option"
is "$OPT_STRING_OPT" ""
is "$OPT_STRING_OPT2" "foo"
is "$OPT_TYPELESS_OPTION" ""
is "$OPT_NOPTION" "no" # negated
is "$OPT_INT_OPT" ""
is "$OPT_INT_OPT2" "42"
is "$OPT_VERSION" ""
# ############################################################################
# Short form.
# ############################################################################
parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v
TEST_NAME="Short form"
is "$OPT_VERSION" "yes"
# ############################################################################
# An unknown option should produce an error.

View File

@@ -90,80 +90,42 @@ See L<"ENVIRONMENT">.
=over
=item --threshold N
=item --string-opt
Max number of C<N> to tolerate. (default: 100)
type: string
=item --variable NAME
String option without a default.
This is the thing to check for. (default: Threads_connected)
=item --string-opt2
=item --cycles N
type: string; default: foo
How many times must the condition be met before the script will fire? (default: 1)
String option with a default.
=item --gdb
=item --typeless-option
Collect GDB stacktraces? (default: no)
Just an option.
=item --oprofile
=item --noption
Collect oprofile data? (default: yes)
default: yes; negatable: yes
=item --strace
=item --int-opt
Collect strace data? (default: no)
type: int
=item --tcpdump
Int option without a default
Collect tcpdump data? (default: yes)
=item --int-opt2
=item --email ADDRESS
type: int; default: 42
Send mail to this list of addresses when the script triggers.
=item --interval SECONDS
This is the interval between checks. (default: 30)
=item --maybe-empty
Result of checks may be empty. (default: no)
If the command you're running to detect the condition is allowed to return
nothing (e.g. a grep line that might not even exist if there's no problem),
then set this to "yes".
=item --collect DIRECTORY
Location of the C<collect> tool. (default: ${HOME}/bin/pt-collect)
=item --dest DIRECTORY
Where to store collected data. (default: ${HOME}/collected/)
=item --duration SECONDS
How long to collect statistics data for? (default: 30)
Make sure that this isn't longer than SLEEP.
=item --sleep SECONDS
How long to sleep after collecting? (default: 300)
=item --pct-threshold PCT
Bail out if the disk is more than this %full. (default: 95)
=item --mb-threshold MEGABYTES
Bail out if the disk has less than this many MB free. (default: 100)
=item --purge DAYS
Remove samples after this many days. (default: 30)
Int option with a default.
=item --version
short form: -v
Print tool's version and exit.
=back