From 65a3ab515767a52e9e3c156259c2fa1be9741fee Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Sat, 21 Jan 2012 13:59:02 -0700 Subject: [PATCH 01/18] Handle values with spaces. Still needs work; one test is failing. --- lib/bash/parse_options.sh | 196 ++++++++++++++++++------------ t/lib/bash/parse_options.sh | 28 ++++- t/lib/samples/bash/config002.conf | 1 + 3 files changed, 144 insertions(+), 81 deletions(-) create mode 100644 t/lib/samples/bash/config002.conf diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 61344f4f..a9286d6a 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -108,9 +108,13 @@ parse_options() { local file="$1" shift - # Change --op=val to --op val because _parse_command_line() needs - # a space-separated list of "op val op val" etc. - local opts=$(echo "$@" | perl -ne 's/--(\S+)=/--$1 /g, print') + # Reset the globals (mostly for testing). + ARGV="" + EXT_ARGV="" + OPT_ERRS=0 + OPT_VERSION="no" + OPT_HELP="no" + PO_DIR="$TMPDIR/po" if [ ! -d "$PO_DIR" ]; then mkdir "$PO_DIR" @@ -129,7 +133,7 @@ parse_options() { _parse_pod "$file" _eval_po _parse_config_files - _parse_command_line $opts # do NOT quote, we want "--op" "val" not "--op val" + _parse_command_line "$@" } _parse_pod() { @@ -230,16 +234,22 @@ _parse_config_files() { local config_files="/etc/percona-toolkit/percona-toolkit.conf /etc/percona-toolkit/$TOOL.conf $HOME/.percona-toolkit.conf $HOME/.$TOOL.conf" for config_file in $config_files; do test -f "$config_file" || continue - - # The config file syntax is just like a command line except there - # is one option per line. In Bash, --foo --bar is the same as - # --foo - # --bar - # So we can simply cat the config file into/as the command line. - # The Perl changes --foo=bar to --foo bar because _parse_command_line() - # needs a space-separated list of "opt val opt val" etc. - _parse_command_line \ - $(cat "$config_file" | perl -ne 's/--(\S+)=/--$1 /g, print') + local dashdash="" + for conf_opt in $(grep '^[^# ]' "$config_file"); do + if [ "$dashdash" ]; then + if [ "$EXT_ARGV" ]; then + EXT_ARGV="$EXT_ARGV $conf_opt" + else + EXT_ARGV="$conf_opt" + fi + else + _parse_command_line "$conf_opt" + if [ $? -eq 1 ]; then + dashdash=1 + EXT_ARGV="" + fi + fi + done done } @@ -254,79 +264,113 @@ _parse_command_line() { # a default value 100, then $OPT_FOO=100 already, but if --foo=500 is # specified on the command line, then we re-eval $OPT_FOO=500 to update # $OPT_FOO. + local opt="" + local val="" + local next_opt_is_val="" + local opt_is_ok="" + local opt_is_negated="" + local real_opt="" + local required_arg="" + for opt in "$@"; do - if [ $# -eq 0 ]; then - break # no more opts - fi - opt=$1 - if [ "$opt" = "--" ]; then - shift - EXT_ARGV="$@" - break - fi - shift - if [ $(expr "$opt" : "-") -eq 0 ]; then - # Option does not begin with a hyphen (-), so treat it as - # a filename, directory, etc. - if [ -z "$ARGV" ]; then - ARGV="$opt" - else - ARGV="$ARGV $opt" - fi - continue - fi - - # Save real opt from cmd line for error messages. - local real_opt="$opt" - - # Strip leading -- or --no- from option. - if $(echo $opt | grep -q '^--no-'); then - neg=1 - opt=$(echo $opt | sed 's/^--no-//') - else - neg=0 - opt=$(echo $opt | sed 's/^-*//') - fi - - # Find the option's spec file. - if [ -f "$TMPDIR/po/$opt" ]; then - spec="$TMPDIR/po/$opt" - else - spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1) - if [ -z "$spec" ]; then - OPT_ERRS=$(($OPT_ERRS + 1)) - echo "Unknown option: $real_opt" >&2 - continue - fi - fi - - # Get the value specified for the option, if any. If the opt's spec - # says it has a type, then it requires a value and that value should - # be the next item ($1). Else, typeless options (like --version) are - # either "yes" if specified, else "no" if negatable and --no-opt. - local required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}') - if [ -n "$required_arg" ]; then - if [ $# -eq 0 ]; then + if [ "$next_opt_is_val" ]; then + next_opt_is_val="" + if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then OPT_ERRS=$(($OPT_ERRS + 1)) echo "$real_opt requires a $required_arg argument" >&2 continue - else - val="$1" - shift fi + val="$opt" + opt_is_ok=1 else - if [ $neg -eq 0 ]; then - val="yes" + if [ "$opt" = "--" ]; then + EXT_ARGV="$@" + return 1 + fi + + # If option does not begin with a hyphen (-), it's a filename, etc. + if [ $(expr "$opt" : "-") -eq 0 ]; then + if [ -z "$ARGV" ]; then + ARGV="$opt" + else + ARGV="$ARGV $opt" + fi + continue + fi + + # Save real opt from cmd line for error messages. + real_opt="$opt" + + # Strip leading -- or --no- from option. + if $(echo $opt | grep -q '^--no-'); then + opt_is_negated=1 + opt=$(echo $opt | sed 's/^--no-//') else - val="no" + opt_is_negated="" + opt=$(echo $opt | sed 's/^-*//') + fi + + # Split opt=val pair. + if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then + val="$(echo $opt | awk -F= '{print $2}')" + opt="$(echo $opt | awk -F= '{print $1}')" + fi + + # Find the option's spec file. + if [ -f "$TMPDIR/po/$opt" ]; then + spec="$TMPDIR/po/$opt" + else + spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1) + if [ -z "$spec" ]; then + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "Unknown option: $real_opt" >&2 + continue + fi + fi + + # Get the value specified for the option, if any. If the opt's spec + # says it has a type, then it requires a value and that value should + # be the next item ($1). Else, typeless options (like --version) are + # either "yes" if specified, else "no" if negatable and --no-opt. + required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}') + if [ "$required_arg" ]; then + # Option takes a value. + if [ "$val" ]; then + opt_is_ok=1 + else + next_opt_is_val=1 + fi + else + # Option does not take a value. + if [ "$val" ]; then + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "Option $real_opt does not take a value" >&2 + continue + fi + if [ "$opt_is_negated" ]; then + val="" + else + val="yes" + fi + opt_is_ok=1 fi fi - # Get and transform the opt's long form. E.g.: -q == --quiet == QUIET. - opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + if [ "$opt_is_ok" ]; then + # Get and transform the opt's long form. E.g.: -q == --quiet == QUIET. + opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) - # Re-eval the option to update its global variable value. - eval "OPT_$opt"="$val" + # Re-eval the option to update its global variable value. + eval "OPT_$opt"="'$val'" + + opt="" + val="" + next_opt_is_val="" + opt_is_ok="" + opt_is_negated="" + real_opt="" + required_arg="" + fi done } diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 0436f406..9938dc73 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=37 +TESTS=44 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -13,7 +13,7 @@ source "$LIB_DIR/parse_options.sh" # Parse options from POD using all default values. # ############################################################################ -parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE +parse_options "$T_LIB_DIR/samples/bash/po001.sh" 2>$TMPFILE is "`cat $TMPFILE`" "" "No warnings or errors" @@ -38,6 +38,8 @@ is "$OPT_NOPTION" "yes" "Default neg option (spec)" is "$OPT_INT_OPT" "50" "Specified int option (spec)" is "$OPT_INT_OPT2" "42" "Default int option with default (spec)" is "$OPT_VERSION" "" "--version (spec)" +is "$ARGV" "" "ARGV" +is "$EXT_ARGV" "" "External ARGV" # ############################################################################ # --option=value should work like --option value. @@ -56,7 +58,7 @@ parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption is "$OPT_STRING_OPT" "" "Default string option (neg)" is "$OPT_STRING_OPT2" "foo" "Default string option with default (neg)" is "$OPT_TYPELESS_OPTION" "" "Default typeless option (neg)" -is "$OPT_NOPTION" "no" "Negated option (neg)" +is "$OPT_NOPTION" "" "Negated option (neg)" is "$OPT_INT_OPT" "" "Default int option (neg)" is "$OPT_INT_OPT2" "42" "Default int option with default (neg)" is "$OPT_VERSION" "" "--version (neg)" @@ -103,7 +105,7 @@ cmd_ok \ TOOL="pt-test" cp "$T_LIB_DIR/samples/bash/config001.conf" "$HOME/.$TOOL.conf" -parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" +parse_options "$T_LIB_DIR/samples/bash/po001.sh" is "$OPT_STRING_OPT" "abc" "Default string option (conf)" is "$OPT_STRING_OPT2" "foo" "Default string option with default (conf)" @@ -112,16 +114,32 @@ is "$OPT_NOPTION" "yes" "Default neg option (conf)" is "$OPT_INT_OPT" "" "Default int option (conf)" is "$OPT_INT_OPT2" "42" "Default int option with default (conf)" is "$OPT_VERSION" "" "--version (conf)" -is "$EXT_ARGV" "--host 127.1 --user daniel" "External ARGV (conf)" +is "$ARGV" "" "ARGV (conf)" +is "$EXT_ARGV" "--host=127.1 --user=daniel" "External ARGV (conf)" # Command line should override config file. parse_options "$T_LIB_DIR/samples/bash/po001.sh" --string-opt zzz is "$OPT_STRING_OPT" "zzz" "Command line overrides config file" +# ############################################################################ +# Option values with spaces. +# ############################################################################ + +# Config file +cp "$T_LIB_DIR/samples/bash/config002.conf" "$HOME/.$TOOL.conf" +parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" +is "$OPT_STRING_OPT" "hello world" "Option value with space (conf)" + rm "$HOME/.$TOOL.conf" TOOL="pt-stalk" +# Command line +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --string-opt "hello world" +is "$OPT_STRING_OPT" "hello world" "Option value with space (cmd line)" +is "$ARGV" "" "ARGV (cmd line)" +is "$EXT_ARGV" "" "External ARGV (cmd line)" + # ############################################################################ # Done # ############################################################################ diff --git a/t/lib/samples/bash/config002.conf b/t/lib/samples/bash/config002.conf new file mode 100644 index 00000000..0321a4d3 --- /dev/null +++ b/t/lib/samples/bash/config002.conf @@ -0,0 +1 @@ +--string-opt "hello world" From 0d348ce28965b0c4d54c238e46243962048bffc0 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 23 Jan 2012 10:30:42 -0700 Subject: [PATCH 02/18] Make parse_options work with everything: cmd line, config files, values with spaces, op val, op=val, etc. --- docs/percona-toolkit.pod | 24 ++++++++++-- lib/bash/parse_options.sh | 61 +++++++++++++++++++++++++++---- t/lib/bash/parse_options.sh | 9 ++++- t/lib/samples/bash/config001.conf | 4 +- t/lib/samples/bash/config002.conf | 6 ++- 5 files changed, 89 insertions(+), 15 deletions(-) diff --git a/docs/percona-toolkit.pod b/docs/percona-toolkit.pod index 42433fb7..3a677489 100644 --- a/docs/percona-toolkit.pod +++ b/docs/percona-toolkit.pod @@ -182,8 +182,8 @@ The syntax of the configuration files is as follows: =item * -Whitespace followed by a hash (#) sign signifies that the rest of the line is a -comment. This is deleted. +Whitespace followed by a hash sign (#) signifies that the rest of the line is a +comment. This is deleted. For example: =item * @@ -200,7 +200,9 @@ Each line is permitted to be in either of the following formats: option option=value -Whitespace around the equals sign is deleted during processing. +Do not prefix the option with C<-->. Do not quote the values, even if +it has spaces; value are literal. Whitespace around the equals sign is +deleted during processing. =item * @@ -214,6 +216,22 @@ program. =back +=head2 EXAMPLE + +This config file for pt-stalk, + + # Config for pt-stalk + variable=Threads_connected + cycles=2 # trigger if problem seen twice in a row + -- + --user daniel + +is equivalent to this command line: + + pt-stalk --variable Threads_connected --cycles 2 -- --user daniel + +Options after C<--> are passed literally to mysql and mysqladmin. + =head2 READ ORDER The tools read several configuration files in order: diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index a9286d6a..2c8e9290 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -26,6 +26,24 @@ # GLOBAL $TMPDIR AND $TOOL MUST BE SET BEFORE USING THIS LIB! # *********************************************************** +# Parsing command line options with Bash is easy until we have to dealt +# with values that have spaces, e.g. --option="hello world". This is +# further complicated by command line vs. config file. From the command +# line, <--option "hello world"> is put into $@ as "--option", "hello world", +# i.e. 2 args. From a config file, is either 2 args +# split on the space, or 1 arg as a whole line. It needs to be 2 args +# split on the = but this isn't possible; see the note before while read +# in _parse_config_files(). Perl tool config files do not work when the +# value is quoted, so we can't quote it either. And in any case, that +# wouldn't work because then the value would include the literal quotes +# because it's a line from a file, not a command line where Bash will +# interpret the quotes and return a single value in the code. So... + +# *************************************************** +# BE CAREFUL MAKING CHANGES TO THIS LIB AND MAKE SURE +# t/lib/bash/parse_options.sh STILL PASSES! +# *************************************************** + set -u # Global variables. These must be global because declare inside a @@ -231,25 +249,52 @@ _eval_po() { } _parse_config_files() { - local config_files="/etc/percona-toolkit/percona-toolkit.conf /etc/percona-toolkit/$TOOL.conf $HOME/.percona-toolkit.conf $HOME/.$TOOL.conf" - for config_file in $config_files; do + for config_file in "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + do + # Next config file if this one doesn't exist. test -f "$config_file" || continue + + # We've hit a -- in the config file, so just append everything + # else to EXT_ARGV. local dashdash="" - for conf_opt in $(grep '^[^# ]' "$config_file"); do + + # We must use while read because values can contain spaces. + # Else, if we for $(grep ...) then a line like "op=hello world" + # will return 2 values: "op=hello" and "world". If we quote + # the command like for "$(grep ...)" then the entire config + # file is returned as 1 value like "opt=hello world\nop2=42". + while read config_opt; do + + # Skip the line if it begins with a # or is blank. + echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue + + # Strip leading and trailing spaces, and spaces around the first =, + # and end-of-line # comments. + config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" + if [ "$dashdash" ]; then + # Previous line was -- so this and subsequent options are + # really external argvs. if [ "$EXT_ARGV" ]; then - EXT_ARGV="$EXT_ARGV $conf_opt" + EXT_ARGV="$EXT_ARGV $config_opt" else - EXT_ARGV="$conf_opt" + EXT_ARGV="$config_opt" fi else - _parse_command_line "$conf_opt" + # Options in a config file are not prefixed with --, + # but command line options are, so one or the other has + # to add or remove the -- prefix. We add it for config + # files rather than trying to strip it from command line + # options because it's a simpler operation here. + _parse_command_line "--$config_opt" + + # _parse_command_line() returns 1 when it sees --. if [ $? -eq 1 ]; then dashdash=1 EXT_ARGV="" fi fi - done + done < "$config_file" done } @@ -283,7 +328,7 @@ _parse_command_line() { val="$opt" opt_is_ok=1 else - if [ "$opt" = "--" ]; then + if [ "$opt" = "--" -o "$opt" = "----" ]; then EXT_ARGV="$@" return 1 fi diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 9938dc73..4a734533 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=44 +TESTS=46 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -49,6 +49,10 @@ parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt=42 is "$OPT_INT_OPT" "42" "Specified int option (--option=value)" +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --string-opt="hello world" + +is "$OPT_STRING_OPT" "hello world" "Specified int option (--option=\"value\")" + # ############################################################################ # Negate an option like --no-option. # ############################################################################ @@ -128,8 +132,11 @@ is "$OPT_STRING_OPT" "zzz" "Command line overrides config file" # Config file cp "$T_LIB_DIR/samples/bash/config002.conf" "$HOME/.$TOOL.conf" + parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" + is "$OPT_STRING_OPT" "hello world" "Option value with space (conf)" +is "$OPT_INT_OPT" "100" "Option = value # comment (conf)" rm "$HOME/.$TOOL.conf" TOOL="pt-stalk" diff --git a/t/lib/samples/bash/config001.conf b/t/lib/samples/bash/config001.conf index 7681362d..f68cf974 100644 --- a/t/lib/samples/bash/config001.conf +++ b/t/lib/samples/bash/config001.conf @@ -1,5 +1,5 @@ ---string-opt=abc ---typeless-option +string-opt=abc +typeless-option -- --host=127.1 --user=daniel diff --git a/t/lib/samples/bash/config002.conf b/t/lib/samples/bash/config002.conf index 0321a4d3..d4a76af5 100644 --- a/t/lib/samples/bash/config002.conf +++ b/t/lib/samples/bash/config002.conf @@ -1 +1,5 @@ ---string-opt "hello world" +# Line comment. +string-opt=hello world + + +int-opt = 100 # Inline comment. From 034f76d77e147dac11c367a58e0e94c9dd068477 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 23 Jan 2012 10:53:41 -0700 Subject: [PATCH 03/18] Fix handling EXT_ARGV. --- lib/bash/parse_options.sh | 58 ++++++++++++++++++------------------- t/lib/bash/parse_options.sh | 12 +++++++- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 2c8e9290..0576bcd7 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -50,6 +50,7 @@ set -u # sub will be scoped locally. ARGV="" # Non-option args (probably input files) EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV OPT_ERRS=0 # How many command line option errors OPT_VERSION="no" # If --version was specified OPT_HELP="no" # If --help was specified @@ -129,6 +130,7 @@ parse_options() { # Reset the globals (mostly for testing). ARGV="" EXT_ARGV="" + HAVE_EXT_ARGV="" OPT_ERRS=0 OPT_VERSION="no" OPT_HELP="no" @@ -254,10 +256,6 @@ _parse_config_files() { # Next config file if this one doesn't exist. test -f "$config_file" || continue - # We've hit a -- in the config file, so just append everything - # else to EXT_ARGV. - local dashdash="" - # We must use while read because values can contain spaces. # Else, if we for $(grep ...) then a line like "op=hello world" # will return 2 values: "op=hello" and "world". If we quote @@ -272,29 +270,19 @@ _parse_config_files() { # and end-of-line # comments. config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" - if [ "$dashdash" ]; then - # Previous line was -- so this and subsequent options are - # really external argvs. - if [ "$EXT_ARGV" ]; then - EXT_ARGV="$EXT_ARGV $config_opt" - else - EXT_ARGV="$config_opt" - fi - else - # Options in a config file are not prefixed with --, - # but command line options are, so one or the other has - # to add or remove the -- prefix. We add it for config - # files rather than trying to strip it from command line - # options because it's a simpler operation here. - _parse_command_line "--$config_opt" - - # _parse_command_line() returns 1 when it sees --. - if [ $? -eq 1 ]; then - dashdash=1 - EXT_ARGV="" - fi + # Options in a config file are not prefixed with --, + # but command line options are, so one or the other has + # to add or remove the -- prefix. We add it for config + # files rather than trying to strip it from command line + # options because it's a simpler operation here. + if ! [ "$HAVE_EXT_ARGV" ]; then + config_opt="--$config_opt" fi + _parse_command_line "$config_opt" done < "$config_file" + + HAVE_EXT_ARGV="" # reset for each file + done } @@ -318,6 +306,21 @@ _parse_command_line() { local required_arg="" for opt in "$@"; do + if [ "$opt" = "--" -o "$opt" = "----" ]; then + HAVE_EXT_ARGV=1 + continue + fi + if [ "$HAVE_EXT_ARGV" ]; then + # Previous line was -- so this and subsequent options are + # really external argvs. + if [ "$EXT_ARGV" ]; then + EXT_ARGV="$EXT_ARGV $opt" + else + EXT_ARGV="$opt" + fi + continue + fi + if [ "$next_opt_is_val" ]; then next_opt_is_val="" if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then @@ -328,11 +331,6 @@ _parse_command_line() { val="$opt" opt_is_ok=1 else - if [ "$opt" = "--" -o "$opt" = "----" ]; then - EXT_ARGV="$@" - return 1 - fi - # If option does not begin with a hyphen (-), it's a filename, etc. if [ $(expr "$opt" : "-") -eq 0 ]; then if [ -z "$ARGV" ]; then diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 4a734533..e60ead5a 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=46 +TESTS=49 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -74,6 +74,16 @@ is "$OPT_VERSION" "" "--version (neg)" parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v is "$OPT_VERSION" "yes" "Short form" +# ############################################################################ +# Command line options plus externals args. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption -- --foo + +is "$OPT_NOPTION" "" "Negated option (--)" +is "$ARGV" "" "ARGV (--)" +is "$EXT_ARGV" "--foo" "External ARGV (--)" + # ############################################################################ # An unknown option should produce an error. # ############################################################################ From cab3bb6eab5ee415821007d0677422934c2a1570 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 23 Jan 2012 10:54:43 -0700 Subject: [PATCH 04/18] Update parse_options.sh in pt-stalk. --- bin/pt-stalk | 281 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 183 insertions(+), 98 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 1968cb85..1c4fdbbd 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -49,21 +49,26 @@ die() { # ########################################################################### + + + set -u -ARGV="" # Non-option args (probably input files) -EXT_ARGV="" # Everything after -- (args for an external command) -OPT_ERRS=0 # How many command line option errors -OPT_VERSION="no" # If --version was specified -OPT_HELP="no" # If --help was specified +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="no" # If --version was specified +OPT_HELP="no" # If --help was specified +PO_DIR="$TMPDIR/po" # Directory with program option spec files usage() { local file="$1" local usage=$(grep '^Usage: ' "$file") - echo $usage >&2 - echo >&2 - echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 + echo $usage + echo + echo "For more information, 'man $TOOL' or 'perldoc $file'." } usage_or_errors() { @@ -77,20 +82,20 @@ usage_or_errors() { if [ "$OPT_HELP" = "yes" ]; then usage "$file" - echo >&2 - echo "Command line options:" >&2 - echo >&2 + echo + echo "Command line options:" + echo for opt in $(ls $TMPDIR/po/); do local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') - echo "--$opt" >&2 - echo " $desc" >&2 - echo >&2 + echo "--$opt" + echo " $desc" + echo done return 1 fi if [ $OPT_ERRS -gt 0 ]; then - echo >&2 + echo usage "$file" return 1 fi @@ -102,54 +107,70 @@ parse_options() { local file="$1" shift - if [ ! -d "$TMPDIR/po/" ]; then - mkdir "$TMPDIR/po/" + ARGV="" + EXT_ARGV="" + HAVE_EXT_ARGV="" + OPT_ERRS=0 + OPT_VERSION="no" + OPT_HELP="no" + PO_DIR="$TMPDIR/po" + + if [ ! -d "$PO_DIR" ]; then + mkdir "$PO_DIR" if [ $? -ne 0 ]; then - echo "Cannot mkdir $TMPDIR/po/" >&2 + echo "Cannot mkdir $PO_DIR" >&2 exit 1 fi fi - rm -rf "$TMPDIR"/po/* + rm -rf "$PO_DIR"/* if [ $? -ne 0 ]; then - echo "Cannot rm -rf $TMPDIR/po/*" >&2 + echo "Cannot rm -rf $PO_DIR/*" >&2 exit 1 fi - - ( - export PO_DIR="$TMPDIR/po" - cat "$file" | perl -ne ' - BEGIN { $/ = ""; } - next unless $_ =~ m/^=head1 OPTIONS/; - while ( defined(my $para = <>) ) { - last if $para =~ m/^=head1/; + + _parse_pod "$file" + _eval_po + _parse_config_files + _parse_command_line "$@" +} + +_parse_pod() { + local file="$1" + + cat "$file" | PO_DIR="$PO_DIR" perl -ne ' + BEGIN { $/ = ""; } + next unless $_ =~ m/^=head1 OPTIONS/; + while ( defined(my $para = <>) ) { + last if $para =~ m/^=head1/; + chomp; + if ( $para =~ m/^=item --(\S+)/ ) { + my $opt = $1; + my $file = "$ENV{PO_DIR}/$opt"; + open my $opt_fh, ">", $file or die "Cannot open $file: $!"; + print $opt_fh "long:$opt\n"; + $para = <>; chomp; - if ( $para =~ m/^=item --(\S+)/ ) { - my $opt = $1; - my $file = "$ENV{PO_DIR}/$opt"; - open my $opt_fh, ">", $file or die "Cannot open $file: $!"; - print $opt_fh "long:$opt\n"; + if ( $para =~ m/^[a-z ]+:/ ) { + map { + chomp; + my ($attrib, $val) = split(/: /, $_); + print $opt_fh "$attrib:$val\n"; + } split(/; /, $para); $para = <>; chomp; - if ( $para =~ m/^[a-z ]+:/ ) { - map { - chomp; - my ($attrib, $val) = split(/: /, $_); - print $opt_fh "$attrib:$val\n"; - } split(/; /, $para); - $para = <>; - chomp; - } - my ($desc) = $para =~ m/^([^?.]+)/; - print $opt_fh "desc:$desc.\n"; - close $opt_fh; } + my ($desc) = $para =~ m/^([^?.]+)/; + print $opt_fh "desc:$desc.\n"; + close $opt_fh; } - last; - ' - ) + } + last; + ' +} - for opt_spec in $(ls "$TMPDIR/po/"); do +_eval_po() { + for opt_spec in $(ls "$PO_DIR"); do local opt="" local default_val="" local neg=0 @@ -175,13 +196,13 @@ parse_options() { fi ;; *) - echo "Invalid attribute in $TMPDIR/po/$opt_spec: $line" >&2 + echo "Invalid attribute in $PO_DIR/$opt_spec: $line" >&2 exit 1 esac - done < "$TMPDIR/po/$opt_spec" + done < "$PO_DIR/$opt_spec" if [ -z "$opt" ]; then - echo "No long attribute in option spec $TMPDIR/po/$opt_spec" >&2 + echo "No long attribute in option spec $PO_DIR/$opt_spec" >&2 exit 1 fi @@ -194,69 +215,133 @@ parse_options() { eval "OPT_${opt}"="$default_val" done +} - for opt; do - if [ $# -eq 0 ]; then - break # no more opts +_parse_config_files() { + for config_file in "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + do + test -f "$config_file" || continue + + while read config_opt; do + + echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue + + config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" + + if ! [ "$HAVE_EXT_ARGV" ]; then + config_opt="--$config_opt" + fi + _parse_command_line "$config_opt" + done < "$config_file" + + HAVE_EXT_ARGV="" # reset for each file + + done +} + +_parse_command_line() { + local opt="" + local val="" + local next_opt_is_val="" + local opt_is_ok="" + local opt_is_negated="" + local real_opt="" + local required_arg="" + + for opt in "$@"; do + if [ "$opt" = "--" -o "$opt" = "----" ]; then + HAVE_EXT_ARGV=1 + continue fi - opt=$1 - if [ "$opt" = "--" ]; then - shift - EXT_ARGV="$@" - break - fi - shift - if [ $(expr "$opt" : "-") -eq 0 ]; then - if [ -z "$ARGV" ]; then - ARGV="$opt" + if [ "$HAVE_EXT_ARGV" ]; then + if [ "$EXT_ARGV" ]; then + EXT_ARGV="$EXT_ARGV $opt" else - ARGV="$ARGV $opt" + EXT_ARGV="$opt" fi continue fi - local real_opt="$opt" - - if $(echo $opt | grep -q '^--no-'); then - neg=1 - opt=$(echo $opt | sed 's/^--no-//') - else - neg=0 - opt=$(echo $opt | sed 's/^-*//') - fi - - if [ -f "$TMPDIR/po/$opt" ]; then - spec="$TMPDIR/po/$opt" - else - spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1) - if [ -z "$spec" ]; then - OPT_ERRS=$(($OPT_ERRS + 1)) - echo "Unknown option: $real_opt" >&2 - continue - fi - fi - - local required_arg=$(cat $spec | awk -F: '/^type:/{print $2}') - if [ -n "$required_arg" ]; then - if [ $# -eq 0 ]; then + if [ "$next_opt_is_val" ]; then + next_opt_is_val="" + if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then OPT_ERRS=$(($OPT_ERRS + 1)) echo "$real_opt requires a $required_arg argument" >&2 continue - else - val="$1" - shift fi + val="$opt" + opt_is_ok=1 else - if [ $neg -eq 0 ]; then - val="yes" + if [ $(expr "$opt" : "-") -eq 0 ]; then + if [ -z "$ARGV" ]; then + ARGV="$opt" + else + ARGV="$ARGV $opt" + fi + continue + fi + + real_opt="$opt" + + if $(echo $opt | grep -q '^--no-'); then + opt_is_negated=1 + opt=$(echo $opt | sed 's/^--no-//') else - val="no" + opt_is_negated="" + opt=$(echo $opt | sed 's/^-*//') + fi + + if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then + val="$(echo $opt | awk -F= '{print $2}')" + opt="$(echo $opt | awk -F= '{print $1}')" + fi + + if [ -f "$TMPDIR/po/$opt" ]; then + spec="$TMPDIR/po/$opt" + else + spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1) + if [ -z "$spec" ]; then + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "Unknown option: $real_opt" >&2 + continue + fi + fi + + required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}') + if [ "$required_arg" ]; then + if [ "$val" ]; then + opt_is_ok=1 + else + next_opt_is_val=1 + fi + else + if [ "$val" ]; then + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "Option $real_opt does not take a value" >&2 + continue + fi + if [ "$opt_is_negated" ]; then + val="" + else + val="yes" + fi + opt_is_ok=1 fi fi - opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + if [ "$opt_is_ok" ]; then + opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) - eval "OPT_$opt"="$val" + eval "OPT_$opt"="'$val'" + + opt="" + val="" + next_opt_is_val="" + opt_is_ok="" + opt_is_negated="" + real_opt="" + required_arg="" + fi done } From 4905e3503c569c8ed6b4a1d815098920ae3818d9 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 11:18:41 -0700 Subject: [PATCH 05/18] Handle --config FILE[,FILE,...] in parse_options.sh. --- lib/bash/parse_options.sh | 69 +++++++++++++++++++++---------- t/lib/bash/parse_options.sh | 24 ++++++++++- t/lib/samples/bash/config003.conf | 2 + 3 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 t/lib/samples/bash/config003.conf diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 0576bcd7..9a71034a 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -1,4 +1,4 @@ -# This program is copyright 2011 Percona Inc. +# This program is copyright 2011-2012 Percona Inc. # Feedback and improvements are welcome. # # THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -22,9 +22,9 @@ # parse_options parses Perl POD options from Bash tools and creates # global variables for each option. -# *********************************************************** +# XXX # GLOBAL $TMPDIR AND $TOOL MUST BE SET BEFORE USING THIS LIB! -# *********************************************************** +# XXX # Parsing command line options with Bash is easy until we have to dealt # with values that have spaces, e.g. --option="hello world". This is @@ -39,21 +39,21 @@ # because it's a line from a file, not a command line where Bash will # interpret the quotes and return a single value in the code. So... -# *************************************************** +# XXX # BE CAREFUL MAKING CHANGES TO THIS LIB AND MAKE SURE # t/lib/bash/parse_options.sh STILL PASSES! -# *************************************************** +# XXX set -u # Global variables. These must be global because declare inside a # sub will be scoped locally. -ARGV="" # Non-option args (probably input files) -EXT_ARGV="" # Everything after -- (args for an external command) -HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV -OPT_ERRS=0 # How many command line option errors -OPT_VERSION="no" # If --version was specified -OPT_HELP="no" # If --help was specified +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="" # If --version was specified +OPT_HELP="" # If --help was specified PO_DIR="$TMPDIR/po" # Directory with program option spec files # Sub: usage @@ -80,13 +80,13 @@ usage() { usage_or_errors() { local file="$1" - if [ "$OPT_VERSION" = "yes" ]; then + if [ "$OPT_VERSION" ]; then local version=$(grep '^pt-[^ ]\+ [0-9]' "$file") echo "$version" return 1 fi - if [ "$OPT_HELP" = "yes" ]; then + if [ "$OPT_HELP" ]; then usage "$file" echo echo "Command line options:" @@ -127,15 +127,18 @@ parse_options() { local file="$1" shift - # Reset the globals (mostly for testing). + # XXX + # Reset all globals else t/lib/bash/parse_options.sh will fail. + # XXX ARGV="" EXT_ARGV="" HAVE_EXT_ARGV="" OPT_ERRS=0 - OPT_VERSION="no" - OPT_HELP="no" + OPT_VERSION="" + OPT_HELP="" PO_DIR="$TMPDIR/po" + # Ready the directory for the program option (po) spec files. if [ ! -d "$PO_DIR" ]; then mkdir "$PO_DIR" if [ $? -ne 0 ]; then @@ -150,9 +153,26 @@ parse_options() { exit 1 fi - _parse_pod "$file" - _eval_po - _parse_config_files + _parse_pod "$file" # Parse POD into program option (po) spec files + _eval_po # Eval po into existence with default values + + # If the first option is --config FILES, then remove it and use + # those files instead of the default config files. + if [ $# -ge 2 ] && [ "$1" = "--config" ]; then + shift # --config + local user_config_files="$1" + shift # that ^ + local old_ifs="$IFS" + IFS="," + for user_config_file in $user_config_files; do + _parse_config_files "$user_config_file" + done + IFS="$old_ifs" + else + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi + + # Finally, parse the command line. _parse_command_line "$@" } @@ -251,8 +271,8 @@ _eval_po() { } _parse_config_files() { - for config_file in "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" - do + + for config_file in "$@"; do # Next config file if this one doesn't exist. test -f "$config_file" || continue @@ -270,6 +290,9 @@ _parse_config_files() { # and end-of-line # comments. config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" + # Skip blank lines. + [ "$config_opt" = "" ] && continue + # Options in a config file are not prefixed with --, # but command line options are, so one or the other has # to add or remove the -- prefix. We add it for config @@ -278,9 +301,11 @@ _parse_config_files() { if ! [ "$HAVE_EXT_ARGV" ]; then config_opt="--$config_opt" fi + _parse_command_line "$config_opt" + done < "$config_file" - + HAVE_EXT_ARGV="" # reset for each file done diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index e60ead5a..8ecfe69d 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=49 +TESTS=63 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -136,6 +136,28 @@ parse_options "$T_LIB_DIR/samples/bash/po001.sh" --string-opt zzz is "$OPT_STRING_OPT" "zzz" "Command line overrides config file" +# User-specified --config +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --config "$T_LIB_DIR/samples/bash/config003.conf" --string-opt bar + +is "$OPT_STRING_OPT" "bar" "--config string option" +is "$OPT_STRING_OPT2" "foo" "--config string option2" +is "$OPT_TYPELESS_OPTION" "" "--config typeless option" +is "$OPT_NOPTION" "yes" "--config negatable option" +is "$OPT_INT_OPT" "123" "--config int option" +is "$OPT_INT_OPT2" "42" "--config int option2" +is "$OPT_VERSION" "" "--config version option" +is "$ARGV" "" "--config ARGV" +is "$EXT_ARGV" "" "--config External ARGV" + +# Multiple --config files, last should take precedence. +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --config $T_LIB_DIR/samples/bash/config001.conf,$T_LIB_DIR/samples/bash/config002.conf + +is "$OPT_STRING_OPT" "hello world" "Two --config string option" +is "$OPT_TYPELESS_OPTION" "yes" "Two --config typeless option" +is "$OPT_INT_OPT" "100" "Two --config int option" +is "$ARGV" "" "Two --config ARGV" +is "$EXT_ARGV" "--host=127.1 --user=daniel" "Two--config External ARGV" + # ############################################################################ # Option values with spaces. # ############################################################################ diff --git a/t/lib/samples/bash/config003.conf b/t/lib/samples/bash/config003.conf new file mode 100644 index 00000000..0ac5a1d5 --- /dev/null +++ b/t/lib/samples/bash/config003.conf @@ -0,0 +1,2 @@ +string-opt=from config file +int-opt=123 From 3c97ae27d1c8abefe64fcea0341b54950e67155f Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 11:50:48 -0700 Subject: [PATCH 06/18] Add and test --config to pt-stalk. --- bin/pt-stalk | 57 ++++++++++++++++++++++--------- t/pt-stalk/pt-stalk.t | 31 ++++++++++++++++- t/pt-stalk/samples/config001.conf | 8 +++++ 3 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 t/pt-stalk/samples/config001.conf diff --git a/bin/pt-stalk b/bin/pt-stalk index 1c4fdbbd..2f0076a2 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -54,12 +54,12 @@ die() { set -u -ARGV="" # Non-option args (probably input files) -EXT_ARGV="" # Everything after -- (args for an external command) -HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV -OPT_ERRS=0 # How many command line option errors -OPT_VERSION="no" # If --version was specified -OPT_HELP="no" # If --help was specified +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="" # If --version was specified +OPT_HELP="" # If --help was specified PO_DIR="$TMPDIR/po" # Directory with program option spec files usage() { @@ -74,13 +74,13 @@ usage() { usage_or_errors() { local file="$1" - if [ "$OPT_VERSION" = "yes" ]; then + if [ "$OPT_VERSION" ]; then local version=$(grep '^pt-[^ ]\+ [0-9]' "$file") echo "$version" return 1 fi - if [ "$OPT_HELP" = "yes" ]; then + if [ "$OPT_HELP" ]; then usage "$file" echo echo "Command line options:" @@ -111,8 +111,8 @@ parse_options() { EXT_ARGV="" HAVE_EXT_ARGV="" OPT_ERRS=0 - OPT_VERSION="no" - OPT_HELP="no" + OPT_VERSION="" + OPT_HELP="" PO_DIR="$TMPDIR/po" if [ ! -d "$PO_DIR" ]; then @@ -129,9 +129,23 @@ parse_options() { exit 1 fi - _parse_pod "$file" - _eval_po - _parse_config_files + _parse_pod "$file" # Parse POD into program option (po) spec files + _eval_po # Eval po into existence with default values + + if [ $# -ge 2 ] && [ "$1" = "--config" ]; then + shift # --config + local user_config_files="$1" + shift # that ^ + local old_ifs="$IFS" + IFS="," + for user_config_file in $user_config_files; do + _parse_config_files "$user_config_file" + done + IFS="$old_ifs" + else + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi + _parse_command_line "$@" } @@ -218,8 +232,8 @@ _eval_po() { } _parse_config_files() { - for config_file in "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" - do + + for config_file in "$@"; do test -f "$config_file" || continue while read config_opt; do @@ -228,12 +242,16 @@ _parse_config_files() { config_opt="$(echo "$config_opt" | sed -e 's/^[ ]*//' -e 's/[ ]*\$//' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')" + [ "$config_opt" = "" ] && continue + if ! [ "$HAVE_EXT_ARGV" ]; then config_opt="--$config_opt" fi + _parse_command_line "$config_opt" + done < "$config_file" - + HAVE_EXT_ARGV="" # reset for each file done @@ -1212,6 +1230,13 @@ Collect strace data. Collect tcpdump data. +=item --config + +type: string + +Read this comma-separated list of config files. If specified, this must be the +first option on the command line. + =item --cycles type: int; default: 5 diff --git a/t/pt-stalk/pt-stalk.t b/t/pt-stalk/pt-stalk.t index d1ac37a2..de18249a 100644 --- a/t/pt-stalk/pt-stalk.t +++ b/t/pt-stalk/pt-stalk.t @@ -10,6 +10,7 @@ use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Test::More; +use Time::HiRes qw(sleep); use PerconaTest; use DSNParser; @@ -23,7 +24,7 @@ if ( !$dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 15; + plan tests => 17; } my $cnf = "/tmp/12345/my.sandbox.cnf"; @@ -168,6 +169,34 @@ like( "Trigger file logs how pt-stalk was ran" ); + +# ############################################################################# +# --config +# ############################################################################# + +diag(`cp $ENV{HOME}/.pt-stalk.conf $ENV{HOME}/.pt-stalk.conf.original 2>/dev/null`); +diag(`cp $trunk/t/pt-stalk/samples/config001.conf $ENV{HOME}/.pt-stalk.conf`); + +system "$trunk/bin/pt-stalk --dest $dest --pid $pid_file >$log_file 2>&1 &"; +sleep 1; +chomp($pid = `cat $pid_file`); +$retval = system("kill $pid 2>/dev/null"); +is( + $retval >> 0, + 0, + "Killed pt-stalk" +); + +$output = `cat $log_file`; +like( + $output, + qr/Check results: Aborted_connects=|variable=Aborted_connects/, + "Read default config file" +); + +diag(`rm $ENV{HOME}/.pt-stalk.conf`); +diag(`cp $ENV{HOME}/.pt-stalk.conf.original $ENV{HOME}/.pt-stalk.conf 2>/dev/null`); + # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-stalk/samples/config001.conf b/t/pt-stalk/samples/config001.conf new file mode 100644 index 00000000..c4ad24e8 --- /dev/null +++ b/t/pt-stalk/samples/config001.conf @@ -0,0 +1,8 @@ +--iterations=1 +--variable=Aborted_connects +--threshold=999999 +-- +-umsandbox +-pmsandbox +--host 127.1 +--port 12345 From c965d7c172b35a1b441f16d22680610be4ec1424 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 12:07:42 -0700 Subject: [PATCH 07/18] Fix POD syntax error. --- bin/pt-stalk | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 7b692fab..b742d071 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -1356,8 +1356,6 @@ options: --trigger processlist --variable State --match statistics --threshold 10 -=back - In addition, you can specify a file that contains your custom trigger function, written in Unix shell script. This can be a wrapper that executes anything you wish. If the argument to --function is a file, then it takes precedence over From 36c14f492a481720286659a96112d70573b1c4f5 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 12:15:29 -0700 Subject: [PATCH 08/18] Don't use grep -q. --- bin/pt-stalk | 2 +- lib/bash/parse_options.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index b742d071..404187ed 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -301,7 +301,7 @@ _parse_command_line() { real_opt="$opt" - if $(echo $opt | grep -q '^--no-'); then + if $(echo $opt | grep '^--no-' >/dev/null); then opt_is_negated=1 opt=$(echo $opt | sed 's/^--no-//') else diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 9a71034a..23443bb4 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -370,7 +370,7 @@ _parse_command_line() { real_opt="$opt" # Strip leading -- or --no- from option. - if $(echo $opt | grep -q '^--no-'); then + if $(echo $opt | grep '^--no-' >/dev/null); then opt_is_negated=1 opt=$(echo $opt | sed 's/^--no-//') else From 42a8e3963569ae581792aa1b57c4f2fa07b81057 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 12:28:41 -0700 Subject: [PATCH 09/18] Use better Bash. --- lib/bash/parse_options.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 23443bb4..bfe3e489 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -222,13 +222,13 @@ _eval_po() { # Evaluate the program options into existence as global variables # transformed like --my-op == $OPT_MY_OP. If an option has a default # value, it's assigned that value. Else, it's value is an empty string. - for opt_spec in $(ls "$PO_DIR"); do + local old_ifs="$IFS" + IFS=":" + for opt_spec in "$PO_DIR"/*; do local opt="" local default_val="" local neg=0 - while read line; do - local key=$(echo $line | cut -d ':' -f 1) - local val=$(echo $line | cut -d ':' -f 2) + while read key val; do case "$key" in long) opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:]) @@ -248,13 +248,13 @@ _eval_po() { fi ;; *) - echo "Invalid attribute in $PO_DIR/$opt_spec: $line" >&2 + echo "Invalid attribute in $opt_spec: $line" >&2 exit 1 esac - done < "$PO_DIR/$opt_spec" + done < "$opt_spec" if [ -z "$opt" ]; then - echo "No long attribute in option spec $PO_DIR/$opt_spec" >&2 + echo "No long attribute in option spec $opt_spec" >&2 exit 1 fi @@ -268,6 +268,8 @@ _eval_po() { # Eval the option into existence as a global variable. eval "OPT_${opt}"="$default_val" done + + IFS="$old_ifs" } _parse_config_files() { From c2fd3f54c2d8becfa6513abefd5ad5e114f6e83e Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 12:49:18 -0700 Subject: [PATCH 10/18] Use "yes" for true, "" for false. Use $PO_DIR instead of $TMP_DIR/po. Add and test Baron's code for 'Options and values after processing arguments'. Make --help exit 0 unless there were errors. --- bin/pt-stalk | 51 ++++++++++++++++++++++--------------- lib/bash/collect.sh | 10 ++++---- lib/bash/parse_options.sh | 10 +++++++- t/lib/bash/collect.sh | 1 + t/lib/bash/parse_options.sh | 6 ++++- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 404187ed..c2748475 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -85,12 +85,20 @@ usage_or_errors() { echo echo "Command line options:" echo - for opt in $(ls $TMPDIR/po/); do + for opt in $(ls "$PO_DIR"); do local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') echo "--$opt" echo " $desc" echo done + echo "Options and values after processing arguments:" + echo + for opt in $(ls "$PO_DIR"); do + local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)" + local varvalue="${!varname}" + printf -- " --%-30s %s" "$opt" "${varvalue:-(No value)}" + echo + done return 1 fi @@ -184,13 +192,13 @@ _parse_pod() { } _eval_po() { - for opt_spec in $(ls "$PO_DIR"); do + local old_ifs="$IFS" + IFS=":" + for opt_spec in "$PO_DIR"/*; do local opt="" local default_val="" local neg=0 - while read line; do - local key=$(echo $line | cut -d ':' -f 1) - local val=$(echo $line | cut -d ':' -f 2) + while read key val; do case "$key" in long) opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:]) @@ -210,13 +218,13 @@ _eval_po() { fi ;; *) - echo "Invalid attribute in $PO_DIR/$opt_spec: $line" >&2 + echo "Invalid attribute in $opt_spec: $line" >&2 exit 1 esac - done < "$PO_DIR/$opt_spec" + done < "$opt_spec" if [ -z "$opt" ]; then - echo "No long attribute in option spec $PO_DIR/$opt_spec" >&2 + echo "No long attribute in option spec $opt_spec" >&2 exit 1 fi @@ -229,6 +237,8 @@ _eval_po() { eval "OPT_${opt}"="$default_val" done + + IFS="$old_ifs" } _parse_config_files() { @@ -574,7 +584,7 @@ collect() { fi fi - if [ "$CMD_GDB" -a "$OPT_COLLECT_GDB" = "yes" -a "$mysqld_pid" ]; then + if [ "$CMD_GDB" -a "$OPT_COLLECT_GDB" -a "$mysqld_pid" ]; then $CMD_GDB \ -ex "set pagination 0" \ -ex "thread apply all bt" \ @@ -613,7 +623,7 @@ collect() { open_tables >> "$d/$p-opentables1" 2>&1 & local tcpdump_pid="" - if [ "$CMD_TCPDUMP" -a "$OPT_COLLECT_TCPDUMP" = "yes" ]; then + if [ "$CMD_TCPDUMP" -a "$OPT_COLLECT_TCPDUMP" ]; then local port=$(awk '/^port/{print $2}' "$d/$p-variables") if [ "$port" ]; then $CMD_TCPDUMP -i any -s 4096 -w "$d/$p-tcpdump" port ${port} & @@ -622,12 +632,12 @@ collect() { fi local have_oprofile="no" - if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" = "yes" ]; then + if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then if $CMD_OPCONTROL --init; then $CMD_OPCONTROL --start --no-vmlinux have_oprofile="yes" fi - elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" = "yes" ]; then + elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then $CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 & local strace_pid=$! fi @@ -726,7 +736,7 @@ collect() { "/path/to/mysqld'" \ > "$d/$p-opreport" fi - elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" = "yes" ]; then + elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then kill -s 2 $strace_pid sleep 1 kill -s 15 $strace_pid @@ -931,7 +941,7 @@ sigtrap() { stalk() { local cycles_true=0 # increment each time check is true, else set to 0 - local matched="no" # set to "yes" when check is true + local matched="" # set to "yes" when check is true local last_prefix="" # prefix of last collection while oktorun; do @@ -944,20 +954,20 @@ stalk() { if [ -z "$value" ]; then # No value. Maybe we failed to connect to MySQL? warn "Detected value is empty; something failed? Trigger exit status: $trg_exit_status" - matched="no" + matched="" cycles_true=0 elif [ $value -gt $OPT_THRESHOLD ]; then matched="yes" cycles_true=$(($cycles_true + 1)) else - matched="no" + matched="" cycles_true=0 fi - local msg="Check results: $OPT_VARIABLE=$value, matched=$matched, cycles_true=$cycles_true" + local msg="Check results: $OPT_VARIABLE=$value, matched=${matched:-no}, cycles_true=$cycles_true" log "$msg" - if [ "$matched" = "yes" -a $cycles_true -ge $OPT_CYCLES ]; then + if [ "$matched" -a $cycles_true -ge $OPT_CYCLES ]; then # ################################################################## # Start collecting, maybe. # ################################################################## @@ -1089,7 +1099,8 @@ if [ "$(basename "$0")" = "pt-stalk" ] \ po_status=$? rm_tmpdir if [ $po_status -ne 0 ]; then - exit $po_status + [ $OPT_ERRS -gt 0 ] && exit 1 + exit 0 fi # Now that we have the cmd line opts, check that we can actually @@ -1097,7 +1108,7 @@ if [ "$(basename "$0")" = "pt-stalk" ] \ [ -n "$(mysql $EXT_ARGV -e 'SELECT 1')" ] \ || die "Cannot connect to MySQL. Check that MySQL is running and that the options after -- are correct." - if [ "$OPT_DAEMONIZE" = "yes" ]; then + if [ "$OPT_DAEMONIZE" ]; then # Check access to the --log file. ( set -e diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 635de955..75f3d69d 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -61,7 +61,7 @@ collect() { # Getting a GDB stacktrace can be an intensive operation, # so do this only if necessary (and possible). - if [ "$CMD_GDB" -a "$OPT_COLLECT_GDB" = "yes" -a "$mysqld_pid" ]; then + if [ "$CMD_GDB" -a "$OPT_COLLECT_GDB" -a "$mysqld_pid" ]; then $CMD_GDB \ -ex "set pagination 0" \ -ex "thread apply all bt" \ @@ -112,7 +112,7 @@ collect() { # If TCP dumping is specified, start that on the server's port. local tcpdump_pid="" - if [ "$CMD_TCPDUMP" -a "$OPT_COLLECT_TCPDUMP" = "yes" ]; then + if [ "$CMD_TCPDUMP" -a "$OPT_COLLECT_TCPDUMP" ]; then local port=$(awk '/^port/{print $2}' "$d/$p-variables") if [ "$port" ]; then $CMD_TCPDUMP -i any -s 4096 -w "$d/$p-tcpdump" port ${port} & @@ -123,12 +123,12 @@ collect() { # Next, start oprofile gathering data during the whole rest of this process. # The --init should be a no-op if it has already been init-ed. local have_oprofile="no" - if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" = "yes" ]; then + if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then if $CMD_OPCONTROL --init; then $CMD_OPCONTROL --start --no-vmlinux have_oprofile="yes" fi - elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" = "yes" ]; then + elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then # Don't run oprofile and strace at the same time. $CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 & local strace_pid=$! @@ -244,7 +244,7 @@ collect() { "/path/to/mysqld'" \ > "$d/$p-opreport" fi - elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" = "yes" ]; then + elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" ]; then kill -s 2 $strace_pid sleep 1 kill -s 15 $strace_pid diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index bfe3e489..3689f82d 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -91,12 +91,20 @@ usage_or_errors() { echo echo "Command line options:" echo - for opt in $(ls $TMPDIR/po/); do + for opt in $(ls "$PO_DIR"); do local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') echo "--$opt" echo " $desc" echo done + echo "Options and values after processing arguments:" + echo + for opt in $(ls "$PO_DIR"); do + local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)" + local varvalue="${!varname}" + printf -- " --%-30s %s" "$opt" "${varvalue:-(No value)}" + echo + done return 1 fi diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index a9138dd7..6ae070d7 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -5,6 +5,7 @@ TESTS=19 TMPFILE="$TEST_TMPDIR/parse-opts-output" TMPDIR="$TEST_TMPDIR" PATH="$PATH:$PERCONA_TOOLKIT_SANDBOX/bin" +TOOL="pt-stalk" mkdir "$TMPDIR/collect" 2>/dev/null diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 8ecfe69d..9c124858 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=63 +TESTS=64 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -105,6 +105,10 @@ cmd_ok \ "grep -q \"For more information, 'man pt-stalk' or 'perldoc\" $TMPFILE" \ "--help" +cmd_ok \ + "grep -q '\-\-string-opt[ ]*(No value)' $TMPFILE" \ + "Options and values after processing arguments" + # Don't interpolate. parse_options "$T_LIB_DIR/samples/bash/po003.sh" --help usage_or_errors "$T_LIB_DIR/samples/bash/po003.sh" >$TMPFILE 2>&1 From b47679cf97df1f18d18cc4e5219c8c134f3e1314 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 13:20:07 -0700 Subject: [PATCH 11/18] Implement --[no]collect. --- bin/pt-stalk | 72 +++++++++++++++++++++++-------------------- t/pt-stalk/pt-stalk.t | 34 +++++++++++++++++++- 2 files changed, 71 insertions(+), 35 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index c2748475..241457e5 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -971,44 +971,48 @@ stalk() { # ################################################################## # Start collecting, maybe. # ################################################################## - local prefix="${OPT_PREFIX:-$(date +%F-%T | tr :- _)}" log "Collect triggered" - # Check if we'll have enough disk space to collect. Disk space - # is also checked every interval while collecting. - local margin="20" # default 20M margin, unless: - if [ -n "$last_prefix" ]; then - margin=$(du -mc "$OPT_DEST"/"$last_prefix"-* | tail -n 1 | awk '{print $1'}) - fi - disk_space "$OPT_DEST" > "$OPT_DEST/$prefix-disk-space" - check_disk_space \ - "$OPT_DEST/$prefix-disk-space" \ - "$OPT_DISK_BYTE_LIMIT" \ - "$OPT_DISK_PCT_LIMIT" \ - "$margin" # real used MB + margin MB - if [ $? -eq 0 ]; then - # There should be enough disk space, so collect. - log "$msg" >> "$OPT_DEST/$prefix-trigger" - log "pt-stalk ran with $RAN_WITH" >> "$OPT_DEST/$prefix-trigger" - last_prefix="$prefix" + # Send email to whomever that collect has been triggered. + if [ "$OPT_NOTIFY_BY_EMAIL" ]; then + echo "$msg on $(hostname)" \ + | mail -s "Collect triggered on $(hostname)" \ + "$OPT_NOTIFY_BY_EMAIL" + fi - # Send email to whomever that collect has been triggered. - if [ "$OPT_NOTIFY_BY_EMAIL" ]; then - echo "$msg on $(hostname)" \ - | mail -s "Collect triggered on $(hostname)" \ - "$OPT_NOTIFY_BY_EMAIL" + if [ "$OPT_COLLECT" ]; then + local prefix="${OPT_PREFIX:-$(date +%F-%T | tr :- _)}" + + # Check if we'll have enough disk space to collect. Disk space + # is also checked every interval while collecting. + local margin="20" # default 20M margin, unless: + if [ -n "$last_prefix" ]; then + margin=$(du -mc "$OPT_DEST"/"$last_prefix"-* | tail -n 1 | awk '{print $1'}) + fi + disk_space "$OPT_DEST" > "$OPT_DEST/$prefix-disk-space" + check_disk_space \ + "$OPT_DEST/$prefix-disk-space" \ + "$OPT_DISK_BYTE_LIMIT" \ + "$OPT_DISK_PCT_LIMIT" \ + "$margin" # real used MB + margin MB + if [ $? -eq 0 ]; then + # There should be enough disk space, so collect. + log "$msg" >> "$OPT_DEST/$prefix-trigger" + log "pt-stalk ran with $RAN_WITH" >> "$OPT_DEST/$prefix-trigger" + last_prefix="$prefix" + + + # Fork and background the collect subroutine which will + # run for --run-time seconds. We (the parent) sleep + # while its collecting (hopefully --sleep is longer than + # --run-time). + ( + collect "$OPT_DEST" "$prefix" + ) >> "$OPT_DEST/$prefix-output" 2>&1 & + else + # There will not be enough disk space, so do not collect. + warn "Collect canceled because there will not be enough disk space after collecting another $margin MB" fi - - # Fork and background the collect subroutine which will - # run for --run-time seconds. We (the parent) sleep - # while its collecting (hopefully --sleep is longer than - # --run-time). - ( - collect "$OPT_DEST" "$prefix" - ) >> "$OPT_DEST/$prefix-output" 2>&1 & - else - # There will not be enough disk space, so do not collect. - warn "Collect canceled because there will not be enough disk space after collecting another $margin MB" fi # ################################################################## diff --git a/t/pt-stalk/pt-stalk.t b/t/pt-stalk/pt-stalk.t index eb491302..6a054899 100644 --- a/t/pt-stalk/pt-stalk.t +++ b/t/pt-stalk/pt-stalk.t @@ -24,7 +24,7 @@ if ( !$dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 17; + plan tests => 20; } my $cnf = "/tmp/12345/my.sandbox.cnf"; @@ -169,6 +169,38 @@ like( "Trigger file logs how pt-stalk was ran" ); +# ########################################################################### +# Triggered but --no-collect. +# ########################################################################### +diag(`rm $pid_file 2>/dev/null`); +diag(`rm $log_file 2>/dev/null`); +diag(`rm $dest/* 2>/dev/null`); + +(undef, $uptime) = $dbh->selectrow_array("SHOW STATUS LIKE 'Uptime'"); +$threshold = $uptime + 2; + +$retval = system("$trunk/bin/pt-stalk --no-collect --iterations 1 --dest $dest --variable Uptime --threshold $threshold --cycles 1 --run-time 1 --pid $pid_file -- --defaults-file=$cnf >$log_file 2>&1"); + +sleep 2; + +$output = `cat $log_file`; +like( + $output, + qr/Collect triggered/, + "Collect triggered" +); + +ok( + ! -f "$dest/*", + "No files collected" +); + +$output = `ps x | grep -v grep | grep 'pt-stalk pt-stalk --iterations 1 --dest $dest'`; +is( + $output, + "", + "pt-stalk is not running" +); # ############################################################################# # --config From 77e5e4c3f20961d3ceda0b60e210f79cd8db2d20 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 13:24:07 -0700 Subject: [PATCH 12/18] Simpler test for +w access to --log. --- bin/pt-stalk | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 241457e5..dc52efde 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -1114,13 +1114,7 @@ if [ "$(basename "$0")" = "pt-stalk" ] \ if [ "$OPT_DAEMONIZE" ]; then # Check access to the --log file. - ( - set -e - touch "$OPT_LOG" - ) - if [ $? -ne 0 ]; then - die "Cannot write to --log $OPT_LOG" - fi + touch "$OPT_LOG" || die "Cannot write to --log $OPT_LOG" # The PID file will at first have our (parent) PID. # This is fine for ensuring that only one of us is From 73bc7cdcf83245452c741f8411990b45ad6e6b0a Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 24 Jan 2012 14:22:19 -0700 Subject: [PATCH 13/18] Change default --dest to /var/lib/pt-stalk. Don't check --dest unless --collect. Fix POD formatting for --function. --- bin/pt-stalk | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index dc52efde..3300101f 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -883,11 +883,6 @@ trg_processlist() { return } -trg_magic() { - echo "TODO" - return -} - oktorun() { if [ $OKTORUN -eq 0 ]; then EXIT_REASON="OKTORUN is false" @@ -1026,7 +1021,9 @@ stalk() { fi # Purge old collect file between checks. - purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" + if [ -d "$OPT_DEST" ]; then + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" + fi done } @@ -1041,23 +1038,8 @@ main() { # Bash 4 has $BASHPID but we can't rely on that. Consequently, # we don't know our own PID. See the usage of $! below. RAN_WITH="--function=$OPT_FUNCTION --variable=$OPT_VARIABLE --threshold=$OPT_THRESHOLD --match=$OPT_MATCH --cycles=$OPT_CYCLES --interval=$OPT_INTERVAL --iterations=$OPT_ITERATIONS --run-time=$OPT_RUN_TIME --sleep=$OPT_SLEEP --dest=$OPT_DEST --prefix=$OPT_PREFIX --notify-by-email=$OPT_NOTIFY_BY_EMAIL --log=$OPT_LOG --pid=$OPT_PID" - log "Starting $0 $RAN_WITH" - # Make sure the collection dir exists. - if [ ! -d "$OPT_DEST" ]; then - mkdir -p "$OPT_DEST" || die "Cannot make --dest $OPT_DEST" - fi - # Check access to the --dest dir. By setting -x in the subshell, - # if either command fails, the subshell will exit immediately and - # $? will be non-zero. - ( - set -e - touch "$OPT_DEST/test" - rm "$OPT_DEST/test" - ) - if [ $? -ne 0 ]; then - die "Cannot read and write files to --dest $OPT_DEST" - fi + log "Starting $0 $RAN_WITH" # Test if we have root; warn if not, but it isn't critical. if [ "$(id -u)" != "0" ]; then @@ -1112,6 +1094,25 @@ if [ "$(basename "$0")" = "pt-stalk" ] \ [ -n "$(mysql $EXT_ARGV -e 'SELECT 1')" ] \ || die "Cannot connect to MySQL. Check that MySQL is running and that the options after -- are correct." + # Check existence and access to the --dest dir if we're collecting. + if [ "$OPT_COLLECT" ]; then + if [ ! -d "$OPT_DEST" ]; then + mkdir -p "$OPT_DEST" || die "Cannot make --dest $OPT_DEST" + fi + + # Check access to the --dest dir. By setting -x in the subshell, + # if either command fails, the subshell will exit immediately and + # $? will be non-zero. + ( + set -e + touch "$OPT_DEST/test" + rm "$OPT_DEST/test" + ) + if [ $? -ne 0 ]; then + die "Cannot read and write files to --dest $OPT_DEST" + fi + fi + if [ "$OPT_DAEMONIZE" ]; then # Check access to the --log file. touch "$OPT_LOG" || die "Cannot write to --log $OPT_LOG" @@ -1307,7 +1308,7 @@ its output as specified in --log. =item --dest -type: string; default: ${HOME}/collected +type: string; default: /var/lib/pt-stalk Where to store the diagnostic data. Each time the tool collects data, it writes to a new set of files, which are named with the current system timestamp. @@ -1365,6 +1366,8 @@ options: --trigger processlist --variable State --match statistics --threshold 10 +=back + In addition, you can specify a file that contains your custom trigger function, written in Unix shell script. This can be a wrapper that executes anything you wish. If the argument to --function is a file, then it takes precedence over @@ -1388,8 +1391,6 @@ MySQL options mentioned in the L<"SYNOPSIS"> above. The plugin should not alter the tool's existing global variables. Prefix any plugin-specific global variables with "PLUGIN_" or make them local. -=back - =item --help Print help and exit. From 81caf6addcdd467fc1c975e5c5e8764bc3ba2d7b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 26 Jan 2012 10:44:55 -0700 Subject: [PATCH 14/18] Convert size \d+[KMGT] opts in parse_options.sh. --- lib/bash/parse_options.sh | 19 +++++++++++++++++++ t/lib/bash/parse_options.sh | 30 +++++++++++++++++++++++++++++- t/lib/samples/bash/po004.sh | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 t/lib/samples/bash/po004.sh diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 3689f82d..a33e68f8 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -236,6 +236,7 @@ _eval_po() { local opt="" local default_val="" local neg=0 + local size=0 while read key val; do case "$key" in long) @@ -247,6 +248,7 @@ _eval_po() { "short form") ;; type) + [ "$val" = "size" ] && size=1 ;; desc) ;; @@ -273,6 +275,11 @@ _eval_po() { fi fi + # Convert sizes. + if [ $size -eq 1 -a -n "$default_val" ]; then + default_val=$(size_to_bytes $default_val) + fi + # Eval the option into existence as a global variable. eval "OPT_${opt}"="$default_val" done @@ -339,6 +346,7 @@ _parse_command_line() { local opt_is_negated="" local real_opt="" local required_arg="" + local spec="" for opt in "$@"; do if [ "$opt" = "--" -o "$opt" = "----" ]; then @@ -438,6 +446,11 @@ _parse_command_line() { # Get and transform the opt's long form. E.g.: -q == --quiet == QUIET. opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + # Convert sizes. + if grep "^type:size" "$spec" >/dev/null; then + val=$(size_to_bytes $val) + fi + # Re-eval the option to update its global variable value. eval "OPT_$opt"="'$val'" @@ -448,10 +461,16 @@ _parse_command_line() { opt_is_negated="" real_opt="" required_arg="" + spec="" fi done } +size_to_bytes() { + local size="$1" + echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};' +} + # ########################################################################### # End parse_options package # ########################################################################### diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 9c124858..b66f8584 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=64 +TESTS=72 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -183,6 +183,34 @@ is "$OPT_STRING_OPT" "hello world" "Option value with space (cmd line)" is "$ARGV" "" "ARGV (cmd line)" is "$EXT_ARGV" "" "External ARGV (cmd line)" +# ############################################################################ +# Size options. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1T +is "$OPT_DISK_BYTES_FREE" "1099511627776" "Size: 1T" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1G +is "$OPT_DISK_BYTES_FREE" "1073741824" "Size: 1G" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1M +is "$OPT_DISK_BYTES_FREE" "1048576" "Size: 1M" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1K +is "$OPT_DISK_BYTES_FREE" "1024" "Size: 1K" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1k +is "$OPT_DISK_BYTES_FREE" "1024" "Size: 1k" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 1 +is "$OPT_DISK_BYTES_FREE" "1" "Size: 1" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" --disk-bytes-free 100M +is "$OPT_DISK_BYTES_FREE" "104857600" "Size: 100M" + +parse_options "$T_LIB_DIR/samples/bash/po004.sh" +is "$OPT_DISK_BYTES_FREE" "104857600" "Size: 100M default" + # ############################################################################ # Done # ############################################################################ diff --git a/t/lib/samples/bash/po004.sh b/t/lib/samples/bash/po004.sh new file mode 100644 index 00000000..0574f221 --- /dev/null +++ b/t/lib/samples/bash/po004.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +: + +# ############################################################################ +# Documentation +# ############################################################################ +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-stalk - Wait for a condition to occur then begin collecting data. + +=head1 OPTIONS + +=over + +=item --disk-bytes-free + +type: size; default: 100M + +Fall apart if there's less than this many bytes free on the disk. + +=item --help + +Print help. + +=back + +=head1 ENVIRONMENT + +No env vars used. + +=cut + +DOCUMENTATION From d7d5381863b1509b585929e8181acab61e7181ff Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 26 Jan 2012 12:19:47 -0700 Subject: [PATCH 15/18] Make check_disk_space() use bytes (given new size type options with are converted to bytes, 1k=>1024). --- lib/bash/safeguards.sh | 57 ++++++++++++++++++++++------------------ t/lib/bash/safeguards.sh | 35 +++++++++++++++++------- 2 files changed, 57 insertions(+), 35 deletions(-) diff --git a/lib/bash/safeguards.sh b/lib/bash/safeguards.sh index 97db3842..91402bb6 100644 --- a/lib/bash/safeguards.sh +++ b/lib/bash/safeguards.sh @@ -1,4 +1,4 @@ -# This program is copyright 2011 Percona Inc. +# This program is copyright 2011-2012 Percona Inc. # Feedback and improvements are welcome. # # THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -33,46 +33,53 @@ disk_space() { # Sub: check_disk_space # Check if there is or will be enough disk space. Input is a file # with output from , i.e. `df -P -k`. The df output -# must use 1k blocks, but the mb arg from the user is in MB. +# must use 1k blocks, which should be POSIX standard. # # Arguments: -# file - File with output from . -# mb - Minimum MB free. -# pc - Minimum percent free. -# mb_margin - Add this many MB to the real MB used. +# file - File with output from . +# min_free_bytes - Minimum free bytes. +# min_free_pct - Minimum free percentage. +# bytes_margin - Add this many bytes to the real bytes used. # # Returns: # 0 if there is/will be enough disk space, else 1. check_disk_space() { local file="$1" - local mb="${2:-0}" - local pc="${3:-0}" - local mb_margin="${4:-0}" + local min_free_bytes="${2:-0}" + local min_free_pct="${3:-0}" + local bytes_margin="${4:-0}" - # Convert MB to KB because the df output should be in 1k blocks. - local kb=$(($mb * 1024)) - local kb_margin=$(($mb_margin * 1024)) + # Real/actual bytes used and bytes free. + local used_bytes=$(cat "$file" | awk '/^\//{print $3 * 1024}'); + local free_bytes=$(cat "$file" | awk '/^\//{print $4 * 1024}'); + local pct_used=$(cat "$file" | awk '/^\//{print $5}' | sed -e 's/%//g'); + local pct_free=$((100 - $pct_used)) - local kb_used=$(cat "$file" | awk '/^\//{print $3}'); - local kb_free=$(cat "$file" | awk '/^\//{print $4}'); - local pc_used=$(cat "$file" | awk '/^\//{print $5}' | sed -e 's/%//g'); + # Report the real values to the user. + local real_free_bytes=$free_bytes + local real_pct_free=$pct_free - if [ "$kb_margin" -gt "0" ]; then - local kb_total=$(($kb_used + $kb_free)) + # If there's a margin, we need to adjust the real values. + if [ $bytes_margin -gt 0 ]; then + used_bytes=$(($used_bytes + $bytes_margin)) + free_bytes=$(($free_bytes - $bytes_margin)) + pct_used=$(awk "BEGIN { printf(\"%d\", ($used_bytes/($used_bytes + $free_bytes)) * 100) }") - kb_used=$(($kb_used + $kb_margin)) - kb_free=$(($kb_free - $kb_margin)) - pc_used=$(awk "BEGIN { printf(\"%d\", $kb_used/$kb_total * 100) }") + pct_free=$((100 - $pct_used)) fi - local pc_free=$((100 - $pc_used)) + if [ $free_bytes -lt $min_free_bytes -o $pct_free -lt $min_free_pct ]; then + warn "Not enough free disk space: + Limit: ${min_free_pct}% free, ${min_free_bytes} bytes free + Actual: ${real_pct_free}% free, ${real_free_bytes} bytes free (- $bytes_margin bytes margin) +" + # Print the df that we used. + cat "$file" >&2 - if [ "$kb_free" -le "$kb" -o "$pc_free" -le "$pc" ]; then - warn "Not enough free disk space: ${pc_free}% free, ${kb_free} KB free; wanted more than ${pc}% free or ${kb} KB free" - return 1 + return 1 # not enough disk space fi - return 0 + return 0 # disk space is OK } # ########################################################################### diff --git a/t/lib/bash/safeguards.sh b/t/lib/bash/safeguards.sh index c874498f..cf678d90 100644 --- a/t/lib/bash/safeguards.sh +++ b/t/lib/bash/safeguards.sh @@ -18,36 +18,51 @@ is \ "2" \ "2-line df output" -check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 >$TMPDIR/out 2>&1 +# Filesystem 1024-blocks Used Available Capacity Mounted on +# /dev/disk0s2 118153176 94409664 23487512 81% / +# +# Those values are in Kb, so: +# used = 94409664 (94.4G) = 96_675_495_936 bytes +# free = 23487512 (23.4G) = 24_051_212_288 bytes +# pct free = 100 - 81 = 19 % + +# want free - 100, 18 < 19, so this should be ok. +check_disk_space "$SAMPLE/diskspace001.txt" 24051212188 18 >$TMPDIR/out 2>&1 is "$?" "0" "Enough disk space" is \ "`cat $TMPDIR/out`" \ "" \ "No output if enough disk space" -check_disk_space "$SAMPLE/diskspace001.txt" 24000 18 >$TMPDIR/out 2>&1 +# want free - 100 is ok, but 20 < 19 is not. +check_disk_space "$SAMPLE/diskspace001.txt" 24051212188 20 >$TMPDIR/out 2>&1 +is "$?" "1" "Not enough % free" + +# want free + 100, so this should fail +# (real free is 100 bytes under what we want) +check_disk_space "$SAMPLE/diskspace001.txt" 24051212388 18 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough MB free" cmd_ok \ - "grep -q '19% free, 23487512 KB free; wanted more than 18% free or 24576000 KB free' $TMPDIR/out" \ + "grep -q 'Actual: 19% free, 24051212288 bytes free (- 0 bytes margin)' $TMPDIR/out" \ "Warning if not enough disk space" -check_disk_space "$SAMPLE/diskspace001.txt" 22000 19 >$TMPDIR/out 2>&1 -is "$?" "1" "Not enough % free" - # ########################################################################### # Check with a margin (amount we plan to use in the future). # ########################################################################### -check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 100 +# want free - 100 + 50 margin, so effectively want free - 50 is ok. +check_disk_space "$SAMPLE/diskspace001.txt" 24051212188 18 50 is "$?" "0" "Enough disk space with margin" -check_disk_space "$SAMPLE/diskspace001.txt" 23000 18 100 >$TMPDIR/out 2>&1 +# want free - 100 + 101 margin, so real free is 1 byte under what we want. +check_disk_space "$SAMPLE/diskspace001.txt" 24051212188 18 101 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough MB free with margin" -check_disk_space "$SAMPLE/diskspace001.txt" 100 5 20000 >$TMPDIR/out 2>&1 +# want free - 100 + 50 margin ok but %free will be 19 which is < 25. +check_disk_space "$SAMPLE/diskspace001.txt" 24051212188 25 50 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough % free with margin" cmd_ok \ - "grep -q '3% free,' $TMPDIR/out" \ + "grep -q 'Actual:[ ]*19% free,' $TMPDIR/out" \ "Calculates % free with margin" # ########################################################################### From 05ac0be1361ed344296aa328947462125a251fe0 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 26 Jan 2012 12:45:44 -0700 Subject: [PATCH 16/18] Change --disk-byte-limit to --disk-bytes-free and --disk-pct-limit to --disk-pct-free. --- bin/pt-stalk | 96 ++++++++++------ lib/bash/collect.sh | 4 +- t/lib/bash/collect.sh | 4 +- t/lib/samples/bash/po002.sh | 212 ------------------------------------ 4 files changed, 65 insertions(+), 251 deletions(-) delete mode 100644 t/lib/samples/bash/po002.sh diff --git a/bin/pt-stalk b/bin/pt-stalk index 3300101f..a431e1a9 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -198,6 +198,7 @@ _eval_po() { local opt="" local default_val="" local neg=0 + local size=0 while read key val; do case "$key" in long) @@ -209,6 +210,7 @@ _eval_po() { "short form") ;; type) + [ "$val" = "size" ] && size=1 ;; desc) ;; @@ -235,6 +237,10 @@ _eval_po() { fi fi + if [ $size -eq 1 -a -n "$default_val" ]; then + default_val=$(size_to_bytes $default_val) + fi + eval "OPT_${opt}"="$default_val" done @@ -275,6 +281,7 @@ _parse_command_line() { local opt_is_negated="" local real_opt="" local required_arg="" + local spec="" for opt in "$@"; do if [ "$opt" = "--" -o "$opt" = "----" ]; then @@ -360,6 +367,10 @@ _parse_command_line() { if [ "$opt_is_ok" ]; then opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + if grep "^type:size" "$spec" >/dev/null; then + val=$(size_to_bytes $val) + fi + eval "OPT_$opt"="'$val'" opt="" @@ -369,10 +380,16 @@ _parse_command_line() { opt_is_negated="" real_opt="" required_arg="" + spec="" fi done } +size_to_bytes() { + local size="$1" + echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};' +} + # ########################################################################### # End parse_options package # ########################################################################### @@ -458,33 +475,37 @@ disk_space() { check_disk_space() { local file="$1" - local mb="${2:-0}" - local pc="${3:-0}" - local mb_margin="${4:-0}" + local min_free_bytes="${2:-0}" + local min_free_pct="${3:-0}" + local bytes_margin="${4:-0}" - local kb=$(($mb * 1024)) - local kb_margin=$(($mb_margin * 1024)) + local used_bytes=$(cat "$file" | awk '/^\//{print $3 * 1024}'); + local free_bytes=$(cat "$file" | awk '/^\//{print $4 * 1024}'); + local pct_used=$(cat "$file" | awk '/^\//{print $5}' | sed -e 's/%//g'); + local pct_free=$((100 - $pct_used)) - local kb_used=$(cat "$file" | awk '/^\//{print $3}'); - local kb_free=$(cat "$file" | awk '/^\//{print $4}'); - local pc_used=$(cat "$file" | awk '/^\//{print $5}' | sed -e 's/%//g'); + local real_free_bytes=$free_bytes + local real_pct_free=$pct_free - if [ "$kb_margin" -gt "0" ]; then - local kb_total=$(($kb_used + $kb_free)) + if [ $bytes_margin -gt 0 ]; then + used_bytes=$(($used_bytes + $bytes_margin)) + free_bytes=$(($free_bytes - $bytes_margin)) + pct_used=$(awk "BEGIN { printf(\"%d\", ($used_bytes/($used_bytes + $free_bytes)) * 100) }") - kb_used=$(($kb_used + $kb_margin)) - kb_free=$(($kb_free - $kb_margin)) - pc_used=$(awk "BEGIN { printf(\"%d\", $kb_used/$kb_total * 100) }") + pct_free=$((100 - $pct_used)) fi - local pc_free=$((100 - $pc_used)) + if [ $free_bytes -lt $min_free_bytes -o $pct_free -lt $min_free_pct ]; then + warn "Not enough free disk space: + Limit: ${min_free_pct}% free, ${min_free_bytes} bytes free + Actual: ${real_pct_free}% free, ${real_free_bytes} bytes free (- $bytes_margin bytes margin) +" + cat "$file" >&2 - if [ "$kb_free" -le "$kb" -o "$pc_free" -le "$pc" ]; then - warn "Not enough free disk space: ${pc_free}% free, ${kb_free} KB free; wanted more than ${pc}% free or ${kb} KB free" - return 1 + return 1 # not enough disk space fi - return 0 + return 0 # disk space is OK } # ########################################################################### @@ -674,8 +695,8 @@ collect() { disk_space $d > $d/$p-disk-space check_disk_space \ $d/$p-disk-space \ - "$OPT_DISK_BYTE_LIMIT" \ - "$OPT_DISK_PCT_LIMIT" \ + "$OPT_DISK_BYTES_FREE" \ + "$OPT_DISK_PCT_FREE" \ || break sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}') @@ -980,16 +1001,16 @@ stalk() { # Check if we'll have enough disk space to collect. Disk space # is also checked every interval while collecting. - local margin="20" # default 20M margin, unless: + local margin="20971520" # default 20M margin, unless: if [ -n "$last_prefix" ]; then margin=$(du -mc "$OPT_DEST"/"$last_prefix"-* | tail -n 1 | awk '{print $1'}) fi disk_space "$OPT_DEST" > "$OPT_DEST/$prefix-disk-space" check_disk_space \ "$OPT_DEST/$prefix-disk-space" \ - "$OPT_DISK_BYTE_LIMIT" \ - "$OPT_DISK_PCT_LIMIT" \ - "$margin" # real used MB + margin MB + "$OPT_DISK_BYTES_FREE" \ + "$OPT_DISK_PCT_FREE" \ + "$margin" if [ $? -eq 0 ]; then # There should be enough disk space, so collect. log "$msg" >> "$OPT_DEST/$prefix-trigger" @@ -1313,29 +1334,34 @@ type: string; default: /var/lib/pt-stalk Where to store the diagnostic data. Each time the tool collects data, it writes to a new set of files, which are named with the current system timestamp. -=item --disk-byte-limit +=item --disk-bytes-free -type: int; default: 100 +type: size; default: 100M -Don't collect data unless the destination disk has this much free space. This -prevents the tool from filling up the disk with diagnostic data. +Don't collect data if the disk has less than this much free space. +This prevents the tool from filling up the disk with diagnostic data. -If the destination directory contains a previously captured sample of data, the -tool will measure its size and use that as an estimate of how much data is +If the L<"--dest"> directory contains a previously captured sample of data, +the tool will measure its size and use that as an estimate of how much data is likely to be gathered this time, too. It will then be even more pessimistic, and will refuse to collect data unless the disk has enough free space to hold the sample and still have the desired amount of free space. For example, if you'd like 100MB of free space and the previous diagnostic sample consumed 100MB, the tool won't collect any data unless the disk has 200MB free. -=item --disk-pct-limit +Valid size value suffixes are k, M, G, and T. + +=item --disk-pct-free type: int; default: 5 -Don't collect data unless the disk has at least this percent free space. This -option works similarly to --disk-byte-limit, but specifies a percentage margin -of safety instead of a byte margin of safety. The tool honors both options, and -will not collect any data unless both margins are satisfied. +Don't collect data if the disk has less than this percent free space. +This prevents the tool from filling up the disk with diagnostic data. + +This option works similarly to L<"--disk-bytes-free"> but specifies a +percentage margin of safety instead of a bytes margin of safety. +The tool honors both options, and will not collect any data unless both +margins are satisfied. =item --function diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 75f3d69d..41d43052 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -178,8 +178,8 @@ collect() { disk_space $d > $d/$p-disk-space check_disk_space \ $d/$p-disk-space \ - "$OPT_DISK_BYTE_LIMIT" \ - "$OPT_DISK_PCT_LIMIT" \ + "$OPT_DISK_BYTES_FREE" \ + "$OPT_DISK_PCT_FREE" \ || break # Synchronize ourselves onto the clock tick, so the sleeps are 1-second diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 6ae070d7..b944fa3b 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -15,7 +15,7 @@ source "$LIB_DIR/safeguards.sh" source "$LIB_DIR/alt_cmds.sh" source "$LIB_DIR/collect.sh" -parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf +parse_options "$BIN_DIR/pt-stalk" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf # Prefix (with path) for the collect files. local p="$TMPDIR/collect/2011_12_05" @@ -112,7 +112,7 @@ is "$iters" "1" "1 iteration/1s run time" # Try longer run time. # ########################################################################### -parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 2 -- --defaults-file=/tmp/12345/my.sandbox.cnf +parse_options "$BIN_DIR/pt-stalk" --run-time 2 -- --defaults-file=/tmp/12345/my.sandbox.cnf rm $TMPDIR/collect/* diff --git a/t/lib/samples/bash/po002.sh b/t/lib/samples/bash/po002.sh deleted file mode 100644 index 63a8672b..00000000 --- a/t/lib/samples/bash/po002.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env bash - -: - -# ############################################################################ -# Documentation -# ############################################################################ -:<<'DOCUMENTATION' -=pod - -=head1 NAME - -pt-stalk - Wait for a condition to occur then begin collecting data. - -=head1 OPTIONS - -=over - -=item --collect - -default: yes; negatable: yes - -Collect system information. - -=item --collect-gdb - -Collect GDB stacktraces. - -=item --collect-oprofile - -Collect oprofile data. - -=item --collect-strace - -Collect strace data. - -=item --collect-tcpdump - -Collect tcpdump data. - -=item --cycles - -type: int; default: 5 - -Number of times condition must be met before triggering collection. - -=item --daemonize - -default: yes; negatable: yes - -Daemonize the tool. - -=item --dest - -type: string - -Where to store collected data. - -=item --disk-byte-limit - -type: int; default: 100 - -Exit if the disk has less than this many MB free. - -=item --disk-pct-limit - -type: int; default: 5 - -Exit if the disk is less than this %full. - -=item --execute-command - -type: string; default: pt-collect - -Location of the C tool. - -=item --function - -type: string; default: status - -Built-in function name or plugin file name which returns the value of C. - -Possible values are: - -=over - -=item * status - -Grep the value of C from C. - -=item * processlist - -Count the number of processes in C whose -C column matches C. For example: - - TRIGGER_FUNCTION="processlist" \ - VARIABLE="State" \ - MATCH="statistics" \ - THRESHOLD="10" - -The above triggers when more than 10 processes are in the "statistics" state. -C must be specified for this trigger function. - -=item * magic - -TODO - -=item * plugin file name - -A plugin file allows you to specify a custom trigger function. The plugin -file must contain a function called C. For example: - - trg_plugin() { - # Do some stuff. - echo "$value" - } - -The last output if the function (its "return value") must be a number. -This number is compared to C. All L<"ENVIRONMENT"> variables -are available to the function. - -Do not alter the tool's existing global variables. Prefix any plugin-specific -global variables with "PLUGIN_". - -=back - -=item --help - -Print help and exit. - -=item --interval - -type: int; default: 1 - -Interval between checks. - -=item --iterations - -type: int - -Exit after triggering C this many times. By default, the tool -will collect as many times as it's triggered. - -=item --log - -type: string; default: /var/log/pt-stalk.log - -Print all output to this file when daemonized. - -=item --match - -type: string - -Match pattern for C L<"--function">. - -=item --notify-by-email - -type: string - -Send mail to this list of addresses when C triggers. - -=item --pid FILE - -type: string; default: /var/run/pt-stalk.pid - -Create a PID file when daemonized. - -=item --retention-time - -type: int; default: 30 - -Remove samples after this many days. - -=item --run-time - -type: int; default: 30 - -How long to collect statistics data for? - -Make sure that this isn't longer than SLEEP. - -=item --sleep - -type: int; default: 300 - -How long to sleep after collecting? - -=item --threshold N - -type: int; default: 25 - -Max number of C to tolerate. - -=item --variable NAME - -type: string; default: Threads_running - -This is the thing to check for. - -=item --version - -Print tool's version and exit. - -=back - -=head1 ENVIRONMENT - -No env vars used. - -=cut - -DOCUMENTATION From 6b017da6fee7fc06cf82f76ae237188f78f81141 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 26 Jan 2012 12:59:48 -0700 Subject: [PATCH 17/18] Use new pretty --help. --- bin/pt-stalk | 34 ++++++++++++++++++++++++++++------ lib/bash/parse_options.sh | 34 ++++++++++++++++++++++++++++------ t/lib/bash/parse_options.sh | 6 +++++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index a431e1a9..97d4ba59 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -85,12 +85,34 @@ usage_or_errors() { echo echo "Command line options:" echo - for opt in $(ls "$PO_DIR"); do - local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') - echo "--$opt" - echo " $desc" - echo - done + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $lcol = 20; # Allow this much space for option names. + my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide. + my $name; + while ( <> ) { + my $line = $_; + chomp $line; + if ( $line =~ s/^long:/ --/ ) { + $name = $line; + } + elsif ( $line =~ s/^desc:// ) { + $line =~ s/ +$//mg; + my @lines = grep { $_ } + $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g; + if ( length($name) >= $lcol ) { + print $name, "\n", (q{ } x $lcol); + } + else { + printf "%-${lcol}s", $name; + } + print join("\n" . (q{ } x $lcol), @lines); + print "\n"; + } + } + ' "$PO_DIR"/* + echo echo "Options and values after processing arguments:" echo for opt in $(ls "$PO_DIR"); do diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index a33e68f8..6320c0f3 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -91,12 +91,34 @@ usage_or_errors() { echo echo "Command line options:" echo - for opt in $(ls "$PO_DIR"); do - local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') - echo "--$opt" - echo " $desc" - echo - done + perl -e ' + use strict; + use warnings FATAL => qw(all); + my $lcol = 20; # Allow this much space for option names. + my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide. + my $name; + while ( <> ) { + my $line = $_; + chomp $line; + if ( $line =~ s/^long:/ --/ ) { + $name = $line; + } + elsif ( $line =~ s/^desc:// ) { + $line =~ s/ +$//mg; + my @lines = grep { $_ } + $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g; + if ( length($name) >= $lcol ) { + print $name, "\n", (q{ } x $lcol); + } + else { + printf "%-${lcol}s", $name; + } + print join("\n" . (q{ } x $lcol), @lines); + print "\n"; + } + } + ' "$PO_DIR"/* + echo echo "Options and values after processing arguments:" echo for opt in $(ls "$PO_DIR"); do diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index b66f8584..5afa53b5 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=72 +TESTS=73 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -105,6 +105,10 @@ cmd_ok \ "grep -q \"For more information, 'man pt-stalk' or 'perldoc\" $TMPFILE" \ "--help" +cmd_ok \ + "grep -q ' --string-opt2[ ]*String option with a default.' $TMPFILE" \ + "Command line options" + cmd_ok \ "grep -q '\-\-string-opt[ ]*(No value)' $TMPFILE" \ "Options and values after processing arguments" From 81ae556f8b9bd40b61912f2e7ddebcd80b1b674c Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 26 Jan 2012 13:04:23 -0700 Subject: [PATCH 18/18] Use local IFS. --- bin/pt-stalk | 9 ++------- lib/bash/parse_options.sh | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 97d4ba59..cb6d872c 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -166,12 +166,10 @@ parse_options() { shift # --config local user_config_files="$1" shift # that ^ - local old_ifs="$IFS" - IFS="," + local IFS="," for user_config_file in $user_config_files; do _parse_config_files "$user_config_file" done - IFS="$old_ifs" else _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" fi @@ -214,8 +212,7 @@ _parse_pod() { } _eval_po() { - local old_ifs="$IFS" - IFS=":" + local IFS=":" for opt_spec in "$PO_DIR"/*; do local opt="" local default_val="" @@ -265,8 +262,6 @@ _eval_po() { eval "OPT_${opt}"="$default_val" done - - IFS="$old_ifs" } _parse_config_files() { diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 6320c0f3..3c2ac8c9 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -192,12 +192,10 @@ parse_options() { shift # --config local user_config_files="$1" shift # that ^ - local old_ifs="$IFS" - IFS="," + local IFS="," for user_config_file in $user_config_files; do _parse_config_files "$user_config_file" done - IFS="$old_ifs" else _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" fi @@ -252,8 +250,7 @@ _eval_po() { # Evaluate the program options into existence as global variables # transformed like --my-op == $OPT_MY_OP. If an option has a default # value, it's assigned that value. Else, it's value is an empty string. - local old_ifs="$IFS" - IFS=":" + local IFS=":" for opt_spec in "$PO_DIR"/*; do local opt="" local default_val="" @@ -305,8 +302,6 @@ _eval_po() { # Eval the option into existence as a global variable. eval "OPT_${opt}"="$default_val" done - - IFS="$old_ifs" } _parse_config_files() {