From 1ae8661271ba7813a835f806dd12dab8cae8a9ac Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 26 Oct 2011 10:50:55 -0600 Subject: [PATCH 01/27] First Bash parse_options lib with tests. --- lib/bash/parse_options.sh | 119 +++++++++++++ t/lib/bash.t | 23 +++ t/lib/bash/parse_options.sh | 25 +++ t/lib/samples/bash/dummy.sh | 4 + t/lib/samples/bash/po001.sh | 340 ++++++++++++++++++++++++++++++++++++ 5 files changed, 511 insertions(+) create mode 100644 lib/bash/parse_options.sh create mode 100755 t/lib/bash.t create mode 100644 t/lib/bash/parse_options.sh create mode 100644 t/lib/samples/bash/dummy.sh create mode 100644 t/lib/samples/bash/po001.sh diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh new file mode 100644 index 00000000..e9003cad --- /dev/null +++ b/lib/bash/parse_options.sh @@ -0,0 +1,119 @@ +# Print usage (--help). +usage() { + local file=$1 + + local usage=$(grep '^Usage: ' $file) + local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help) + + if [ "${OPT_ERR}" ]; then + echo "Error: ${OPT_ERR}" >&2 + fi + echo $usage >&2 + echo >&2 + echo "Options:" >&2 + echo >&2 + cat $TMPDIR/help >&2 + echo >&2 + echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 +} + +# Parse command line options. +declare -a ARGV # non-option args (probably input files) +declare EXT_ARGV # everything after -- (args for an external command) +parse_options() { + local file=$1 + shift + + local opt="" + local val="" + local default="" + local version="" + local i=0 + + awk ' + /^=head1 OPTIONS/ { + getline + while ($0 !~ /^=head1/) { + if ($0 ~ /^=item --.*/) { + long_opt=substr($2, 3, length($2) - 2) + short_opt="" + required_arg="" + + if ($3) { + if ($3 ~ /-[a-z]/) + short_opt=substr($3, 3, length($3) - 3) + else + required_arg=$3 + } + + if ($4 ~ /[A-Z]/) + required_arg=$4 + + getline # blank line + getline # short description line + + if ($0 ~ /default: /) { + i=index($0, "default: ") + default=substr($0, i + 9, length($0) - (i + 9)) + } + else + default="" + + print long_opt "," short_opt "," required_arg "," default + } + getline + } + exit + }' $file > $TMPDIR/options + + while read spec; do + opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + default=$(echo $spec | cut -d',' -f4) + eval "$opt"="$default" + done < <(cat $TMPDIR/options) + + for opt; do + if [ $# -eq 0 ]; then + break + fi + opt=$1 + if [ "$opt" = "--" ]; then + shift + EXT_ARGV="$@" + break + fi + if [ "$opt" = "--version" ]; then + version=$(grep '^pt-[^ ]\+ [0-9]' $0) + echo "$version" + exit 0 + fi + if [ "$opt" = "--help" ]; then + usage $file + exit 0 + fi + shift + if [ $(expr "$opt" : "-") -eq 0 ]; then + ARGV[i]="$opt" + i=$((i+1)) + continue + fi + opt=$(echo $opt | sed 's/^-*//') + spec=$(grep -E "^$opt,|,$opt," "$TMPDIR/options") + if [ -z "$spec" ]; then + die "Unknown option: $opt" + fi + opt=$(echo $spec | cut -d',' -f1) + required_arg=$(echo $spec | cut -d',' -f3) + val=1 + if [ -n "$required_arg" ]; then + if [ $# -eq 0 ]; then + die "--$opt requires a $required_arg argument" + else + val="$1" + shift + fi + fi + opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:]) + eval "$opt"="$val" + done +} diff --git a/t/lib/bash.t b/t/lib/bash.t new file mode 100755 index 00000000..aaf1fe28 --- /dev/null +++ b/t/lib/bash.t @@ -0,0 +1,23 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); + +use PerconaTest; + +my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/; +push @ARGV, "$trunk/t/lib/bash/*.sh" unless @ARGV; + +$ENV{LIB_DIR} = "$trunk/lib/bash"; +$ENV{T_LIB_DIR} = "$trunk/t/lib"; + +system("$trunk/util/test-bash-functions $trunk/t/lib/samples/bash/dummy.sh @ARGV"); + +exit; diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh new file mode 100644 index 00000000..00c8de65 --- /dev/null +++ b/t/lib/bash/parse_options.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +TESTS=17 + +source "$LIB_DIR/parse_options.sh" + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" + +is "$THRESHOLD" "100" +is "$VARIABLE" "Threads_connected" +is "$CYCLES" "1" +is "$GDB" "no" +is "$OPROFILE" "yes" +is "$STRACE" "no" +is "$TCPDUMP" "yes" +is "$EMAIL" "" +is "$INTERVAL" "30" +is "$MAYBE_EMPTY" "no" +is "$COLLECT" "${HOME}/bin/pt-collect" +is "$DEST" "${HOME}/collected/" +is "$DURATION" "30" +is "$SLEEP" "300" +is "$PCT_THRESHOLD" "95" +is "$MB_THRESHOLD" "100" +is "$PURGE" "30" diff --git a/t/lib/samples/bash/dummy.sh b/t/lib/samples/bash/dummy.sh new file mode 100644 index 00000000..a7eea1ee --- /dev/null +++ b/t/lib/samples/bash/dummy.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# This is a dummy script for testing the Bash libs. t/lib/bashLibs.t +# calls "util/test-bash-functions dummy.sh ". diff --git a/t/lib/samples/bash/po001.sh b/t/lib/samples/bash/po001.sh new file mode 100644 index 00000000..0fdd943d --- /dev/null +++ b/t/lib/samples/bash/po001.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash + +# This is a fake script for testing the parse_options.sh lib. +exit 0; + +# ############################################################################ +# Documentation +# ############################################################################ +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-stalk - Wait for a condition to occur then begin collecting data. + +=head1 SYNOPSIS + +Usage: pt-stalk [OPTIONS] [-- MYSQL_OPTIONS] + +pt-stalk watches for a condition to become true, and when it does, executes +a script. By default it executes L, but that can be customized. +This tool is useful for gathering diagnostic data when an infrequent event +occurs, so an expert person can review the data later. + +=head1 RISKS + +The following section is included to inform users about the potential risks, +whether known or unknown, of using this tool. The two main categories of risks +are those created by the nature of the tool (e.g. read-only tools vs. read-write +tools) and those created by bugs. + +pt-stalk is a read-only tool. It should be very low-risk. + +At the time of this release, we know of no bugs that could cause serious harm +to users. + +The authoritative source for updated information is always the online issue +tracking system. Issues that affect this tool will be marked as such. You can +see a list of such issues at the following URL: +L. + +See also L<"BUGS"> for more information on filing bugs and getting help. + +=head1 DESCRIPTION + +Although pt-stalk comes pre-configured to do a specific thing, in general +this tool is just a skeleton script for the following flow of actions: + +=over + +=item 1. + +Loop infinitely, sleeping between iterations. + +=item 2. + +In each iteration, run some command and get the output. + +=item 3. + +If the command fails or the output is larger than the threshold, +execute the collection script; but do not execute if the destination disk +is too full. + +=back + +By default, the tool is configured to execute mysqladmin extended-status and +extract the value of the Threads_connected variable; if this is greater than +100, it runs the collection script. This is really just placeholder code, +and almost certainly needs to be customized! + +If the tool does execute the collection script, it will wait for a while +before checking and executing again. This is to prevent a continuous +condition from causing a huge number of executions to fire off. + +The name 'stalk' is because 'watch' is already taken, and 'stalk' is fun. + +=head1 CONFIGURING + +If the file F exists in the current working directory, then +L<"ENVIRONMENT"> variables are imported from it. For example, the config +file has the format: + + INTERVAL=10 + GDB=yes + +See L<"ENVIRONMENT">. + +=head1 OPTIONS + +=over + +=item --threshold N + +Max number of C to tolerate. (default: 100) + +=item --variable NAME + +This is the thing to check for. (default: Threads_connected) + +=item --cycles N + +How many times must the condition be met before the script will fire? (default: 1) + +=item --gdb + +Collect GDB stacktraces? (default: no) + +=item --oprofile + +Collect oprofile data? (default: yes) + +=item --strace + +Collect strace data? (default: no) + +=item --tcpdump + +Collect tcpdump data? (default: yes) + +=item --email ADDRESS + +Send mail to this list of addresses when the script triggers. + +=item --interval SECONDS + +This is the interval between checks. (default: 30) + +=item --maybe-empty + +Result of checks may be empty. (default: no) +If the command you're running to detect the condition is allowed to return +nothing (e.g. a grep line that might not even exist if there's no problem), +then set this to "yes". + +=item --collect DIRECTORY + +Location of the C tool. (default: ${HOME}/bin/pt-collect) + +=item --dest DIRECTORY + +Where to store collected data. (default: ${HOME}/collected/) + +=item --duration SECONDS + +How long to collect statistics data for? (default: 30) +Make sure that this isn't longer than SLEEP. + +=item --sleep SECONDS + +How long to sleep after collecting? (default: 300) + +=item --pct-threshold PCT + +Bail out if the disk is more than this %full. (default: 95) + +=item --mb-threshold MEGABYTES + +Bail out if the disk has less than this many MB free. (default: 100) + +=item --purge DAYS + +Remove samples after this many days. (default: 30) + +=item --version + +Print tool's version and exit. + +=back + +=head1 ENVIRONMENT + +The following environment variables configure how, what, and when the tool +runs. They are all optional and can be specified either on the command line +or in the F config file (see L<"CONFIGURING">). + +=over + +=item THRESHOLD (default 100) + +This is the max number of we want to tolerate. + +=item VARIABLE (default Threads_connected} + +This is the thing to check for. + +=item CYCLES (default 1) + +How many times must the condition be met before the script will fire? + +=item GDB (default no) + +Collect GDB stacktraces? + +=item OPROFILE (default yes) + +Collect oprofile data? + +=item STRACE (default no) + +Collect strace data? + +=item TCPDUMP (default yes) + +Collect tcpdump data? + +=item EMAIL + +Send mail to this list of addresses when the script triggers. + +=item MYSQLOPTIONS + +Any options to pass to mysql/mysqladmin, such as -u, -p, etc + +=item INTERVAL (default 30) + +This is the interval between checks. + +=item MAYBE_EMPTY (default no) + +If the command you're running to detect the condition is allowed to return +nothing (e.g. a grep line that might not even exist if there's no problem), +then set this to "yes". + +=item COLLECT (default ${HOME}/bin/pt-collect) + +This is the location of the 'collect' script. + +=item DEST (default ${HOME}/collected/) + +This is where to store the collected data. + +=item DURATION (default 30) + +How long to collect statistics data for? Make sure that this isn't longer +than SLEEP. + +=item SLEEP (default DURATION * 10) + +How long to sleep after collecting? + +=item PCT_THRESHOLD (default 95) + +Bail out if the disk is more than this %full. + +=item MB_THRESHOLD (default 100) + +Bail out if the disk has less than this many MB free. + +=item PURGE (default 30) + +Remove samples after this many days. + +=back + +=head1 SYSTEM REQUIREMENTS + +This tool requires Bash v3 or newer. + +=head1 BUGS + +For a list of known bugs, see L. + +Please report bugs at L. +Include the following information in your bug report: + +=over + +=item * Complete command-line used to run the tool + +=item * Tool L<"--version"> + +=item * MySQL version of all servers involved + +=item * Output from the tool including STDERR + +=item * Input files (log/dump/config files, etc.) + +=back + +If possible, include debugging output by running the tool with C; +see L<"ENVIRONMENT">. + +=head1 DOWNLOADING + +Visit L to download the +latest release of Percona Toolkit. Or, get the latest release from the +command line: + + wget percona.com/get/percona-toolkit.tar.gz + + wget percona.com/get/percona-toolkit.rpm + + wget percona.com/get/percona-toolkit.deb + +You can also get individual tools from the latest release: + + wget percona.com/get/TOOL + +Replace C with the name of any tool. + +=head1 AUTHORS + +Baron Schwartz, Justin Swanhart, and Fernando Ipar + +=head1 ABOUT PERCONA TOOLKIT + +This tool is part of Percona Toolkit, a collection of advanced command-line +tools developed by Percona for MySQL support and consulting. Percona Toolkit +was forked from two projects in June, 2011: Maatkit and Aspersa. Those +projects were created by Baron Schwartz and developed primarily by him and +Daniel Nichter, both of whom are employed by Percona. Visit +L for more software developed by Percona. + +=head1 COPYRIGHT, LICENSE, AND WARRANTY + +This program is copyright 2010-2011 Baron Schwartz, 2011 Percona Inc. +Feedback and improvements are welcome. + +THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +systems, you can issue `man perlgpl' or `man perlartistic' to read these +licenses. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +=head1 VERSION + +pt-stalk 1.0.1 + +=cut + +DOCUMENTATION From 1ec666de0eba1428b18f17434160e5839fb7ae51 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 27 Oct 2011 11:56:58 -0600 Subject: [PATCH 02/27] Add Bash libs tmpdir and log_warn_die. Docu parse_options. Don't use TMPDIR in test-bash-functions; add ok() to test-bash-functions. --- lib/bash/log_warn_die.sh | 45 ++++++++++++++++++++++++++ lib/bash/parse_options.sh | 66 ++++++++++++++++++++++++++++++++++++--- lib/bash/tmpdir.sh | 50 +++++++++++++++++++++++++++++ t/lib/bash/tmpdir.sh | 44 ++++++++++++++++++++++++++ util/test-bash-functions | 23 +++++++++----- 5 files changed, 215 insertions(+), 13 deletions(-) create mode 100644 lib/bash/log_warn_die.sh create mode 100644 lib/bash/tmpdir.sh create mode 100644 t/lib/bash/tmpdir.sh diff --git a/lib/bash/log_warn_die.sh b/lib/bash/log_warn_die.sh new file mode 100644 index 00000000..ef60daa3 --- /dev/null +++ b/lib/bash/log_warn_die.sh @@ -0,0 +1,45 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin log_warn_die lib +# ########################################################################### + +# Library: log_warn_die +# log_warn_die provides standard log(), warn(), and die() subs. + +set -u + +EXIT_STATUS=0 + +log() { + TS=$(date +%F-%T | tr :- _); + echo "$TS $1" +} + +warn() { + log "$1" >&2 + EXIT_STATUS=$((EXIT_STATUS | 1)) +} + +die() { + warn "$1" + exit 1 +} + +# ########################################################################### +# End log_warn_die lib +# ########################################################################### diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index e9003cad..8b6b31d2 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -1,11 +1,53 @@ -# Print usage (--help). +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin parse_options lib +# ########################################################################### + +# Library: parse_options +# parse_options parses Perl POD options from Bash tools and creates +# global variables for each option. + +set -u + +# Global variables. These must be global because declare inside a +# sub will be scoped locally. +declare -a ARGV # non-option args (probably input files) +declare EXT_ARGV # everything after -- (args for an external command) + +# Sub: usage +# Print usage (--help) and list the program's options. +# +# Arguments: +# file - Program file with Perl POD which has usage and options. +# +# Required Global Variables: +# TIMDIR - Temp directory. +# TOOL - Tool's name. +# +# Optional Global Variables: +# OPT_ERR - Command line option error message. usage() { local file=$1 local usage=$(grep '^Usage: ' $file) local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help) - if [ "${OPT_ERR}" ]; then + if [ "$OPT_ERR" ]; then echo "Error: ${OPT_ERR}" >&2 fi echo $usage >&2 @@ -17,9 +59,19 @@ usage() { echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 } -# Parse command line options. -declare -a ARGV # non-option args (probably input files) -declare EXT_ARGV # everything after -- (args for an external command) +# Sub: parse_options +# Parse Perl POD options from a program file. +# +# Arguments: +# file - Program file with Perl POD options. +# +# Required Global Variables: +# TIMDIR - Temp directory. +# +# Declared Global Variables: +# This sub decalres a global var for each option by uppercasing the +# option, removing the option's leading --, changing all - to _, and +# prefixing with "OPT_". E.g. --foo-bar becomes OPT_FOO_BAR. parse_options() { local file=$1 shift @@ -117,3 +169,7 @@ parse_options() { eval "$opt"="$val" done } + +# ########################################################################### +# End parse_options lib +# ########################################################################### diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh new file mode 100644 index 00000000..208d883b --- /dev/null +++ b/lib/bash/tmpdir.sh @@ -0,0 +1,50 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin tmpdir lib +# ########################################################################### + +# Library: tmpdir +# tmpdir make a secure temporary directory using mktemp. + +set -u + +TMPDIR="" +OPT_TMPDIR={OPT_TMPDIR:""} + +set_TMPDIR() { + if [ -n "$OPT_TMPDIR" ]; then + TMPDIR="$OPT_TMPDIR" + if [ ! -d "$TMPDIR" ]; then + mkdir $TMPDIR || die "Cannot make $TMPDIR" + fi + else + local tool=`basename $0` + TMPDIR=`mktemp -d /tmp/${tool}.XXXXX` || die "Cannot make secure tmpdir" + fi +} + +rm_TMPDIR() { + if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then + rm -rf $TMPDIR + fi + TMPDIR="" +} + +# ########################################################################### +# End tmpdir lib +# ########################################################################### diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh new file mode 100644 index 00000000..1a7d5809 --- /dev/null +++ b/t/lib/bash/tmpdir.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +TESTS=9 + +source "$LIB_DIR/log_warn_die.sh" +source "$LIB_DIR/tmpdir.sh" + +TEST_NAME="TMPDIR not defined" +is "$TMPDIR" "" + +TEST_NAME="set_TMPDIR makes secure tmpdir" +set_TMPDIR +ok "test -d $TMPDIR" + +tmpdir=$TMPDIR; + +TEST_NAME="rm_TMPDIR" +rm_TMPDIR +ok "test ! -d $tmpdir" + +TEST_NAME="rm_TMPDIR resets TMPDIR" +is "$TMPDIR" "" + +# --tmpdir +OPT_TMPDIR="/tmp/use--tmpdir" + +TEST_NAME="TMPDIR not defined" +is "$TMPDIR" "" + +TEST_NAME="--tmpdir does not exist yet" +ok "test ! -d $OPT_TMPDIR" + +set_TMPDIR +TEST_NAME="set_TMPDIR uses --tmpdir" +is "$TMPDIR" "/tmp/use--tmpdir" + +TEST_NAME="set_TMPDIR creates --tmpdir" +ok "test -d $TMPDIR" + +tmpdir=$TMPDIR; + +TEST_NAME="rm_TMPDIR removes --tmpdir" +rm_TMPDIR +ok "test ! -d $tmpdir" diff --git a/util/test-bash-functions b/util/test-bash-functions index 1ffb0075..901e818e 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -34,9 +34,10 @@ die() { # Paths # ############################################################################ -TMPDIR="/tmp/percona-toolkit.test" -if [ ! -d $TMPDIR ]; then - mkdir $TMPDIR +# Do not use TMPDIR because the tools use it for their own secure tmpdir. +TEST_TMPDIR="/tmp/percona-toolkit.test" +if [ ! -d $TEST_TMPDIR ]; then + mkdir $TEST_TMPDIR fi # ############################################################################ @@ -69,7 +70,7 @@ load_tests() { # Source a test file to run whatever it contains (hopefully tests!). run_test() { local t=$1 # test file name, e.g. "group-by-all-01" for pt-diskstats - rm -rf $TMPDIR/* >/dev/null 2>&1 + rm -rf $TEST_TMPDIR/* >/dev/null 2>&1 TEST_NUMBER=1 # test number in this test file @@ -98,8 +99,8 @@ result() { echo "not ok $testno - $TEST_FILE $test_name" failed_tests=$(( failed_tests + 1)) echo "# Failed '$test_command'" >&2 - if [ -f $TMPDIR/failed_result ]; then - cat $TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2 + if [ -f $TEST_TMPDIR/failed_result ]; then + cat $TEST_TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2 fi fi testno=$((testno + 1)) @@ -115,7 +116,7 @@ no_diff() { local got=$1 local expected=$2 test_command="diff $got $expected" - eval $test_command > $TMPDIR/failed_result 2>&1 + eval $test_command > $TEST_TMPDIR/failed_result 2>&1 result $? } @@ -127,6 +128,12 @@ is() { result $? } +ok() { + local test_command=$1 + $test_command + result $? +} + # ############################################################################ # Script starts here # ############################################################################ @@ -157,6 +164,6 @@ for t in "${tests[@]}"; do run_test $t done -rm -rf $TMPDIR +rm -rf $TEST_TMPDIR exit $failed_tests From f2b644ba723d26e753a5db7d8254eb7fe1f48529 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 28 Oct 2011 11:08:59 -0600 Subject: [PATCH 03/27] More parse_options tests. Make --opt yes have value "yes" instead of "1". Change ok() to cmd_ok() in test-bash-functions. Mimic Perl modulue headers in Bash libs so update-modules will work with the latter. --- lib/bash/log_warn_die.sh | 7 +-- lib/bash/parse_options.sh | 18 +++---- lib/bash/tmpdir.sh | 23 +++++++-- t/lib/bash.t | 0 t/lib/bash/parse_options.sh | 94 +++++++++++++++++++++++++++++-------- t/lib/bash/tmpdir.sh | 10 ++-- util/test-bash-functions | 4 +- 7 files changed, 115 insertions(+), 41 deletions(-) mode change 100755 => 100644 t/lib/bash.t diff --git a/lib/bash/log_warn_die.sh b/lib/bash/log_warn_die.sh index ef60daa3..6d5ca2e4 100644 --- a/lib/bash/log_warn_die.sh +++ b/lib/bash/log_warn_die.sh @@ -15,14 +15,15 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin log_warn_die lib +# log_warn_die package # ########################################################################### -# Library: log_warn_die +# Package: log_warn_die # log_warn_die provides standard log(), warn(), and die() subs. set -u +# Global variables. EXIT_STATUS=0 log() { @@ -41,5 +42,5 @@ die() { } # ########################################################################### -# End log_warn_die lib +# End log_warn_die package # ########################################################################### diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 8b6b31d2..b94ea52b 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -15,10 +15,10 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin parse_options lib +# parse_options package # ########################################################################### -# Library: parse_options +# Package: parse_options # parse_options parses Perl POD options from Bash tools and creates # global variables for each option. @@ -36,7 +36,7 @@ declare EXT_ARGV # everything after -- (args for an external command) # file - Program file with Perl POD which has usage and options. # # Required Global Variables: -# TIMDIR - Temp directory. +# TIMDIR - Temp directory set by . # TOOL - Tool's name. # # Optional Global Variables: @@ -66,9 +66,9 @@ usage() { # file - Program file with Perl POD options. # # Required Global Variables: -# TIMDIR - Temp directory. +# TIMDIR - Temp directory set by . # -# Declared Global Variables: +# Set Global Variables: # This sub decalres a global var for each option by uppercasing the # option, removing the option's leading --, changing all - to _, and # prefixing with "OPT_". E.g. --foo-bar becomes OPT_FOO_BAR. @@ -121,7 +121,7 @@ parse_options() { while read spec; do opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:]) default=$(echo $spec | cut -d',' -f4) - eval "$opt"="$default" + eval "OPT_${opt}"="$default" done < <(cat $TMPDIR/options) for opt; do @@ -156,7 +156,7 @@ parse_options() { fi opt=$(echo $spec | cut -d',' -f1) required_arg=$(echo $spec | cut -d',' -f3) - val=1 + val="yes" if [ -n "$required_arg" ]; then if [ $# -eq 0 ]; then die "--$opt requires a $required_arg argument" @@ -166,10 +166,10 @@ parse_options() { fi fi opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:]) - eval "$opt"="$val" + eval "OPT_${opt}"="$val" done } # ########################################################################### -# End parse_options lib +# End parse_options package # ########################################################################### diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index 208d883b..143ec1df 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -15,17 +15,26 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin tmpdir lib +# tmpdir package # ########################################################################### -# Library: tmpdir +# Package: tmpdir # tmpdir make a secure temporary directory using mktemp. set -u +# Global variables. TMPDIR="" OPT_TMPDIR={OPT_TMPDIR:""} +# Sub: set_TMPDIR +# Create a secure tmpdir and set TMPDIR. +# +# Optional Global Variables: +# OPT_TMPDIR - User-specified --tmpdir (default none). +# +# Set Global Variables: +# TMPDIR - Absolute path of secure temp directory. set_TMPDIR() { if [ -n "$OPT_TMPDIR" ]; then TMPDIR="$OPT_TMPDIR" @@ -38,6 +47,14 @@ set_TMPDIR() { fi } +# Sub: rm_TMPDIR +# Remove the tmpdir and unset TMPDIR. +# +# Optional Global Variables: +# TMPDIR - TMPDIR set by . +# +# Set Global Variables: +# TMPDIR - Set to "". rm_TMPDIR() { if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then rm -rf $TMPDIR @@ -46,5 +63,5 @@ rm_TMPDIR() { } # ########################################################################### -# End tmpdir lib +# End tmpdir package # ########################################################################### diff --git a/t/lib/bash.t b/t/lib/bash.t old mode 100755 new mode 100644 diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 00c8de65..a5daeffe 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,25 +1,81 @@ #!/usr/bin/env bash -TESTS=17 +TESTS=37 +TMPFILE="$TEST_TMPDIR/parse-opts-output" + +source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/parse_options.sh" -parse_options "$T_LIB_DIR/samples/bash/po001.sh" +# ############################################################################ +# Parse options from POD using all default values. +# ############################################################################ -is "$THRESHOLD" "100" -is "$VARIABLE" "Threads_connected" -is "$CYCLES" "1" -is "$GDB" "no" -is "$OPROFILE" "yes" -is "$STRACE" "no" -is "$TCPDUMP" "yes" -is "$EMAIL" "" -is "$INTERVAL" "30" -is "$MAYBE_EMPTY" "no" -is "$COLLECT" "${HOME}/bin/pt-collect" -is "$DEST" "${HOME}/collected/" -is "$DURATION" "30" -is "$SLEEP" "300" -is "$PCT_THRESHOLD" "95" -is "$MB_THRESHOLD" "100" -is "$PURGE" "30" +parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE + +TEST_NAME="No warnings or errors" +is "`cat $TMPFILE`" "" + +TEST_NAME="Default opts" +is "$OPT_THRESHOLD" "100" +is "$OPT_VARIABLE" "Threads_connected" +is "$OPT_CYCLES" "1" +is "$OPT_GDB" "no" +is "$OPT_OPROFILE" "yes" +is "$OPT_STRACE" "no" +is "$OPT_TCPDUMP" "yes" +is "$OPT_EMAIL" "" +is "$OPT_INTERVAL" "30" +is "$OPT_MAYBE_EMPTY" "no" +is "$OPT_COLLECT" "${HOME}/bin/pt-collect" +is "$OPT_DEST" "${HOME}/collected/" +is "$OPT_DURATION" "30" +is "$OPT_SLEEP" "300" +is "$OPT_PCT_THRESHOLD" "95" +is "$OPT_MB_THRESHOLD" "100" +is "$OPT_PURGE" "30" + +# ############################################################################ +# Specify some opts, but use default values for the rest. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --threshold 50 --gdb yes --email user@example.com + +TEST_NAME="User-specified opts with defaults" +is "$OPT_THRESHOLD" "50" # specified +is "$OPT_VARIABLE" "Threads_connected" +is "$OPT_CYCLES" "1" +is "$OPT_GDB" "yes" # specified +is "$OPT_OPROFILE" "yes" +is "$OPT_STRACE" "no" +is "$OPT_TCPDUMP" "yes" +is "$OPT_EMAIL" "user@example.com" # specified +is "$OPT_INTERVAL" "30" +is "$OPT_MAYBE_EMPTY" "no" +is "$OPT_COLLECT" "${HOME}/bin/pt-collect" +is "$OPT_DEST" "${HOME}/collected/" +is "$OPT_DURATION" "30" +is "$OPT_SLEEP" "300" +is "$OPT_PCT_THRESHOLD" "95" +is "$OPT_MB_THRESHOLD" "100" +is "$OPT_PURGE" "30" + +# ############################################################################ +# An unknown option should produce an error. +# ############################################################################ + +# Have to call this in a subshell because the error will cause an exit. +( + parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 +) +local err=$? +TEST_NAME="Non-zero exit on unknown option" +is "$err" "1" + +TEST_NAME="Error on unknown option" +cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" + +# ############################################################################ +# Done +# ############################################################################ +exit diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index 1a7d5809..83708652 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -10,13 +10,13 @@ is "$TMPDIR" "" TEST_NAME="set_TMPDIR makes secure tmpdir" set_TMPDIR -ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; TEST_NAME="rm_TMPDIR" rm_TMPDIR -ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" TEST_NAME="rm_TMPDIR resets TMPDIR" is "$TMPDIR" "" @@ -28,17 +28,17 @@ TEST_NAME="TMPDIR not defined" is "$TMPDIR" "" TEST_NAME="--tmpdir does not exist yet" -ok "test ! -d $OPT_TMPDIR" +cmd_ok "test ! -d $OPT_TMPDIR" set_TMPDIR TEST_NAME="set_TMPDIR uses --tmpdir" is "$TMPDIR" "/tmp/use--tmpdir" TEST_NAME="set_TMPDIR creates --tmpdir" -ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; TEST_NAME="rm_TMPDIR removes --tmpdir" rm_TMPDIR -ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" diff --git a/util/test-bash-functions b/util/test-bash-functions index 901e818e..9a10a782 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -128,9 +128,9 @@ is() { result $? } -ok() { +cmd_ok() { local test_command=$1 - $test_command + eval $test_command result $? } From 8cb5f6b600638d15c3cdfa3027b2d726509f5759 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 31 Oct 2011 11:25:56 -0600 Subject: [PATCH 04/27] Add default OPT_ERR and fix default OPT_TMPDIR. --- lib/bash/parse_options.sh | 1 + lib/bash/tmpdir.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index b94ea52b..037902c6 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -28,6 +28,7 @@ set -u # sub will be scoped locally. declare -a ARGV # non-option args (probably input files) declare EXT_ARGV # everything after -- (args for an external command) +OPT_ERR=${OPT_ERR:""} # Sub: usage # Print usage (--help) and list the program's options. diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index 143ec1df..b0983f44 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -25,7 +25,7 @@ set -u # Global variables. TMPDIR="" -OPT_TMPDIR={OPT_TMPDIR:""} +OPT_TMPDIR=${OPT_TMPDIR:""} # Sub: set_TMPDIR # Create a secure tmpdir and set TMPDIR. From 42675e33b891bfde05ac967772e32bcea96d01fb Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 1 Nov 2011 09:26:58 -0600 Subject: [PATCH 05/27] Fix update-modules to work with Bash tools and libs. --- util/update-modules | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/util/update-modules b/util/update-modules index fb69fcb7..b976e07e 100755 --- a/util/update-modules +++ b/util/update-modules @@ -49,7 +49,11 @@ file_is_modified() { pkgs_in_tool() { local tool=$1 - pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1) + if [ "$tool_lang" = "perl" ]; then + pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1) + else + pkgs=$(grep '^# [a-z_]* package' $tool | awk '{print $2}') + fi } replace_pkg_in_tool() { @@ -72,7 +76,8 @@ replace_pkg_in_tool() { head -n $pkg_start_line $tool_file > $tmp_file - echo "# $pkg package + if [ "$tool_lang" = "perl" ]; then + echo "# $pkg package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/$pkg.pm @@ -80,11 +85,24 @@ replace_pkg_in_tool() { # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### {" >> $tmp_file + else + echo "# $pkg package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/bash/$pkg.sh +# t/lib/bash/$pkg.sh +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +" >> $tmp_file + fi $BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file - echo "} -# ########################################################################### + if [ "$tool_lang" = "perl" ]; then + echo "}" + fi + +echo "# ########################################################################### # End $pkg package" >> $tmp_file tail -n +$pkg_end_line $tool_file >> $tmp_file @@ -97,14 +115,19 @@ replace_pkg_in_tool() { # ############################################################################ tool_file=$1 + if [ -z "$tool_file" ]; then die "Usage: $0 TOOL [MODULE...]" fi + if [ ! -f $tool_file ]; then die "$tool_file does not exist" fi -if [ -z "$(head -n 1 $tool_file | grep perl)" ]; then - die "$tool_file is not a Perl tool" + +if [ -n "$(head -n 1 $tool_file | grep perl)" ]; then + tool_lang="perl" +else + tool_lang="bash" fi tool=$(basename $tool_file) @@ -124,11 +147,17 @@ for pkg in $pkgs; do continue fi - pkg_file="$BRANCH/lib/$pkg.pm" + if [ "$tool_lang" = "perl" ]; then + pkg_file="$BRANCH/lib/$pkg.pm" + else + pkg_file="$BRANCH/lib/bash/$pkg.sh" + fi + if [ ! -f $pkg_file ]; then warn "$pkg_file does not exist" continue fi + if file_is_modified $pkg_file; then warn "$pkg_file has uncommitted changes" continue From 173085e823273f8dff8b45a3de2d7dd09c70e9f7 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 28 Nov 2011 11:16:23 -0700 Subject: [PATCH 06/27] Rename tmpdir funcs. Include pid in tmpdir name. --- lib/bash/tmpdir.sh | 14 ++++++++------ t/lib/bash/tmpdir.sh | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index b0983f44..6349bf02 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -27,7 +27,7 @@ set -u TMPDIR="" OPT_TMPDIR=${OPT_TMPDIR:""} -# Sub: set_TMPDIR +# Sub: mk_tmpdir # Create a secure tmpdir and set TMPDIR. # # Optional Global Variables: @@ -35,7 +35,7 @@ OPT_TMPDIR=${OPT_TMPDIR:""} # # Set Global Variables: # TMPDIR - Absolute path of secure temp directory. -set_TMPDIR() { +mk_tmpdir() { if [ -n "$OPT_TMPDIR" ]; then TMPDIR="$OPT_TMPDIR" if [ ! -d "$TMPDIR" ]; then @@ -43,19 +43,21 @@ set_TMPDIR() { fi else local tool=`basename $0` - TMPDIR=`mktemp -d /tmp/${tool}.XXXXX` || die "Cannot make secure tmpdir" + local pid="$$" + TMPDIR=`mktemp -d /tmp/${tool}.${pid}.XXXXX` \ + || die "Cannot make secure tmpdir" fi } -# Sub: rm_TMPDIR +# Sub: rm_tmpdir # Remove the tmpdir and unset TMPDIR. # # Optional Global Variables: -# TMPDIR - TMPDIR set by . +# TMPDIR - TMPDIR set by . # # Set Global Variables: # TMPDIR - Set to "". -rm_TMPDIR() { +rm_tmpdir() { if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then rm -rf $TMPDIR fi diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index 83708652..4698b04e 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -8,17 +8,17 @@ source "$LIB_DIR/tmpdir.sh" TEST_NAME="TMPDIR not defined" is "$TMPDIR" "" -TEST_NAME="set_TMPDIR makes secure tmpdir" -set_TMPDIR +TEST_NAME="mk_tmpdir makes secure tmpdir" +mk_tmpdir cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; -TEST_NAME="rm_TMPDIR" -rm_TMPDIR +TEST_NAME="rm_tmpdir" +rm_tmpdir cmd_ok "test ! -d $tmpdir" -TEST_NAME="rm_TMPDIR resets TMPDIR" +TEST_NAME="rm_tmpdir resets TMPDIR" is "$TMPDIR" "" # --tmpdir @@ -30,15 +30,15 @@ is "$TMPDIR" "" TEST_NAME="--tmpdir does not exist yet" cmd_ok "test ! -d $OPT_TMPDIR" -set_TMPDIR -TEST_NAME="set_TMPDIR uses --tmpdir" +mk_tmpdir +TEST_NAME="mk_tmpdir uses --tmpdir" is "$TMPDIR" "/tmp/use--tmpdir" -TEST_NAME="set_TMPDIR creates --tmpdir" +TEST_NAME="mk_tmpdir creates --tmpdir" cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; -TEST_NAME="rm_TMPDIR removes --tmpdir" -rm_TMPDIR +TEST_NAME="rm_tmpdir removes --tmpdir" +rm_tmpdir cmd_ok "test ! -d $tmpdir" From 75d2c99d45be8cc391161adf9f9516e593ceec08 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 2 Dec 2011 11:31:31 -0700 Subject: [PATCH 07/27] Parse option specs, allow negating with --no-, add more tests. --- lib/bash/parse_options.sh | 172 +++++++++++++++++++++++++----------- t/lib/bash/parse_options.sh | 77 ++++++++-------- t/lib/samples/bash/po001.sh | 74 ++++------------ 3 files changed, 180 insertions(+), 143 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 037902c6..20953c2e 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -46,17 +46,12 @@ usage() { local file=$1 local usage=$(grep '^Usage: ' $file) - local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help) if [ "$OPT_ERR" ]; then echo "Error: ${OPT_ERR}" >&2 fi echo $usage >&2 echo >&2 - echo "Options:" >&2 - echo >&2 - cat $TMPDIR/help >&2 - echo >&2 echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 } @@ -77,57 +72,100 @@ parse_options() { local file=$1 shift - local opt="" - local val="" - local default="" - local version="" - local i=0 - - awk ' + # Parse the program options (po) from the POD. Each option has + # a spec file like: + # $ cat po/string-opt2 + # long=string-opt2 + # type=string + # default=foo + # That's the spec for --string-opt2. Each line is a key:value pair + # from the option's POD line like "type: string; default: foo". + mkdir $TMPDIR/po/ 2>/dev/null + rm -rf $TMPDIR/po/* + awk -v "po_dir"="$TMPDIR/po" ' /^=head1 OPTIONS/ { getline while ($0 !~ /^=head1/) { if ($0 ~ /^=item --.*/) { - long_opt=substr($2, 3, length($2) - 2) - short_opt="" - required_arg="" - - if ($3) { - if ($3 ~ /-[a-z]/) - short_opt=substr($3, 3, length($3) - 3) - else - required_arg=$3 - } - - if ($4 ~ /[A-Z]/) - required_arg=$4 + long_opt = substr($2, 3, length($2) - 2) + spec_file = po_dir "/" long_opt + trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file getline # blank line - getline # short description line + getline # specs or description - if ($0 ~ /default: /) { - i=index($0, "default: ") - default=substr($0, i + 9, length($0) - (i + 9)) + if ($0 ~ /^[a-z]/ ) { + # spec line like "type: int; default: 100" + print "long:" long_opt "; " $0 | trf + } + else { + # no specs, should be description of option + print "long:" long_opt > spec_file } - else - default="" - - print long_opt "," short_opt "," required_arg "," default } getline } exit - }' $file > $TMPDIR/options + }' $file - while read spec; do - opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:]) - default=$(echo $spec | cut -d',' -f4) - eval "OPT_${opt}"="$default" - done < <(cat $TMPDIR/options) + # Evaluate the program options into existence as global variables + # transformed like --my-op == $OPT_MY_OP. If an option has a default + # value, it's assigned that value. Else, it's value is an empty string. + for opt_spec in $(ls $TMPDIR/po/); do + local opt="" + local default_val="" + local neg=0 + while read line; do + local key=`echo $line | cut -d ':' -f 1` + local val=`echo $line | cut -d ':' -f 2` + case "$key" in + long) + opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:]) + ;; + default) + default_val="$val" + ;; + shortform) + ;; + type) + ;; + negatable) + if [ "$val" = "yes" ]; then + neg=1 + fi + ;; + *) + die "Invalid attribute in $TMPDIR/po/$opt_spec: $line" + esac + done < $TMPDIR/po/$opt_spec + if [ -z "$opt" ]; then + die "No long attribute in option spec $TMPDIR/po/$opt_spec" + fi + + if [ $neg -eq 1 ]; then + if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then + die "Option $opt_spec is negatable but not default: yes" + fi + fi + + eval "OPT_${opt}"="$default_val" + done + + # Parse the command line options. Anything after -- is put into + # EXT_ARGV. Options must begin with one or two hyphens (--help or -h), + # else the item is put into ARGV (it's probably a filename, directory, + # etc.) The program option specs parsed above are used to valid the + # command line options. All options have already been eval'd into + # existence, but we re-eval opts specified on the command line to update + # the corresponding global variable's value. For example, if --foo has + # a default value 100, then $OPT_FOO=100 already, but if --foo=500 is + # specified on the command line, then we re-eval $OPT_FOO=500 to update + # $OPT_FOO. + local i=0 # ARGV index for opt; do if [ $# -eq 0 ]; then - break + break # no more opts fi opt=$1 if [ "$opt" = "--" ]; then @@ -144,30 +182,62 @@ parse_options() { usage $file exit 0 fi - shift + shift if [ $(expr "$opt" : "-") -eq 0 ]; then + # Option does not begin with a hyphen (-), so treat it as + # a filename, directory, etc. ARGV[i]="$opt" i=$((i+1)) continue fi - opt=$(echo $opt | sed 's/^-*//') - spec=$(grep -E "^$opt,|,$opt," "$TMPDIR/options") - if [ -z "$spec" ]; then - die "Unknown option: $opt" + + # Save real opt from cmd line for error messages. + local real_opt="$opt" + + # Strip leading -- or --no- from option. + if $(echo $opt | grep -q '^--no-'); then + neg=1 + opt=$(echo $opt | sed 's/^--no-//') + else + neg=0 + opt=$(echo $opt | sed 's/^-*//') fi - opt=$(echo $spec | cut -d',' -f1) - required_arg=$(echo $spec | cut -d',' -f3) - val="yes" + + # Find the option's spec file. + if [ -f "$TMPDIR/po/$opt" ]; then + spec="$TMPDIR/po/$opt" + else + spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1) + if [ -z "$spec" ]; then + die "Unknown option: $opt" + fi + fi + + # Get the value specified for the option, if any. If the opt's spec + # says it has a type, then it requires a value and that value should + # be the next item ($1). Else, typeless options (like --version) are + # either "yes" if specified, else "no" if negatable and --no-opt. + required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2) if [ -n "$required_arg" ]; then if [ $# -eq 0 ]; then - die "--$opt requires a $required_arg argument" + die "$real_opt requires a $required_arg argument" else val="$1" shift fi + else + if [ $neg -eq 0 ]; then + val="yes" + else + val="no" + fi fi - opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:]) - eval "OPT_${opt}"="$val" + + # Get and transform the opt's long form. E.g.: -q == --quiet == QUIET. + opt=$(cat $spec | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + + # Re-eval the option to update its global variable value. + eval "OPT_$opt"="$val" done } diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index a5daeffe..fcb80850 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=37 +TESTS=24 TMPFILE="$TEST_TMPDIR/parse-opts-output" @@ -11,54 +11,59 @@ source "$LIB_DIR/parse_options.sh" # Parse options from POD using all default values. # ############################################################################ +TMPDIR="$TEST_TMPDIR" parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE TEST_NAME="No warnings or errors" is "`cat $TMPFILE`" "" TEST_NAME="Default opts" -is "$OPT_THRESHOLD" "100" -is "$OPT_VARIABLE" "Threads_connected" -is "$OPT_CYCLES" "1" -is "$OPT_GDB" "no" -is "$OPT_OPROFILE" "yes" -is "$OPT_STRACE" "no" -is "$OPT_TCPDUMP" "yes" -is "$OPT_EMAIL" "" -is "$OPT_INTERVAL" "30" -is "$OPT_MAYBE_EMPTY" "no" -is "$OPT_COLLECT" "${HOME}/bin/pt-collect" -is "$OPT_DEST" "${HOME}/collected/" -is "$OPT_DURATION" "30" -is "$OPT_SLEEP" "300" -is "$OPT_PCT_THRESHOLD" "95" -is "$OPT_MB_THRESHOLD" "100" -is "$OPT_PURGE" "30" +is "$OPT_STRING_OPT" "" +is "$OPT_STRING_OPT2" "foo" +is "$OPT_TYPELESS_OPTION" "" +is "$OPT_NOPTION" "yes" +is "$OPT_INT_OPT" "" +is "$OPT_INT_OPT2" "42" +is "$OPT_VERSION" "" # ############################################################################ # Specify some opts, but use default values for the rest. # ############################################################################ -parse_options "$T_LIB_DIR/samples/bash/po001.sh" --threshold 50 --gdb yes --email user@example.com +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt 50 --typeless-option --string-opt bar TEST_NAME="User-specified opts with defaults" -is "$OPT_THRESHOLD" "50" # specified -is "$OPT_VARIABLE" "Threads_connected" -is "$OPT_CYCLES" "1" -is "$OPT_GDB" "yes" # specified -is "$OPT_OPROFILE" "yes" -is "$OPT_STRACE" "no" -is "$OPT_TCPDUMP" "yes" -is "$OPT_EMAIL" "user@example.com" # specified -is "$OPT_INTERVAL" "30" -is "$OPT_MAYBE_EMPTY" "no" -is "$OPT_COLLECT" "${HOME}/bin/pt-collect" -is "$OPT_DEST" "${HOME}/collected/" -is "$OPT_DURATION" "30" -is "$OPT_SLEEP" "300" -is "$OPT_PCT_THRESHOLD" "95" -is "$OPT_MB_THRESHOLD" "100" -is "$OPT_PURGE" "30" +is "$OPT_STRING_OPT" "bar" # specified +is "$OPT_STRING_OPT2" "foo" +is "$OPT_TYPELESS_OPTION" "yes" # specified +is "$OPT_NOPTION" "yes" +is "$OPT_INT_OPT" "50" # specified +is "$OPT_INT_OPT2" "42" +is "$OPT_VERSION" "" + +# ############################################################################ +# Negate an option like --no-option. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption + +TEST_NAME="Negated option" +is "$OPT_STRING_OPT" "" +is "$OPT_STRING_OPT2" "foo" +is "$OPT_TYPELESS_OPTION" "" +is "$OPT_NOPTION" "no" # negated +is "$OPT_INT_OPT" "" +is "$OPT_INT_OPT2" "42" +is "$OPT_VERSION" "" + +# ############################################################################ +# Short form. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v + +TEST_NAME="Short form" +is "$OPT_VERSION" "yes" # ############################################################################ # An unknown option should produce an error. diff --git a/t/lib/samples/bash/po001.sh b/t/lib/samples/bash/po001.sh index 0fdd943d..fc0c387e 100644 --- a/t/lib/samples/bash/po001.sh +++ b/t/lib/samples/bash/po001.sh @@ -90,80 +90,42 @@ See L<"ENVIRONMENT">. =over -=item --threshold N +=item --string-opt -Max number of C to tolerate. (default: 100) +type: string -=item --variable NAME +String option without a default. -This is the thing to check for. (default: Threads_connected) +=item --string-opt2 -=item --cycles N +type: string; default: foo -How many times must the condition be met before the script will fire? (default: 1) +String option with a default. -=item --gdb +=item --typeless-option -Collect GDB stacktraces? (default: no) +Just an option. -=item --oprofile +=item --noption -Collect oprofile data? (default: yes) +default: yes; negatable: yes -=item --strace +=item --int-opt -Collect strace data? (default: no) +type: int -=item --tcpdump +Int option without a default -Collect tcpdump data? (default: yes) +=item --int-opt2 -=item --email ADDRESS +type: int; default: 42 -Send mail to this list of addresses when the script triggers. - -=item --interval SECONDS - -This is the interval between checks. (default: 30) - -=item --maybe-empty - -Result of checks may be empty. (default: no) -If the command you're running to detect the condition is allowed to return -nothing (e.g. a grep line that might not even exist if there's no problem), -then set this to "yes". - -=item --collect DIRECTORY - -Location of the C tool. (default: ${HOME}/bin/pt-collect) - -=item --dest DIRECTORY - -Where to store collected data. (default: ${HOME}/collected/) - -=item --duration SECONDS - -How long to collect statistics data for? (default: 30) -Make sure that this isn't longer than SLEEP. - -=item --sleep SECONDS - -How long to sleep after collecting? (default: 300) - -=item --pct-threshold PCT - -Bail out if the disk is more than this %full. (default: 95) - -=item --mb-threshold MEGABYTES - -Bail out if the disk has less than this many MB free. (default: 100) - -=item --purge DAYS - -Remove samples after this many days. (default: 30) +Int option with a default. =item --version +short form: -v + Print tool's version and exit. =back From 6764f4cd0651a7e13454c1fa5106c40673025e21 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 2 Dec 2011 14:43:43 -0700 Subject: [PATCH 08/27] Close awk fhs to avoid 'too many open files' error. --- lib/bash/parse_options.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 20953c2e..a5a35bf8 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -97,10 +97,12 @@ parse_options() { if ($0 ~ /^[a-z]/ ) { # spec line like "type: int; default: 100" print "long:" long_opt "; " $0 | trf + close(trf) } else { # no specs, should be description of option print "long:" long_opt > spec_file + close(spec_file) } } getline From c804079c0de62605a9e6cb96a04b25601566c5b2 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 5 Dec 2011 13:48:54 -0700 Subject: [PATCH 09/27] Add Bash libs collect.sh (pt-collect internals), safeguards.sh, and alt_cmds.sh. --- lib/bash/alt_cmds.sh | 34 ++++++ lib/bash/collect.sh | 238 ++++++++++++++++++++++++++++++++++++ lib/bash/safeguards.sh | 51 ++++++++ t/lib/bash/collect.sh | 24 ++++ t/lib/samples/bash/po002.sh | 212 ++++++++++++++++++++++++++++++++ 5 files changed, 559 insertions(+) create mode 100644 lib/bash/alt_cmds.sh create mode 100644 lib/bash/collect.sh create mode 100644 lib/bash/safeguards.sh create mode 100644 t/lib/bash/collect.sh create mode 100644 t/lib/samples/bash/po002.sh diff --git a/lib/bash/alt_cmds.sh b/lib/bash/alt_cmds.sh new file mode 100644 index 00000000..55a41e3a --- /dev/null +++ b/lib/bash/alt_cmds.sh @@ -0,0 +1,34 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# alt_cmds package +# ########################################################################### + +# Package: alt_cmds +# alt_cmds provides alternatives to commands that aren't on all systems. + +set -u + +# seq N, return 1, ..., 5 +_seq() { + local i=$1 + awk "BEGIN { for(i=1; i<=$i; i++) print i; }" +} + +# ########################################################################### +# End alt_cmds package +# ########################################################################### diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh new file mode 100644 index 00000000..7c8b68f9 --- /dev/null +++ b/lib/bash/collect.sh @@ -0,0 +1,238 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# collect package +# ########################################################################### + +# Package: collect +# collect collects system information. + +set -u + +# Global variables. +CMD_GDB=${CMD_GDB:-"gdb"} +CMD_IOSTAT=${CMD_IOSTAT:-"iostat"} +CMD_MPSTAT=${CMD_MPSTAT:-"mpstat"} +CMD_MYSQL=${CMD_MSSQL:-"mysql"} +CMD_MYSQLADMIN=${CMD_MYSQL_ADMIN:-"mysqladmin"} +CMD_OPCONTROL=${CMD_OPCONTROL:-"opcontrol"} +CMD_OPREPORT=${CMD_OPREPORT:-"opreport"} +CMD_PMAP=${CMD_PMAP:-"pmap"} +CMD_STRACE=${CMD_STRACE:-"strace"} +CMD_TCPDUMP=${CMD_TCPDUMP:-"tcpdump"} +CMD_VMSTAT=${CMD_VMSTAT:-"vmstat"} + +collect() { + local d=$1 # directory to save results in + local p=$2 # prefix for each result file + + # Get pidof mysqld; pidof doesn't exist on some systems. We try our best... + local mysqld_pid=$(pidof -s mysqld); + if [ -z "$mysqld_pid" ]; then + mysqld_pid=$(pgrep -o -x mysqld); + fi + if [ -z "$mysqld_pid" ]; then + mysqld_pid=$(ps -eaf | grep 'mysql[d]' | grep -v mysqld_safe | awk '{print $2}' | head -n1); + fi + + # Get memory allocation info before anything else. + if [ -x "$CMD_PMAP" -a "$mysqld_pid" ]; then + if $CMD_PMAP --help 2>&1 | grep -- -x >/dev/null 2>&1 ; then + $CMD_PMAP -x $mysqld_pid > "$d/$p-pmap" + else + # Some pmap's apparently don't support -x (issue 116). + $CMD_PMAP $mysqld_pid > "$d/$p-pmap" + fi + fi + + # Getting a GDB stacktrace can be an intensive operation, + # so do this only if necessary. + if [ "$OPT_COLLECT_GDB" = "yes" -a "$mysqld_pid" ]; then + $CMD_GDB \ + -ex "set pagination 0" \ + -ex "thread apply all bt" \ + --batch -p $mysqld_pid \ + >> "$d/$p-stacktrace" + else + echo "GDB (--collect-gdb) was not enabled" >> "$d/$p-stacktrace" + fi + + # Get MySQL's variables if possible. Then sleep long enough that we probably + # complete SHOW VARIABLES if all's well. (We don't want to run mysql in the + # foreground, because it could hang.) + $CMD_MYSQL "$EXT_ARGV" -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 & + sleep .2 + + # Get the major.minor version number. Version 3.23 doesn't matter for our + # purposes, and other releases have x.x.x* version conventions so far. + local mysql_version="$(awk '/^version[^_]/{print substr($2,1,3)}' "$d/$p-variables")" + + # Is MySQL logging its errors to a file? If so, tail that file. + local mysql_error_log="$(awk '/log_error/{print $2}' "$d/$p-variables")" + if [ -z "$mysql_error_log" -a "$mysqld_pid" ]; then + # Try getting it from the open filehandle... + mysql_error_log="$(ls -l /proc/$mysqld_pid/fd | awk '/ 2 ->/{print $NF}')" + fi + + local tail_error_log_pid="" + if [ "$mysql_error_log" ]; then + echo "The MySQL error log seems to be ${mysql_error_log}" + tail -f "$mysql_error_log" >"$d/$p-log_error" 2>&1 & + tail_error_log_pid=$! + # Send a mysqladmin debug to the server so we can potentially learn about + # locking etc. + $CMD_MYSQLADMIN "$EXT_ARGV" debug + else + echo "Could not find the MySQL error log" + fi + + # Get a sample of these right away, so we can get these without interaction + # with the other commands we're about to run. + local innostat="SHOW /*!40100 ENGINE*/ INNODB STATUS\G" + local proclist="SHOW FULL PROCESSLIST\G" + if [ "${mysql_version}" '>' "5.1" ]; then + local mutex="SHOW ENGINE INNODB MUTEX" + else + local mutex="SHOW MUTEX STATUS" + fi + $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$proclist" >> "$d/$p-processlist1" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables1" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & + + # If TCP dumping is specified, start that on the server's port. + local tcpdump_pid="" + if [ "$OPT_COLLECT_TCPDUMP" = "yes" ]; then + local port=$(awk '/^port/{print $2}' "$d/$p-variables") + if [ "$port" ]; then + $CMD_TCPDUMP -i any -s 4096 -w "$d/$p-tcpdump" port ${port} & + tcpdump_pid=$! + fi + fi + + # Next, start oprofile gathering data during the whole rest of this process. + # The --init should be a no-op if it has already been init-ed. + local have_oprofile="no" + if [ "$OPT_COLLECT_OPROFILE" = "yes" ]; then + if $CMD_OPCONTROL --init; then + $CMD_OPCONTROL --start --no-vmlinux + have_oprofile="yes" + fi + elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then + # Don't run oprofile and strace at the same time. + $CMD_STRACE -T -s 0 -f -p $mysqld_pid > "${DEST}/$d-strace" 2>&1 & + local strace_pid=$! + fi + + # Grab a few general things first. Background all of these so we can start + # them all up as quickly as possible. We use mysqladmin -c even though it is + # buggy and won't stop on its own in 5.1 and newer, because there is a chance + # that we will get and keep a connection to the database; in troubled times + # the database tends to exceed max_connections, so reconnecting in the loop + # tends not to work very well. + ps -eaf >> "$d/$p-ps" 2>&1 & + sysctl -a >> "$d/$p-sysctl" 2>&1 & + top -bn1 >> "$d/$p-top" 2>&1 & + $CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 & + $CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 & + $CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 & + $CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 & + $CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 & + $CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 & + lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 & + $CMD_MYSQLADMIN "$EXT_ARGV" ext -i1 -c$OPT_INTERVAL >> "$d/$p-mysqladmin" 2>&1 & + local mysqladmin_pid=$! + + # This loop gathers data for the rest of the duration, and defines the time + # of the whole job. + echo "Loop start: $(date +'TS %s.%N %F %T')" + for a in $(_seq $OPT_INTERVAL); do + # We check the disk, but don't exit, because we need to stop jobs if we + # need to exit. + disk_space $d > $d/$p-disk-space + check_disk_space \ + $d/$p-disk-space \ + "$OPT_DISK_BYTE_LIMIT" \ + "$OPT_DISK_PCT_LIMIT" \ + || break + + # Synchronize ourselves onto the clock tick, so the sleeps are 1-second + sleep $(date +%s.%N | awk '{print 1 - ($1 % 1)}') + local ts="$(date +"TS %s.%N %F %T")" + + # Collect the stuff for this cycle + (cat /proc/diskstats 2>&1; echo $ts) >> "$d/$p-diskstats" & + (cat /proc/stat 2>&1; echo $ts) >> "$d/$p-procstat" & + (cat /proc/vmstat 2>&1; echo $ts) >> "$d/$p-procvmstat" & + (cat /proc/meminfo 2>&1; echo $ts) >> "$d/$p-meminfo" & + (cat /proc/slabinfo 2>&1; echo $ts) >> "$d/$p-slabinfo" & + (cat /proc/interrupts 2>&1; echo $ts) >> "$d/$p-interrupts" & + (df -h 2>&1; echo $ts) >> "$d/$p-df" & + (netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" & + (netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" & + done + echo "Loop end: $(date +'TS %s.%N %F %T')" + + if [ "$have_oprofile" = "yes" ]; then + $CMD_OPCONTROL --stop + $CMD_OPCONTROL --dump + kill $(pidof oprofiled); # TODO: what if system doesn't have pidof? + $CMD_OPCONTROL --save=pt_collect_$p + + # Attempt to generate a report; if this fails, then just tell the user + # how to generate the report. + local mysqld_path=$(which mysqld); + if [ "$mysqld_path" -a -f "$mysqld_path" ]; then + $CMD_OPREPORT \ + --demangle=smart \ + --symbols \ + --merge tgid \ + session:pt_collect_$p \ + "$mysqld_path" \ + > "$d/$p-opreport" + else + echo "oprofile data saved to pt_collect_$p; you should be able" \ + "to get a report by running something like 'opreport" \ + "--demangle=smart --symbols --merge tgid session:pt_collect_$p" \ + "/path/to/mysqld'" \ + > "$d/$p-opreport" + fi + elif [ "$OPT_COLLECT_STRACE" = "yes" ]; then + kill -s 2 $strace_pid + sleep 1 + kill -s 15 $strace_pid + # Sometimes strace leaves threads/processes in T status. + kill -s 18 $mysqld_pid + fi + + $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$proclist" >> "$d/$p-processlist2" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables2" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & + + # Kill backgrounded tasks. + kill $mysqladmin_pid + [ "$tail_error_log_pid" ] && kill $tail_error_log_pid + [ "$tcpdump_pid" ] && kill $tcpdump_pid + + # Finally, record what system we collected this data from. + hostname > "$d/$p-hostname" +} + +# ########################################################################### +# End tmpdir package +# ########################################################################### diff --git a/lib/bash/safeguards.sh b/lib/bash/safeguards.sh new file mode 100644 index 00000000..216cbeac --- /dev/null +++ b/lib/bash/safeguards.sh @@ -0,0 +1,51 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# safeguards package +# ########################################################################### + +# Package: safeguards +# safeguards is a collection of function to help avoid blowing things up. + +set -u + +disk_space() { + local filesystem=${1:-"$PWD"} + # Filesystem 512-blocks Used Available Capacity Mounted on + # /dev/disk0s2 236306352 190223184 45571168 81% / + df -m -P $filesystem +} + +check_disk_space() { + local file=$1 + local mb=${2:-"0"} + local pct=${3:-"0"} + + local avail=$(cat $file | awk '/^\//{print $4}'); + local full=$(cat $file | awk '/^\//{print $5}' | sed -e 's/%//g'); + if [ "${avail}" -le "$mb" -o "$full" -le "$pct" ]; then + echo "Not enough free space (${full}% full, ${avail}MB free)" + echo "Wanted less than ${pct}% full and more than ${mb}MB" + return 1 + fi + return 0 +} + + +# ########################################################################### +# End safeguards package +# ########################################################################### diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh new file mode 100644 index 00000000..d906011d --- /dev/null +++ b/t/lib/bash/collect.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +TESTS=24 + +TMPFILE="$TEST_TMPDIR/parse-opts-output" +TMPDIR="$TEST_TMPDIR" +PATH="$PATH:$PERCONA_TOOLKIT_SANDBOX/bin" + +mkdir "$TMPDIR/collect" 2>/dev/null + +source "$LIB_DIR/log_warn_die.sh" +source "$LIB_DIR/parse_options.sh" +source "$LIB_DIR/safeguards.sh" +source "$LIB_DIR/alt_cmds.sh" +source "$LIB_DIR/collect.sh" + +parse_options "$T_LIB_DIR/samples/bash/po002.sh" -- --defaults-file=/tmp/12345/my.sandbox.cnf + +collect "$TMPDIR/collect" "2011_12_05" + +# ############################################################################ +# Done +# ############################################################################ +exit diff --git a/t/lib/samples/bash/po002.sh b/t/lib/samples/bash/po002.sh new file mode 100644 index 00000000..63a8672b --- /dev/null +++ b/t/lib/samples/bash/po002.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +: + +# ############################################################################ +# Documentation +# ############################################################################ +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-stalk - Wait for a condition to occur then begin collecting data. + +=head1 OPTIONS + +=over + +=item --collect + +default: yes; negatable: yes + +Collect system information. + +=item --collect-gdb + +Collect GDB stacktraces. + +=item --collect-oprofile + +Collect oprofile data. + +=item --collect-strace + +Collect strace data. + +=item --collect-tcpdump + +Collect tcpdump data. + +=item --cycles + +type: int; default: 5 + +Number of times condition must be met before triggering collection. + +=item --daemonize + +default: yes; negatable: yes + +Daemonize the tool. + +=item --dest + +type: string + +Where to store collected data. + +=item --disk-byte-limit + +type: int; default: 100 + +Exit if the disk has less than this many MB free. + +=item --disk-pct-limit + +type: int; default: 5 + +Exit if the disk is less than this %full. + +=item --execute-command + +type: string; default: pt-collect + +Location of the C tool. + +=item --function + +type: string; default: status + +Built-in function name or plugin file name which returns the value of C. + +Possible values are: + +=over + +=item * status + +Grep the value of C from C. + +=item * processlist + +Count the number of processes in C whose +C column matches C. For example: + + TRIGGER_FUNCTION="processlist" \ + VARIABLE="State" \ + MATCH="statistics" \ + THRESHOLD="10" + +The above triggers when more than 10 processes are in the "statistics" state. +C must be specified for this trigger function. + +=item * magic + +TODO + +=item * plugin file name + +A plugin file allows you to specify a custom trigger function. The plugin +file must contain a function called C. For example: + + trg_plugin() { + # Do some stuff. + echo "$value" + } + +The last output if the function (its "return value") must be a number. +This number is compared to C. All L<"ENVIRONMENT"> variables +are available to the function. + +Do not alter the tool's existing global variables. Prefix any plugin-specific +global variables with "PLUGIN_". + +=back + +=item --help + +Print help and exit. + +=item --interval + +type: int; default: 1 + +Interval between checks. + +=item --iterations + +type: int + +Exit after triggering C this many times. By default, the tool +will collect as many times as it's triggered. + +=item --log + +type: string; default: /var/log/pt-stalk.log + +Print all output to this file when daemonized. + +=item --match + +type: string + +Match pattern for C L<"--function">. + +=item --notify-by-email + +type: string + +Send mail to this list of addresses when C triggers. + +=item --pid FILE + +type: string; default: /var/run/pt-stalk.pid + +Create a PID file when daemonized. + +=item --retention-time + +type: int; default: 30 + +Remove samples after this many days. + +=item --run-time + +type: int; default: 30 + +How long to collect statistics data for? + +Make sure that this isn't longer than SLEEP. + +=item --sleep + +type: int; default: 300 + +How long to sleep after collecting? + +=item --threshold N + +type: int; default: 25 + +Max number of C to tolerate. + +=item --variable NAME + +type: string; default: Threads_running + +This is the thing to check for. + +=item --version + +Print tool's version and exit. + +=back + +=head1 ENVIRONMENT + +No env vars used. + +=cut + +DOCUMENTATION From f6306c9ec0ac61a0f6e44c0201f739e25e46563c Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 6 Dec 2011 13:23:47 -0700 Subject: [PATCH 10/27] Test collect.sh. Use --run-time instead of --interval for collect loop. Fix and require per-test test names in util/test-bash-functions. Fix OPT_ERR in parse_options.sh. --- lib/bash/collect.sh | 2 +- lib/bash/parse_options.sh | 2 +- t/lib/bash/collect.sh | 102 ++++++++++++++++++++++++++++-- t/lib/bash/parse_options.sh | 62 ++++++++---------- t/lib/bash/tmpdir.sh | 27 +++----- t/lib/samples/bash/collect001.txt | 34 ++++++++++ util/test-bash-functions | 23 ++++--- 7 files changed, 182 insertions(+), 70 deletions(-) create mode 100644 t/lib/samples/bash/collect001.txt diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 7c8b68f9..319e3dd6 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -160,7 +160,7 @@ collect() { # This loop gathers data for the rest of the duration, and defines the time # of the whole job. echo "Loop start: $(date +'TS %s.%N %F %T')" - for a in $(_seq $OPT_INTERVAL); do + for a in $(_seq $OPT_RUN_TIME); do # We check the disk, but don't exit, because we need to stop jobs if we # need to exit. disk_space $d > $d/$p-disk-space diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index a5a35bf8..18632762 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -28,7 +28,7 @@ set -u # sub will be scoped locally. declare -a ARGV # non-option args (probably input files) declare EXT_ARGV # everything after -- (args for an external command) -OPT_ERR=${OPT_ERR:""} +OPT_ERR=${OPT_ERR:-""} # Sub: usage # Print usage (--help) and list the program's options. diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index d906011d..ec6b59d1 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=24 +TESTS=19 TMPFILE="$TEST_TMPDIR/parse-opts-output" TMPDIR="$TEST_TMPDIR" @@ -14,11 +14,105 @@ source "$LIB_DIR/safeguards.sh" source "$LIB_DIR/alt_cmds.sh" source "$LIB_DIR/collect.sh" -parse_options "$T_LIB_DIR/samples/bash/po002.sh" -- --defaults-file=/tmp/12345/my.sandbox.cnf +parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf -collect "$TMPDIR/collect" "2011_12_05" +# Prefix (with path) for the collect files. +local p="$TMPDIR/collect/2011_12_05" + +# Default collect, no extras like gdb, tcpdump, etc. +collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 + +# Even if this system doesn't have all the cmds, collect should still +# create all the default files. +ls -1 $TMPDIR/collect | sort > $TMPDIR/collect-files +no_diff \ + $TMPDIR/collect-files \ + $T_LIB_DIR/samples/bash/collect001.txt \ + "Default collect files" + +cmd_ok \ + "grep -q 'Avail' $p-df" \ + "df" + +# hostname is the last thing collected, so if it's ok, +# then the sub reached its end. +is \ + "`cat $p-hostname`" \ + "`hostname`" \ + "hostname" + +cmd_ok \ + "grep -q -i 'buffer pool' $p-innodbstatus1" \ + "innodbstatus1" + +cmd_ok \ + "grep -q -i 'buffer pool' $p-innodbstatus2" \ + "innodbstatus2" + +cmd_ok \ + "grep -q 'error log seems to be /tmp/12345/data/mysqld.log' $p-output" \ + "Finds MySQL error log" + +cmd_ok \ + "grep -q 'Status information:' $p-log_error" \ + "debug" + +cmd_ok \ + "grep -q 'COMMAND[ ]\+PID[ ]\+USER' $p-lsof" \ + "lsof" + +cmd_ok \ + "grep -q 'buf/buf0buf.c' $p-mutex-status1" \ + "mutex-status1" + +cmd_ok \ + "grep -q 'buf/buf0buf.c' $p-mutex-status2" \ + "mutex-status2" + +cmd_ok \ + "grep -q '^| Uptime' $p-mysqladmin" \ + "mysqladmin ext" + +cmd_ok \ + "grep -qP 'Database\tTable\tIn_use' $p-opentables1" \ + "opentables1" + +cmd_ok \ + "grep -qP 'Database\tTable\t\In_use' $p-opentables2" \ + "opentables2" + +cmd_ok \ + "grep -q '1. row' $p-processlist1" \ + "processlist1" + +cmd_ok \ + "grep -q '1. row' $p-processlist2" \ + "processlist2" + +cmd_ok \ + "grep -q 'mysqld' $p-ps" \ + "ps" + +cmd_ok \ + "grep -qP '^warning_count\t\d' $p-variables" \ + "variables" + +local iters=$(cat $p-df | grep -c '^TS ') +is "$iters" "1" "1 iteration/1s run time" + +# ########################################################################### +# Try longer run time. +# ########################################################################### + +parse_options "$T_LIB_DIR/samples/bash/po002.sh" --run-time 2 -- --defaults-file=/tmp/12345/my.sandbox.cnf + +rm $TMPDIR/collect/* + +collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 + +local iters=$(cat $p-df | grep -c '^TS ') +is "$iters" "2" "2 iteration/2s run time" # ############################################################################ # Done # ############################################################################ -exit diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index fcb80850..ce69ec71 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=24 +TESTS=25 TMPFILE="$TEST_TMPDIR/parse-opts-output" @@ -14,17 +14,15 @@ source "$LIB_DIR/parse_options.sh" TMPDIR="$TEST_TMPDIR" parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE -TEST_NAME="No warnings or errors" -is "`cat $TMPFILE`" "" +is "`cat $TMPFILE`" "" "No warnings or errors" -TEST_NAME="Default opts" -is "$OPT_STRING_OPT" "" -is "$OPT_STRING_OPT2" "foo" -is "$OPT_TYPELESS_OPTION" "" -is "$OPT_NOPTION" "yes" -is "$OPT_INT_OPT" "" -is "$OPT_INT_OPT2" "42" -is "$OPT_VERSION" "" +is "$OPT_STRING_OPT" "" "Default string option" +is "$OPT_STRING_OPT2" "foo" "Default string option with default" +is "$OPT_TYPELESS_OPTION" "" "Default typeless option" +is "$OPT_NOPTION" "yes" "Defailt neg option" +is "$OPT_INT_OPT" "" "Default int option" +is "$OPT_INT_OPT2" "42" "Default int option with default" +is "$OPT_VERSION" "" "--version" # ############################################################################ # Specify some opts, but use default values for the rest. @@ -32,14 +30,13 @@ is "$OPT_VERSION" "" parse_options "$T_LIB_DIR/samples/bash/po001.sh" --int-opt 50 --typeless-option --string-opt bar -TEST_NAME="User-specified opts with defaults" -is "$OPT_STRING_OPT" "bar" # specified -is "$OPT_STRING_OPT2" "foo" -is "$OPT_TYPELESS_OPTION" "yes" # specified -is "$OPT_NOPTION" "yes" -is "$OPT_INT_OPT" "50" # specified -is "$OPT_INT_OPT2" "42" -is "$OPT_VERSION" "" +is "$OPT_STRING_OPT" "bar" "Specified string option (spec)" +is "$OPT_STRING_OPT2" "foo" "Default string option with default (spec)" +is "$OPT_TYPELESS_OPTION" "yes" "Specified typeless option (spec)" +is "$OPT_NOPTION" "yes" "Default neg option (spec)" +is "$OPT_INT_OPT" "50" "Specified int option (spec)" +is "$OPT_INT_OPT2" "42" "Default int option with default (spec)" +is "$OPT_VERSION" "" "--version (spec)" # ############################################################################ # Negate an option like --no-option. @@ -47,23 +44,20 @@ is "$OPT_VERSION" "" parse_options "$T_LIB_DIR/samples/bash/po001.sh" --no-noption -TEST_NAME="Negated option" -is "$OPT_STRING_OPT" "" -is "$OPT_STRING_OPT2" "foo" -is "$OPT_TYPELESS_OPTION" "" -is "$OPT_NOPTION" "no" # negated -is "$OPT_INT_OPT" "" -is "$OPT_INT_OPT2" "42" -is "$OPT_VERSION" "" +is "$OPT_STRING_OPT" "" "Default string option (neg)" +is "$OPT_STRING_OPT2" "foo" "Default string option with default (net)" +is "$OPT_TYPELESS_OPTION" "" "Default typeless option (neg)" +is "$OPT_NOPTION" "no" "Negated option (neg)" +is "$OPT_INT_OPT" "" "Default int option (neg)" +is "$OPT_INT_OPT2" "42" "Default int option with default (neg)" +is "$OPT_VERSION" "" "--version (neg)" # ############################################################################ # Short form. # ############################################################################ parse_options "$T_LIB_DIR/samples/bash/po001.sh" -v - -TEST_NAME="Short form" -is "$OPT_VERSION" "yes" +is "$OPT_VERSION" "yes" "Short form" # ############################################################################ # An unknown option should produce an error. @@ -74,13 +68,9 @@ is "$OPT_VERSION" "yes" parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 ) local err=$? -TEST_NAME="Non-zero exit on unknown option" -is "$err" "1" - -TEST_NAME="Error on unknown option" -cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" +is "$err" "1" "Non-zero exit on unknown option" +cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" "Error on unknown option" # ############################################################################ # Done # ############################################################################ -exit diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index 4698b04e..a5b86823 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -5,40 +5,31 @@ TESTS=9 source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/tmpdir.sh" -TEST_NAME="TMPDIR not defined" -is "$TMPDIR" "" +is "$TMPDIR" "" "TMPDIR not defined" -TEST_NAME="mk_tmpdir makes secure tmpdir" mk_tmpdir -cmd_ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" "mk_tmpdir makes secure tmpdir" tmpdir=$TMPDIR; -TEST_NAME="rm_tmpdir" rm_tmpdir -cmd_ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" "rm_tmpdir" -TEST_NAME="rm_tmpdir resets TMPDIR" -is "$TMPDIR" "" +is "$TMPDIR" "" "rm_tmpdir resets TMPDIR" # --tmpdir OPT_TMPDIR="/tmp/use--tmpdir" -TEST_NAME="TMPDIR not defined" -is "$TMPDIR" "" +is "$TMPDIR" "" "TMPDIR not defined" -TEST_NAME="--tmpdir does not exist yet" -cmd_ok "test ! -d $OPT_TMPDIR" +cmd_ok "test ! -d $OPT_TMPDIR" "--tmpdir does not exist yet" mk_tmpdir -TEST_NAME="mk_tmpdir uses --tmpdir" -is "$TMPDIR" "/tmp/use--tmpdir" +is "$TMPDIR" "/tmp/use--tmpdir" "mk_tmpdir uses --tmpdir" -TEST_NAME="mk_tmpdir creates --tmpdir" -cmd_ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" "mk_tmpdir creates --tmpdir" tmpdir=$TMPDIR; -TEST_NAME="rm_tmpdir removes --tmpdir" rm_tmpdir -cmd_ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" "rm_tmpdir removes --tmpdir" diff --git a/t/lib/samples/bash/collect001.txt b/t/lib/samples/bash/collect001.txt new file mode 100644 index 00000000..0976e680 --- /dev/null +++ b/t/lib/samples/bash/collect001.txt @@ -0,0 +1,34 @@ +2011_12_05-df +2011_12_05-disk-space +2011_12_05-diskstats +2011_12_05-hostname +2011_12_05-innodbstatus1 +2011_12_05-innodbstatus2 +2011_12_05-interrupts +2011_12_05-iostat +2011_12_05-iostat-overall +2011_12_05-log_error +2011_12_05-lsof +2011_12_05-meminfo +2011_12_05-mpstat +2011_12_05-mpstat-overall +2011_12_05-mutex-status1 +2011_12_05-mutex-status2 +2011_12_05-mysqladmin +2011_12_05-netstat +2011_12_05-netstat_s +2011_12_05-opentables1 +2011_12_05-opentables2 +2011_12_05-output +2011_12_05-processlist1 +2011_12_05-processlist2 +2011_12_05-procstat +2011_12_05-procvmstat +2011_12_05-ps +2011_12_05-slabinfo +2011_12_05-stacktrace +2011_12_05-sysctl +2011_12_05-top +2011_12_05-variables +2011_12_05-vmstat +2011_12_05-vmstat-overall diff --git a/util/test-bash-functions b/util/test-bash-functions index 9a10a782..1a14a0de 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -48,6 +48,7 @@ fi load_tests() { local test_files="$@" local i=0 + local n_tests=0 for t in $test_files; do # Return unless the test file is bash. There may be other types of # files in the tool's test dir. @@ -57,14 +58,16 @@ load_tests() { head -n 1 $t | grep -q bash || continue tests[$i]=$t + i=$((i + 1)) + number_of_tests=$(grep --max-count 1 '^TESTS=[0-9]' $t | cut -d'=' -f2) if [ -z "$number_of_tests" ]; then - i=$(( i + 1 )) + n_tests=$(( $n_tests + 1 )) else - i=$(( i + $number_of_tests )) + n_tests=$(( $n_tests + $number_of_tests )) fi done - echo "1..$i" + echo "1..$n_tests" } # Source a test file to run whatever it contains (hopefully tests!). @@ -72,8 +75,6 @@ run_test() { local t=$1 # test file name, e.g. "group-by-all-01" for pt-diskstats rm -rf $TEST_TMPDIR/* >/dev/null 2>&1 - TEST_NUMBER=1 # test number in this test file - # Tests assume that they're being ran from their own dir, so they access # sample files like "samples/foo.txt". So cd to the dir of the test file # and run it. But the test file may have been given as a relative path, @@ -92,7 +93,7 @@ run_test() { # Print a TAP-style test result. result() { local result=$1 - local test_name=${TEST_NAME:-"$TEST_NUMBER"} + local test_name=$2 if [ $result -eq 0 ]; then echo "ok $testno - $TEST_FILE $test_name" else @@ -104,7 +105,6 @@ result() { fi fi testno=$((testno + 1)) - TEST_NUMBER=$((TEST_NUMBER + 1)) return $result } @@ -115,23 +115,26 @@ result() { no_diff() { local got=$1 local expected=$2 + local test_name=$3 test_command="diff $got $expected" eval $test_command > $TEST_TMPDIR/failed_result 2>&1 - result $? + result $? "$test_name" } is() { local got=$1 local expected=$2 + local test_name=$3 test_command="\"$got\" == \"$expected\"" test "$got" = "$expected" - result $? + result $? "$test_name" } cmd_ok() { local test_command=$1 + local test_name=$2 eval $test_command - result $? + result $? "$test_name" } # ############################################################################ From f624220b40b575034318c1684338e70c50c0fde7 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 6 Dec 2011 13:36:45 -0700 Subject: [PATCH 11/27] Fix typo. --- lib/bash/collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 319e3dd6..e4050fef 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -234,5 +234,5 @@ collect() { } # ########################################################################### -# End tmpdir package +# End collect package # ########################################################################### From 578896c217288db26fbf94fac05f1e98175b92fe Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 7 Dec 2011 10:37:10 -0700 Subject: [PATCH 12/27] Add daemon.sh. --- lib/bash/daemon.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++ t/lib/bash/daemon.sh | 72 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 lib/bash/daemon.sh create mode 100644 t/lib/bash/daemon.sh diff --git a/lib/bash/daemon.sh b/lib/bash/daemon.sh new file mode 100644 index 00000000..91528b3d --- /dev/null +++ b/lib/bash/daemon.sh @@ -0,0 +1,75 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# daemon package +# ########################################################################### + +# Package: daemon +# daemon handles daemon related tasks like checking a PID file. + +set -u + +# Sub: make_pid_file +# Check and make a PID file. +# +# Arguments: +# file - File to write PID to. +# pid - PID to write into file. +# +# Required Global Variables: +# ON_EXIT - String to append "remove_pid_file file". +make_pid_file() { + local file=$1 + local pid=$2 + + # Yes there's a race condition here, between checking if the file exists + # and creating it, but it's not important enough to handle. + + if [ -f "$file" ]; then + # PID file already exists. See if the pid it contains is still running. + # If yes, then die. Else, the pid file is stale and we can reclaim it. + local old_pid=$(cat $file) + if [ -z "$old_pid" ]; then + # PID file is empty, so be safe and die since we can't check a + # non-existent pid. + die "PID file $file already exists but it is empty" + else + kill -0 $old_pid 2>/dev/null + if [ $? -eq 0 ]; then + die "PID file $file already exists and its PID ($old_pid) is running" + else + echo "Overwriting PID file $file because its PID ($old_pid)" \ + "is not running" + fi + fi + fi + + # PID file doesn't exist, or it does but its pid is stale. + echo "$pid" > $file + ON_EXIT="$ON_EXIT; remove_pid_file $file" +} + +remove_pid_file() { + local file=$1 + if [ -f "$file" ]; then + rm $file + fi +} + +# ########################################################################### +# End daemon package +# ########################################################################### diff --git a/t/lib/bash/daemon.sh b/t/lib/bash/daemon.sh new file mode 100644 index 00000000..b5946629 --- /dev/null +++ b/t/lib/bash/daemon.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +TESTS=8 + +TMPDIR="$TEST_TMPDIR" +ON_EXIT=":" +local file="$TMPDIR/pid-file" + +source "$LIB_DIR/log_warn_die.sh" +source "$LIB_DIR/daemon.sh" + +cmd_ok \ + "test ! -f $file" \ + "PID file doesn't exist" + +make_pid_file $file $$ + +cmd_ok \ + "test -f $file" \ + "PID file created" + +local pid=`cat $file` +is \ + "$pid" \ + "$$" \ + "Correct PID" + +remove_pid_file $file + +cmd_ok \ + "test ! -f $file" \ + "PID file removed" + +is \ + "$ON_EXIT" \ + ":; remove_pid_file $file" \ + "Sets ON_EXIT to remove PID file" + +# ########################################################################### +# PID file already exists and proc is running. +# ########################################################################### +ON_EXIT="" +echo $$ > $file + +( + make_pid_file $file $$ >$TMPDIR/output 2>&1 +) + +cmd_ok \ + "grep -q \"PID file /tmp/percona-toolkit.test/pid-file already exists and its PID ($$) is running\" $TMPDIR/output" \ + "Does not overwrite PID file is PID is running" + +echo 999999 > $file + +make_pid_file $file $$ >$TMPDIR/output 2>&1 + +cmd_ok \ + "grep -q 'Overwriting PID file /tmp/percona-toolkit.test/pid-file because its PID (999999) is not running' $TMPDIR/output" \ + "Overwrites PID file if PID is not running" + +pid=`cat $file` +is \ + "$pid" \ + "$$" \ + "Correct PID" + +rm $file +rm $TMPDIR/output + +# ########################################################################### +# Done. +# ########################################################################### From c486bf8461563ff214295f96f7ad8fc6e84a1c36 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 7 Dec 2011 10:48:47 -0700 Subject: [PATCH 13/27] Remove ON_EXIT. --- lib/bash/daemon.sh | 4 ---- t/lib/bash/daemon.sh | 9 +-------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/bash/daemon.sh b/lib/bash/daemon.sh index 91528b3d..1fe823b8 100644 --- a/lib/bash/daemon.sh +++ b/lib/bash/daemon.sh @@ -29,9 +29,6 @@ set -u # Arguments: # file - File to write PID to. # pid - PID to write into file. -# -# Required Global Variables: -# ON_EXIT - String to append "remove_pid_file file". make_pid_file() { local file=$1 local pid=$2 @@ -60,7 +57,6 @@ make_pid_file() { # PID file doesn't exist, or it does but its pid is stale. echo "$pid" > $file - ON_EXIT="$ON_EXIT; remove_pid_file $file" } remove_pid_file() { diff --git a/t/lib/bash/daemon.sh b/t/lib/bash/daemon.sh index b5946629..76ec4d0b 100644 --- a/t/lib/bash/daemon.sh +++ b/t/lib/bash/daemon.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash -TESTS=8 +TESTS=7 TMPDIR="$TEST_TMPDIR" -ON_EXIT=":" local file="$TMPDIR/pid-file" source "$LIB_DIR/log_warn_die.sh" @@ -31,15 +30,9 @@ cmd_ok \ "test ! -f $file" \ "PID file removed" -is \ - "$ON_EXIT" \ - ":; remove_pid_file $file" \ - "Sets ON_EXIT to remove PID file" - # ########################################################################### # PID file already exists and proc is running. # ########################################################################### -ON_EXIT="" echo $$ > $file ( From 876e3533d002ce51f769f5e6b1f42c39a6095d06 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 7 Dec 2011 11:58:33 -0700 Subject: [PATCH 14/27] Use LC_ALL=C to fix broken awk on some systems. --- lib/bash/parse_options.sh | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 18632762..dfecf2d7 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -82,33 +82,39 @@ parse_options() { # from the option's POD line like "type: string; default: foo". mkdir $TMPDIR/po/ 2>/dev/null rm -rf $TMPDIR/po/* - awk -v "po_dir"="$TMPDIR/po" ' - /^=head1 OPTIONS/ { - getline - while ($0 !~ /^=head1/) { - if ($0 ~ /^=item --.*/) { - long_opt = substr($2, 3, length($2) - 2) - spec_file = po_dir "/" long_opt - trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file + ( + # awk is stupid on some systems (e.g. Ubuntu 10) such that + # /^[a-z]/ incorrectly matches "Foo". This fixes that. + export LC_ALL="C" - getline # blank line - getline # specs or description - - if ($0 ~ /^[a-z]/ ) { - # spec line like "type: int; default: 100" - print "long:" long_opt "; " $0 | trf - close(trf) - } - else { - # no specs, should be description of option - print "long:" long_opt > spec_file - close(spec_file) - } - } + awk -v "po_dir"="$TMPDIR/po" ' + /^=head1 OPTIONS/ { getline - } - exit - }' $file + while ($0 !~ /^=head1/) { + if ($0 ~ /^=item --.*/) { + long_opt = substr($2, 3, length($2) - 2) + spec_file = po_dir "/" long_opt + trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file + + getline # blank line + getline # specs or description + + if ($0 ~ /^[a-z]/ ) { + # spec line like "type: int; default: 100" + print "long:" long_opt "; " $0 | trf + close(trf) + } + else { + # no specs, should be description of option + print "long:" long_opt > spec_file + close(spec_file) + } + } + getline + } + exit + }' $file + ) # Evaluate the program options into existence as global variables # transformed like --my-op == $OPT_MY_OP. If an option has a default From 0cfef9134ccff52e7f071abf01f8b5b5f2d7727b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 8 Dec 2011 11:45:16 -0700 Subject: [PATCH 15/27] Collect proclist in loop. --- lib/bash/collect.sh | 44 +++++++++++++++++-------------- t/lib/bash/collect.sh | 10 +++---- t/lib/samples/bash/collect001.txt | 3 +-- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index e4050fef..3f2cb792 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -103,14 +103,12 @@ collect() { # Get a sample of these right away, so we can get these without interaction # with the other commands we're about to run. local innostat="SHOW /*!40100 ENGINE*/ INNODB STATUS\G" - local proclist="SHOW FULL PROCESSLIST\G" if [ "${mysql_version}" '>' "5.1" ]; then local mutex="SHOW ENGINE INNODB MUTEX" else local mutex="SHOW MUTEX STATUS" fi $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$proclist" >> "$d/$p-processlist1" 2>&1 & $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables1" 2>&1 & $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & @@ -139,28 +137,32 @@ collect() { fi # Grab a few general things first. Background all of these so we can start - # them all up as quickly as possible. We use mysqladmin -c even though it is - # buggy and won't stop on its own in 5.1 and newer, because there is a chance - # that we will get and keep a connection to the database; in troubled times - # the database tends to exceed max_connections, so reconnecting in the loop - # tends not to work very well. - ps -eaf >> "$d/$p-ps" 2>&1 & - sysctl -a >> "$d/$p-sysctl" 2>&1 & - top -bn1 >> "$d/$p-top" 2>&1 & - $CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 & - $CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 & - $CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 & - $CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 & - $CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 & - $CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 & - lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 & - $CMD_MYSQLADMIN "$EXT_ARGV" ext -i1 -c$OPT_INTERVAL >> "$d/$p-mysqladmin" 2>&1 & + # them all up as quickly as possible. + ps -eaf >> "$d/$p-ps" 2>&1 & + sysctl -a >> "$d/$p-sysctl" 2>&1 & + top -bn1 >> "$d/$p-top" 2>&1 & + $CMD_VMSTAT 1 $OPT_INTERVAL >> "$d/$p-vmstat" 2>&1 & + $CMD_VMSTAT $OPT_INTERVAL 2 >> "$d/$p-vmstat-overall" 2>&1 & + $CMD_IOSTAT -dx 1 $OPT_INTERVAL >> "$d/$p-iostat" 2>&1 & + $CMD_IOSTAT -dx $OPT_INTERVAL 2 >> "$d/$p-iostat-overall" 2>&1 & + $CMD_MPSTAT -P ALL 1 $OPT_INTERVAL >> "$d/$p-mpstat" 2>&1 & + $CMD_MPSTAT -P ALL $OPT_INTERVAL 1 >> "$d/$p-mpstat-overall" 2>&1 & + lsof -nP -p $mysqld_pid -bw >> "$d/$p-lsof" 2>&1 & + + # Collect multiple snapshots of the status variables. We use + # mysqladmin -c even though it is buggy and won't stop on its + # own in 5.1 and newer, because there is a chance that we will + # get and keep a connection to the database; in troubled times + # the database tends to exceed max_connections, so reconnecting + # in the loop tends not to work very well. + $CMD_MYSQLADMIN "$EXT_ARGV" ext -i1 -c$OPT_RUN_TIME \ + >> "$d/$p-mysqladmin" 2>&1 & local mysqladmin_pid=$! # This loop gathers data for the rest of the duration, and defines the time # of the whole job. echo "Loop start: $(date +'TS %s.%N %F %T')" - for a in $(_seq $OPT_RUN_TIME); do + for loopno in $(_seq $OPT_RUN_TIME); do # We check the disk, but don't exit, because we need to stop jobs if we # need to exit. disk_space $d > $d/$p-disk-space @@ -184,6 +186,9 @@ collect() { (df -h 2>&1; echo $ts) >> "$d/$p-df" & (netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" & (netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" & + + ($CMD_MYSQL "$EXT_ARGV" -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \ + >> "$d/$p-processlist" done echo "Loop end: $(date +'TS %s.%N %F %T')" @@ -220,7 +225,6 @@ collect() { fi $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$proclist" >> "$d/$p-processlist2" 2>&1 & $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables2" 2>&1 & $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index ec6b59d1..ad059e29 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=19 +TESTS=18 TMPFILE="$TEST_TMPDIR/parse-opts-output" TMPDIR="$TEST_TMPDIR" @@ -82,12 +82,8 @@ cmd_ok \ "opentables2" cmd_ok \ - "grep -q '1. row' $p-processlist1" \ - "processlist1" - -cmd_ok \ - "grep -q '1. row' $p-processlist2" \ - "processlist2" + "grep -q '1. row' $p-processlist" \ + "processlist" cmd_ok \ "grep -q 'mysqld' $p-ps" \ diff --git a/t/lib/samples/bash/collect001.txt b/t/lib/samples/bash/collect001.txt index 0976e680..f76b8ecb 100644 --- a/t/lib/samples/bash/collect001.txt +++ b/t/lib/samples/bash/collect001.txt @@ -20,8 +20,7 @@ 2011_12_05-opentables1 2011_12_05-opentables2 2011_12_05-output -2011_12_05-processlist1 -2011_12_05-processlist2 +2011_12_05-processlist 2011_12_05-procstat 2011_12_05-procvmstat 2011_12_05-ps From 1c495936c895da76a3f3d1d5571eb4d77e58333b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 8 Dec 2011 12:05:20 -0700 Subject: [PATCH 16/27] Get Open_tables if <= 1000. --- lib/bash/collect.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 3f2cb792..f594e5f9 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -108,9 +108,9 @@ collect() { else local mutex="SHOW MUTEX STATUS" fi - $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables1" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & + open_tables >> "$d/$p-opentables1" 2>&1 & # If TCP dumping is specified, start that on the server's port. local tcpdump_pid="" @@ -224,9 +224,9 @@ collect() { kill -s 18 $mysqld_pid fi - $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' >> "$d/$p-opentables2" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & + $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & + open_tables >> "$d/$p-opentables2" 2>&1 & # Kill backgrounded tasks. kill $mysqladmin_pid @@ -237,6 +237,15 @@ collect() { hostname > "$d/$p-hostname" } +open_tables() { + local open_tables=$($CMD_MYSQLADMIN "$EXT_ARGV" ext | grep "Open_tables" | awk '{print $4}') + if [ -n "$open_tables" -a $open_tables -le 1000 ]; then + $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' 2>&1 & + else + echo "Too many open tables: $open_tables" + fi +} + # ########################################################################### # End collect package # ########################################################################### From 9ec71db008349edca984d80c8eaad837bd159625 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 8 Dec 2011 13:33:50 -0700 Subject: [PATCH 17/27] Collect INFORMATION_SCHEMA.INNODB_LOCK_WAITS data. --- lib/bash/collect.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index f594e5f9..afd6d0ba 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -159,6 +159,13 @@ collect() { >> "$d/$p-mysqladmin" 2>&1 & local mysqladmin_pid=$! + local have_lock_waits_table=0 + $MYSQL_CMD "$EXT_ARGV" -e "SHOW TABLES FROM INFORMATION_SCHEMA" \ + | grep -qi "INNODB_LOCK_WAITS" + if [ $? -eq 0 ]; then + have_lock_waits_table=1 + fi + # This loop gathers data for the rest of the duration, and defines the time # of the whole job. echo "Loop start: $(date +'TS %s.%N %F %T')" @@ -189,6 +196,10 @@ collect() { ($CMD_MYSQL "$EXT_ARGV" -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \ >> "$d/$p-processlist" + + if [ $have_lock_waits_table -eq 1 ]; then + (lock_waits 2>&1; echo $ts) >>"$d/$p-lock-waits" + fi done echo "Loop end: $(date +'TS %s.%N %F %T')" @@ -246,6 +257,39 @@ open_tables() { fi } +lock_waits() { + local sql1="SELECT + CONCAT('thread ', b.trx_mysql_thread_id, ' from ', p.host) AS who_blocks, + IF(p.command = \"Sleep\", p.time, 0) AS idle_in_trx, + MAX(TIMESTAMPDIFF(SECOND, r.trx_wait_started, CURRENT_TIMESTAMP)) AS max_wait_time, + COUNT(*) AS num_waiters + FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS AS w + INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS b ON b.trx_id = w.blocking_trx_id + INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS r ON r.trx_id = w.requesting_trx_id + LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id + GROUP BY who_blocks ORDER BY num_waiters DESC\G" + $CMD_MYSQL "$EXT_ARGV" -e "$sql1" + + local sql2="SELECT + r.trx_id AS waiting_trx_id, + r.trx_mysql_thread_id AS waiting_thread, + TIMESTAMPDIFF(SECOND, r.trx_wait_started, CURRENT_TIMESTAMP) AS wait_time, + r.trx_query AS waiting_query, + l.lock_table AS waiting_table_lock, + b.trx_id AS blocking_trx_id, b.trx_mysql_thread_id AS blocking_thread, + SUBSTRING(p.host, 1, INSTR(p.host, ':') - 1) AS blocking_host, + SUBSTRING(p.host, INSTR(p.host, ':') +1) AS blocking_port, + IF(p.command = \"Sleep\", p.time, 0) AS idle_in_trx, + b.trx_query AS blocking_query + FROM INFORMATION_SCHEMA.INNODB_LOCK_WAITS AS w + INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS b ON b.trx_id = w.blocking_trx_id + INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS r ON r.trx_id = w.requesting_trx_id + INNER JOIN INFORMATION_SCHEMA.INNODB_LOCKS AS l ON w.requested_lock_id = l.lock_id + LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id + ORDER BY wait_time DESC\G" + $CMD_MYSQL "$EXT_ARGV" -e "$sql2" +} + # ########################################################################### # End collect package # ########################################################################### From 36e88c8e40e866b160c7a07352aca651175109cb Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 8 Dec 2011 13:49:44 -0700 Subject: [PATCH 18/27] Fix typo. --- lib/bash/collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index afd6d0ba..4b01b37e 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -160,7 +160,7 @@ collect() { local mysqladmin_pid=$! local have_lock_waits_table=0 - $MYSQL_CMD "$EXT_ARGV" -e "SHOW TABLES FROM INFORMATION_SCHEMA" \ + $CMD_MYSQL "$EXT_ARGV" -e "SHOW TABLES FROM INFORMATION_SCHEMA" \ | grep -qi "INNODB_LOCK_WAITS" if [ $? -eq 0 ]; then have_lock_waits_table=1 From 90fbfaf41f2caab97eeef526980618861f2c4c81 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 8 Dec 2011 14:25:35 -0700 Subject: [PATCH 19/27] Don't quote $EXT_ARGV so multiple opts work. --- lib/bash/collect.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 4b01b37e..9acf2a0c 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -74,7 +74,7 @@ collect() { # Get MySQL's variables if possible. Then sleep long enough that we probably # complete SHOW VARIABLES if all's well. (We don't want to run mysql in the # foreground, because it could hang.) - $CMD_MYSQL "$EXT_ARGV" -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 & + $CMD_MYSQL $EXT_ARGV -e 'SHOW GLOBAL VARIABLES' >> "$d/$p-variables" 2>&1 & sleep .2 # Get the major.minor version number. Version 3.23 doesn't matter for our @@ -95,7 +95,7 @@ collect() { tail_error_log_pid=$! # Send a mysqladmin debug to the server so we can potentially learn about # locking etc. - $CMD_MYSQLADMIN "$EXT_ARGV" debug + $CMD_MYSQLADMIN $EXT_ARGV debug else echo "Could not find the MySQL error log" fi @@ -108,9 +108,9 @@ collect() { else local mutex="SHOW MUTEX STATUS" fi - $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & - open_tables >> "$d/$p-opentables1" 2>&1 & + $CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus1" 2>&1 & + $CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status1" 2>&1 & + open_tables >> "$d/$p-opentables1" 2>&1 & # If TCP dumping is specified, start that on the server's port. local tcpdump_pid="" @@ -155,12 +155,11 @@ collect() { # get and keep a connection to the database; in troubled times # the database tends to exceed max_connections, so reconnecting # in the loop tends not to work very well. - $CMD_MYSQLADMIN "$EXT_ARGV" ext -i1 -c$OPT_RUN_TIME \ - >> "$d/$p-mysqladmin" 2>&1 & + $CMD_MYSQLADMIN $EXT_ARGV ext -i1 -c$OPT_RUN_TIME >>"$d/$p-mysqladmin" 2>&1 & local mysqladmin_pid=$! local have_lock_waits_table=0 - $CMD_MYSQL "$EXT_ARGV" -e "SHOW TABLES FROM INFORMATION_SCHEMA" \ + $CMD_MYSQL $EXT_ARGV -e "SHOW TABLES FROM INFORMATION_SCHEMA" \ | grep -qi "INNODB_LOCK_WAITS" if [ $? -eq 0 ]; then have_lock_waits_table=1 @@ -194,7 +193,7 @@ collect() { (netstat -antp 2>&1; echo $ts) >> "$d/$p-netstat" & (netstat -s 2>&1; echo $ts) >> "$d/$p-netstat_s" & - ($CMD_MYSQL "$EXT_ARGV" -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \ + ($CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G" 2>&1; echo $ts) \ >> "$d/$p-processlist" if [ $have_lock_waits_table -eq 1 ]; then @@ -235,9 +234,9 @@ collect() { kill -s 18 $mysqld_pid fi - $CMD_MYSQL "$EXT_ARGV" -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & - $CMD_MYSQL "$EXT_ARGV" -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & - open_tables >> "$d/$p-opentables2" 2>&1 & + $CMD_MYSQL $EXT_ARGV -e "$innostat" >> "$d/$p-innodbstatus2" 2>&1 & + $CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" 2>&1 & + open_tables >> "$d/$p-opentables2" 2>&1 & # Kill backgrounded tasks. kill $mysqladmin_pid @@ -249,9 +248,9 @@ collect() { } open_tables() { - local open_tables=$($CMD_MYSQLADMIN "$EXT_ARGV" ext | grep "Open_tables" | awk '{print $4}') + local open_tables=$($CMD_MYSQLADMIN $EXT_ARGV ext | grep "Open_tables" | awk '{print $4}') if [ -n "$open_tables" -a $open_tables -le 1000 ]; then - $CMD_MYSQL "$EXT_ARGV" -e 'SHOW OPEN TABLES' 2>&1 & + $CMD_MYSQL $EXT_ARGV -e 'SHOW OPEN TABLES' 2>&1 & else echo "Too many open tables: $open_tables" fi @@ -268,7 +267,7 @@ lock_waits() { INNER JOIN INFORMATION_SCHEMA.INNODB_TRX AS r ON r.trx_id = w.requesting_trx_id LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id GROUP BY who_blocks ORDER BY num_waiters DESC\G" - $CMD_MYSQL "$EXT_ARGV" -e "$sql1" + $CMD_MYSQL $EXT_ARGV -e "$sql1" local sql2="SELECT r.trx_id AS waiting_trx_id, @@ -287,7 +286,7 @@ lock_waits() { INNER JOIN INFORMATION_SCHEMA.INNODB_LOCKS AS l ON w.requested_lock_id = l.lock_id LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id ORDER BY wait_time DESC\G" - $CMD_MYSQL "$EXT_ARGV" -e "$sql2" + $CMD_MYSQL $EXT_ARGV -e "$sql2" } # ########################################################################### From 92c8635c4a423df22fdf0f50f771d91815a56509 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 14 Dec 2011 16:12:43 -0700 Subject: [PATCH 20/27] Fix and test safeguards.sh. Implement disk space+margin check. --- lib/bash/safeguards.sh | 43 +++++++++++++++++++------ t/lib/bash/safeguards.sh | 50 +++++++++++++++++++++++++++++ t/lib/samples/bash/diskspace001.txt | 2 ++ 3 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 t/lib/bash/safeguards.sh create mode 100644 t/lib/samples/bash/diskspace001.txt diff --git a/lib/bash/safeguards.sh b/lib/bash/safeguards.sh index 216cbeac..5fd5850f 100644 --- a/lib/bash/safeguards.sh +++ b/lib/bash/safeguards.sh @@ -25,27 +25,50 @@ set -u disk_space() { local filesystem=${1:-"$PWD"} - # Filesystem 512-blocks Used Available Capacity Mounted on - # /dev/disk0s2 236306352 190223184 45571168 81% / - df -m -P $filesystem + # Filesystem 1M-blocks Used Available Capacity Mounted on + # /dev/disk0s2 115383 92637 22496 81% / + df -m $filesystem } +# Sub: check_disk_space +# Check if there is or will be enough disk space. +# +# Arguments: +# file - File with output from . +# mb - Minimum MB free. +# pc - Minimum percent free. +# margin - Add this many MB to the real MB used. +# +# Returns: +# 0 if there is/will be enough disk space, else 1. check_disk_space() { local file=$1 local mb=${2:-"0"} - local pct=${3:-"0"} + local pc=${3:-"0"} + local margin=${4:-"0"} - local avail=$(cat $file | awk '/^\//{print $4}'); - local full=$(cat $file | awk '/^\//{print $5}' | sed -e 's/%//g'); - if [ "${avail}" -le "$mb" -o "$full" -le "$pct" ]; then - echo "Not enough free space (${full}% full, ${avail}MB free)" - echo "Wanted less than ${pct}% full and more than ${mb}MB" + local mb_used=$(cat $file | awk '/^\//{print $3}'); + local mb_free=$(cat $file | awk '/^\//{print $4}'); + local pc_used=$(cat $file | awk '/^\//{print $5}' | sed -e 's/%//g'); + + if [ "$margin" -gt "0" ]; then + local mb_total=$(($mb_used + $mb_free)) + + mb_used=$(($mb_used + $margin)) + mb_free=$(($mb_free - $margin)) + pc_used=$(awk "BEGIN { printf(\"%d\", $mb_used/$mb_total * 100) }") + fi + + local pc_free=$((100 - $pc_used)) + + if [ "$mb_free" -le "$mb" -o "$pc_free" -le "$pc" ]; then + warn "Not enough free disk space: ${pc_free}% free, ${mb_free} MB free; wanted more than ${pc}% free or ${mb} MB free" return 1 fi + return 0 } - # ########################################################################### # End safeguards package # ########################################################################### diff --git a/t/lib/bash/safeguards.sh b/t/lib/bash/safeguards.sh new file mode 100644 index 00000000..7bbc3ed1 --- /dev/null +++ b/t/lib/bash/safeguards.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +TESTS=10 + +source "$LIB_DIR/log_warn_die.sh" +source "$LIB_DIR/safeguards.sh" + +TMPDIR="$TEST_TMPDIR" +SAMPLE="$T_LIB_DIR/samples/bash" + +disk_space "/" > $TMPDIR/df-out +cmd_ok \ + "grep -q Avail $TMPDIR/df-out" \ + "disk_space()" + +check_disk_space "$SAMPLE/diskspace001.txt" 22495 18 >$TMPDIR/out 2>&1 +is "$?" "0" "Enough disk space" +is \ + "`cat $TMPDIR/out`" \ + "" \ + "No output if enough disk space" + +check_disk_space "$SAMPLE/diskspace001.txt" 22496 18 >$TMPDIR/out 2>&1 +is "$?" "1" "Not enough MB free" +cmd_ok \ + "grep -q '19% free, 22496 MB free; wanted more than 18% free or 22496 MB free' $TMPDIR/out" \ + "Warning if not enough disk space" + +check_disk_space "$SAMPLE/diskspace001.txt" 22495 19 >$TMPDIR/out 2>&1 +is "$?" "1" "Not enough % free" + +# ########################################################################### +# Check with a margin (amount we plan to use in the future). +# ########################################################################### + +check_disk_space "$SAMPLE/diskspace001.txt" 22395 18 100 +is "$?" "0" "Enough disk space with margin" + +check_disk_space "$SAMPLE/diskspace001.txt" 22396 18 100 >$TMPDIR/out 2>&1 +is "$?" "1" "Not enough MB free with margin" + +check_disk_space "$SAMPLE/diskspace001.txt" 100 5 20000 >$TMPDIR/out 2>&1 +is "$?" "1" "Not enough % free with margin" +cmd_ok \ + "grep -q '3% free,' $TMPDIR/out" \ + "Calculates % free with margin" + +# ########################################################################### +# Done +# ########################################################################### diff --git a/t/lib/samples/bash/diskspace001.txt b/t/lib/samples/bash/diskspace001.txt new file mode 100644 index 00000000..acc1d512 --- /dev/null +++ b/t/lib/samples/bash/diskspace001.txt @@ -0,0 +1,2 @@ +Filesystem 1M-blocks Used Available Capacity Mounted on +/dev/disk0s2 115383 92637 22496 81% / From f3ac1b20c228c0ddd1f8b81010f68efd4e0b0154 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 15 Dec 2011 12:01:22 -0700 Subject: [PATCH 21/27] Add usage_or_errors() to parse_options.sh. --- lib/bash/parse_options.sh | 71 ++++++++++++++++++++++++++----------- t/lib/bash/parse_options.sh | 12 ++++--- 2 files changed, 57 insertions(+), 26 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index dfecf2d7..7c0f5a97 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -26,9 +26,11 @@ set -u # Global variables. These must be global because declare inside a # sub will be scoped locally. -declare -a ARGV # non-option args (probably input files) -declare EXT_ARGV # everything after -- (args for an external command) -OPT_ERR=${OPT_ERR:-""} +declare -a ARGV # non-option args (probably input files) +declare EXT_ARGV # everything after -- (args for an external command) +declare -a OPT_ERRS # errors while parsing options, for usage_or_errors() +OPT_VERSION="no" +OPT_HELP="no" # Sub: usage # Print usage (--help) and list the program's options. @@ -46,15 +48,43 @@ usage() { local file=$1 local usage=$(grep '^Usage: ' $file) - - if [ "$OPT_ERR" ]; then - echo "Error: ${OPT_ERR}" >&2 - fi echo $usage >&2 echo >&2 echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 } +usage_or_errors() { + local file=$1 + + if [ "$OPT_VERSION" = "yes" ]; then + local version=$(grep '^pt-[^ ]\+ [0-9]' $file) + echo "$version" + return 0 + fi + + if [ "$OPT_HELP" = "yes" ]; then + usage "$file" + return 0 + fi + + local n_errs=${#OPT_ERRS[*]} + if [ $n_errs -gt 0 ]; then + local i=0 + echo "Errors parsing command line options:" >&2 + echo >&2 + while [ $i -lt $n_errs ]; do + echo " * ${OPT_ERRS[$i]}" >&2 + i=$(($i + 1)) + done + echo >&2 + usage $file + return 1 + fi + + # No --help, --version, or errors. + return 0 +} + # Sub: parse_options # Parse Perl POD options from a program file. # @@ -143,17 +173,20 @@ parse_options() { fi ;; *) - die "Invalid attribute in $TMPDIR/po/$opt_spec: $line" + echo "Invalid attribute in $TMPDIR/po/$opt_spec: $line" >&2 + exit 1 esac done < $TMPDIR/po/$opt_spec if [ -z "$opt" ]; then - die "No long attribute in option spec $TMPDIR/po/$opt_spec" + echo "No long attribute in option spec $TMPDIR/po/$opt_spec" >&2 + exit 1 fi if [ $neg -eq 1 ]; then if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then - die "Option $opt_spec is negatable but not default: yes" + echo "Option $opt_spec is negatable but not default: yes" >&2 + exit 1 fi fi @@ -171,6 +204,7 @@ parse_options() { # specified on the command line, then we re-eval $OPT_FOO=500 to update # $OPT_FOO. local i=0 # ARGV index + local j=0 # OPT_ERRS index for opt; do if [ $# -eq 0 ]; then break # no more opts @@ -181,15 +215,6 @@ parse_options() { EXT_ARGV="$@" break fi - if [ "$opt" = "--version" ]; then - version=$(grep '^pt-[^ ]\+ [0-9]' $0) - echo "$version" - exit 0 - fi - if [ "$opt" = "--help" ]; then - usage $file - exit 0 - fi shift if [ $(expr "$opt" : "-") -eq 0 ]; then # Option does not begin with a hyphen (-), so treat it as @@ -217,7 +242,9 @@ parse_options() { else spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1) if [ -z "$spec" ]; then - die "Unknown option: $opt" + OPT_ERRS[j]="Unknown option: $real_opt" + j=$((j+1)) + continue fi fi @@ -228,7 +255,9 @@ parse_options() { required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2) if [ -n "$required_arg" ]; then if [ $# -eq 0 ]; then - die "$real_opt requires a $required_arg argument" + OPT_ERRS[j]="$real_opt requires a $required_arg argument" + j=$((j+1)) + continue else val="$1" shift diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index ce69ec71..9ece27ce 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=25 +TESTS=26 TMPFILE="$TEST_TMPDIR/parse-opts-output" @@ -11,6 +11,7 @@ source "$LIB_DIR/parse_options.sh" # Parse options from POD using all default values. # ############################################################################ +TOOL="pt-stalk" TMPDIR="$TEST_TMPDIR" parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE @@ -64,12 +65,13 @@ is "$OPT_VERSION" "yes" "Short form" # ############################################################################ # Have to call this in a subshell because the error will cause an exit. -( - parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 -) +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 +is "`cat $TMPFILE`" "" "No warnings or errors yet" + +usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1 local err=$? is "$err" "1" "Non-zero exit on unknown option" -cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" "Error on unknown option" +cmd_ok "grep -q 'Unknown option: --foo' $TMPFILE" "Error on unknown option" # ############################################################################ # Done From a172a3f2a7d7b4c9d105d322f541e1b500cc85a5 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 15 Dec 2011 12:12:18 -0700 Subject: [PATCH 22/27] Return 0 only if no --help, --verion, or opt errors. --- lib/bash/parse_options.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 7c0f5a97..18e66412 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -59,12 +59,12 @@ usage_or_errors() { if [ "$OPT_VERSION" = "yes" ]; then local version=$(grep '^pt-[^ ]\+ [0-9]' $file) echo "$version" - return 0 + return 1 fi if [ "$OPT_HELP" = "yes" ]; then usage "$file" - return 0 + return 1 fi local n_errs=${#OPT_ERRS[*]} From bb315948cd555ede71b0a9df4f9c4d0735ef71a0 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 19 Dec 2011 10:22:42 -0700 Subject: [PATCH 23/27] Parse disk space using df -P -k. --- lib/bash/safeguards.sh | 42 ++++++++++++++++------------- t/lib/bash/safeguards.sh | 19 ++++++++----- t/lib/samples/bash/diskspace001.txt | 4 +-- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/lib/bash/safeguards.sh b/lib/bash/safeguards.sh index 5fd5850f..1e6df342 100644 --- a/lib/bash/safeguards.sh +++ b/lib/bash/safeguards.sh @@ -25,19 +25,21 @@ set -u disk_space() { local filesystem=${1:-"$PWD"} - # Filesystem 1M-blocks Used Available Capacity Mounted on - # /dev/disk0s2 115383 92637 22496 81% / - df -m $filesystem + # Filesystem 1024-blocks Used Available Capacity Mounted on + # /dev/disk0s2 118153176 94409664 23487512 81% / + df -P -k $filesystem } # Sub: check_disk_space -# Check if there is or will be enough disk space. +# Check if there is or will be enough disk space. Input is a file +# with output from , i.e. `df -P -k`. The df output +# must use 1k blocks, but the mb arg from the user is in MB. # # Arguments: -# file - File with output from . -# mb - Minimum MB free. -# pc - Minimum percent free. -# margin - Add this many MB to the real MB used. +# file - File with output from . +# mb - Minimum MB free. +# pc - Minimum percent free. +# mb_margin - Add this many MB to the real MB used. # # Returns: # 0 if there is/will be enough disk space, else 1. @@ -45,24 +47,28 @@ check_disk_space() { local file=$1 local mb=${2:-"0"} local pc=${3:-"0"} - local margin=${4:-"0"} + local mb_margin=${4:-"0"} - local mb_used=$(cat $file | awk '/^\//{print $3}'); - local mb_free=$(cat $file | awk '/^\//{print $4}'); + # Convert MB to KB because the df output should be in 1k blocks. + local kb=$(($mb * 1024)) + local kb_margin=$(($mb_margin * 1024)) + + local kb_used=$(cat $file | awk '/^\//{print $3}'); + local kb_free=$(cat $file | awk '/^\//{print $4}'); local pc_used=$(cat $file | awk '/^\//{print $5}' | sed -e 's/%//g'); - if [ "$margin" -gt "0" ]; then - local mb_total=$(($mb_used + $mb_free)) + if [ "$kb_margin" -gt "0" ]; then + local kb_total=$(($kb_used + $kb_free)) - mb_used=$(($mb_used + $margin)) - mb_free=$(($mb_free - $margin)) - pc_used=$(awk "BEGIN { printf(\"%d\", $mb_used/$mb_total * 100) }") + kb_used=$(($kb_used + $kb_margin)) + kb_free=$(($kb_free - $kb_margin)) + pc_used=$(awk "BEGIN { printf(\"%d\", $kb_used/$kb_total * 100) }") fi local pc_free=$((100 - $pc_used)) - if [ "$mb_free" -le "$mb" -o "$pc_free" -le "$pc" ]; then - warn "Not enough free disk space: ${pc_free}% free, ${mb_free} MB free; wanted more than ${pc}% free or ${mb} MB free" + if [ "$kb_free" -le "$kb" -o "$pc_free" -le "$pc" ]; then + warn "Not enough free disk space: ${pc_free}% free, ${kb_free} KB free; wanted more than ${pc}% free or ${kb} KB free" return 1 fi diff --git a/t/lib/bash/safeguards.sh b/t/lib/bash/safeguards.sh index 7bbc3ed1..c874498f 100644 --- a/t/lib/bash/safeguards.sh +++ b/t/lib/bash/safeguards.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=10 +TESTS=11 source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/safeguards.sh" @@ -13,30 +13,35 @@ cmd_ok \ "grep -q Avail $TMPDIR/df-out" \ "disk_space()" -check_disk_space "$SAMPLE/diskspace001.txt" 22495 18 >$TMPDIR/out 2>&1 +is \ + "`wc -l $TMPDIR/df-out | awk '{print $1}'`" \ + "2" \ + "2-line df output" + +check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 >$TMPDIR/out 2>&1 is "$?" "0" "Enough disk space" is \ "`cat $TMPDIR/out`" \ "" \ "No output if enough disk space" -check_disk_space "$SAMPLE/diskspace001.txt" 22496 18 >$TMPDIR/out 2>&1 +check_disk_space "$SAMPLE/diskspace001.txt" 24000 18 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough MB free" cmd_ok \ - "grep -q '19% free, 22496 MB free; wanted more than 18% free or 22496 MB free' $TMPDIR/out" \ + "grep -q '19% free, 23487512 KB free; wanted more than 18% free or 24576000 KB free' $TMPDIR/out" \ "Warning if not enough disk space" -check_disk_space "$SAMPLE/diskspace001.txt" 22495 19 >$TMPDIR/out 2>&1 +check_disk_space "$SAMPLE/diskspace001.txt" 22000 19 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough % free" # ########################################################################### # Check with a margin (amount we plan to use in the future). # ########################################################################### -check_disk_space "$SAMPLE/diskspace001.txt" 22395 18 100 +check_disk_space "$SAMPLE/diskspace001.txt" 22000 18 100 is "$?" "0" "Enough disk space with margin" -check_disk_space "$SAMPLE/diskspace001.txt" 22396 18 100 >$TMPDIR/out 2>&1 +check_disk_space "$SAMPLE/diskspace001.txt" 23000 18 100 >$TMPDIR/out 2>&1 is "$?" "1" "Not enough MB free with margin" check_disk_space "$SAMPLE/diskspace001.txt" 100 5 20000 >$TMPDIR/out 2>&1 diff --git a/t/lib/samples/bash/diskspace001.txt b/t/lib/samples/bash/diskspace001.txt index acc1d512..c68af8c9 100644 --- a/t/lib/samples/bash/diskspace001.txt +++ b/t/lib/samples/bash/diskspace001.txt @@ -1,2 +1,2 @@ -Filesystem 1M-blocks Used Available Capacity Mounted on -/dev/disk0s2 115383 92637 22496 81% / +Filesystem 1024-blocks Used Available Capacity Mounted on +/dev/disk0s2 118153176 94409664 23487512 81% / From 8439962b45a46830cf281ca2c4044b8f3f08f5f9 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 19 Dec 2011 10:44:36 -0700 Subject: [PATCH 24/27] Make tests more flexible. --- t/lib/bash/collect.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index ad059e29..9d9249fa 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -62,11 +62,11 @@ cmd_ok \ "lsof" cmd_ok \ - "grep -q 'buf/buf0buf.c' $p-mutex-status1" \ + "grep -q 'buf0buf.c' $p-mutex-status1" \ "mutex-status1" cmd_ok \ - "grep -q 'buf/buf0buf.c' $p-mutex-status2" \ + "grep -q 'buf0buf.c' $p-mutex-status2" \ "mutex-status2" cmd_ok \ @@ -90,7 +90,7 @@ cmd_ok \ "ps" cmd_ok \ - "grep -qP '^warning_count\t\d' $p-variables" \ + "grep -qP '^wait_timeout\t\d' $p-variables" \ "variables" local iters=$(cat $p-df | grep -c '^TS ') From 3266e0aeca7d1a0e2f2c560464db6703a5dc7bf9 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 19 Dec 2011 11:29:21 -0700 Subject: [PATCH 25/27] Don't use OPT_TMPDIR in tmpdir.sh. Define EXT_ARGV in parse_options.sh. --- lib/bash/parse_options.sh | 6 +++--- lib/bash/tmpdir.sh | 15 ++++++++------- t/lib/bash/tmpdir.sh | 22 ++++++++++++++-------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 18e66412..9eaccb06 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -26,9 +26,9 @@ set -u # Global variables. These must be global because declare inside a # sub will be scoped locally. -declare -a ARGV # non-option args (probably input files) -declare EXT_ARGV # everything after -- (args for an external command) -declare -a OPT_ERRS # errors while parsing options, for usage_or_errors() +declare -a ARGV # non-option args (probably input files) +EXT_ARGV="" # everything after -- (args for an external command) +declare -a OPT_ERRS # errors while parsing options, for usage_or_errors() OPT_VERSION="no" OPT_HELP="no" diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index 6349bf02..d1b9e1b7 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -25,22 +25,23 @@ set -u # Global variables. TMPDIR="" -OPT_TMPDIR=${OPT_TMPDIR:""} # Sub: mk_tmpdir # Create a secure tmpdir and set TMPDIR. # -# Optional Global Variables: -# OPT_TMPDIR - User-specified --tmpdir (default none). +# Optional Arguments: +# dir - User-specified tmpdir (default none). # # Set Global Variables: # TMPDIR - Absolute path of secure temp directory. mk_tmpdir() { - if [ -n "$OPT_TMPDIR" ]; then - TMPDIR="$OPT_TMPDIR" - if [ ! -d "$TMPDIR" ]; then - mkdir $TMPDIR || die "Cannot make $TMPDIR" + local dir=${1:-""} + + if [ -n "$dir" ]; then + if [ ! -d "$dir" ]; then + mkdir $dir || die "Cannot make tmpdir $dir" fi + TMPDIR="$dir" else local tool=`basename $0` local pid="$$" diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index a5b86823..55098f4d 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -17,19 +17,25 @@ cmd_ok "test ! -d $tmpdir" "rm_tmpdir" is "$TMPDIR" "" "rm_tmpdir resets TMPDIR" -# --tmpdir -OPT_TMPDIR="/tmp/use--tmpdir" +# ########################################################################### +# User-specified tmpdir. +# ########################################################################### + +local dir="/tmp/use--tmpdir" is "$TMPDIR" "" "TMPDIR not defined" -cmd_ok "test ! -d $OPT_TMPDIR" "--tmpdir does not exist yet" +cmd_ok "test ! -d $dir" "--tmpdir does not exist yet" -mk_tmpdir -is "$TMPDIR" "/tmp/use--tmpdir" "mk_tmpdir uses --tmpdir" +mk_tmpdir $dir +is "$TMPDIR" "$dir" "mk_tmpdir uses --tmpdir" -cmd_ok "test -d $TMPDIR" "mk_tmpdir creates --tmpdir" - -tmpdir=$TMPDIR; +cmd_ok "test -d $dir" "mk_tmpdir creates --tmpdir" rm_tmpdir + cmd_ok "test ! -d $tmpdir" "rm_tmpdir removes --tmpdir" + +# ########################################################################### +# Done +# ########################################################################### From 9baea8b7b6ee5e5629210567780c82e41bf7a633 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 19 Dec 2011 12:51:47 -0700 Subject: [PATCH 26/27] Convert parse_option to sh, use Perl instead of awk, implement --help. --- lib/bash/parse_options.sh | 109 +++++++++++++++++---------------- t/lib/bash/parse_options.sh | 13 +++- t/lib/samples/bash/help001.txt | 30 +++++++++ t/lib/samples/bash/po001.sh | 8 ++- 4 files changed, 104 insertions(+), 56 deletions(-) create mode 100644 t/lib/samples/bash/help001.txt diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 9eaccb06..008af21a 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -26,11 +26,11 @@ set -u # Global variables. These must be global because declare inside a # sub will be scoped locally. -declare -a ARGV # non-option args (probably input files) -EXT_ARGV="" # everything after -- (args for an external command) -declare -a OPT_ERRS # errors while parsing options, for usage_or_errors() -OPT_VERSION="no" -OPT_HELP="no" +ARGV="" # Non-option args (probably input files) +EXT_ARGV="" # Everything after -- (args for an external command) +OPT_ERRS=0 # How many command line option errors +OPT_VERSION="no" # If --version was specified +OPT_HELP="no" # If --help was specified # Sub: usage # Print usage (--help) and list the program's options. @@ -64,18 +64,19 @@ usage_or_errors() { if [ "$OPT_HELP" = "yes" ]; then usage "$file" + echo >&2 + echo "Command line options:" >&2 + echo >&2 + for opt in $(ls $TMPDIR/po/); do + local desc=$(cat $TMPDIR/po/$opt | grep '^desc:' | sed -e 's/^desc://') + echo "--$opt" >&2 + echo " $desc" >&2 + echo >&2 + done return 1 fi - local n_errs=${#OPT_ERRS[*]} - if [ $n_errs -gt 0 ]; then - local i=0 - echo "Errors parsing command line options:" >&2 - echo >&2 - while [ $i -lt $n_errs ]; do - echo " * ${OPT_ERRS[$i]}" >&2 - i=$(($i + 1)) - done + if [ $OPT_ERRS -gt 0 ]; then echo >&2 usage $file return 1 @@ -113,37 +114,36 @@ parse_options() { mkdir $TMPDIR/po/ 2>/dev/null rm -rf $TMPDIR/po/* ( - # awk is stupid on some systems (e.g. Ubuntu 10) such that - # /^[a-z]/ incorrectly matches "Foo". This fixes that. - export LC_ALL="C" - - awk -v "po_dir"="$TMPDIR/po" ' - /^=head1 OPTIONS/ { - getline - while ($0 !~ /^=head1/) { - if ($0 ~ /^=item --.*/) { - long_opt = substr($2, 3, length($2) - 2) - spec_file = po_dir "/" long_opt - trf = "sed -e \"s/[ ]//g\" | tr \";\" \"\n\" > " spec_file - - getline # blank line - getline # specs or description - - if ($0 ~ /^[a-z]/ ) { - # spec line like "type: int; default: 100" - print "long:" long_opt "; " $0 | trf - close(trf) - } - else { - # no specs, should be description of option - print "long:" long_opt > spec_file - close(spec_file) - } + export PO_DIR="$TMPDIR/po" + cat $file | perl -ne ' + BEGIN { $/ = ""; } + next unless $_ =~ m/^=head1 OPTIONS/; + while ( defined(my $para = <>) ) { + last if $para =~ m/^=head1/; + chomp; + if ( $para =~ m/^=item --(\S+)/ ) { + my $opt = $1; + my $file = "$ENV{PO_DIR}/$opt"; + open my $opt_fh, ">", $file or die "Cannot open $file: $!"; + printf $opt_fh "long:$opt\n"; + $para = <>; + chomp; + if ( $para =~ m/^[a-z ]+:/ ) { + map { + chomp; + my ($attrib, $val) = split(/: /, $_); + printf $opt_fh "$attrib:$val\n"; + } split(/; /, $para); + $para = <>; + chomp; } - getline + my ($desc) = $para =~ m/^([^.]+)/; + printf $opt_fh "desc:$desc.\n"; + close $opt_fh; } - exit - }' $file + } + last; + ' ) # Evaluate the program options into existence as global variables @@ -163,10 +163,12 @@ parse_options() { default) default_val="$val" ;; - shortform) + "short form") ;; type) ;; + desc) + ;; negatable) if [ "$val" = "yes" ]; then neg=1 @@ -203,8 +205,6 @@ parse_options() { # a default value 100, then $OPT_FOO=100 already, but if --foo=500 is # specified on the command line, then we re-eval $OPT_FOO=500 to update # $OPT_FOO. - local i=0 # ARGV index - local j=0 # OPT_ERRS index for opt; do if [ $# -eq 0 ]; then break # no more opts @@ -219,8 +219,11 @@ parse_options() { if [ $(expr "$opt" : "-") -eq 0 ]; then # Option does not begin with a hyphen (-), so treat it as # a filename, directory, etc. - ARGV[i]="$opt" - i=$((i+1)) + if [ -z "$ARGV" ]; then + ARGV="$opt" + else + ARGV="$ARGV $opt" + fi continue fi @@ -240,10 +243,10 @@ parse_options() { if [ -f "$TMPDIR/po/$opt" ]; then spec="$TMPDIR/po/$opt" else - spec=$(grep "^shortform:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1) + spec=$(grep "^short form:-$opt\$" $TMPDIR/po/* | cut -d ':' -f 1) if [ -z "$spec" ]; then - OPT_ERRS[j]="Unknown option: $real_opt" - j=$((j+1)) + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "Unknown option: $real_opt" >&2 continue fi fi @@ -255,8 +258,8 @@ parse_options() { required_arg=$(cat $spec | grep '^type:' | cut -d':' -f2) if [ -n "$required_arg" ]; then if [ $# -eq 0 ]; then - OPT_ERRS[j]="$real_opt requires a $required_arg argument" - j=$((j+1)) + OPT_ERRS=$(($OPT_ERRS + 1)) + echo "$real_opt requires a $required_arg argument" >&2 continue else val="$1" diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 9ece27ce..176c824c 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -66,12 +66,21 @@ is "$OPT_VERSION" "yes" "Short form" # Have to call this in a subshell because the error will cause an exit. parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 -is "`cat $TMPFILE`" "" "No warnings or errors yet" +cmd_ok "grep -q 'Unknown option: --foo' $TMPFILE" "Error on unknown option" usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1 local err=$? is "$err" "1" "Non-zero exit on unknown option" -cmd_ok "grep -q 'Unknown option: --foo' $TMPFILE" "Error on unknown option" + +# ########################################################################### +# --help +# ########################################################################### +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --help +usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1 +no_diff \ + "$TMPFILE" \ + "$T_LIB_DIR/samples/bash/help001.txt" \ + "--help" # ############################################################################ # Done diff --git a/t/lib/samples/bash/help001.txt b/t/lib/samples/bash/help001.txt new file mode 100644 index 00000000..65dc71b2 --- /dev/null +++ b/t/lib/samples/bash/help001.txt @@ -0,0 +1,30 @@ +Usage: pt-stalk [OPTIONS] [-- MYSQL_OPTIONS] + +For more information, 'man pt-stalk' or 'perldoc /Users/daniel/p/bash-tool-libs/t/lib/samples/bash/po001.sh'. + +Command line options: + +--help + Print help and exit. + +--int-opt + Int option without a default. + +--int-opt2 + Int option with a default. + +--noption + Negatable option. + +--string-opt + String option without a default. + +--string-opt2 + String option with a default. + +--typeless-option + Just an option. + +--version + Print tool's version and exit. + diff --git a/t/lib/samples/bash/po001.sh b/t/lib/samples/bash/po001.sh index fc0c387e..15b71d25 100644 --- a/t/lib/samples/bash/po001.sh +++ b/t/lib/samples/bash/po001.sh @@ -110,11 +110,13 @@ Just an option. default: yes; negatable: yes +Negatable option. + =item --int-opt type: int -Int option without a default +Int option without a default. =item --int-opt2 @@ -128,6 +130,10 @@ short form: -v Print tool's version and exit. +=item --help + +Print help and exit. + =back =head1 ENVIRONMENT From 0056014339954f0ad9c37d8283858e8650d0e3ca Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 19 Dec 2011 12:55:14 -0700 Subject: [PATCH 27/27] Let desc sentence be terminated by period of question mark. --- lib/bash/parse_options.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 008af21a..e4915247 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -137,7 +137,7 @@ parse_options() { $para = <>; chomp; } - my ($desc) = $para =~ m/^([^.]+)/; + my ($desc) = $para =~ m/^([^?.]+)/; printf $opt_fh "desc:$desc.\n"; close $opt_fh; }