From 1ae8661271ba7813a835f806dd12dab8cae8a9ac Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 26 Oct 2011 10:50:55 -0600 Subject: [PATCH 1/5] First Bash parse_options lib with tests. --- lib/bash/parse_options.sh | 119 +++++++++++++ t/lib/bash.t | 23 +++ t/lib/bash/parse_options.sh | 25 +++ t/lib/samples/bash/dummy.sh | 4 + t/lib/samples/bash/po001.sh | 340 ++++++++++++++++++++++++++++++++++++ 5 files changed, 511 insertions(+) create mode 100644 lib/bash/parse_options.sh create mode 100755 t/lib/bash.t create mode 100644 t/lib/bash/parse_options.sh create mode 100644 t/lib/samples/bash/dummy.sh create mode 100644 t/lib/samples/bash/po001.sh diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh new file mode 100644 index 00000000..e9003cad --- /dev/null +++ b/lib/bash/parse_options.sh @@ -0,0 +1,119 @@ +# Print usage (--help). +usage() { + local file=$1 + + local usage=$(grep '^Usage: ' $file) + local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help) + + if [ "${OPT_ERR}" ]; then + echo "Error: ${OPT_ERR}" >&2 + fi + echo $usage >&2 + echo >&2 + echo "Options:" >&2 + echo >&2 + cat $TMPDIR/help >&2 + echo >&2 + echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 +} + +# Parse command line options. +declare -a ARGV # non-option args (probably input files) +declare EXT_ARGV # everything after -- (args for an external command) +parse_options() { + local file=$1 + shift + + local opt="" + local val="" + local default="" + local version="" + local i=0 + + awk ' + /^=head1 OPTIONS/ { + getline + while ($0 !~ /^=head1/) { + if ($0 ~ /^=item --.*/) { + long_opt=substr($2, 3, length($2) - 2) + short_opt="" + required_arg="" + + if ($3) { + if ($3 ~ /-[a-z]/) + short_opt=substr($3, 3, length($3) - 3) + else + required_arg=$3 + } + + if ($4 ~ /[A-Z]/) + required_arg=$4 + + getline # blank line + getline # short description line + + if ($0 ~ /default: /) { + i=index($0, "default: ") + default=substr($0, i + 9, length($0) - (i + 9)) + } + else + default="" + + print long_opt "," short_opt "," required_arg "," default + } + getline + } + exit + }' $file > $TMPDIR/options + + while read spec; do + opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:]) + default=$(echo $spec | cut -d',' -f4) + eval "$opt"="$default" + done < <(cat $TMPDIR/options) + + for opt; do + if [ $# -eq 0 ]; then + break + fi + opt=$1 + if [ "$opt" = "--" ]; then + shift + EXT_ARGV="$@" + break + fi + if [ "$opt" = "--version" ]; then + version=$(grep '^pt-[^ ]\+ [0-9]' $0) + echo "$version" + exit 0 + fi + if [ "$opt" = "--help" ]; then + usage $file + exit 0 + fi + shift + if [ $(expr "$opt" : "-") -eq 0 ]; then + ARGV[i]="$opt" + i=$((i+1)) + continue + fi + opt=$(echo $opt | sed 's/^-*//') + spec=$(grep -E "^$opt,|,$opt," "$TMPDIR/options") + if [ -z "$spec" ]; then + die "Unknown option: $opt" + fi + opt=$(echo $spec | cut -d',' -f1) + required_arg=$(echo $spec | cut -d',' -f3) + val=1 + if [ -n "$required_arg" ]; then + if [ $# -eq 0 ]; then + die "--$opt requires a $required_arg argument" + else + val="$1" + shift + fi + fi + opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:]) + eval "$opt"="$val" + done +} diff --git a/t/lib/bash.t b/t/lib/bash.t new file mode 100755 index 00000000..aaf1fe28 --- /dev/null +++ b/t/lib/bash.t @@ -0,0 +1,23 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); + +use PerconaTest; + +my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/; +push @ARGV, "$trunk/t/lib/bash/*.sh" unless @ARGV; + +$ENV{LIB_DIR} = "$trunk/lib/bash"; +$ENV{T_LIB_DIR} = "$trunk/t/lib"; + +system("$trunk/util/test-bash-functions $trunk/t/lib/samples/bash/dummy.sh @ARGV"); + +exit; diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh new file mode 100644 index 00000000..00c8de65 --- /dev/null +++ b/t/lib/bash/parse_options.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +TESTS=17 + +source "$LIB_DIR/parse_options.sh" + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" + +is "$THRESHOLD" "100" +is "$VARIABLE" "Threads_connected" +is "$CYCLES" "1" +is "$GDB" "no" +is "$OPROFILE" "yes" +is "$STRACE" "no" +is "$TCPDUMP" "yes" +is "$EMAIL" "" +is "$INTERVAL" "30" +is "$MAYBE_EMPTY" "no" +is "$COLLECT" "${HOME}/bin/pt-collect" +is "$DEST" "${HOME}/collected/" +is "$DURATION" "30" +is "$SLEEP" "300" +is "$PCT_THRESHOLD" "95" +is "$MB_THRESHOLD" "100" +is "$PURGE" "30" diff --git a/t/lib/samples/bash/dummy.sh b/t/lib/samples/bash/dummy.sh new file mode 100644 index 00000000..a7eea1ee --- /dev/null +++ b/t/lib/samples/bash/dummy.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# This is a dummy script for testing the Bash libs. t/lib/bashLibs.t +# calls "util/test-bash-functions dummy.sh ". diff --git a/t/lib/samples/bash/po001.sh b/t/lib/samples/bash/po001.sh new file mode 100644 index 00000000..0fdd943d --- /dev/null +++ b/t/lib/samples/bash/po001.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash + +# This is a fake script for testing the parse_options.sh lib. +exit 0; + +# ############################################################################ +# Documentation +# ############################################################################ +:<<'DOCUMENTATION' +=pod + +=head1 NAME + +pt-stalk - Wait for a condition to occur then begin collecting data. + +=head1 SYNOPSIS + +Usage: pt-stalk [OPTIONS] [-- MYSQL_OPTIONS] + +pt-stalk watches for a condition to become true, and when it does, executes +a script. By default it executes L, but that can be customized. +This tool is useful for gathering diagnostic data when an infrequent event +occurs, so an expert person can review the data later. + +=head1 RISKS + +The following section is included to inform users about the potential risks, +whether known or unknown, of using this tool. The two main categories of risks +are those created by the nature of the tool (e.g. read-only tools vs. read-write +tools) and those created by bugs. + +pt-stalk is a read-only tool. It should be very low-risk. + +At the time of this release, we know of no bugs that could cause serious harm +to users. + +The authoritative source for updated information is always the online issue +tracking system. Issues that affect this tool will be marked as such. You can +see a list of such issues at the following URL: +L. + +See also L<"BUGS"> for more information on filing bugs and getting help. + +=head1 DESCRIPTION + +Although pt-stalk comes pre-configured to do a specific thing, in general +this tool is just a skeleton script for the following flow of actions: + +=over + +=item 1. + +Loop infinitely, sleeping between iterations. + +=item 2. + +In each iteration, run some command and get the output. + +=item 3. + +If the command fails or the output is larger than the threshold, +execute the collection script; but do not execute if the destination disk +is too full. + +=back + +By default, the tool is configured to execute mysqladmin extended-status and +extract the value of the Threads_connected variable; if this is greater than +100, it runs the collection script. This is really just placeholder code, +and almost certainly needs to be customized! + +If the tool does execute the collection script, it will wait for a while +before checking and executing again. This is to prevent a continuous +condition from causing a huge number of executions to fire off. + +The name 'stalk' is because 'watch' is already taken, and 'stalk' is fun. + +=head1 CONFIGURING + +If the file F exists in the current working directory, then +L<"ENVIRONMENT"> variables are imported from it. For example, the config +file has the format: + + INTERVAL=10 + GDB=yes + +See L<"ENVIRONMENT">. + +=head1 OPTIONS + +=over + +=item --threshold N + +Max number of C to tolerate. (default: 100) + +=item --variable NAME + +This is the thing to check for. (default: Threads_connected) + +=item --cycles N + +How many times must the condition be met before the script will fire? (default: 1) + +=item --gdb + +Collect GDB stacktraces? (default: no) + +=item --oprofile + +Collect oprofile data? (default: yes) + +=item --strace + +Collect strace data? (default: no) + +=item --tcpdump + +Collect tcpdump data? (default: yes) + +=item --email ADDRESS + +Send mail to this list of addresses when the script triggers. + +=item --interval SECONDS + +This is the interval between checks. (default: 30) + +=item --maybe-empty + +Result of checks may be empty. (default: no) +If the command you're running to detect the condition is allowed to return +nothing (e.g. a grep line that might not even exist if there's no problem), +then set this to "yes". + +=item --collect DIRECTORY + +Location of the C tool. (default: ${HOME}/bin/pt-collect) + +=item --dest DIRECTORY + +Where to store collected data. (default: ${HOME}/collected/) + +=item --duration SECONDS + +How long to collect statistics data for? (default: 30) +Make sure that this isn't longer than SLEEP. + +=item --sleep SECONDS + +How long to sleep after collecting? (default: 300) + +=item --pct-threshold PCT + +Bail out if the disk is more than this %full. (default: 95) + +=item --mb-threshold MEGABYTES + +Bail out if the disk has less than this many MB free. (default: 100) + +=item --purge DAYS + +Remove samples after this many days. (default: 30) + +=item --version + +Print tool's version and exit. + +=back + +=head1 ENVIRONMENT + +The following environment variables configure how, what, and when the tool +runs. They are all optional and can be specified either on the command line +or in the F config file (see L<"CONFIGURING">). + +=over + +=item THRESHOLD (default 100) + +This is the max number of we want to tolerate. + +=item VARIABLE (default Threads_connected} + +This is the thing to check for. + +=item CYCLES (default 1) + +How many times must the condition be met before the script will fire? + +=item GDB (default no) + +Collect GDB stacktraces? + +=item OPROFILE (default yes) + +Collect oprofile data? + +=item STRACE (default no) + +Collect strace data? + +=item TCPDUMP (default yes) + +Collect tcpdump data? + +=item EMAIL + +Send mail to this list of addresses when the script triggers. + +=item MYSQLOPTIONS + +Any options to pass to mysql/mysqladmin, such as -u, -p, etc + +=item INTERVAL (default 30) + +This is the interval between checks. + +=item MAYBE_EMPTY (default no) + +If the command you're running to detect the condition is allowed to return +nothing (e.g. a grep line that might not even exist if there's no problem), +then set this to "yes". + +=item COLLECT (default ${HOME}/bin/pt-collect) + +This is the location of the 'collect' script. + +=item DEST (default ${HOME}/collected/) + +This is where to store the collected data. + +=item DURATION (default 30) + +How long to collect statistics data for? Make sure that this isn't longer +than SLEEP. + +=item SLEEP (default DURATION * 10) + +How long to sleep after collecting? + +=item PCT_THRESHOLD (default 95) + +Bail out if the disk is more than this %full. + +=item MB_THRESHOLD (default 100) + +Bail out if the disk has less than this many MB free. + +=item PURGE (default 30) + +Remove samples after this many days. + +=back + +=head1 SYSTEM REQUIREMENTS + +This tool requires Bash v3 or newer. + +=head1 BUGS + +For a list of known bugs, see L. + +Please report bugs at L. +Include the following information in your bug report: + +=over + +=item * Complete command-line used to run the tool + +=item * Tool L<"--version"> + +=item * MySQL version of all servers involved + +=item * Output from the tool including STDERR + +=item * Input files (log/dump/config files, etc.) + +=back + +If possible, include debugging output by running the tool with C; +see L<"ENVIRONMENT">. + +=head1 DOWNLOADING + +Visit L to download the +latest release of Percona Toolkit. Or, get the latest release from the +command line: + + wget percona.com/get/percona-toolkit.tar.gz + + wget percona.com/get/percona-toolkit.rpm + + wget percona.com/get/percona-toolkit.deb + +You can also get individual tools from the latest release: + + wget percona.com/get/TOOL + +Replace C with the name of any tool. + +=head1 AUTHORS + +Baron Schwartz, Justin Swanhart, and Fernando Ipar + +=head1 ABOUT PERCONA TOOLKIT + +This tool is part of Percona Toolkit, a collection of advanced command-line +tools developed by Percona for MySQL support and consulting. Percona Toolkit +was forked from two projects in June, 2011: Maatkit and Aspersa. Those +projects were created by Baron Schwartz and developed primarily by him and +Daniel Nichter, both of whom are employed by Percona. Visit +L for more software developed by Percona. + +=head1 COPYRIGHT, LICENSE, AND WARRANTY + +This program is copyright 2010-2011 Baron Schwartz, 2011 Percona Inc. +Feedback and improvements are welcome. + +THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +systems, you can issue `man perlgpl' or `man perlartistic' to read these +licenses. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +=head1 VERSION + +pt-stalk 1.0.1 + +=cut + +DOCUMENTATION From 1ec666de0eba1428b18f17434160e5839fb7ae51 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 27 Oct 2011 11:56:58 -0600 Subject: [PATCH 2/5] Add Bash libs tmpdir and log_warn_die. Docu parse_options. Don't use TMPDIR in test-bash-functions; add ok() to test-bash-functions. --- lib/bash/log_warn_die.sh | 45 ++++++++++++++++++++++++++ lib/bash/parse_options.sh | 66 ++++++++++++++++++++++++++++++++++++--- lib/bash/tmpdir.sh | 50 +++++++++++++++++++++++++++++ t/lib/bash/tmpdir.sh | 44 ++++++++++++++++++++++++++ util/test-bash-functions | 23 +++++++++----- 5 files changed, 215 insertions(+), 13 deletions(-) create mode 100644 lib/bash/log_warn_die.sh create mode 100644 lib/bash/tmpdir.sh create mode 100644 t/lib/bash/tmpdir.sh diff --git a/lib/bash/log_warn_die.sh b/lib/bash/log_warn_die.sh new file mode 100644 index 00000000..ef60daa3 --- /dev/null +++ b/lib/bash/log_warn_die.sh @@ -0,0 +1,45 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin log_warn_die lib +# ########################################################################### + +# Library: log_warn_die +# log_warn_die provides standard log(), warn(), and die() subs. + +set -u + +EXIT_STATUS=0 + +log() { + TS=$(date +%F-%T | tr :- _); + echo "$TS $1" +} + +warn() { + log "$1" >&2 + EXIT_STATUS=$((EXIT_STATUS | 1)) +} + +die() { + warn "$1" + exit 1 +} + +# ########################################################################### +# End log_warn_die lib +# ########################################################################### diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index e9003cad..8b6b31d2 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -1,11 +1,53 @@ -# Print usage (--help). +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin parse_options lib +# ########################################################################### + +# Library: parse_options +# parse_options parses Perl POD options from Bash tools and creates +# global variables for each option. + +set -u + +# Global variables. These must be global because declare inside a +# sub will be scoped locally. +declare -a ARGV # non-option args (probably input files) +declare EXT_ARGV # everything after -- (args for an external command) + +# Sub: usage +# Print usage (--help) and list the program's options. +# +# Arguments: +# file - Program file with Perl POD which has usage and options. +# +# Required Global Variables: +# TIMDIR - Temp directory. +# TOOL - Tool's name. +# +# Optional Global Variables: +# OPT_ERR - Command line option error message. usage() { local file=$1 local usage=$(grep '^Usage: ' $file) local opts=$(grep -A 2 '^=item --' $file | sed -e 's/^=item //' -e 's/^\([A-Z]\)/ \1/' -e 's/^--$//' > $TMPDIR/help) - if [ "${OPT_ERR}" ]; then + if [ "$OPT_ERR" ]; then echo "Error: ${OPT_ERR}" >&2 fi echo $usage >&2 @@ -17,9 +59,19 @@ usage() { echo "For more information, 'man $TOOL' or 'perldoc $file'." >&2 } -# Parse command line options. -declare -a ARGV # non-option args (probably input files) -declare EXT_ARGV # everything after -- (args for an external command) +# Sub: parse_options +# Parse Perl POD options from a program file. +# +# Arguments: +# file - Program file with Perl POD options. +# +# Required Global Variables: +# TIMDIR - Temp directory. +# +# Declared Global Variables: +# This sub decalres a global var for each option by uppercasing the +# option, removing the option's leading --, changing all - to _, and +# prefixing with "OPT_". E.g. --foo-bar becomes OPT_FOO_BAR. parse_options() { local file=$1 shift @@ -117,3 +169,7 @@ parse_options() { eval "$opt"="$val" done } + +# ########################################################################### +# End parse_options lib +# ########################################################################### diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh new file mode 100644 index 00000000..208d883b --- /dev/null +++ b/lib/bash/tmpdir.sh @@ -0,0 +1,50 @@ +# This program is copyright 2011 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# Begin tmpdir lib +# ########################################################################### + +# Library: tmpdir +# tmpdir make a secure temporary directory using mktemp. + +set -u + +TMPDIR="" +OPT_TMPDIR={OPT_TMPDIR:""} + +set_TMPDIR() { + if [ -n "$OPT_TMPDIR" ]; then + TMPDIR="$OPT_TMPDIR" + if [ ! -d "$TMPDIR" ]; then + mkdir $TMPDIR || die "Cannot make $TMPDIR" + fi + else + local tool=`basename $0` + TMPDIR=`mktemp -d /tmp/${tool}.XXXXX` || die "Cannot make secure tmpdir" + fi +} + +rm_TMPDIR() { + if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then + rm -rf $TMPDIR + fi + TMPDIR="" +} + +# ########################################################################### +# End tmpdir lib +# ########################################################################### diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh new file mode 100644 index 00000000..1a7d5809 --- /dev/null +++ b/t/lib/bash/tmpdir.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +TESTS=9 + +source "$LIB_DIR/log_warn_die.sh" +source "$LIB_DIR/tmpdir.sh" + +TEST_NAME="TMPDIR not defined" +is "$TMPDIR" "" + +TEST_NAME="set_TMPDIR makes secure tmpdir" +set_TMPDIR +ok "test -d $TMPDIR" + +tmpdir=$TMPDIR; + +TEST_NAME="rm_TMPDIR" +rm_TMPDIR +ok "test ! -d $tmpdir" + +TEST_NAME="rm_TMPDIR resets TMPDIR" +is "$TMPDIR" "" + +# --tmpdir +OPT_TMPDIR="/tmp/use--tmpdir" + +TEST_NAME="TMPDIR not defined" +is "$TMPDIR" "" + +TEST_NAME="--tmpdir does not exist yet" +ok "test ! -d $OPT_TMPDIR" + +set_TMPDIR +TEST_NAME="set_TMPDIR uses --tmpdir" +is "$TMPDIR" "/tmp/use--tmpdir" + +TEST_NAME="set_TMPDIR creates --tmpdir" +ok "test -d $TMPDIR" + +tmpdir=$TMPDIR; + +TEST_NAME="rm_TMPDIR removes --tmpdir" +rm_TMPDIR +ok "test ! -d $tmpdir" diff --git a/util/test-bash-functions b/util/test-bash-functions index 1ffb0075..901e818e 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -34,9 +34,10 @@ die() { # Paths # ############################################################################ -TMPDIR="/tmp/percona-toolkit.test" -if [ ! -d $TMPDIR ]; then - mkdir $TMPDIR +# Do not use TMPDIR because the tools use it for their own secure tmpdir. +TEST_TMPDIR="/tmp/percona-toolkit.test" +if [ ! -d $TEST_TMPDIR ]; then + mkdir $TEST_TMPDIR fi # ############################################################################ @@ -69,7 +70,7 @@ load_tests() { # Source a test file to run whatever it contains (hopefully tests!). run_test() { local t=$1 # test file name, e.g. "group-by-all-01" for pt-diskstats - rm -rf $TMPDIR/* >/dev/null 2>&1 + rm -rf $TEST_TMPDIR/* >/dev/null 2>&1 TEST_NUMBER=1 # test number in this test file @@ -98,8 +99,8 @@ result() { echo "not ok $testno - $TEST_FILE $test_name" failed_tests=$(( failed_tests + 1)) echo "# Failed '$test_command'" >&2 - if [ -f $TMPDIR/failed_result ]; then - cat $TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2 + if [ -f $TEST_TMPDIR/failed_result ]; then + cat $TEST_TMPDIR/failed_result | sed -e 's/^/# /' -e '30q' >&2 fi fi testno=$((testno + 1)) @@ -115,7 +116,7 @@ no_diff() { local got=$1 local expected=$2 test_command="diff $got $expected" - eval $test_command > $TMPDIR/failed_result 2>&1 + eval $test_command > $TEST_TMPDIR/failed_result 2>&1 result $? } @@ -127,6 +128,12 @@ is() { result $? } +ok() { + local test_command=$1 + $test_command + result $? +} + # ############################################################################ # Script starts here # ############################################################################ @@ -157,6 +164,6 @@ for t in "${tests[@]}"; do run_test $t done -rm -rf $TMPDIR +rm -rf $TEST_TMPDIR exit $failed_tests From f2b644ba723d26e753a5db7d8254eb7fe1f48529 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 28 Oct 2011 11:08:59 -0600 Subject: [PATCH 3/5] More parse_options tests. Make --opt yes have value "yes" instead of "1". Change ok() to cmd_ok() in test-bash-functions. Mimic Perl modulue headers in Bash libs so update-modules will work with the latter. --- lib/bash/log_warn_die.sh | 7 +-- lib/bash/parse_options.sh | 18 +++---- lib/bash/tmpdir.sh | 23 +++++++-- t/lib/bash.t | 0 t/lib/bash/parse_options.sh | 94 +++++++++++++++++++++++++++++-------- t/lib/bash/tmpdir.sh | 10 ++-- util/test-bash-functions | 4 +- 7 files changed, 115 insertions(+), 41 deletions(-) mode change 100755 => 100644 t/lib/bash.t diff --git a/lib/bash/log_warn_die.sh b/lib/bash/log_warn_die.sh index ef60daa3..6d5ca2e4 100644 --- a/lib/bash/log_warn_die.sh +++ b/lib/bash/log_warn_die.sh @@ -15,14 +15,15 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin log_warn_die lib +# log_warn_die package # ########################################################################### -# Library: log_warn_die +# Package: log_warn_die # log_warn_die provides standard log(), warn(), and die() subs. set -u +# Global variables. EXIT_STATUS=0 log() { @@ -41,5 +42,5 @@ die() { } # ########################################################################### -# End log_warn_die lib +# End log_warn_die package # ########################################################################### diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index 8b6b31d2..b94ea52b 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -15,10 +15,10 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin parse_options lib +# parse_options package # ########################################################################### -# Library: parse_options +# Package: parse_options # parse_options parses Perl POD options from Bash tools and creates # global variables for each option. @@ -36,7 +36,7 @@ declare EXT_ARGV # everything after -- (args for an external command) # file - Program file with Perl POD which has usage and options. # # Required Global Variables: -# TIMDIR - Temp directory. +# TIMDIR - Temp directory set by . # TOOL - Tool's name. # # Optional Global Variables: @@ -66,9 +66,9 @@ usage() { # file - Program file with Perl POD options. # # Required Global Variables: -# TIMDIR - Temp directory. +# TIMDIR - Temp directory set by . # -# Declared Global Variables: +# Set Global Variables: # This sub decalres a global var for each option by uppercasing the # option, removing the option's leading --, changing all - to _, and # prefixing with "OPT_". E.g. --foo-bar becomes OPT_FOO_BAR. @@ -121,7 +121,7 @@ parse_options() { while read spec; do opt=$(echo $spec | cut -d',' -f1 | sed 's/-/_/g' | tr [:lower:] [:upper:]) default=$(echo $spec | cut -d',' -f4) - eval "$opt"="$default" + eval "OPT_${opt}"="$default" done < <(cat $TMPDIR/options) for opt; do @@ -156,7 +156,7 @@ parse_options() { fi opt=$(echo $spec | cut -d',' -f1) required_arg=$(echo $spec | cut -d',' -f3) - val=1 + val="yes" if [ -n "$required_arg" ]; then if [ $# -eq 0 ]; then die "--$opt requires a $required_arg argument" @@ -166,10 +166,10 @@ parse_options() { fi fi opt=$(echo $opt | sed 's/-/_/g' | tr [:lower:] [:upper:]) - eval "$opt"="$val" + eval "OPT_${opt}"="$val" done } # ########################################################################### -# End parse_options lib +# End parse_options package # ########################################################################### diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index 208d883b..143ec1df 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -15,17 +15,26 @@ # this program; if not, write to the Free Software Foundation, Inc., 59 Temple # Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### -# Begin tmpdir lib +# tmpdir package # ########################################################################### -# Library: tmpdir +# Package: tmpdir # tmpdir make a secure temporary directory using mktemp. set -u +# Global variables. TMPDIR="" OPT_TMPDIR={OPT_TMPDIR:""} +# Sub: set_TMPDIR +# Create a secure tmpdir and set TMPDIR. +# +# Optional Global Variables: +# OPT_TMPDIR - User-specified --tmpdir (default none). +# +# Set Global Variables: +# TMPDIR - Absolute path of secure temp directory. set_TMPDIR() { if [ -n "$OPT_TMPDIR" ]; then TMPDIR="$OPT_TMPDIR" @@ -38,6 +47,14 @@ set_TMPDIR() { fi } +# Sub: rm_TMPDIR +# Remove the tmpdir and unset TMPDIR. +# +# Optional Global Variables: +# TMPDIR - TMPDIR set by . +# +# Set Global Variables: +# TMPDIR - Set to "". rm_TMPDIR() { if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then rm -rf $TMPDIR @@ -46,5 +63,5 @@ rm_TMPDIR() { } # ########################################################################### -# End tmpdir lib +# End tmpdir package # ########################################################################### diff --git a/t/lib/bash.t b/t/lib/bash.t old mode 100755 new mode 100644 diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 00c8de65..a5daeffe 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,25 +1,81 @@ #!/usr/bin/env bash -TESTS=17 +TESTS=37 +TMPFILE="$TEST_TMPDIR/parse-opts-output" + +source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/parse_options.sh" -parse_options "$T_LIB_DIR/samples/bash/po001.sh" +# ############################################################################ +# Parse options from POD using all default values. +# ############################################################################ -is "$THRESHOLD" "100" -is "$VARIABLE" "Threads_connected" -is "$CYCLES" "1" -is "$GDB" "no" -is "$OPROFILE" "yes" -is "$STRACE" "no" -is "$TCPDUMP" "yes" -is "$EMAIL" "" -is "$INTERVAL" "30" -is "$MAYBE_EMPTY" "no" -is "$COLLECT" "${HOME}/bin/pt-collect" -is "$DEST" "${HOME}/collected/" -is "$DURATION" "30" -is "$SLEEP" "300" -is "$PCT_THRESHOLD" "95" -is "$MB_THRESHOLD" "100" -is "$PURGE" "30" +parse_options "$T_LIB_DIR/samples/bash/po001.sh" "" 2>$TMPFILE + +TEST_NAME="No warnings or errors" +is "`cat $TMPFILE`" "" + +TEST_NAME="Default opts" +is "$OPT_THRESHOLD" "100" +is "$OPT_VARIABLE" "Threads_connected" +is "$OPT_CYCLES" "1" +is "$OPT_GDB" "no" +is "$OPT_OPROFILE" "yes" +is "$OPT_STRACE" "no" +is "$OPT_TCPDUMP" "yes" +is "$OPT_EMAIL" "" +is "$OPT_INTERVAL" "30" +is "$OPT_MAYBE_EMPTY" "no" +is "$OPT_COLLECT" "${HOME}/bin/pt-collect" +is "$OPT_DEST" "${HOME}/collected/" +is "$OPT_DURATION" "30" +is "$OPT_SLEEP" "300" +is "$OPT_PCT_THRESHOLD" "95" +is "$OPT_MB_THRESHOLD" "100" +is "$OPT_PURGE" "30" + +# ############################################################################ +# Specify some opts, but use default values for the rest. +# ############################################################################ + +parse_options "$T_LIB_DIR/samples/bash/po001.sh" --threshold 50 --gdb yes --email user@example.com + +TEST_NAME="User-specified opts with defaults" +is "$OPT_THRESHOLD" "50" # specified +is "$OPT_VARIABLE" "Threads_connected" +is "$OPT_CYCLES" "1" +is "$OPT_GDB" "yes" # specified +is "$OPT_OPROFILE" "yes" +is "$OPT_STRACE" "no" +is "$OPT_TCPDUMP" "yes" +is "$OPT_EMAIL" "user@example.com" # specified +is "$OPT_INTERVAL" "30" +is "$OPT_MAYBE_EMPTY" "no" +is "$OPT_COLLECT" "${HOME}/bin/pt-collect" +is "$OPT_DEST" "${HOME}/collected/" +is "$OPT_DURATION" "30" +is "$OPT_SLEEP" "300" +is "$OPT_PCT_THRESHOLD" "95" +is "$OPT_MB_THRESHOLD" "100" +is "$OPT_PURGE" "30" + +# ############################################################################ +# An unknown option should produce an error. +# ############################################################################ + +# Have to call this in a subshell because the error will cause an exit. +( + parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 +) +local err=$? +TEST_NAME="Non-zero exit on unknown option" +is "$err" "1" + +TEST_NAME="Error on unknown option" +cmd_ok "grep -q 'Unknown option: foo' $TMPFILE" + +# ############################################################################ +# Done +# ############################################################################ +exit diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index 1a7d5809..83708652 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -10,13 +10,13 @@ is "$TMPDIR" "" TEST_NAME="set_TMPDIR makes secure tmpdir" set_TMPDIR -ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; TEST_NAME="rm_TMPDIR" rm_TMPDIR -ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" TEST_NAME="rm_TMPDIR resets TMPDIR" is "$TMPDIR" "" @@ -28,17 +28,17 @@ TEST_NAME="TMPDIR not defined" is "$TMPDIR" "" TEST_NAME="--tmpdir does not exist yet" -ok "test ! -d $OPT_TMPDIR" +cmd_ok "test ! -d $OPT_TMPDIR" set_TMPDIR TEST_NAME="set_TMPDIR uses --tmpdir" is "$TMPDIR" "/tmp/use--tmpdir" TEST_NAME="set_TMPDIR creates --tmpdir" -ok "test -d $TMPDIR" +cmd_ok "test -d $TMPDIR" tmpdir=$TMPDIR; TEST_NAME="rm_TMPDIR removes --tmpdir" rm_TMPDIR -ok "test ! -d $tmpdir" +cmd_ok "test ! -d $tmpdir" diff --git a/util/test-bash-functions b/util/test-bash-functions index 901e818e..9a10a782 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -128,9 +128,9 @@ is() { result $? } -ok() { +cmd_ok() { local test_command=$1 - $test_command + eval $test_command result $? } From 8cb5f6b600638d15c3cdfa3027b2d726509f5759 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 31 Oct 2011 11:25:56 -0600 Subject: [PATCH 4/5] Add default OPT_ERR and fix default OPT_TMPDIR. --- lib/bash/parse_options.sh | 1 + lib/bash/tmpdir.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index b94ea52b..037902c6 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -28,6 +28,7 @@ set -u # sub will be scoped locally. declare -a ARGV # non-option args (probably input files) declare EXT_ARGV # everything after -- (args for an external command) +OPT_ERR=${OPT_ERR:""} # Sub: usage # Print usage (--help) and list the program's options. diff --git a/lib/bash/tmpdir.sh b/lib/bash/tmpdir.sh index 143ec1df..b0983f44 100644 --- a/lib/bash/tmpdir.sh +++ b/lib/bash/tmpdir.sh @@ -25,7 +25,7 @@ set -u # Global variables. TMPDIR="" -OPT_TMPDIR={OPT_TMPDIR:""} +OPT_TMPDIR=${OPT_TMPDIR:""} # Sub: set_TMPDIR # Create a secure tmpdir and set TMPDIR. From 42675e33b891bfde05ac967772e32bcea96d01fb Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 1 Nov 2011 09:26:58 -0600 Subject: [PATCH 5/5] Fix update-modules to work with Bash tools and libs. --- util/update-modules | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/util/update-modules b/util/update-modules index fb69fcb7..b976e07e 100755 --- a/util/update-modules +++ b/util/update-modules @@ -49,7 +49,11 @@ file_is_modified() { pkgs_in_tool() { local tool=$1 - pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1) + if [ "$tool_lang" = "perl" ]; then + pkgs=$(grep '^package [A-Za-z]*;' $tool | cut -d' ' -f2 | cut -d';' -f1) + else + pkgs=$(grep '^# [a-z_]* package' $tool | awk '{print $2}') + fi } replace_pkg_in_tool() { @@ -72,7 +76,8 @@ replace_pkg_in_tool() { head -n $pkg_start_line $tool_file > $tmp_file - echo "# $pkg package + if [ "$tool_lang" = "perl" ]; then + echo "# $pkg package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/$pkg.pm @@ -80,11 +85,24 @@ replace_pkg_in_tool() { # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### {" >> $tmp_file + else + echo "# $pkg package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/bash/$pkg.sh +# t/lib/bash/$pkg.sh +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +" >> $tmp_file + fi $BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file - echo "} -# ########################################################################### + if [ "$tool_lang" = "perl" ]; then + echo "}" + fi + +echo "# ########################################################################### # End $pkg package" >> $tmp_file tail -n +$pkg_end_line $tool_file >> $tmp_file @@ -97,14 +115,19 @@ replace_pkg_in_tool() { # ############################################################################ tool_file=$1 + if [ -z "$tool_file" ]; then die "Usage: $0 TOOL [MODULE...]" fi + if [ ! -f $tool_file ]; then die "$tool_file does not exist" fi -if [ -z "$(head -n 1 $tool_file | grep perl)" ]; then - die "$tool_file is not a Perl tool" + +if [ -n "$(head -n 1 $tool_file | grep perl)" ]; then + tool_lang="perl" +else + tool_lang="bash" fi tool=$(basename $tool_file) @@ -124,11 +147,17 @@ for pkg in $pkgs; do continue fi - pkg_file="$BRANCH/lib/$pkg.pm" + if [ "$tool_lang" = "perl" ]; then + pkg_file="$BRANCH/lib/$pkg.pm" + else + pkg_file="$BRANCH/lib/bash/$pkg.sh" + fi + if [ ! -f $pkg_file ]; then warn "$pkg_file does not exist" continue fi + if file_is_modified $pkg_file; then warn "$pkg_file has uncommitted changes" continue