From 048ad9493c02184fe4e2a05dc3c4112c1300b3c6 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 25 Aug 2011 11:08:57 -0600 Subject: [PATCH 01/34] Various test tweaks for stability. --- sandbox/test-env | 2 +- t/pt-fk-error-logger/basics.t | 5 ++- t/pt-heartbeat/basics.t | 14 ++++--- t/pt-index-usage/save_results.t | 37 ++++++++++++------- t/pt-online-schema-change/basics.t | 2 +- t/pt-query-digest/daemon.t | 31 +++++++++------- t/pt-query-digest/execute.t | 3 ++ t/pt-query-digest/mirror.t | 32 ++++++++-------- t/pt-query-digest/processlist.t | 6 +++ .../samples/slow018_execute_report_2.txt | 2 +- 10 files changed, 80 insertions(+), 54 deletions(-) diff --git a/sandbox/test-env b/sandbox/test-env index 023d0427..67a99860 100755 --- a/sandbox/test-env +++ b/sandbox/test-env @@ -9,7 +9,7 @@ err() { echo for msg; do - echo "$msg" + echo "$msg" >&2 done } diff --git a/t/pt-fk-error-logger/basics.t b/t/pt-fk-error-logger/basics.t index 5c7f8d39..c4890d83 100644 --- a/t/pt-fk-error-logger/basics.t +++ b/t/pt-fk-error-logger/basics.t @@ -9,6 +9,7 @@ BEGIN { use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); +use Time::HiRes qw(sleep); use Test::More; use PerconaTest; @@ -44,6 +45,7 @@ $sb->load_file('master', 't/pt-fk-error-logger/samples/fke_tbl.sql', 'test'); # Then get and save that fke. output(sub { pt_fk_error_logger::main('h=127.1,P=12345,u=msandbox,p=msandbox', '--dest', 'h=127.1,P=12345,D=test,t=foreign_key_errors'); } ); +sleep 0.1; # And then test that it was actually saved. my $today = $dbh->selectall_arrayref('SELECT NOW()')->[0]->[0]; @@ -64,6 +66,7 @@ like( # Check again to make sure that the same fke isn't saved twice. my $first_ts = $fke->[0]->[0]; output(sub { pt_fk_error_logger::main('h=127.1,P=12345,u=msandbox,p=msandbox', '--dest', 'h=127.1,P=12345,D=test,t=foreign_key_errors'); } ); +sleep 0.1; $fke = $dbh->selectall_arrayref('SELECT * FROM test.foreign_key_errors'); is( $fke->[0]->[0], # Timestamp @@ -84,6 +87,7 @@ eval { $dbh->do('DELETE FROM parent WHERE id = 2'); # Causes foreign key error. }; output( sub { pt_fk_error_logger::main('h=127.1,P=12345,u=msandbox,p=msandbox', '--dest', 'h=127.1,P=12345,D=test,t=foreign_key_errors'); } ); +sleep 0.1; $fke = $dbh->selectall_arrayref('SELECT * FROM test.foreign_key_errors'); like( $fke->[1]->[1], # Error @@ -99,7 +103,6 @@ is( # ########################################################################## # Test printing the errors. # ########################################################################## -sleep 1; $dbh->do('USE test'); eval { $dbh->do('DELETE FROM parent WHERE id = 2'); # Causes foreign key error. diff --git a/t/pt-heartbeat/basics.t b/t/pt-heartbeat/basics.t index 60dccf3b..5a634412 100644 --- a/t/pt-heartbeat/basics.t +++ b/t/pt-heartbeat/basics.t @@ -29,12 +29,13 @@ else { $sb->create_dbs($dbh, ['test']); my $output; -my $cnf = '/tmp/12345/my.sandbox.cnf'; -my $cmd = "$trunk/bin/pt-heartbeat -F $cnf "; -my $pid_file = "/tmp/__pt-heartbeat-test.pid"; +my $cnf = '/tmp/12345/my.sandbox.cnf'; +my $cmd = "$trunk/bin/pt-heartbeat -F $cnf "; +my $pid_file = "/tmp/__pt-heartbeat-test.pid"; +my $sent_file = "/tmp/pt-heartbeat-sentinel"; my $ps_grep_cmd = "ps x | grep pt-heartbeat | grep daemonize | grep -v grep"; -`rm /tmp/pt-heartbeat-sentinel 2>/dev/null`; +`rm $sent_file 2>/dev/null`; $dbh->do('drop table if exists test.heartbeat'); $dbh->do(q{CREATE TABLE test.heartbeat ( @@ -102,8 +103,8 @@ like($output, qr/Successfully created/, 'Created sentinel'); sleep(2); $output = `$ps_grep_cmd`; unlike($output, qr/$cmd/, 'It is not running'); -ok(-f '/tmp/pt-heartbeat-sentinel', 'Sentinel file is there'); -unlink('/tmp/pt-heartbeat-sentinel'); +ok(-f $sent_file, 'Sentinel file is there'); +unlink($sent_file); $dbh->do('drop table if exists test.heartbeat'); # This will kill it # ############################################################################# @@ -137,5 +138,6 @@ like( # ############################################################################# # Done. # ############################################################################# +`rm $pid_file $sent_file 2>/dev/null`; $sb->wipe_clean($dbh); exit; diff --git a/t/pt-index-usage/save_results.t b/t/pt-index-usage/save_results.t index 19c74da2..1a310d95 100644 --- a/t/pt-index-usage/save_results.t +++ b/t/pt-index-usage/save_results.t @@ -116,24 +116,33 @@ is_deeply( "Table access counts" ); -# EXPLAIN results differ a little between 5.0 and 5.1. 5.1 is smarter. -my $res = $sandbox_version ge '5.1' ? - # v5.1 and newer - [ +$rows = $dbh->selectall_arrayref("select * from mk.indexes order by db, tbl"); +# EXPLAIN results differ a little between 5.0 and 5.1, and sometimes 5.1 acts +# like 5.0. So here we detect which verison MySQL is acting like and future +# tests will select between 2 possibilities based on exp_plan. Note: both +# possibilities are correct; they're variants of the same result. +my $res; +my $exp_plan; +if ( $rows->[0]->[3] == 1 ) { + # Usually v5.1 and newer + $res = [ [qw(sakila actor idx_actor_last_name 1)], [qw(sakila actor PRIMARY 3)], [qw(sakila address idx_fk_city_id 0)], [qw(sakila address PRIMARY 0)], - ] - : # v5.0 and older - [ + ]; + $exp_plan = '5.1'; # acting like 5.1 +} +else { + # Usually v5.0 and older, but somtimes 5.1. + $res = [ [qw(sakila actor idx_actor_last_name 2)], [qw(sakila actor PRIMARY 2)], [qw(sakila address idx_fk_city_id 0)], [qw(sakila address PRIMARY 0)], ]; - -$rows = $dbh->selectall_arrayref("select * from mk.indexes order by db, tbl"); + $exp_plan = '5.0'; # acting like 5.0 +} is_deeply( $rows, $res, @@ -161,7 +170,7 @@ is_deeply( ); $rows = $dbh->selectall_arrayref("select query_id, db, tbl, idx, sample, cnt from index_usage iu left join queries q using (query_id) order by db, tbl, idx"); -$res = $sandbox_version ge '5.1' ? +$res = $exp_plan eq '5.1' ? # v5.1 and newer [ [ @@ -212,7 +221,7 @@ is_deeply( ); $rows = $dbh->selectall_arrayref("select db,tbl,idx,alt_idx,sample from index_alternatives a left join queries q using (query_id)"); -$res = $sandbox_version ge '5.1' ? +$res = $exp_plan eq '5.1' ? [[qw(sakila actor PRIMARY idx_actor_last_name), "select * from sakila.actor where last_name like 'A%' order by actor_id"]] : []; @@ -239,7 +248,7 @@ is_deeply( ); # EXPLAIN results differ a little between 5.0 and 5.1. 5.1 is smarter. -$res = $sandbox_version ge '5.1' ? +$res = $exp_plan eq '5.1' ? # v5.1 and newer [ [qw(sakila actor idx_actor_last_name 2)], @@ -283,7 +292,7 @@ is_deeply( ); $rows = $dbh->selectall_arrayref("select query_id, db, tbl, idx, sample, cnt from index_usage iu left join queries q using (query_id) order by db, tbl, idx"); -$res = $sandbox_version ge '5.1' ? +$res = $exp_plan eq '5.1' ? # v5.1 and newer [ [ @@ -334,7 +343,7 @@ is_deeply( ); $rows = $dbh->selectall_arrayref("select db,tbl,idx,alt_idx,sample from index_alternatives a left join queries q using (query_id)"); -$res = $sandbox_version ge '5.1' ? +$res = $exp_plan eq '5.1' ? [[qw(sakila actor PRIMARY idx_actor_last_name), "select * from sakila.actor where last_name like 'A%' order by actor_id"]] : []; diff --git a/t/pt-online-schema-change/basics.t b/t/pt-online-schema-change/basics.t index c39c9526..7e461580 100644 --- a/t/pt-online-schema-change/basics.t +++ b/t/pt-online-schema-change/basics.t @@ -116,7 +116,7 @@ is( # ############################################################################# # No --alter and --drop-old-table. # ############################################################################# -$dbh->do('drop table mkosc.__old_a'); # from previous run +$dbh->do('drop table if exists mkosc.__old_a'); # from previous run $sb->load_file('master', "t/pt-online-schema-change/samples/small_table.sql"); output( diff --git a/t/pt-query-digest/daemon.t b/t/pt-query-digest/daemon.t index bf03713b..3f82df4a 100644 --- a/t/pt-query-digest/daemon.t +++ b/t/pt-query-digest/daemon.t @@ -9,6 +9,7 @@ BEGIN { use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); +use Time::HiRes qw(sleep); use Test::More tests => 6; use PerconaTest; @@ -21,17 +22,20 @@ my $dbh = $sb->get_dbh_for('master'); my $output; +my $pid_file = '/tmp/pt-query-digest.test.pid'; +`rm $pid_file >/dev/null 2>&1`; + # ######################################################################### # Issue 391: Add --pid option to all scripts # ######################################################################### -`touch /tmp/pt-script.pid`; -$output = `$trunk/bin/pt-query-digest $trunk/commont/t/samples/slow002.txt --pid /tmp/pt-script.pid 2>&1`; +`touch $pid_file`; +$output = `$trunk/bin/pt-query-digest $trunk/commont/t/samples/slow002.txt --pid $pid_file 2>&1`; like( $output, - qr{PID file /tmp/pt-script.pid already exists}, + qr{PID file $pid_file already exists}, 'Dies if PID file already exists (--pid without --daemonize) (issue 391)' ); -`rm -rf /tmp/pt-script.pid`; +`rm $pid_file >/dev/null 2>&1`; # ######################################################################### # Daemonizing and pid creation @@ -39,22 +43,22 @@ like( SKIP: { skip "Cannot connect to sandbox master", 5 unless $dbh; - my $cmd = "$trunk/bin/pt-query-digest --daemonize --pid /tmp/pt-query-digest.pid --processlist h=127.1,P=12345,u=msandbox,p=msandbox --log /dev/null"; + my $cmd = "$trunk/bin/pt-query-digest --daemonize --pid $pid_file --processlist h=127.1,P=12345,u=msandbox,p=msandbox --log /dev/null"; `$cmd`; - $output = `ps -eaf | grep -v grep | grep pt-query-digest`; + $output = `ps xw | grep -v grep | grep '$cmd'`; like($output, qr/$cmd/, 'It is running'); - ok(-f '/tmp/pt-query-digest.pid', 'PID file created'); + ok(-f $pid_file, 'PID file created'); - my ($pid) = $output =~ /\s+(\d+)\s+/; - $output = `cat /tmp/pt-query-digest.pid`; + my ($pid) = $output =~ /^\s*(\d+)/; + chomp($output = `cat $pid_file`); is($output, $pid, 'PID file has correct PID'); kill 15, $pid; - sleep 1; - $output = `ps -eaf | grep pt-query-digest | grep daemonize`; - unlike($output, qr/$trunk\/pt-query-digest\/pt-query-digest/, 'It is not running'); + sleep 0.25; + $output = `ps xw | grep -v grep | grep '$cmd'`; + is($output, "", 'It is not running'); ok( - !-f '/tmp/pt-query-digest.pid', + !-f $pid_file, 'Removes its PID file' ); }; @@ -62,4 +66,5 @@ SKIP: { # ############################################################################# # Done. # ############################################################################# +`rm $pid_file >/dev/null 2>&1`; exit; diff --git a/t/pt-query-digest/execute.t b/t/pt-query-digest/execute.t index a181c0b2..3a677a55 100644 --- a/t/pt-query-digest/execute.t +++ b/t/pt-query-digest/execute.t @@ -68,6 +68,8 @@ $cnf .= ',D=test'; # TODO: This test is a PITA because every time the mqd output # changes the -n of tail has to be adjusted. +# + # We tail to get everything from "Exec orig" onward. The lines # above have the real execution time will will vary. The last 18 lines # are sufficient to see that it actually executed without errors. @@ -77,6 +79,7 @@ ok( "$trunk/t/lib/samples/slowlogs/slow018.txt") }, 't/pt-query-digest/samples/slow018_execute_report_2.txt', trf => 'tail -n 30', + sed => ["-e 's/s ##*/s/g'"], ), '--execute with default database' ); diff --git a/t/pt-query-digest/mirror.t b/t/pt-query-digest/mirror.t index b3ba8e9f..5445e2de 100644 --- a/t/pt-query-digest/mirror.t +++ b/t/pt-query-digest/mirror.t @@ -10,6 +10,7 @@ use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Test::More; +use Time::HiRes qw(sleep); use PerconaTest; use DSNParser; @@ -33,6 +34,9 @@ else { my $output; my $cmd; +my $pid_file = '/tmp/pt-query-digest.test.pid'; +`rm -rf $pid_file >/dev/null`; + # ########################################################################## # Tests for swapping --processlist and --execute # ########################################################################## @@ -41,35 +45,29 @@ $dbh2->do('set global read_only=1'); $cmd = "$trunk/bin/pt-query-digest " . "--processlist h=127.1,P=12345,u=msandbox,p=msandbox " . "--execute h=127.1,P=12346,u=msandbox,p=msandbox --mirror 1 " - . "--pid foobar"; -# --pid actually does nothing because the script is not daemonizing. -# I include it for the identifier (foobar) so that we can more easily -# grep the PID below. Otherwise, a ps | grep mk-query-digest will -# match this test script and any vi mk-query-digest[.t] that may happen -# to be running. + . "--pid $pid_file"; $ENV{MKDEBUG}=1; `$cmd > /tmp/read_only.txt 2>&1 &`; $ENV{MKDEBUG}=0; -sleep 5; +sleep 3; $dbh1->do('select sleep(1)'); sleep 1; $dbh1->do('set global read_only=1'); $dbh2->do('set global read_only=0'); $dbh1->do('select sleep(1)'); -sleep 2; -$output = `ps -eaf | grep mk-query-diges[t] | grep foobar | awk '{print \$2}'`; -kill 15, $output =~ m/(\d+)/g; sleep 1; +chomp(my $pid = `cat $pid_file`); +kill 15, $pid; +sleep 0.25; # Verify that it's dead... -$output = `ps -eaf | grep mk-query-diges[t] | grep foobar`; -if ( $output =~ m/digest/ ) { - $output = `ps -eaf | grep mk-query-diges[t] | grep foobar`; -} -unlike($output, qr/mk-query-digest/, 'It is stopped now'); +$output = `ps x | grep '^[ ]*$pid'`; +is( + $output, + '', + 'It is stopped now' +); -$dbh1->do('set global read_only=0'); -$dbh2->do('set global read_only=1'); $output = `grep read_only /tmp/read_only.txt`; # Sample output: # # main:3619 6897 read_only on execute for --execute: 1 (want 1) diff --git a/t/pt-query-digest/processlist.t b/t/pt-query-digest/processlist.t index 7bd56e4c..041c4005 100644 --- a/t/pt-query-digest/processlist.t +++ b/t/pt-query-digest/processlist.t @@ -55,6 +55,12 @@ my $output = output( ); ($exec) = $output =~ m/^(# Exec time.+?)$/ms; +# The end of the line is like "786ms 3s". The 2nd to last value is +# stddev which can vary slightly depending on the real exec time. The +# other int values should always round to the correct values. 786ms is +# the usual stddev. -- stddev doesn't matter much. It's the other vals +# that indicate that --processlist works. +$exec =~ s/(\S+) 3s$/786ms 3s/; ok( no_diff( $exec, diff --git a/t/pt-query-digest/samples/slow018_execute_report_2.txt b/t/pt-query-digest/samples/slow018_execute_report_2.txt index 3890383f..dc7626a3 100644 --- a/t/pt-query-digest/samples/slow018_execute_report_2.txt +++ b/t/pt-query-digest/samples/slow018_execute_report_2.txt @@ -10,7 +10,7 @@ # Query_time distribution # 1us # 10us -# 100us ################################################################ +# 100us # 1ms # 10ms # 100ms From 0c785d63b2c8157d81ea362b8644d646e244c67a Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 25 Aug 2011 11:44:56 -0600 Subject: [PATCH 02/34] Fix pt-index-usage/basics.t by fixing pt-heartbeat/standard_options.t. Make pt-kill/basics.t dump output if tests fail. --- t/pt-heartbeat/standard_options.t | 3 ++- t/pt-kill/basics.t | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/t/pt-heartbeat/standard_options.t b/t/pt-heartbeat/standard_options.t index 3a168dcd..8a459ce4 100644 --- a/t/pt-heartbeat/standard_options.t +++ b/t/pt-heartbeat/standard_options.t @@ -48,6 +48,7 @@ like( `rm -rf /tmp/mk-script.pid`; # ############################################################################# -# Done. +# Doe. # ############################################################################# +$sb->wipe_clean($dbh); exit; diff --git a/t/pt-kill/basics.t b/t/pt-kill/basics.t index 918bef1a..a82e199b 100644 --- a/t/pt-kill/basics.t +++ b/t/pt-kill/basics.t @@ -11,6 +11,8 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use Test::More; +use Data::Dumper; + use PerconaTest; use Sandbox; require "$trunk/bin/pt-kill"; @@ -32,7 +34,7 @@ my $cmd = "$trunk/bin/pt-kill -F $cnf -h 127.1"; # Shell out to a sleep(10) query and try to capture the query. # Backticks don't work here. -system("/tmp/12345/use -h127.1 -P12345 -umsandbox -pmsandbox -e 'select sleep(5)' >/dev/null&"); +system("/tmp/12345/use -h127.1 -P12345 -umsandbox -pmsandbox -e 'select sleep(5)' >/dev/null &"); $output = `$cmd --busy-time 1s --print --run-time 10`; @@ -47,7 +49,10 @@ $output = `$cmd --busy-time 1s --print --run-time 10`; # 2009-05-27T22:19:47 KILL 5 (Query 8 sec) select sleep(10) # 2009-05-27T22:19:48 KILL 5 (Query 9 sec) select sleep(10) my @times = $output =~ m/\(Query (\d+) sec\)/g; -ok(@times > 2 && @times < 7, "There were 2 to 5 captures"); +ok( + @times > 2 && @times < 7, + "There were 2 to 5 captures" +) or print STDERR Dumper($output); # This is to catch a bad bug where there wasn't any sleep time when # --iterations was 0, and another bug when --run-time was not respected. @@ -56,7 +61,10 @@ ok(@times > 2 && @times < 7, "There were 2 to 5 captures"); system("/tmp/12345/use -h127.1 -P12345 -umsandbox -pmsandbox -e 'select sleep(10)' >/dev/null&"); $output = `$cmd --busy-time 1s --print --run-time 11s`; @times = $output =~ m/\(Query (\d+) sec\)/g; -ok(@times > 7 && @times < 12, 'Approximately 9 or 10 captures with --iterations 0'); +ok( + @times > 7 && @times < 12, + 'Approximately 9 or 10 captures with --iterations 0' +) or print STDERR Dumper($output); # ############################################################################ From 9492b8dce65ce984565cc27cf20c598ac9e24d7b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 25 Aug 2011 11:54:10 -0600 Subject: [PATCH 03/34] Make check-dev-env print Perl version. --- util/check-dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/check-dev-env b/util/check-dev-env index 7072d569..633ff2a6 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -30,6 +30,8 @@ use Test::More; use Time::HiRes; use Time::Local; +print `perl --version | grep 'v5'`; + my $file = __FILE__; my $m = `cat $file | grep '^use'`; my @modules = map { m/use (.+?);/; $1 } split("\n", $m); From 8c27f618b7ad677fcf4ac33be334c084cf51c9a9 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 7 Feb 2012 14:53:01 -0700 Subject: [PATCH 04/34] Make check-dev-env check all modules without dying. --- util/check-dev-env | 74 ++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/util/check-dev-env b/util/check-dev-env index 633ff2a6..78b06945 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -1,44 +1,54 @@ #!/usr/bin/env perl -# This pseudo-script is for developers to see if their box has all -# the modules necessary for testing Percona Toolkit. Any missing -# modules will cause an error like "Can't locate Foo.pm in @INC ...". -# Else the version for each module used by this script will be printed. +use warnings FATAL => 'all'; +use English; + +# This script is for developers to see if their box has all the modules +# necessary for testing Percona Toolkit. Any missing modules are printed +# to STDERR and installed modules are printed to STDOUT. # # In addition to these modules, many non-standard programs are needed # for other tasks, like building packages, writing test coverage, etc. # # Exits 0 if all modules are installed, else exits non-zero. -use Data::Dumper; -use DBD::mysql; -use DBI; -use Digest::Crc32; -use Digest::MD5; -use File::Basename; -use File::Find; -use File::Spec; -use File::Temp; -use Getopt::Long; -use IO::File; -use IO::Uncompress::Inflate; -use List::Util; -use POSIX; -use Socket; -use Term::ReadKey; -use Test::More; -use Time::HiRes; -use Time::Local; +my @modules = qw( + Data::Dumper + DBD::mysql + DBI + Digest::Crc32 + Digest::MD5 + File::Basename + File::Find + File::Spec + File::Temp + Getopt::Long + IO::File + IO::Uncompress::Inflate + List::Util + POSIX + Socket + Term::ReadKey + Test::More + Time::HiRes + Time::Local +); -print `perl --version | grep 'v5'`; +my $exit_status = 0; +my $fmt = "%-23s %s\n"; -my $file = __FILE__; -my $m = `cat $file | grep '^use'`; -my @modules = map { m/use (.+?);/; $1 } split("\n", $m); +printf $fmt, "Perl", `perl -v | perl -ne '/v([\\d\\.]+)/ && print \$1'`; -foreach my $module ( @modules ) { - my $version = "${module}::VERSION"; - print "$module " . ${$version} . "\n"; -} +foreach my $module (@modules) { + my $version = "Not installed"; + eval "require $module"; + if ( $EVAL_ERROR ) { + $exit_status = 1; + } + else { + $version = ${"${module}::VERSION"}; + } + printf $fmt, $module, $version; +} -exit 0; +exit $exit_status; From 0659100b6bcf9687b7d22e529f8b0fbc2fe34ded Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 7 Feb 2012 15:06:06 -0700 Subject: [PATCH 05/34] Don't require Digest::Crc32 for testing. --- t/lib/Transformers.t | 22 ++++++++-------------- util/check-dev-env | 1 - 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/t/lib/Transformers.t b/t/lib/Transformers.t index 30a1f81b..99db6ac0 100644 --- a/t/lib/Transformers.t +++ b/t/lib/Transformers.t @@ -102,21 +102,15 @@ is(make_checksum('hello world'), '93CB22BB8F5ACDC3', 'make_checksum'); # ############################################################################# # crc32() tests. # ############################################################################# -eval { - require Digest::Crc32; -}; -SKIP: { - skip "Digest::Crc32 is not installed", 1 if $EVAL_ERROR; - - my $crc = new Digest::Crc32(); - # Our crc32 should match the one from which we copied it. - is( - crc32('Hello world'), - $crc->strcrc32("Hello world"), - "crc32" - ); -}; +# Our crc32 is copied from Digest::Crc32 which is not a common module, +# so we don't rely on it being installed. Instead, I just copied the +# original value from Digest::Crc32 into this test. +is( + crc32('Hello world'), # our crc32() + '2346098258', # original value from Digest::Crc32::strcrc32() + "crc32" +); # ############################################################################# # any_unix_timestamp() tests. diff --git a/util/check-dev-env b/util/check-dev-env index 78b06945..f15eb537 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -16,7 +16,6 @@ my @modules = qw( Data::Dumper DBD::mysql DBI - Digest::Crc32 Digest::MD5 File::Basename File::Find From 1c5eb84cd57003177c9743881339c70bbcd7d1de Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 7 Feb 2012 15:15:44 -0700 Subject: [PATCH 06/34] Don't require Term::ReadKey for testing. --- util/check-dev-env | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/util/check-dev-env b/util/check-dev-env index f15eb537..1f66877f 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -3,14 +3,12 @@ use warnings FATAL => 'all'; use English; -# This script is for developers to see if their box has all the modules -# necessary for testing Percona Toolkit. Any missing modules are printed -# to STDERR and installed modules are printed to STDOUT. +# This script checks if all modules necessary for testing Percona Toolkit +# are installed. It exits 0 if all modules are installed, else it exits +# non-zero. # -# In addition to these modules, many non-standard programs are needed -# for other tasks, like building packages, writing test coverage, etc. -# -# Exits 0 if all modules are installed, else exits non-zero. +# This check is just for testing Percona Toolkit. Other dev tasks, like +# building release packages, require other modules and programs. my @modules = qw( Data::Dumper @@ -26,8 +24,7 @@ my @modules = qw( IO::Uncompress::Inflate List::Util POSIX - Socket - Term::ReadKey + Socket Test::More Time::HiRes Time::Local @@ -36,6 +33,7 @@ my @modules = qw( my $exit_status = 0; my $fmt = "%-23s %s\n"; +# Not a module but we want to know the Perl version. printf $fmt, "Perl", `perl -v | perl -ne '/v([\\d\\.]+)/ && print \$1'`; foreach my $module (@modules) { From 2e171dfb8ce492b199ebd5f297238ea37d534993 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 9 Feb 2012 11:42:48 -0700 Subject: [PATCH 07/34] Make IO::Uncompress::Inflate an optional module for testing. --- util/check-dev-env | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/util/check-dev-env b/util/check-dev-env index 1f66877f..a1c51261 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -10,7 +10,7 @@ use English; # This check is just for testing Percona Toolkit. Other dev tasks, like # building release packages, require other modules and programs. -my @modules = qw( +my @required_modules = qw( Data::Dumper DBD::mysql DBI @@ -21,7 +21,6 @@ my @modules = qw( File::Temp Getopt::Long IO::File - IO::Uncompress::Inflate List::Util POSIX Socket @@ -30,13 +29,18 @@ my @modules = qw( Time::Local ); +# CentOS doesn't seem to have this in its repo. +my @optional_modules = qw( + IO::Uncompress::Inflate +); + my $exit_status = 0; -my $fmt = "%-23s %s\n"; +my $fmt = "%-23s %8s %s\n"; # Not a module but we want to know the Perl version. -printf $fmt, "Perl", `perl -v | perl -ne '/v([\\d\\.]+)/ && print \$1'`; +printf $fmt, "Perl", `perl -v | perl -ne '/v([\\d\\.]+)/ && print \$1'`, ""; -foreach my $module (@modules) { +foreach my $module (@required_modules) { my $version = "Not installed"; eval "require $module"; if ( $EVAL_ERROR ) { @@ -45,7 +49,16 @@ foreach my $module (@modules) { else { $version = ${"${module}::VERSION"}; } - printf $fmt, $module, $version; + printf $fmt, $module, $version, ""; +} + +foreach my $module (@optional_modules) { + my $version = "Not installed"; + eval "require $module"; + if ( !$EVAL_ERROR ) { + $version = ${"${module}::VERSION"}; + } + printf $fmt, $module, $version, "MySQLProtocolParser, ProtocolParser" } exit $exit_status; From 6bb7756c09cfbb13367f99dc5ef9594780609ea1 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 9 Feb 2012 11:54:49 -0700 Subject: [PATCH 08/34] Use 'NA' instead of 'Not installed' to keep output pretty. --- util/check-dev-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/check-dev-env b/util/check-dev-env index a1c51261..662cae9d 100755 --- a/util/check-dev-env +++ b/util/check-dev-env @@ -41,7 +41,7 @@ my $fmt = "%-23s %8s %s\n"; printf $fmt, "Perl", `perl -v | perl -ne '/v([\\d\\.]+)/ && print \$1'`, ""; foreach my $module (@required_modules) { - my $version = "Not installed"; + my $version = "NA"; eval "require $module"; if ( $EVAL_ERROR ) { $exit_status = 1; @@ -53,7 +53,7 @@ foreach my $module (@required_modules) { } foreach my $module (@optional_modules) { - my $version = "Not installed"; + my $version = "NA"; eval "require $module"; if ( !$EVAL_ERROR ) { $version = ${"${module}::VERSION"}; From c943f7a0da073ae4535852ae0127ba67bf945f8a Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 08:58:10 -0700 Subject: [PATCH 09/34] Debug Daemon.t failures. --- lib/PerconaTest.pm | 8 ++++++++ t/lib/Daemon.t | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index 65d49209..5b6e5aaa 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -617,6 +617,14 @@ sub get_master_binlog_pos { return $ms->{position}; } +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + 1; } # ########################################################################### diff --git a/t/lib/Daemon.t b/t/lib/Daemon.t index 611b5460..f20437c9 100644 --- a/t/lib/Daemon.t +++ b/t/lib/Daemon.t @@ -15,6 +15,8 @@ use Daemon; use OptionParser; use PerconaTest; +use constant PTDEVDEBUG => $ENV{PTDEVDEBUG} || 0; + my $o = new OptionParser(file => "$trunk/t/lib/samples/daemonizes.pl"); my $d = new Daemon(o=>$o); @@ -30,7 +32,7 @@ sub rm_tmp_files() { rm_tmp_files(); my $cmd = "$trunk/t/lib/samples/daemonizes.pl"; -my $ret_val = system("$cmd 2 --daemonize --pid $pid_file"); +my $ret_val = system("$cmd 2 --daemonize --pid $pid_file >/dev/null 2>&1"); die 'Cannot test Daemon.pm because t/daemonizes.pl is not working' unless $ret_val == 0; @@ -108,6 +110,8 @@ SKIP: { my $proc_fd_0 = -l "/proc/$pid/0" ? "/proc/$pid/0" : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" : die "Cannot find fd 0 symlink in /proc/$pid"; + PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, + 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); my $stdin = readlink $proc_fd_0; is( $stdin, @@ -136,7 +140,7 @@ SKIP: { # pid-file is still running # ########################################################################## rm_tmp_files(); -system("$cmd 5 --daemonize --pid $pid_file 2>&1"); +system("$cmd 5 --daemonize --pid $pid_file >/dev/null 2>&1"); PerconaTest::wait_for_files($pid_file); chomp($pid = `cat $pid_file`); kill 9, $pid; From e9f632559246c582c90f58507aedfbb3629fa1eb Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 09:31:47 -0700 Subject: [PATCH 10/34] Add hires timestamp to PTDEVDEBUG. Debug daemonize.pl. Make wait_until() defaults shorter. --- lib/PerconaTest.pm | 13 ++++++++----- t/lib/Daemon.t | 12 ++++++++---- t/lib/samples/daemonizes.pl | 19 ++++++++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index 5b6e5aaa..4943d7ba 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -31,10 +31,10 @@ package PerconaTest; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); -use constant PTDEBUG => $ENV{PTDEBUG} || 0; +use constant PTDEVDEBUG => $ENV{PTDEVDEBUG} || 0; use Test::More; -use Time::HiRes qw(sleep); +use Time::HiRes qw(sleep time); use POSIX qw(signal_h); use Data::Dumper; $Data::Dumper::Indent = 1; @@ -220,14 +220,16 @@ sub parse_file { # Wait until code returns true. sub wait_until { my ( $code, $t, $max_t ) = @_; - $t ||= .5; - $max_t ||= 10; + $t ||= .25; + $max_t ||= 5; my $slept = 0; while ( $slept <= $max_t ) { return 1 if $code->(); + PTDEVDEBUG && _d('wait_until sleeping', $t); sleep $t; $slept += $t; + PTDEVDEBUG && _d('wait_until slept', $slept, 'of', $max_t); } return 0; } @@ -622,7 +624,8 @@ sub _d { @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; + my $t = sprintf '%.3f', time; + print STDERR "# $package:$line $PID $t ", join(' ', @_), "\n"; } 1; diff --git a/t/lib/Daemon.t b/t/lib/Daemon.t index f20437c9..21faee35 100644 --- a/t/lib/Daemon.t +++ b/t/lib/Daemon.t @@ -54,17 +54,19 @@ ok(! -f $pid_file, 'Removes PID file upon exit'); # ############################################################################ rm_tmp_files(); -system("$cmd 2 --daemonize --log $log_file"); +system("$cmd 0 --daemonize --log $log_file"); PerconaTest::wait_for_files($log_file); ok(-f $log_file, 'Log file exists'); -sleep 2; $output = `cat $log_file`; like($output, qr/STDOUT\nSTDERR\n/, 'STDOUT and STDERR went to log file'); +my $log_size = -s $log_file; +PTDEVDEBUG && PerconaTest::_d('log size', $log_size); + # Check that the log file is appended to. system("$cmd 0 --daemonize --log $log_file"); -PerconaTest::wait_for_files($log_file); +PerconaTest::wait_until(sub { -s $log_file > $log_size }); $output = `cat $log_file`; like( $output, @@ -82,7 +84,7 @@ ok( 'PID file already exists' ); -$output = `PTDEBUG=1 $cmd 0 --daemonize --pid $pid_file 2>&1`; +$output = `$cmd 2 --daemonize --pid $pid_file 2>&1`; like( $output, qr{The PID file $pid_file already exists}, @@ -126,6 +128,8 @@ SKIP: { $proc_fd_0 = -l "/proc/$pid/0" ? "/proc/$pid/0" : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" : die "Cannot find fd 0 symlink in /proc/$pid"; + PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, + 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); $stdin = readlink $proc_fd_0; like( $stdin, diff --git a/t/lib/samples/daemonizes.pl b/t/lib/samples/daemonizes.pl index 33053294..b9319487 100755 --- a/t/lib/samples/daemonizes.pl +++ b/t/lib/samples/daemonizes.pl @@ -11,9 +11,10 @@ BEGIN { use strict; use warnings FATAL => 'all'; - use English qw(-no_match_vars); -use constant PTDEBUG => $ENV{PTDEBUG}; +use constant PTDEVDEBUG => $ENV{PTDEVDEBUG}; + +use Time::HiRes qw(sleep); use Daemon; use OptionParser; @@ -23,7 +24,8 @@ my $o = new OptionParser(file => "$trunk/t/lib/samples/daemonizes.pl"); $o->get_specs(); $o->get_opts(); -if ( scalar @ARGV < 1 ) { +my ($sleep_time) = shift @ARGV; +if ( !defined $sleep_time ) { $o->save_error('No SLEEP_TIME specified'); } @@ -31,15 +33,22 @@ $o->usage_or_errors(); my $daemon; if ( $o->get('daemonize') ) { + PTDEVDEBUG && PerconaTest::_d('daemonizing'); + + $OUTPUT_AUTOFLUSH = 1; + $daemon = new Daemon(o=>$o); $daemon->daemonize(); + PTDEVDEBUG && PerconaTest::_d('daemonized'); print "STDOUT\n"; print STDERR "STDERR\n"; - sleep $ARGV[0]; + PTDEVDEBUG && PerconaTest::_d('daemon sleep', $sleep_time); + sleep $sleep_time; } +PTDEVDEBUG && PerconaTest::_d('daemon done'); exit; # ############################################################################ @@ -50,7 +59,7 @@ exit; =head1 SYNOPSIS -Usage: daemonizes.pl SLEEP_TIME [ARGS] +Usage: daemonizes.pl SLEEP_TIME daemonizes.pl daemonizes, prints to STDOUT and STDERR, sleeps and exits. From c67adc1a8fa8b4a48e1d2a17ebe0f40e16f9127d Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 10:20:40 -0700 Subject: [PATCH 11/34] More Daemon debugging. --- lib/Daemon.pm | 10 ++++++++-- lib/PerconaTest.pm | 9 ++++----- t/lib/Daemon.t | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/Daemon.pm b/lib/Daemon.pm index 9d55a858..0b8b0a32 100644 --- a/lib/Daemon.pm +++ b/lib/Daemon.pm @@ -55,11 +55,12 @@ sub daemonize { PTDEBUG && _d('About to fork and daemonize'); defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR"; if ( $pid ) { - PTDEBUG && _d('I am the parent and now I die'); + PTDEBUG && _d('Parent PID', $PID, 'exiting after forking child PID',$pid); exit; } # I'm daemonized now. + PTDEBUG && _d('Daemonizing child PID', $PID); $self->{PID_owner} = $PID; $self->{child} = 1; @@ -73,12 +74,14 @@ sub daemonize { # Only reopen STDIN to /dev/null if it's a tty. It may be a pipe, # in which case we don't want to break it. if ( -t STDIN ) { + PTDEBUG && _d('STDIN is a terminal; redirecting from /dev/null'); close STDIN; open STDIN, '/dev/null' or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; } if ( $self->{log_file} ) { + PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file}); close STDOUT; open STDOUT, '>>', $self->{log_file} or die "Cannot open log file $self->{log_file}: $OS_ERROR"; @@ -93,18 +96,21 @@ sub daemonize { } else { if ( -t STDOUT ) { + PTDEBUG && _d('No log file and STDOUT is a terminal;', + 'redirecting to /dev/null'); close STDOUT; open STDOUT, '>', '/dev/null' or die "Cannot reopen STDOUT to /dev/null: $OS_ERROR"; } if ( -t STDERR ) { + PTDEBUG && _d('No log file and STDERR is a terminal;', + 'redirecting to /dev/null'); close STDERR; open STDERR, '>', '/dev/null' or die "Cannot reopen STDERR to /dev/null: $OS_ERROR"; } } - PTDEBUG && _d('I am the child and now I live daemonized'); return; } diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index 4943d7ba..2d02e90a 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -270,8 +270,6 @@ sub wait_for_table { } return 1; }, - 0.25, - 15, ); } @@ -280,12 +278,13 @@ sub wait_for_files { return wait_until( sub { foreach my $file (@files) { - return 0 if ! -f $file; + if ( ! -f $file ) { + PTDEVDEBUG && _d('Waiting for file', $file); + return 0; + } } return 1; }, - 0.25, - 15, ); } diff --git a/t/lib/Daemon.t b/t/lib/Daemon.t index 21faee35..6a35b117 100644 --- a/t/lib/Daemon.t +++ b/t/lib/Daemon.t @@ -113,7 +113,7 @@ SKIP: { : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" : die "Cannot find fd 0 symlink in /proc/$pid"; PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, - 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); + 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`, `lsof -p $pid`); my $stdin = readlink $proc_fd_0; is( $stdin, @@ -129,7 +129,7 @@ SKIP: { : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" : die "Cannot find fd 0 symlink in /proc/$pid"; PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, - 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); + 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`, `lsof -p $pid`); $stdin = readlink $proc_fd_0; like( $stdin, From 0539775d2400654fb28695110fef88d9f6de3705 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 10:54:02 -0700 Subject: [PATCH 12/34] Always close STDIN if --daemonize. --- lib/Daemon.pm | 19 +++++++++++-------- t/lib/Daemon.t | 40 ++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/lib/Daemon.pm b/lib/Daemon.pm index 0b8b0a32..acc9e1fb 100644 --- a/lib/Daemon.pm +++ b/lib/Daemon.pm @@ -71,14 +71,17 @@ sub daemonize { $OUTPUT_AUTOFLUSH = 1; - # Only reopen STDIN to /dev/null if it's a tty. It may be a pipe, - # in which case we don't want to break it. - if ( -t STDIN ) { - PTDEBUG && _d('STDIN is a terminal; redirecting from /dev/null'); - close STDIN; - open STDIN, '/dev/null' - or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; - } + # We used to only reopen STDIN to /dev/null if it's a tty because + # otherwise it may be a pipe, in which case we didn't want to break + # it. However, Perl -t is not reliable. This is true and false on + # various boxes even when the same code is ran, or it depends on if + # the code is ran via cron, Jenkins, etc. Since there should be no + # sane reason to `foo | pt-tool --daemonize` for a tool that reads + # STDIN, we now just always close STDIN. + PTDEBUG && _d('Redirecting STDIN to /dev/null'); + close STDIN; + open STDIN, '/dev/null' + or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; if ( $self->{log_file} ) { PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file}); diff --git a/t/lib/Daemon.t b/t/lib/Daemon.t index 6a35b117..810e03e3 100644 --- a/t/lib/Daemon.t +++ b/t/lib/Daemon.t @@ -9,7 +9,7 @@ BEGIN { use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); -use Test::More tests => 22; +use Test::More tests => 21; use Time::HiRes qw(sleep); use Daemon; use OptionParser; @@ -103,8 +103,8 @@ unlike( # ########################################################################## rm_tmp_files(); SKIP: { - skip 'No /proc', 2 unless -d '/proc'; - skip 'No fd in /proc', 2 unless -l "/proc/$PID/0" || -l "/proc/$PID/fd/0"; + skip 'No /proc', 1 unless -d '/proc'; + skip 'No fd in /proc', 1 unless -l "/proc/$PID/0" || -l "/proc/$PID/fd/0"; system("$cmd 1 --daemonize --pid $pid_file --log $log_file"); PerconaTest::wait_for_files($pid_file); @@ -113,29 +113,29 @@ SKIP: { : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" : die "Cannot find fd 0 symlink in /proc/$pid"; PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, - 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`, `lsof -p $pid`); + 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); my $stdin = readlink $proc_fd_0; is( $stdin, '/dev/null', - 'Reopens STDIN to /dev/null if not piped', + 'Reopens STDIN to /dev/null' ); - sleep 1; - system("echo foo | $cmd 1 --daemonize --pid $pid_file --log $log_file"); - PerconaTest::wait_for_files($pid_file, $log_file); - chomp($pid = `cat $pid_file`); - $proc_fd_0 = -l "/proc/$pid/0" ? "/proc/$pid/0" - : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" - : die "Cannot find fd 0 symlink in /proc/$pid"; - PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, - 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`, `lsof -p $pid`); - $stdin = readlink $proc_fd_0; - like( - $stdin, - qr/pipe/, - 'Does not reopen STDIN to /dev/null when piped', - ); +# sleep 1; +# system("echo foo | $cmd 1 --daemonize --pid $pid_file --log $log_file"); +# PerconaTest::wait_for_files($pid_file, $log_file); +# chomp($pid = `cat $pid_file`); +# $proc_fd_0 = -l "/proc/$pid/0" ? "/proc/$pid/0" +# : -l "/proc/$pid/fd/0" ? "/proc/$pid/fd/0" +# : die "Cannot find fd 0 symlink in /proc/$pid"; +# PTDEVDEBUG && PerconaTest::_d('pid_file', $pid_file, +# 'pid', $pid, 'proc_fd_0', $proc_fd_0, `ls -l $proc_fd_0`); +# $stdin = readlink $proc_fd_0; +# like( +# $stdin, +# qr/pipe/, +# 'Does not reopen STDIN to /dev/null when piped', +# ); sleep 1; }; From 1c57fe8ee2082108af0ffd07daa1e91767b9a5e1 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 11:02:28 -0700 Subject: [PATCH 13/34] Conditionalize lsof test in collect.sh. --- t/lib/bash/collect.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 624e6dd9..a40a1dfb 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -73,9 +73,13 @@ else is "1" "1" "SKIP Can't determine MySQL 5.0 error log" fi -cmd_ok \ - "grep -q 'COMMAND[ ]\+PID[ ]\+USER' $p-lsof" \ - "lsof" +if [ "$(which lsof 2>/dev/null)" ]; then + cmd_ok \ + "grep -q 'COMMAND[ ]\+PID[ ]\+USER' $p-lsof" \ + "lsof" +else + is "1" "1" "SKIP lsof not in PATH" +fi cmd_ok \ "grep -q 'buf0buf.c' $p-mutex-status1" \ From da051b33717af76b26bc54dc292b21442ed9c481 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 11:19:23 -0700 Subject: [PATCH 14/34] Conditionalize IO::Uncompress::Inflate test. Make test names shorter, uniform. --- lib/PerconaTest.pm | 16 ++-- t/lib/MySQLProtocolParser.t | 148 ++++++++++++++++++------------------ t/lib/SlowLogParser.t | 2 +- 3 files changed, 86 insertions(+), 80 deletions(-) diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index 2d02e90a..04e3e75a 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -324,17 +324,18 @@ sub test_log_parser { close $fh; }; + my ($base_file_name) = $args{file} =~ m/([^\/]+)$/; is( $EVAL_ERROR, '', - "No error on $args{file}" + "$base_file_name: no errors" ); if ( defined $args{result} ) { is_deeply( \@e, $args{result}, - $args{file} + "$base_file_name: results" ) or print "Got: ", Dumper(\@e); } @@ -342,7 +343,7 @@ sub test_log_parser { is( scalar @e, $args{num_events}, - "$args{file} num_events" + "$base_file_name: $args{num_events} events" ); } @@ -380,17 +381,18 @@ sub test_protocol_parser { close $fh; }; + my ($base_file_name) = $args{file} =~ m/([^\/]+)$/; is( $EVAL_ERROR, '', - "No error on $args{file}" + "$base_file_name: no errors" ); - + if ( defined $args{result} ) { is_deeply( \@e, $args{result}, - $args{file} . ($args{desc} ? ": $args{desc}" : '') + "$base_file_name: " . ($args{desc} || "results") ) or print "Got: ", Dumper(\@e); } @@ -398,7 +400,7 @@ sub test_protocol_parser { is( scalar @e, $args{num_events}, - "$args{file} num_events" + "$base_file_name: $args{num_events} events" ); } diff --git a/t/lib/MySQLProtocolParser.t b/t/lib/MySQLProtocolParser.t index 54b7f9d7..5b3fc54c 100644 --- a/t/lib/MySQLProtocolParser.t +++ b/t/lib/MySQLProtocolParser.t @@ -15,7 +15,7 @@ use MySQLProtocolParser; use TcpdumpParser; use PerconaTest; -my $sample = "t/lib/samples/tcpdump/"; +my $sample = "t/lib/samples/tcpdump"; my $tcpdump = new TcpdumpParser(); my $protocol; # Create a new MySQLProtocolParser for each test. @@ -487,79 +487,83 @@ test_protocol_parser( ], ); -# Check data decompression. -$protocol = new MySQLProtocolParser( - server => '127.0.0.1', - port => '12345', -); -test_protocol_parser( - parser => $tcpdump, - protocol => $protocol, - file => "$sample/tcpdump015.txt", - desc => 'compressed data', - result => [ - { - Error_no => 'none', - No_good_index_used => 'No', - No_index_used => 'No', - Query_time => '0.006415', - Rows_affected => 0, - Thread_id => 20, - Warning_count => 0, - arg => 'administrator command: Connect', - bytes => 30, - cmd => 'Admin', - db => 'mysql', - host => '127.0.0.1', - ip => '127.0.0.1', - port => '44489', - pos_in_log => 664, - ts => '090612 08:39:05.316805', - user => 'msandbox', - }, - { - Error_no => 'none', - No_good_index_used => 'No', - No_index_used => 'Yes', - Query_time => '0.002884', - Rows_affected => 0, - Thread_id => 20, - Warning_count => 0, - arg => 'select * from help_relation', - bytes => 27, - cmd => 'Query', - db => 'mysql', - host => '127.0.0.1', - ip => '127.0.0.1', - port => '44489', - pos_in_log => 1637, - ts => '090612 08:39:08.428913', - user => 'msandbox', - }, - { - Error_no => 'none', - No_good_index_used => 'No', - No_index_used => 'No', - Query_time => '0.000000', - Rows_affected => 0, - Thread_id => 20, - Warning_count => 0, - arg => 'administrator command: Quit', - bytes => 27, - cmd => 'Admin', - db => 'mysql', - host => '127.0.0.1', - ip => '127.0.0.1', - port => '44489', - pos_in_log => 15782, - ts => '090612 08:39:09.145334', - user => 'msandbox', - }, - ], -); +eval { require IO::Uncompress::Inflate; }; +SKIP: { + skip "IO::Uncompress::Inflate not installed", 2 if $EVAL_ERROR; + + # Check data decompression. + $protocol = new MySQLProtocolParser( + server => '127.0.0.1', + port => '12345', + ); + test_protocol_parser( + parser => $tcpdump, + protocol => $protocol, + file => "$sample/tcpdump015.txt", + desc => 'compressed data', + result => [ + { + Error_no => 'none', + No_good_index_used => 'No', + No_index_used => 'No', + Query_time => '0.006415', + Rows_affected => 0, + Thread_id => 20, + Warning_count => 0, + arg => 'administrator command: Connect', + bytes => 30, + cmd => 'Admin', + db => 'mysql', + host => '127.0.0.1', + ip => '127.0.0.1', + port => '44489', + pos_in_log => 664, + ts => '090612 08:39:05.316805', + user => 'msandbox', + }, + { + Error_no => 'none', + No_good_index_used => 'No', + No_index_used => 'Yes', + Query_time => '0.002884', + Rows_affected => 0, + Thread_id => 20, + Warning_count => 0, + arg => 'select * from help_relation', + bytes => 27, + cmd => 'Query', + db => 'mysql', + host => '127.0.0.1', + ip => '127.0.0.1', + port => '44489', + pos_in_log => 1637, + ts => '090612 08:39:08.428913', + user => 'msandbox', + }, + { + Error_no => 'none', + No_good_index_used => 'No', + No_index_used => 'No', + Query_time => '0.000000', + Rows_affected => 0, + Thread_id => 20, + Warning_count => 0, + arg => 'administrator command: Quit', + bytes => 27, + cmd => 'Admin', + db => 'mysql', + host => '127.0.0.1', + ip => '127.0.0.1', + port => '44489', + pos_in_log => 15782, + ts => '090612 08:39:09.145334', + user => 'msandbox', + }, + ], + ); +} # TCP retransmission. -# Check data decompression. $protocol = new MySQLProtocolParser( server => '10.55.200.15', ); diff --git a/t/lib/SlowLogParser.t b/t/lib/SlowLogParser.t index ce58e7ed..c332ff19 100644 --- a/t/lib/SlowLogParser.t +++ b/t/lib/SlowLogParser.t @@ -15,7 +15,7 @@ use SlowLogParser; use PerconaTest; my $p = new SlowLogParser; -my $sample = "t/lib/samples/slowlogs/"; +my $sample = "t/lib/samples/slowlogs"; # Check that I can parse a slow log in the default slow log format. test_log_parser( From 9cfb71dff4743fa5580e74867988b83ddb3b4233 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:22:44 -0700 Subject: [PATCH 15/34] Make CompareResults.t wait for its tables. --- t/lib/CompareResults.t | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/t/lib/CompareResults.t b/t/lib/CompareResults.t index 09906f05..b92706d6 100644 --- a/t/lib/CompareResults.t +++ b/t/lib/CompareResults.t @@ -107,7 +107,8 @@ sub get_id { # Test the checksum method. # ############################################################################# -diag(`/tmp/12345/use < $trunk/t/lib/samples/compare-results.sql`); +$sb->load_file('master', "t/lib/samples/compare-results.sql"); +PerconaTest::wait_for_table($dbh1, "test.t3", "f > 1"); $cr = new CompareResults( method => 'checksum', @@ -132,21 +133,6 @@ isa_ok($cr, 'CompareResults'); }, ); -$i = 0; -PerconaTest::wait_until( - sub { - my $r; - eval { - $r = $dbh1->selectrow_arrayref('SHOW TABLES FROM test LIKE "dropme"'); - }; - return 1 if ($r->[0] || '') eq 'dropme'; - diag('Waiting for CREATE TABLE...') unless $i++; - return 0; - }, - 0.5, - 30, -); - is_deeply( $dbh1->selectrow_arrayref('SHOW TABLES FROM test LIKE "dropme"'), ['dropme'], @@ -326,9 +312,10 @@ is_deeply( # ############################################################################# my $tmpdir = '/tmp/mk-upgrade-res'; +diag(`rm -rf $tmpdir 2>/dev/null; mkdir $tmpdir`); -diag(`/tmp/12345/use < $trunk/t/lib/samples/compare-results.sql`); -diag(`rm -rf $tmpdir; mkdir $tmpdir`); +$sb->load_file('master', "t/lib/samples/compare-results.sql"); +PerconaTest::wait_for_table($dbh1, "test.t3", "f > 1"); $cr = new CompareResults( method => 'rows', @@ -351,21 +338,6 @@ isa_ok($cr, 'CompareResults'); }, ); -$i = 0; -PerconaTest::wait_until( - sub { - my $r; - eval { - $r = $dbh1->selectrow_arrayref('SHOW TABLES FROM test LIKE "dropme"'); - }; - return 1 if ($r->[0] || '') eq 'dropme'; - diag('Waiting for CREATE TABLE...') unless $i++; - return 0; - }, - 0.5, - 30, -); - is_deeply( $dbh1->selectrow_arrayref('SHOW TABLES FROM test LIKE "dropme"'), ['dropme'], From 072f69022ca3d9d3741a0806a9f8067025f9d73c Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:36:00 -0700 Subject: [PATCH 16/34] Add jenkins-test script for simplier build step. --- sandbox/jenkins-test | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 sandbox/jenkins-test diff --git a/sandbox/jenkins-test b/sandbox/jenkins-test new file mode 100644 index 00000000..9a985a93 --- /dev/null +++ b/sandbox/jenkins-test @@ -0,0 +1,62 @@ +#!/bin/sh + +###################################### +# Source our alt_cmds.sh for _seq(). # +###################################### +. lib/bash/alt_cmds.sh + +############## +# Set modes. # +############## +set +u +set -e + +################################## +# Check for needed Perl modules. # +################################## +util/check-dev-env + +##################################### +# Install barebones MySQL binaries. # +##################################### +if ! [ -d "mysql-${MYSQL}-barebones" ]; then + wget -q http://hackmysql.com/mysql-${MYSQL}-barebones.tar.gz + tar xvfz mysql-${MYSQL}-barebones.tar.gz +fi + +########################## +# Set required env vars. # +########################## +export PERCONA_TOOLKIT_BRANCH="$PWD" +export PERCONA_TOOLKIT_SANDBOX="$PWD/mysql-${MYSQL}-barebones" +export PATH="$PATH:/usr/sbin:$PWD/mysql-${MYSQL}-barebones/bin" + +############################# +# Check and start test env. # +############################# +sandbox/test-env checkconfig +sandbox/test-env restart + +####################### +# Set debug env vars. # +####################### +if [ "$DEBUG_CODE" = "true" ]; then + export PTDEBUG=1 +fi + +if [ "$DEBUG_TEST" = "true" ]; then + export PTDEVDEBUG=1 +fi + +################## +# Run the tests. # +################## +ITERATIONS={ITERATIONS:-1} +for iter in $(_seq $ITERATIONS); do + $TEST_CMD +done + +############# +# Clean up. # +############# +sandbox/test-env stop From fd9700757c2b115d40ae708027b58493ca1984bc Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:42:53 -0700 Subject: [PATCH 17/34] Make jenkins-test e executable and -x. --- sandbox/jenkins-test | 5 +++++ 1 file changed, 5 insertions(+) mode change 100644 => 100755 sandbox/jenkins-test diff --git a/sandbox/jenkins-test b/sandbox/jenkins-test old mode 100644 new mode 100755 index 9a985a93..fc2468e7 --- a/sandbox/jenkins-test +++ b/sandbox/jenkins-test @@ -10,6 +10,7 @@ ############## set +u set -e +set -x ################################## # Check for needed Perl modules. # @@ -34,8 +35,10 @@ export PATH="$PATH:/usr/sbin:$PWD/mysql-${MYSQL}-barebones/bin" ############################# # Check and start test env. # ############################# +set +x sandbox/test-env checkconfig sandbox/test-env restart +set -x ####################### # Set debug env vars. # @@ -59,4 +62,6 @@ done ############# # Clean up. # ############# +set +x sandbox/test-env stop +set -x From 815b406a40c0460b5402f7988d22081674a19fc0 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:48:24 -0700 Subject: [PATCH 18/34] Add PTDEVDEBUG statements to PerconaTest::wait_for_table(). --- lib/PerconaTest.pm | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index 04e3e75a..8c076840 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -264,9 +264,15 @@ sub wait_for_table { sub { my $r; eval { $r = $dbh->selectrow_arrayref($sql); }; - return 0 if $EVAL_ERROR; - if ( $where ) { - return 0 unless $r && @$r; + if ( $EVAL_ERROR ) { + PTDEVDEBUG && _d('Waiting on', $dbh, 'for table', $tbl, + 'error:', $EVAL_ERROR); + return 0; + } + if ( $where && (!$r || !scalar @$r) ) { + PTDEVDEBUG && _d('Waiting on', $dbh, 'for table', $tbl, + 'WHERE', $where); + return 0; } return 1; }, From 721b29851832ca6b36866e50edac24c702ca49d0 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:52:56 -0700 Subject: [PATCH 19/34] Fix Bash syntax error in jenkins-test. --- sandbox/jenkins-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandbox/jenkins-test b/sandbox/jenkins-test index fc2468e7..94f9a794 100755 --- a/sandbox/jenkins-test +++ b/sandbox/jenkins-test @@ -54,7 +54,7 @@ fi ################## # Run the tests. # ################## -ITERATIONS={ITERATIONS:-1} +ITERATIONS="${ITERATIONS:-1}" for iter in $(_seq $ITERATIONS); do $TEST_CMD done From 8b81b2398f7c4cfaea6320002fe18382184b7466 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 12:59:54 -0700 Subject: [PATCH 20/34] Debug hack in CompareResults.t. --- t/lib/CompareResults.t | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/lib/CompareResults.t b/t/lib/CompareResults.t index b92706d6..488809ec 100644 --- a/t/lib/CompareResults.t +++ b/t/lib/CompareResults.t @@ -158,6 +158,10 @@ ok( "checksum: Query_time doesn't exist before execute()" ); +# XXX +diag(`/tmp/12345/use -e "show tables from test"`); +diag(`/tmp/12346/use -e "show tables from test"`); + proc('execute'); ok( From b6dc9d3832084edbbc6724f9beb361eba6f00433 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 13:22:11 -0700 Subject: [PATCH 21/34] More debug hacks. --- t/lib/CompareResults.t | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/t/lib/CompareResults.t b/t/lib/CompareResults.t index 488809ec..9cba8e89 100644 --- a/t/lib/CompareResults.t +++ b/t/lib/CompareResults.t @@ -110,6 +110,10 @@ sub get_id { $sb->load_file('master', "t/lib/samples/compare-results.sql"); PerconaTest::wait_for_table($dbh1, "test.t3", "f > 1"); +# XXX +warn "1", `/tmp/12345/use -e "show tables from test"`; +warn `/tmp/12346/use -e "show tables from test"`; + $cr = new CompareResults( method => 'checksum', 'base-dir' => '/dev/null', # not used with checksum method @@ -139,6 +143,10 @@ is_deeply( 'checksum: temp table exists' ); +# XXX +warn "2", `/tmp/12345/use -e "show tables from test"`; +warn `/tmp/12346/use -e "show tables from test"`; + proc('before_execute', db=>'test', 'temp-table'=>'dropme'); is( @@ -159,8 +167,8 @@ ok( ); # XXX -diag(`/tmp/12345/use -e "show tables from test"`); -diag(`/tmp/12346/use -e "show tables from test"`); +warn "3", `/tmp/12345/use -e "show tables from test"`; +warn `/tmp/12346/use -e "show tables from test"`; proc('execute'); From d0e3fb38ca335da5d8df5030b13fda0aea80c584 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 13:30:15 -0700 Subject: [PATCH 22/34] More debug hacks. --- t/lib/CompareResults.t | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/t/lib/CompareResults.t b/t/lib/CompareResults.t index 9cba8e89..8578718b 100644 --- a/t/lib/CompareResults.t +++ b/t/lib/CompareResults.t @@ -46,8 +46,6 @@ else { plan tests => 56; } -$sb->create_dbs($dbh1, ['test']); - Transformers->import(qw(make_checksum)); my $vp = new VersionParser(); @@ -108,7 +106,7 @@ sub get_id { # ############################################################################# $sb->load_file('master', "t/lib/samples/compare-results.sql"); -PerconaTest::wait_for_table($dbh1, "test.t3", "f > 1"); +PerconaTest::wait_for_table($dbh2, "test.t3", "f > 1"); # XXX warn "1", `/tmp/12345/use -e "show tables from test"`; @@ -327,7 +325,7 @@ my $tmpdir = '/tmp/mk-upgrade-res'; diag(`rm -rf $tmpdir 2>/dev/null; mkdir $tmpdir`); $sb->load_file('master', "t/lib/samples/compare-results.sql"); -PerconaTest::wait_for_table($dbh1, "test.t3", "f > 1"); +PerconaTest::wait_for_table($dbh2, "test.t3", "f > 1"); $cr = new CompareResults( method => 'rows', From 810ce4b02d0cce3b90792202b750b61aaa5c0211 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 13:37:52 -0700 Subject: [PATCH 23/34] Fixed CompareResults.t. --- t/lib/CompareResults.t | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/t/lib/CompareResults.t b/t/lib/CompareResults.t index 8578718b..042e5824 100644 --- a/t/lib/CompareResults.t +++ b/t/lib/CompareResults.t @@ -108,10 +108,6 @@ sub get_id { $sb->load_file('master', "t/lib/samples/compare-results.sql"); PerconaTest::wait_for_table($dbh2, "test.t3", "f > 1"); -# XXX -warn "1", `/tmp/12345/use -e "show tables from test"`; -warn `/tmp/12346/use -e "show tables from test"`; - $cr = new CompareResults( method => 'checksum', 'base-dir' => '/dev/null', # not used with checksum method @@ -141,10 +137,6 @@ is_deeply( 'checksum: temp table exists' ); -# XXX -warn "2", `/tmp/12345/use -e "show tables from test"`; -warn `/tmp/12346/use -e "show tables from test"`; - proc('before_execute', db=>'test', 'temp-table'=>'dropme'); is( @@ -164,10 +156,6 @@ ok( "checksum: Query_time doesn't exist before execute()" ); -# XXX -warn "3", `/tmp/12345/use -e "show tables from test"`; -warn `/tmp/12346/use -e "show tables from test"`; - proc('execute'); ok( From 2c65aa85ae261b82ad958140e9371d084ff559d9 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 14:03:15 -0700 Subject: [PATCH 24/34] Add debug hack to collect.sh. --- t/lib/bash/collect.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index a40a1dfb..cf108ff5 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -23,6 +23,10 @@ local p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 +# XXX +ls $TMPDIR/collect +df + # Even if this system doesn't have all the cmds, collect should still # have created some files for cmds that (hopefully) all systems have. ls -1 $TMPDIR/collect | sort > $TMPDIR/collect-files From ab6bafe3ed811bd70aef27d1699ad3c8457e3b87 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 14:22:17 -0700 Subject: [PATCH 25/34] Remove debug hack from collect.sh. --- t/lib/bash/collect.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index cf108ff5..a40a1dfb 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -23,10 +23,6 @@ local p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 -# XXX -ls $TMPDIR/collect -df - # Even if this system doesn't have all the cmds, collect should still # have created some files for cmds that (hopefully) all systems have. ls -1 $TMPDIR/collect | sort > $TMPDIR/collect-files From c800ba8fa1ddb20b04d69c5ad021cd4c27759964 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 14:50:38 -0700 Subject: [PATCH 26/34] Add test debug hack back to collect.sh. --- t/lib/bash/collect.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index a40a1dfb..1b0a7a9f 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -21,7 +21,11 @@ parse_options "$BIN_DIR/pt-stalk" --run-time 1 -- --defaults-file=/tmp/12345/my. local p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. -collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 +set -x +collect "$TMPDIR/collect" "2011_12_05" # > $p-output 2>&1 +set +x +# XXX +ls $TMPDIR/collect # Even if this system doesn't have all the cmds, collect should still # have created some files for cmds that (hopefully) all systems have. From 845933b10ba3258407159975c04cd21aaab09091 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 14 Feb 2012 15:06:27 -0700 Subject: [PATCH 27/34] Repeat Jenkins test even if an iter fails. --- sandbox/jenkins-test | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sandbox/jenkins-test b/sandbox/jenkins-test index 94f9a794..1c25104f 100755 --- a/sandbox/jenkins-test +++ b/sandbox/jenkins-test @@ -54,9 +54,13 @@ fi ################## # Run the tests. # ################## +EXIT_STATUS=0 ITERATIONS="${ITERATIONS:-1}" for iter in $(_seq $ITERATIONS); do - $TEST_CMD + ( + $TEST_CMD + ) + EXIT_STATUS=$(($? | 0)) done ############# @@ -65,3 +69,5 @@ done set +x sandbox/test-env stop set -x + +exit $EXIT_STATUS From 7016b26f09a60cb3c68bbe9885b9eff70bff3d46 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 16 Feb 2012 10:52:28 -0700 Subject: [PATCH 28/34] More test debug hacks. --- t/lib/bash/collect.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 1b0a7a9f..4f50f633 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -22,9 +22,10 @@ local p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. set -x -collect "$TMPDIR/collect" "2011_12_05" # > $p-output 2>&1 +collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 set +x # XXX +cat $p-output ls $TMPDIR/collect # Even if this system doesn't have all the cmds, collect should still @@ -46,6 +47,9 @@ cmd_ok \ "grep -q '\-hostname\$' $TMPDIR/collect-files" \ "Collected hostname" +# XXX +df && cat $p-df + cmd_ok \ "grep -q 'Avail' $p-df" \ "df" @@ -97,6 +101,9 @@ cmd_ok \ "grep -q '^| Uptime' $p-mysqladmin" \ "mysqladmin ext" +# XXX +cat $p-opentables* + cmd_ok \ "grep -qP 'Database\tTable\tIn_use' $p-opentables1" \ "opentables1" From 13623102e1f7d36e97a94c02f962aa49400f1a74 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 16 Feb 2012 11:03:23 -0700 Subject: [PATCH 29/34] Debug test-env. --- sandbox/jenkins-test | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sandbox/jenkins-test b/sandbox/jenkins-test index 1c25104f..6a3e707a 100755 --- a/sandbox/jenkins-test +++ b/sandbox/jenkins-test @@ -35,10 +35,8 @@ export PATH="$PATH:/usr/sbin:$PWD/mysql-${MYSQL}-barebones/bin" ############################# # Check and start test env. # ############################# -set +x -sandbox/test-env checkconfig -sandbox/test-env restart -set -x +sandbox/test-env checkconfig || exit 1 +sandbox/test-env restart || exit 1 ####################### # Set debug env vars. # From 65bf6291841250a9ce5daaa6563e671ddca266c5 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 21 Feb 2012 14:23:41 -0700 Subject: [PATCH 30/34] Remove test debug hacks. --- t/lib/bash/collect.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 4f50f633..a40a1dfb 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -21,12 +21,7 @@ parse_options "$BIN_DIR/pt-stalk" --run-time 1 -- --defaults-file=/tmp/12345/my. local p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. -set -x collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 -set +x -# XXX -cat $p-output -ls $TMPDIR/collect # Even if this system doesn't have all the cmds, collect should still # have created some files for cmds that (hopefully) all systems have. @@ -47,9 +42,6 @@ cmd_ok \ "grep -q '\-hostname\$' $TMPDIR/collect-files" \ "Collected hostname" -# XXX -df && cat $p-df - cmd_ok \ "grep -q 'Avail' $p-df" \ "df" @@ -101,9 +93,6 @@ cmd_ok \ "grep -q '^| Uptime' $p-mysqladmin" \ "mysqladmin ext" -# XXX -cat $p-opentables* - cmd_ok \ "grep -qP 'Database\tTable\tIn_use' $p-opentables1" \ "opentables1" From b99293d1b31bbaedd41653a674e2f8c3e8d5e01d Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 23 Feb 2012 13:58:57 -0700 Subject: [PATCH 31/34] Simplify and individuate Bash tests so prove reports failures where they happen. --- t/lib/bash.t | 25 ---------- t/lib/bash/alt_cmds.sh | 2 +- t/lib/bash/alt_cmds.t | 1 + t/lib/bash/collect.sh | 8 ++-- t/lib/bash/collect.t | 1 + t/lib/bash/daemon.sh | 6 +-- t/lib/bash/daemon.t | 1 + t/lib/bash/log_warn_die.sh | 2 +- t/lib/bash/log_warn_die.t | 1 + t/lib/bash/parse_options.sh | 4 +- t/lib/bash/parse_options.t | 1 + t/lib/bash/safeguards.sh | 2 +- t/lib/bash/safeguards.t | 1 + t/lib/bash/tmpdir.sh | 4 +- t/lib/bash/tmpdir.t | 1 + t/lib/samples/bash/dummy.sh | 4 -- util/test-bash-functions | 94 ++++++++++++++++++++++--------------- 17 files changed, 77 insertions(+), 81 deletions(-) delete mode 100644 t/lib/bash.t create mode 120000 t/lib/bash/alt_cmds.t create mode 120000 t/lib/bash/collect.t create mode 120000 t/lib/bash/daemon.t create mode 120000 t/lib/bash/log_warn_die.t create mode 120000 t/lib/bash/parse_options.t create mode 120000 t/lib/bash/safeguards.t create mode 120000 t/lib/bash/tmpdir.t delete mode 100644 t/lib/samples/bash/dummy.sh diff --git a/t/lib/bash.t b/t/lib/bash.t deleted file mode 100644 index 68706d3c..00000000 --- a/t/lib/bash.t +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env perl - -BEGIN { - die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" - unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; - unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; -}; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); - -use PerconaTest; - -my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/; -push @ARGV, "$trunk/t/lib/bash/*.sh" unless @ARGV; - -$ENV{BIN_DIR} = "$trunk/bin"; -$ENV{LIB_DIR} = "$trunk/lib/bash"; -$ENV{T_LIB_DIR} = "$trunk/t/lib"; -$ENV{SANDBOX_VERSION} = "$sandbox_version"; - -system("$trunk/util/test-bash-functions $trunk/t/lib/samples/bash/dummy.sh @ARGV"); - -exit; diff --git a/t/lib/bash/alt_cmds.sh b/t/lib/bash/alt_cmds.sh index 5d674bc0..b37e7321 100644 --- a/t/lib/bash/alt_cmds.sh +++ b/t/lib/bash/alt_cmds.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=1 +plan 1 source "$LIB_DIR/alt_cmds.sh" diff --git a/t/lib/bash/alt_cmds.t b/t/lib/bash/alt_cmds.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/alt_cmds.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index a40a1dfb..02c8c027 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=20 +plan 20 TMPFILE="$TEST_TMPDIR/parse-opts-output" TMPDIR="$TEST_TMPDIR" @@ -18,7 +18,7 @@ source "$LIB_DIR/collect.sh" parse_options "$BIN_DIR/pt-stalk" --run-time 1 -- --defaults-file=/tmp/12345/my.sandbox.cnf # Prefix (with path) for the collect files. -local p="$TMPDIR/collect/2011_12_05" +p="$TMPDIR/collect/2011_12_05" # Default collect, no extras like gdb, tcpdump, etc. collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 @@ -113,7 +113,7 @@ cmd_ok \ "grep -qP '^wait_timeout\t\d' $p-variables" \ "variables" -local iters=$(cat $p-df | grep -c '^TS ') +iters=$(cat $p-df | grep -c '^TS ') is "$iters" "1" "1 iteration/1s run time" empty_files=0 @@ -140,7 +140,7 @@ rm $TMPDIR/collect/* collect "$TMPDIR/collect" "2011_12_05" > $p-output 2>&1 -local iters=$(cat $p-df | grep -c '^TS ') +iters=$(cat $p-df | grep -c '^TS ') is "$iters" "2" "2 iteration/2s run time" # ############################################################################ diff --git a/t/lib/bash/collect.t b/t/lib/bash/collect.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/collect.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/daemon.sh b/t/lib/bash/daemon.sh index 0ffad457..c46d560f 100644 --- a/t/lib/bash/daemon.sh +++ b/t/lib/bash/daemon.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -TESTS=9 +plan 9 TMPDIR="$TEST_TMPDIR" -local file="$TMPDIR/pid-file" +file="$TMPDIR/pid-file" source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/daemon.sh" @@ -18,7 +18,7 @@ cmd_ok \ "test -f $file" \ "PID file created" -local pid=`cat $file` +pid=`cat $file` is \ "$pid" \ "$$" \ diff --git a/t/lib/bash/daemon.t b/t/lib/bash/daemon.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/daemon.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/log_warn_die.sh b/t/lib/bash/log_warn_die.sh index c3f4ed74..b31c1308 100644 --- a/t/lib/bash/log_warn_die.sh +++ b/t/lib/bash/log_warn_die.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=6 +plan 6 source "$LIB_DIR/log_warn_die.sh" diff --git a/t/lib/bash/log_warn_die.t b/t/lib/bash/log_warn_die.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/log_warn_die.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index 8c462eff..f9536d65 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=78 +plan 78 TMPFILE="$TEST_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -93,7 +93,7 @@ parse_options "$T_LIB_DIR/samples/bash/po001.sh" --foo >$TMPFILE 2>&1 cmd_ok "grep -q 'Unknown option: --foo' $TMPFILE" "Error on unknown option" usage_or_errors "$T_LIB_DIR/samples/bash/po001.sh" >$TMPFILE 2>&1 -local err=$? +err=$? is "$err" "1" "Non-zero exit on unknown option" # ########################################################################### diff --git a/t/lib/bash/parse_options.t b/t/lib/bash/parse_options.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/parse_options.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/safeguards.sh b/t/lib/bash/safeguards.sh index cf678d90..adcd3b62 100644 --- a/t/lib/bash/safeguards.sh +++ b/t/lib/bash/safeguards.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=11 +plan 11 source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/safeguards.sh" diff --git a/t/lib/bash/safeguards.t b/t/lib/bash/safeguards.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/safeguards.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/bash/tmpdir.sh b/t/lib/bash/tmpdir.sh index 55098f4d..78f15f92 100644 --- a/t/lib/bash/tmpdir.sh +++ b/t/lib/bash/tmpdir.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -TESTS=9 +plan 9 source "$LIB_DIR/log_warn_die.sh" source "$LIB_DIR/tmpdir.sh" @@ -21,7 +21,7 @@ is "$TMPDIR" "" "rm_tmpdir resets TMPDIR" # User-specified tmpdir. # ########################################################################### -local dir="/tmp/use--tmpdir" +dir="/tmp/use--tmpdir" is "$TMPDIR" "" "TMPDIR not defined" diff --git a/t/lib/bash/tmpdir.t b/t/lib/bash/tmpdir.t new file mode 120000 index 00000000..edb04c13 --- /dev/null +++ b/t/lib/bash/tmpdir.t @@ -0,0 +1 @@ +../../../util/test-bash-functions \ No newline at end of file diff --git a/t/lib/samples/bash/dummy.sh b/t/lib/samples/bash/dummy.sh deleted file mode 100644 index a7eea1ee..00000000 --- a/t/lib/samples/bash/dummy.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# This is a dummy script for testing the Bash libs. t/lib/bashLibs.t -# calls "util/test-bash-functions dummy.sh ". diff --git a/util/test-bash-functions b/util/test-bash-functions index c6c0cd61..73795fcc 100755 --- a/util/test-bash-functions +++ b/util/test-bash-functions @@ -14,21 +14,26 @@ die() { exit 255 } -( - if [ -n "$PERCONA_TOOLKIT_BRANCH" ]; then - BRANCH=$PERCONA_TOOLKIT_BRANCH - cd $BRANCH - else - while [ ! -f Makefile.PL ] && [ $(pwd) != "/" ]; do - cd .. - done - if [ ! -f Makefile.PL ]; then - die "Cannot find the root directory of the Percona Toolkit branch" - exit 1 - fi - BRANCH=`pwd` +cwd="$PWD" +if [ -n "$PERCONA_TOOLKIT_BRANCH" ]; then + BRANCH=$PERCONA_TOOLKIT_BRANCH + cd $BRANCH +else + while [ ! -f Makefile.PL ] && [ $(pwd) != "/" ]; do + cd .. + done + if [ ! -f Makefile.PL ]; then + die "Cannot find the root directory of the Percona Toolkit branch" + exit 1 fi -) + BRANCH="$PWD" +fi +cd "$cwd" + +BIN_DIR="$BRANCH/bin"; +LIB_DIR="$BRANCH/lib/bash"; +T_LIB_DIR="$BRANCH/t/lib"; +SANDBOX_VERSION="$($BRANCH/sandbox/test-env version)" # ############################################################################ # Paths @@ -108,6 +113,13 @@ result() { return $result } +plan() { + local n_tests=${1:-""} + if [ "$n_tests" ]; then + echo "1..$n_tests" + fi +} + # # The following subs are for the test files to call. # @@ -141,32 +153,38 @@ cmd_ok() { # Script starts here # ############################################################################ -if [ $# -lt 2 ]; then - die "Usage: test-back-functions FILE TESTS" -fi - -# Check and source the bash file. This is the code being tested. -# All its global vars and subs will be imported. -bash_file=$1 -shift -if [ ! -f "$bash_file" ]; then - die "$bash_file does not exist" -fi -head -n1 $bash_file | grep -q -E 'bash|sh' || die "$bash_file is not a bash file" -source $bash_file - -# Load (count) the tests so that we can write a TAP test plan like 1..5 -# for expecting 5 tests. Perl prove needs this. -declare -a tests -load_tests "$@" - -# Run the test files. testno=1 failed_tests=0 -for t in "${tests[@]}"; do - run_test $t -done + +if [ $# -eq 0 ]; then + TEST_FILE=$(basename "$0") + TEST="${TEST_FILE%".t"}" + source "$BRANCH/t/lib/bash/$TEST.sh" +else + if [ $# -lt 2 ]; then + die "Usage: test-bash-functions FILE TESTS" + fi + + # Check and source the bash file. This is the code being tested. + # All its global vars and subs will be imported. + bash_file=$1 + shift + if [ ! -f "$bash_file" ]; then + die "$bash_file does not exist" + fi + head -n1 $bash_file | grep -q -E 'bash|sh' || die "$bash_file is not a bash file" + source $bash_file + + # Load (count) the tests so that we can write a TAP test plan like 1..5 + # for expecting 5 tests. Perl prove needs this. + declare -a tests + load_tests "$@" + + # Run the test files. + for t in "${tests[@]}"; do + run_test $t + done +fi rm -rf $TEST_TMPDIR - exit $failed_tests From 6620595fac0176c55efc8fa042fb4b16efb39d5b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 24 Feb 2012 09:18:18 -0700 Subject: [PATCH 32/34] Make collect.t mysqladmin debug test reliable. --- t/lib/bash/collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 02c8c027..325a3299 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -67,7 +67,7 @@ cmd_ok \ if [[ "$SANDBOX_VERSION" > "5.0" ]]; then cmd_ok \ - "grep -q 'Status information:' $p-log_error" \ + "grep -q 'Memory status:' $p-log_error" \ "debug" else is "1" "1" "SKIP Can't determine MySQL 5.0 error log" From b9f6c14e30b1d197a97168c088f683d5c9bbcc3c Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 24 Feb 2012 09:39:49 -0700 Subject: [PATCH 33/34] Make collect.t mysqladmin debug test more reliable. --- t/lib/bash/collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/lib/bash/collect.sh b/t/lib/bash/collect.sh index 325a3299..397b902e 100644 --- a/t/lib/bash/collect.sh +++ b/t/lib/bash/collect.sh @@ -67,7 +67,7 @@ cmd_ok \ if [[ "$SANDBOX_VERSION" > "5.0" ]]; then cmd_ok \ - "grep -q 'Memory status:' $p-log_error" \ + "grep -qE 'Memory status|Open streams|Begin safemalloc' $p-log_error" \ "debug" else is "1" "1" "SKIP Can't determine MySQL 5.0 error log" From 6e94c39072aa4f6f7675f02d5c5279f7e87c9ea4 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Fri, 24 Feb 2012 11:11:07 -0700 Subject: [PATCH 34/34] Use Perl instead of awk to avoid 32-bit limitation. --- lib/bash/safeguards.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/bash/safeguards.sh b/lib/bash/safeguards.sh index edd9ed26..59e54292 100644 --- a/lib/bash/safeguards.sh +++ b/lib/bash/safeguards.sh @@ -50,9 +50,10 @@ check_disk_space() { local bytes_margin="${4:-0}" # Real/actual bytes used and bytes free. - local used_bytes=$(cat "$file" | awk '/^\//{printf("%d",$3 * 1024)}'); - local free_bytes=$(cat "$file" | awk '/^\//{printf("%d",$4 * 1024)}'); - local pct_used=$(cat "$file" | awk '/^\//{print $5}' | sed -e 's/%//g'); + + local used_bytes=$(perl -ane 'm!^/! && print $F[2] * 1024' "$file") + local free_bytes=$(perl -ane 'm!^/! && print $F[3] * 1024' "$file") + local pct_used=$(perl -ane 'm!^/! && print ($F[4] =~ m/(\d+)/)' "$file") local pct_free=$((100 - $pct_used)) # Report the real values to the user. @@ -63,7 +64,7 @@ check_disk_space() { if [ $bytes_margin -gt 0 ]; then used_bytes=$(($used_bytes + $bytes_margin)) free_bytes=$(($free_bytes - $bytes_margin)) - pct_used=$(awk "BEGIN { printf(\"%d\", ($used_bytes/($used_bytes + $free_bytes)) * 100) }") + pct_used=$(perl -e "print int(($used_bytes/($used_bytes + $free_bytes)) * 100)") pct_free=$((100 - $pct_used)) fi