diff --git a/Changelog b/Changelog index 58cf1058..2823771d 100644 --- a/Changelog +++ b/Changelog @@ -1,5 +1,23 @@ Changelog for Percona Toolkit +v2.2.13 released 2015-01-26 + + + * Feature 1391240: pt-kill added query fingerprint hash to output + * Fixed bug 1402668: pt-mysql-summary fails on cluster in Donor/Desynced status + * Fixed bug 1396870: pt-online-schema-change CTRL+C leaves terminal in inconsistent state + * Fixed bug 1396868: pt-online-schema-change --ask-pass option error + * Fixed bug 1266869: pt-stalk fails to start if $HOME environment variable is not set + * Fixed bug 1019479: pt-table-checksum does not work with sql_mode ONLY_FULL_GROUP_BY + * Fixed bug 1394934: pt-table-checksum error in debug mode + * Fixed bug 1321297: pt-table-checksum reports diffs on timestamp columns in 5.5 vs 5.6 + * Fixed bug 1399789: pt-table-checksum fails to find pxc nodes when wsrep_node_incoming_address is set to AUTO + * Fixed bug 1388870: pt-table-checksum has some errors with different time zones + * Fixed bug 1408375: vulnerable to MITM attack which would allow exfiltration of MySQL configuration information via --version-check + * Fixed bug 1404298: missing MySQL5.7 test files for pt-table-checksum + * Fixed bug 1403900: added sandbox and fixed sakila test db for 5.7 + + v2.2.12 released 2014-11-14 diff --git a/Makefile.PL b/Makefile.PL index abdf18f7..0ccb4e4a 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -2,7 +2,7 @@ use ExtUtils::MakeMaker; WriteMakefile( NAME => 'percona-toolkit', - VERSION => '2.2.12', + VERSION => '2.2.13', EXE_FILES => [ ], MAN1PODS => { 'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1p', diff --git a/bin/pt-align b/bin/pt-align index b9adcc2e..6bb356a0 100755 --- a/bin/pt-align +++ b/bin/pt-align @@ -1312,7 +1312,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -1331,6 +1331,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-align 2.2.12 +pt-align 2.2.13 =cut diff --git a/bin/pt-archiver b/bin/pt-archiver index 874a2234..b2c4f83d 100755 --- a/bin/pt-archiver +++ b/bin/pt-archiver @@ -43,7 +43,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -4421,7 +4421,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4943,11 +4944,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -5384,6 +5386,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -7884,7 +7891,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -7903,6 +7910,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-archiver 2.2.12 +pt-archiver 2.2.13 =cut diff --git a/bin/pt-config-diff b/bin/pt-config-diff index 4de976fa..56330e79 100755 --- a/bin/pt-config-diff +++ b/bin/pt-config-diff @@ -43,7 +43,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -2295,7 +2295,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -2311,7 +2311,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -2393,13 +2393,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -2412,11 +2439,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -4169,7 +4193,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4691,11 +4716,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -5132,6 +5158,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -5734,7 +5765,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates. +This program is copyright 2011-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -5752,6 +5783,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-config-diff 2.2.12 +pt-config-diff 2.2.13 =cut diff --git a/bin/pt-deadlock-logger b/bin/pt-deadlock-logger index e9140b02..5235a9d2 100755 --- a/bin/pt-deadlock-logger +++ b/bin/pt-deadlock-logger @@ -42,7 +42,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -2639,7 +2639,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -2655,7 +2655,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -2737,13 +2737,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -2756,11 +2783,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -3234,7 +3258,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -3756,11 +3781,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4197,6 +4223,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -5523,7 +5554,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -5542,6 +5573,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-deadlock-logger 2.2.12 +pt-deadlock-logger 2.2.13 =cut diff --git a/bin/pt-diskstats b/bin/pt-diskstats index 76e0cb14..96ac0dfa 100755 --- a/bin/pt-diskstats +++ b/bin/pt-diskstats @@ -38,7 +38,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3828,7 +3828,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4350,11 +4351,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4791,6 +4793,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -5560,7 +5567,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -5579,6 +5586,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-diskstats 2.2.12 +pt-diskstats 2.2.13 =cut diff --git a/bin/pt-duplicate-key-checker b/bin/pt-duplicate-key-checker index c38f6e86..6600b7c9 100755 --- a/bin/pt-duplicate-key-checker +++ b/bin/pt-duplicate-key-checker @@ -39,7 +39,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3845,7 +3845,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4367,11 +4368,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4808,6 +4810,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -5581,7 +5588,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -5600,6 +5607,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-duplicate-key-checker 2.2.12 +pt-duplicate-key-checker 2.2.13 =cut diff --git a/bin/pt-fifo-split b/bin/pt-fifo-split index 74245a3b..0dff4447 100755 --- a/bin/pt-fifo-split +++ b/bin/pt-fifo-split @@ -1601,7 +1601,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -1620,6 +1620,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-fifo-split 2.2.12 +pt-fifo-split 2.2.13 =cut diff --git a/bin/pt-find b/bin/pt-find index 9d953277..fb1c9ea9 100755 --- a/bin/pt-find +++ b/bin/pt-find @@ -35,7 +35,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -2572,7 +2572,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -3094,11 +3095,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -3535,6 +3537,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -4965,7 +4972,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -4984,6 +4991,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-find 2.2.12 +pt-find 2.2.13 =cut diff --git a/bin/pt-fingerprint b/bin/pt-fingerprint index c08c20c9..67e60c01 100755 --- a/bin/pt-fingerprint +++ b/bin/pt-fingerprint @@ -2193,7 +2193,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates. +This program is copyright 2011-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -2211,6 +2211,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-fingerprint 2.2.12 +pt-fingerprint 2.2.13 =cut diff --git a/bin/pt-fk-error-logger b/bin/pt-fk-error-logger index 14b43289..e7783618 100755 --- a/bin/pt-fk-error-logger +++ b/bin/pt-fk-error-logger @@ -37,7 +37,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -1791,7 +1791,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -1807,7 +1807,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -1889,13 +1889,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -1908,11 +1935,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -2739,7 +2763,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -3261,11 +3286,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -3702,6 +3728,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -4510,7 +4541,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates. +This program is copyright 2011-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -4528,6 +4559,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-fk-error-logger 2.2.12 +pt-fk-error-logger 2.2.13 =cut diff --git a/bin/pt-heartbeat b/bin/pt-heartbeat index 855ded6f..baf50777 100755 --- a/bin/pt-heartbeat +++ b/bin/pt-heartbeat @@ -38,7 +38,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3744,7 +3744,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4266,11 +4267,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4707,6 +4709,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -6197,7 +6204,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2007-2014 Percona LLC and/or its affiliates, +This program is copyright 2007-2015 Percona LLC and/or its affiliates, 2006 Proven Scaling LLC and Six Apart Ltd. Feedback and improvements are welcome. @@ -6218,6 +6225,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-heartbeat 2.2.12 +pt-heartbeat 2.2.13 =cut diff --git a/bin/pt-index-usage b/bin/pt-index-usage index 1185ade7..85815f4c 100755 --- a/bin/pt-index-usage +++ b/bin/pt-index-usage @@ -45,7 +45,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -5249,7 +5249,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -5771,11 +5772,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -6212,6 +6214,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -7529,7 +7536,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -7548,6 +7555,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-index-usage 2.2.12 +pt-index-usage 2.2.13 =cut diff --git a/bin/pt-ioprofile b/bin/pt-ioprofile index 41e3f3b7..d1e73584 100755 --- a/bin/pt-ioprofile +++ b/bin/pt-ioprofile @@ -201,7 +201,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -1103,7 +1106,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -1122,7 +1125,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-ioprofile 2.2.12 +pt-ioprofile 2.2.13 =cut diff --git a/bin/pt-kill b/bin/pt-kill index c7e5482b..3e2e6256 100755 --- a/bin/pt-kill +++ b/bin/pt-kill @@ -47,7 +47,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -4007,7 +4007,7 @@ sub get_connected_slaves { die "You do not have the PROCESS privilege"; } - $sql = 'SHOW PROCESSLIST'; + $sql = 'SHOW FULL PROCESSLIST'; PTDEBUG && _d($dbh, $sql); grep { $_->{command} =~ m/Binlog Dump/i } map { # Lowercase the column names @@ -5158,7 +5158,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -5174,7 +5174,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -5256,13 +5256,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -5275,11 +5302,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -5551,7 +5575,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -6073,11 +6098,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -6514,6 +6540,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -6567,6 +6598,7 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use POSIX qw(setsid); use List::Util qw(max); +use Digest::MD5 qw(md5_hex); use Data::Dumper; $Data::Dumper::Indent = 1; @@ -7101,6 +7133,11 @@ sub main { $query->{Id}, ($query->{Command} || 'NULL'), $query->{Time}, ($query->{Info} || 'NULL'); } + if ( $o->get('query-id') ) { + my $fp = $qr->fingerprint($query->{'Info'}); + my $chksm = Transformers::make_checksum($fp); + print "Query ID: 0x$chksm\n"; + } if ( $o->get('execute-command') ) { exec_cmd($o->get('execute-command')); msg('Executed ' . $o->get('execute-command')); @@ -7487,6 +7524,7 @@ pt-kill does not provide any safeguards so code carefully! It is permissible for the code to have side effects (to alter C<$event>). + =item --group-by type: string @@ -7590,6 +7628,20 @@ short form: -P; type: int Port number to use for connection. +=item --query-id + +Prints an ID of the query that was just killed. This is +equivalent to the "ID" output of pt-query-digest. This allows +cross-referencing the output of both tools. + +Example: + + Query ID 0xE9800998ECF8427E + +Note that this is a digest (or hash) of the query's "fingerprint", +so queries of the same form but with different values will have the same ID. +See pt-query-digest for more information. + =item --run-time type: time @@ -8183,7 +8235,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2009-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -8202,6 +8254,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-kill 2.2.12 +pt-kill 2.2.13 =cut diff --git a/bin/pt-mext b/bin/pt-mext index 088ca107..83e19040 100755 --- a/bin/pt-mext +++ b/bin/pt-mext @@ -242,7 +242,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -779,7 +782,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -798,7 +801,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-mext 2.2.12 +pt-mext 2.2.13 =cut diff --git a/bin/pt-mysql-summary b/bin/pt-mysql-summary index df639463..b0fe56d9 100755 --- a/bin/pt-mysql-summary +++ b/bin/pt-mysql-summary @@ -203,7 +203,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -2397,7 +2400,7 @@ check_mysql () { # Now that we have the cmd line opts, check that we can actually # connect to MySQL. - [ -n "$(mysql $EXT_ARGV -e 'SELECT 1')" ] \ + [ -n "$(mysql $EXT_ARGV -e 'SHOW STATUS')" ] \ || die "Cannot connect to MySQL. Check that MySQL is running and that the options after -- are correct." } @@ -3066,7 +3069,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -3085,7 +3088,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-mysql-summary 2.2.12 +pt-mysql-summary 2.2.13 =cut diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 67aa5343..7ddc6a0f 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -40,6 +40,7 @@ BEGIN { HTTP::Micro VersionCheck Percona::XtraDB::Cluster + ReadKeyMini )); } @@ -54,7 +55,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3755,7 +3756,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -3771,7 +3772,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -3853,13 +3854,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -3872,11 +3900,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -6552,7 +6577,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -7074,11 +7100,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -7515,6 +7542,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -7662,10 +7694,7 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -7757,6 +7786,162 @@ sub _d { # End Percona::XtraDB::Cluster package # ########################################################################### +# ########################################################################### +# ReadKeyMini package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/ReadKeyMini.pm +# t/lib/ReadKeyMini.t +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +{ + +BEGIN { + +package ReadKeyMini; +BEGIN { $INC{"ReadKeyMini.pm"} ||= 1 } + +use warnings; +use strict; +use English qw(-no_match_vars); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +use POSIX qw( :termios_h ); +use Fcntl qw( F_SETFL F_GETFL ); + +use base qw( Exporter ); + +BEGIN { + our @EXPORT_OK = qw( GetTerminalSize ReadMode ); + *ReadMode = *Term::ReadKey::ReadMode = \&_ReadMode; + *GetTerminalSize = *Term::ReadKey::GetTerminalSize = \&_GetTerminalSize; +} + +my %modes = ( + original => 0, + restore => 0, + normal => 1, + noecho => 2, + cbreak => 3, + raw => 4, + 'ultra-raw' => 5, +); + +{ + my $fd_stdin = fileno(STDIN); + my $flags; + unless ( $PerconaTest::DONT_RESTORE_STDIN ) { + $flags = fcntl(STDIN, F_GETFL, 0) + or warn "Error getting STDIN flags with fcntl: $OS_ERROR"; + } + my $term = POSIX::Termios->new(); + $term->getattr($fd_stdin); + my $oterm = $term->getlflag(); + my $echo = ECHO | ECHOK | ICANON; + my $noecho = $oterm & ~$echo; + + sub _ReadMode { + my $mode = $modes{ $_[0] }; + if ( $mode == $modes{normal} ) { + cooked(); + } + elsif ( $mode == $modes{cbreak} || $mode == $modes{noecho} ) { + cbreak( $mode == $modes{noecho} ? $noecho : $oterm ); + } + else { + die("ReadMore('$_[0]') not supported"); + } + } + + sub cbreak { + my ($lflag) = $_[0] || $noecho; + $term->setlflag($lflag); + $term->setcc( VTIME, 1 ); + $term->setattr( $fd_stdin, TCSANOW ); + } + + sub cooked { + $term->setlflag($oterm); + $term->setcc( VTIME, 0 ); + $term->setattr( $fd_stdin, TCSANOW ); + if ( !$PerconaTest::DONT_RESTORE_STDIN ) { + fcntl(STDIN, F_SETFL, int($flags)) + or warn "Error restoring STDIN flags with fcntl: $OS_ERROR"; + } + } + + END { cooked() } +} + +sub readkey { + my $key = ''; + cbreak(); + sysread(STDIN, $key, 1); + my $timeout = 0.1; + if ( $key eq "\033" ) { + my $x = ''; + STDIN->blocking(0); + sysread(STDIN, $x, 2); + STDIN->blocking(1); + $key .= $x; + redo if $key =~ /\[[0-2](?:[0-9];)?$/ + } + cooked(); + return $key; +} + + +BEGIN { + eval { no warnings; local $^W; require 'sys/ioctl.ph' }; + if ( !defined &TIOCGWINSZ ) { + *TIOCGWINSZ = sub () { + $^O eq 'linux' ? 0x005413 + : $^O eq 'solaris' ? 0x005468 + : 0x40087468; + }; + } +} + +sub _GetTerminalSize { + if ( @_ ) { + die "My::Term::ReadKey doesn't implement GetTerminalSize with arguments"; + } + + my $cols = $ENV{COLUMNS} || 80; + my $rows = $ENV{LINES} || 24; + + if ( open( TTY, "+<", "/dev/tty" ) ) { # Got a tty + my $winsize = ''; + if ( ioctl( TTY, &TIOCGWINSZ, $winsize ) ) { + ( $rows, $cols, my ( $xpixel, $ypixel ) ) = unpack( 'S4', $winsize ); + return ( $cols, $rows, $xpixel, $ypixel ); + } + } + + if ( $rows = `tput lines 2>/dev/null` ) { + chomp($rows); + chomp($cols = `tput cols`); + } + elsif ( my $stty = `stty -a 2>/dev/null` ) { + ($rows, $cols) = $stty =~ /([0-9]+) rows; ([0-9]+) columns;/; + } + else { + ($cols, $rows) = @ENV{qw( COLUMNS LINES )}; + $cols ||= 80; + $rows ||= 24; + } + + return ( $cols, $rows ); +} + +} + +1; +} +# ########################################################################### +# End ReadKeyMini package +# ########################################################################### + # ########################################################################### # This is a combination of modules and programs in one -- a runnable module. # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last @@ -10484,6 +10669,12 @@ sub sig_int { my ( $signal ) = @_; $oktorun = 0; # flag for cleanup tasks print STDERR "# Exiting on SIG$signal.\n"; + # restore terminal to normal state in case CTL+C issued while + # asking for password + # https://bugs.launchpad.net/percona-toolkit/+bug/1396870 + # note: just including ReadKeyMini seems to solve the bug, + # but lets use it explicitly so we don't forget why we need it + ReadKeyMini::ReadMode 0; exit 1; } @@ -11598,7 +11789,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates. +This program is copyright 2011-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -11616,6 +11807,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-online-schema-change 2.2.12 +pt-online-schema-change 2.2.13 =cut diff --git a/bin/pt-pmp b/bin/pt-pmp index 63504fbe..6a56910b 100755 --- a/bin/pt-pmp +++ b/bin/pt-pmp @@ -244,7 +244,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -873,7 +876,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -892,7 +895,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-pmp 2.2.12 +pt-pmp 2.2.13 =cut diff --git a/bin/pt-query-digest b/bin/pt-query-digest index cdf6ba6b..697288ca 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -64,7 +64,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -11833,7 +11833,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -12355,11 +12356,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -12796,6 +12798,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -16599,7 +16606,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2008-2014 Percona LLC and/or its affiliates. +This program is copyright 2008-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -16617,6 +16624,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-query-digest 2.2.12 +pt-query-digest 2.2.13 =cut diff --git a/bin/pt-show-grants b/bin/pt-show-grants index 9e4c02e3..6a0e34e6 100755 --- a/bin/pt-show-grants +++ b/bin/pt-show-grants @@ -2395,7 +2395,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -2414,6 +2414,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-show-grants 2.2.12 +pt-show-grants 2.2.13 =cut diff --git a/bin/pt-sift b/bin/pt-sift index e21a4a2e..a87a9f3b 100755 --- a/bin/pt-sift +++ b/bin/pt-sift @@ -242,7 +242,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -1221,7 +1224,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -1240,7 +1243,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-sift 2.2.12 +pt-sift 2.2.13 =cut diff --git a/bin/pt-slave-delay b/bin/pt-slave-delay index a9c4d063..87d04b7f 100755 --- a/bin/pt-slave-delay +++ b/bin/pt-slave-delay @@ -40,7 +40,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3097,7 +3097,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -3619,11 +3620,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4060,6 +4062,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -4850,7 +4857,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Sergey Zhuravle and Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -4869,6 +4876,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-slave-delay 2.2.12 +pt-slave-delay 2.2.13 =cut diff --git a/bin/pt-slave-find b/bin/pt-slave-find index 568e9dfb..cd447fff 100755 --- a/bin/pt-slave-find +++ b/bin/pt-slave-find @@ -4323,7 +4323,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -4342,6 +4342,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-slave-find 2.2.12 +pt-slave-find 2.2.13 =cut diff --git a/bin/pt-slave-restart b/bin/pt-slave-restart index a336b3f7..c319a369 100755 --- a/bin/pt-slave-restart +++ b/bin/pt-slave-restart @@ -41,7 +41,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -3746,7 +3746,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4268,11 +4269,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4709,6 +4711,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -5918,7 +5925,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -5937,6 +5944,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-slave-restart 2.2.12 +pt-slave-restart 2.2.13 =cut diff --git a/bin/pt-stalk b/bin/pt-stalk index 653614fc..4b084187 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -255,7 +255,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -2219,7 +2222,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -2238,7 +2241,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-stalk 2.2.12 +pt-stalk 2.2.13 =cut diff --git a/bin/pt-summary b/bin/pt-summary index 0582e253..364c9947 100755 --- a/bin/pt-summary +++ b/bin/pt-summary @@ -210,7 +210,10 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi _parse_command_line "${@:-""}" @@ -2674,7 +2677,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2010-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -2693,7 +2696,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-summary 2.2.12 +pt-summary 2.2.13 =cut diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 3adee83e..295af447 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -57,7 +57,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -332,7 +332,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -854,11 +855,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -1295,6 +1297,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -3533,7 +3540,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -3549,7 +3556,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -3631,13 +3638,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -3650,11 +3684,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -3812,10 +3843,7 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -5714,8 +5742,8 @@ sub make_row_checksum { $query = join(', ', map { my $col = $_; - if ( $col =~ m/\+ 0/ ) { - my ($real_col) = /^(\S+)/; + if ( $col =~ m/UNIX_TIMESTAMP/ ) { + my ($real_col) = /^UNIX_TIMESTAMP\((.+?)\)/; $col .= " AS $real_col"; } elsif ( $col =~ m/TRIM/ ) { @@ -5815,7 +5843,7 @@ sub get_checksum_columns { my $type = $tbl_struct->{type_for}->{$_}; my $result = $q->quote($_); if ( $type eq 'timestamp' ) { - $result .= ' + 0'; + $result = "UNIX_TIMESTAMP($result)"; } elsif ( $float_precision && $type =~ m/float|double/ ) { $result = "ROUND($result, $float_precision)"; @@ -9131,6 +9159,26 @@ sub main { return if $o->get('explain'); my $sql; + # https://bugs.launchpad.net/percona-toolkit/+bug/1019479 + # sql_mode ONLY_FULL_GROUP_BY often raises error even when query is + # safe and deterministic. It's best to turn it off for the session + # at this point. + $sql = 'SELECT @@SQL_MODE'; + PTDEBUG && _d($dbh, $sql); + my ($sql_mode) = eval { $dbh->selectrow_array($sql) }; + if ( $EVAL_ERROR ) { + die "Error getting the current SQL_MODE: $EVAL_ERROR"; + } + $sql_mode =~ s/ONLY_FULL_GROUP_BY//i; + $sql = qq[SET SQL_MODE='$sql_mode']; + PTDEBUG && _d($dbh, $sql); + eval { $dbh->do($sql) }; + if ( $EVAL_ERROR ) { + die "Error setting SQL_MODE" + . ": $EVAL_ERROR"; + } + + # https://bugs.launchpad.net/percona-toolkit/+bug/919352 # The tool shouldn't blindly attempt to change binlog_format; # instead, it should check if it's already set to STATEMENT. @@ -9323,10 +9371,8 @@ sub main { my %seen_ids; for my $cxn ($master_cxn, @$slaves) { my $dbh = $cxn->dbh(); - # if it's a cluster node we use its incoming address as id ( see https://bugs.launchpad.net/percona-toolkit/+bug/1217466 ) - my $sql = $cluster->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($cxn, $dbh, $sql); - my ($id) = $dbh->selectrow_array($sql); + # get server/node unique id ( https://bugs.launchpad.net/percona-toolkit/+bug/1217466 ) + my $id = $cxn->get_id(); $seen_ids{$id}++; } @@ -12723,7 +12769,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -12742,6 +12788,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-table-checksum 2.2.12 +pt-table-checksum 2.2.13 =cut diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 9d389748..36da0d28 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -55,7 +55,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -8605,7 +8605,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -9127,11 +9128,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -9568,6 +9570,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -12749,7 +12756,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -12768,6 +12775,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-table-sync 2.2.12 +pt-table-sync 2.2.13 =cut diff --git a/bin/pt-table-usage b/bin/pt-table-usage index 4d86326a..35a689d9 100755 --- a/bin/pt-table-usage +++ b/bin/pt-table-usage @@ -7553,7 +7553,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2012-2014 Percona LLC and/or its affiliates. +This program is copyright 2012-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -7571,6 +7571,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-table-usage 2.2.12 +pt-table-usage 2.2.13 =cut diff --git a/bin/pt-upgrade b/bin/pt-upgrade index cf04c82b..fb9441bd 100755 --- a/bin/pt-upgrade +++ b/bin/pt-upgrade @@ -61,7 +61,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -2464,7 +2464,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -2480,7 +2480,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -2562,13 +2562,40 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -2581,11 +2608,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { @@ -3545,7 +3569,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4067,11 +4092,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4508,6 +4534,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -11240,7 +11271,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2009-2014 Percona LLC and/or its affiliates. +This program is copyright 2009-2015 Percona LLC and/or its affiliates. Feedback and improvements are welcome. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -11259,6 +11290,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-upgrade 2.2.12 +pt-upgrade 2.2.13 =cut diff --git a/bin/pt-variable-advisor b/bin/pt-variable-advisor index a966883a..73052e4d 100755 --- a/bin/pt-variable-advisor +++ b/bin/pt-variable-advisor @@ -44,7 +44,7 @@ BEGIN { { package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; @@ -4004,7 +4004,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { my $fh = $self->{fh}; @@ -4526,11 +4527,12 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; # optimistic, but... + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); @@ -4967,6 +4969,11 @@ sub get_from_mysql { return; } + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + $item->{vars} = ['version_comment', 'version']; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { @@ -6120,7 +6127,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2010-2014 Percona LLC and/or its affiliates. +This program is copyright 2010-2015 Percona LLC and/or its affiliates. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF @@ -6138,6 +6145,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-variable-advisor 2.2.12 +pt-variable-advisor 2.2.13 =cut diff --git a/bin/pt-visual-explain b/bin/pt-visual-explain index 6379ebad..6eed88c7 100755 --- a/bin/pt-visual-explain +++ b/bin/pt-visual-explain @@ -3232,7 +3232,7 @@ software from Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY -This program is copyright 2011-2014 Percona LLC and/or its affiliates, +This program is copyright 2011-2015 Percona LLC and/or its affiliates, 2007-2011 Baron Schwartz. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -3251,6 +3251,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -pt-visual-explain 2.2.12 +pt-visual-explain 2.2.13 =cut diff --git a/config/deb/changelog b/config/deb/changelog index d825775d..3ea10bfe 100644 --- a/config/deb/changelog +++ b/config/deb/changelog @@ -1,3 +1,21 @@ +percona-toolkit (2.2.13) unstable; urgency=low + + * Feature 1391240: pt-kill added query fingerprint hash to output + * Fixed bug 1402668: pt-mysql-summary fails on cluster in Donor/Desynced status + * Fixed bug 1396870: pt-online-schema-change CTRL+C leaves terminal in inconsistent state + * Fixed bug 1396868: pt-online-schema-change --ask-pass option error + * Fixed bug 1266869: pt-stalk fails to start if $HOME environment variable is not set + * Fixed bug 1019479: pt-table-checksum does not work with sql_mode ONLY_FULL_GROUP_BY + * Fixed bug 1394934: pt-table-checksum error in debug mode + * Fixed bug 1321297: pt-table-checksum reports diffs on timestamp columns in 5.5 vs 5.6 + * Fixed bug 1399789: pt-table-checksum fails to find pxc nodes when wsrep_node_incoming_address is set to AUTO + * Fixed bug 1388870: pt-table-checksum has some errors with different time zones + * Fixed bug 1408375: vulnerable to MITM attack which would allow exfiltration of MySQL configuration information via --version-check + * Fixed bug 1404298: missing MySQL5.7 test files for pt-table-checksum + * Fixed bug 1403900: added sandbox and fixed sakila test db for 5.7 + + -- Percona Toolkit Developers Fri, 23 Jan 2015 10:08:15 +0000 + percona-toolkit (2.2.12) unstable; urgency=low * Fixed bug 1376561: pt-archiver is not able to archive all the rows when a table has a hash partition diff --git a/docs/percona-toolkit.pod b/docs/percona-toolkit.pod index 5e2427ff..b58a476a 100644 --- a/docs/percona-toolkit.pod +++ b/docs/percona-toolkit.pod @@ -538,7 +538,7 @@ Many people have contributed code over the years. See each tool's =head1 COPYRIGHT, LICENSE, AND WARRANTY -Percona Toolkit is copyright 2011-2014 Percona LLC and/or its affiliates, et al. +Percona Toolkit is copyright 2011-2015 Percona LLC and/or its affiliates, et al. See each program's documentation for complete copyright notices. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED @@ -557,6 +557,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION -Percona Toolkit v2.2.12 released 2014-11-11 +Percona Toolkit v2.2.13 released 2015-01-23 =cut diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 16a3a72f..4dd6d7f8 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,52 @@ Release Notes ************* +v2.2.13 released 2015-01-26 +=========================== + +Percona Toolkit 2.2.13 has been released. This release contains one new feature and twelve bug fixes. + +New Features: + +* pt-kill now supports new ``--query-id`` option. This option can be used to print a query fingerprint hash after killing a query to enable the cross-referencing with the pt-query-digest output. This option can be used along with ``--print`` option as well. + +Bugs Fixed: + +* Fixed bug 1019479: pt-table-checksum now works with ``ONLY_FULL_GROUP_BY`` sql_mode. + +* Fixed bug 1394934: running pt-table-checksum in debug mode would cause an error. + +* Fixed bug 1396868: regression introduced in Percona Toolkit 2.2.12 caused pt-online-schema-change not to honor ``--ask-pass`` option. + +* Fixed bug 1399789: pt-table-checksum would fail to find Percona XtraDB Cluster nodes when variable ``wsrep_node_incoming_address`` was set to ``AUTO``. + +* Fixed bug 1408375: Percona Toolkit was vulnerable to MITM attack which could allow exfiltration of MySQL configuration information via ``--version-check`` option. This vulnerability was logged as `CVE 2015-1027 _` + +* Fixed bug 1321297: pt-table-checksum was reporting differences on timestamp columns with replication from 5.5 to 5.6 server version, although the data was identical. + +* Fixed bug 1388870: pt-table-checksum was showing differences if the master and slave were in different time zone. + +* Fixed bug 1402668: pt-mysql-summary would exit if Percona XtraDB Cluster was in ``Donor/Desynced`` state. + +* Fixed bug 1266869: pt-stalk would fail to start if ``$HOME`` environment variable was not set. + +Changelog +--------- + +* Feature 1391240: pt-kill added query fingerprint hash to output +* Fixed bug 1402668: pt-mysql-summary fails on cluster in Donor/Desynced status +* Fixed bug 1396870: pt-online-schema-change CTRL+C leaves terminal in inconsistent state +* Fixed bug 1396868: pt-online-schema-change --ask-pass option error +* Fixed bug 1266869: pt-stalk fails to start if $HOME environment variable is not set +* Fixed bug 1019479: pt-table-checksum does not work with sql_mode ONLY_FULL_GROUP_BY +* Fixed bug 1394934: pt-table-checksum error in debug mode +* Fixed bug 1321297: pt-table-checksum reports diffs on timestamp columns in 5.5 vs 5.6 +* Fixed bug 1399789: pt-table-checksum fails to find pxc nodes when wsrep_node_incoming_address is set to AUTO +* Fixed bug 1388870: pt-table-checksum has some errors with different time zones +* Fixed bug 1408375: vulnerable to MITM attack which would allow exfiltration of MySQL configuration information via --version-check +* Fixed bug 1404298: missing MySQL5.7 test files for pt-table-checksum +* Fixed bug 1403900: added sandbox and fixed sakila test db for 5.7 + v2.2.12 released 2014-11-14 =========================== diff --git a/lib/Cxn.pm b/lib/Cxn.pm index b8dc7168..4fbc1a8b 100644 --- a/lib/Cxn.pm +++ b/lib/Cxn.pm @@ -108,7 +108,7 @@ sub new { set => $args{set}, NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, dbh_set => 0, - ask_pass => $args{ask_pass}, + ask_pass => $o->get('ask-pass'), DSNParser => $dp, is_cluster_node => undef, parent => $args{parent}, @@ -125,7 +125,7 @@ sub connect { my $dbh = $self->{dbh}; if ( !$dbh || !$dbh->ping() ) { # Ask for password once. - if ( $self->{ask_pass} && !$self->{asked_for_pass} ) { + if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p} ) { $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); $self->{asked_for_pass} = 1; } @@ -226,15 +226,45 @@ sub name { return $self->{hostname} || $self->{dsn_name} || 'unknown host'; } +# This returns the server_id. +# For cluster nodes, since server_id is unreliable, we use a combination of +# variables to create an id string that is unique. +sub get_id { + my ($self, $cxn) = @_; + + $cxn ||= $self; + + my $unique_id; + if ($cxn->is_cluster_node()) { # for cluster we concatenate various variables to maximize id 'uniqueness' across versions + my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; + my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); + PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); + $unique_id = $wsrep_local_index."|"; + foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { + my $sql = "SHOW VARIABLES LIKE '$val'"; + PTDEBUG && _d($cxn->name, $sql); + my (undef, $val) = $cxn->dbh->selectrow_array($sql); + $unique_id .= "|$val"; + } + } else { + my $sql = 'SELECT @@SERVER_ID'; + PTDEBUG && _d($sql); + $unique_id = $cxn->dbh->selectrow_array($sql); + } + PTDEBUG && _d("Generated unique id for cluster:", $unique_id); + return $unique_id; +} + + # This is used to help remove_duplicate_cxns detect cluster nodes # (which often have unreliable server_id's) sub is_cluster_node { my ($self, $cxn) = @_; + $cxn ||= $self; my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'"; PTDEBUG && _d($cxn->name, $sql); my $row = $cxn->dbh->selectrow_arrayref($sql); - PTDEBUG && _d(Dumper($row)); return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0; } @@ -257,14 +287,8 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - # Very often cluster nodes are configured with matching server_id's - # So in that case we'll use its incoming address as its unique identifier - # Note: this relies on "seen_ids" being populated using the same strategy - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { diff --git a/lib/HTTP/Micro.pm b/lib/HTTP/Micro.pm index 859d5ddd..9cc35853 100644 --- a/lib/HTTP/Micro.pm +++ b/lib/HTTP/Micro.pm @@ -237,7 +237,8 @@ sub _split_url { ref($self->{fh}) eq 'IO::Socket::SSL' or die(qq/SSL connection failed for $host\n/); if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ); + $self->{fh}->verify_hostname( $host, $ssl_verify_args ) + or die(qq/SSL certificate not valid for $host\n/); } else { # Can't use $self->{fh}->verify_hostname because the IO::Socket::SSL diff --git a/lib/Percona/Toolkit.pm b/lib/Percona/Toolkit.pm index 4dc19a75..494e2ace 100644 --- a/lib/Percona/Toolkit.pm +++ b/lib/Percona/Toolkit.pm @@ -18,7 +18,7 @@ # ########################################################################### package Percona::Toolkit; -our $VERSION = '2.2.12'; +our $VERSION = '2.2.13'; use strict; use warnings FATAL => 'all'; diff --git a/lib/Percona/XtraDB/Cluster.pm b/lib/Percona/XtraDB/Cluster.pm index c8ad96ce..836e3e49 100644 --- a/lib/Percona/XtraDB/Cluster.pm +++ b/lib/Percona/XtraDB/Cluster.pm @@ -137,13 +137,7 @@ sub remove_duplicate_cxns { my @trimmed_cxns; for my $cxn ( @cxns ) { - my $dbh = $cxn->dbh(); - # Very often cluster nodes are configured with matching server_id's - # So in that case we'll use its incoming address as its unique identifier - # Note: This relies on "seen_ids" being populated using the same strategy - my $sql = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id}; - PTDEBUG && _d($sql); - my ($id) = $dbh->selectrow_array($sql); + my $id = $cxn->get_id(); PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id); if ( ! $seen_ids->{$id}++ ) { diff --git a/lib/RowChecksum.pm b/lib/RowChecksum.pm index a8e584dc..3d6e814a 100644 --- a/lib/RowChecksum.pm +++ b/lib/RowChecksum.pm @@ -82,10 +82,10 @@ sub make_row_checksum { $query = join(', ', map { my $col = $_; - if ( $col =~ m/\+ 0/ ) { + if ( $col =~ m/UNIX_TIMESTAMP/ ) { # Alias col name back to itself else its name becomes # "col + 0" instead of just "col". - my ($real_col) = /^(\S+)/; + my ($real_col) = /^UNIX_TIMESTAMP\((.+?)\)/; $col .= " AS $real_col"; } elsif ( $col =~ m/TRIM/ ) { @@ -216,7 +216,7 @@ sub get_checksum_columns { my $type = $tbl_struct->{type_for}->{$_}; my $result = $q->quote($_); if ( $type eq 'timestamp' ) { - $result .= ' + 0'; + $result = "UNIX_TIMESTAMP($result)"; } elsif ( $float_precision && $type =~ m/float|double/ ) { $result = "ROUND($result, $float_precision)"; diff --git a/lib/VersionCheck.pm b/lib/VersionCheck.pm index 18294fae..616df25a 100644 --- a/lib/VersionCheck.pm +++ b/lib/VersionCheck.pm @@ -138,17 +138,17 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - # Get the list of program to check from Percona. Try using - # https first; fallback to http if that fails (probably because - # IO::Socket::SSL isn't installed). - my $protocol = 'https'; # optimistic, but... + # Skip Version Check altogether if SSL not available + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); - $protocol = 'http'; + PTDEBUG && _d("SSL not available, won't run version_check"); + return; } PTDEBUG && _d('Using', $protocol); + # Get list of programs to check from Percona. my $advice = pingback( instances => $instances_to_check, protocol => $protocol, @@ -644,6 +644,13 @@ sub get_from_mysql { return; } + # Only allow version variables to be reported + # So in case of MITM attack, we don't report sensitive data + if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { + @{$item->{vars}} = grep { $_ eq 'version' || $_ eq 'version_comment' } @{$item->{vars}}; + } + + my @versions; my %version_for; foreach my $instance ( @$instances ) { diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index b07074ec..eee9e154 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -213,7 +213,11 @@ parse_options() { _parse_config_files "$user_config_file" done else - _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" + # conditional in case $HOME isn't set; e.g. tool launched from init + if [ "${HOME:-}" ]; then + _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf" + fi fi # Finally, parse the command line. diff --git a/sandbox/sakila.sql b/sandbox/sakila.sql index 6dee8c2a..94b5216e 100644 --- a/sandbox/sakila.sql +++ b/sandbox/sakila.sql @@ -6,6 +6,7 @@ SET NAMES utf8; SET UNIQUE_CHECKS=0; SET FOREIGN_KEY_CHECKS=0; + CREATE TABLE `actor` ( `actor_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT, `first_name` varchar(45) NOT NULL, @@ -370,7 +371,7 @@ SELECT film.film_id AS FID, film.title AS title, film.description AS description FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id JOIN film_actor ON film.film_id = film_actor.film_id JOIN actor ON film_actor.actor_id = actor.actor_id -GROUP BY film.film_id; +GROUP BY film.film_id, category.name; CREATE VIEW nicer_but_slower_film_list AS @@ -381,7 +382,7 @@ SELECT film.film_id AS FID, film.title AS title, film.description AS description FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id JOIN film_actor ON film.film_id = film_actor.film_id JOIN actor ON film_actor.actor_id = actor.actor_id -GROUP BY film.film_id; +GROUP BY film.film_id, category.name; CREATE VIEW staff_list AS diff --git a/sandbox/servers/5.7/data.tar.gz b/sandbox/servers/5.7/data.tar.gz new file mode 100644 index 00000000..fb36f610 Binary files /dev/null and b/sandbox/servers/5.7/data.tar.gz differ diff --git a/sandbox/servers/5.7/my.sandbox.cnf b/sandbox/servers/5.7/my.sandbox.cnf new file mode 100644 index 00000000..5cbe3f5d --- /dev/null +++ b/sandbox/servers/5.7/my.sandbox.cnf @@ -0,0 +1,29 @@ +[client] +user = msandbox +password = msandbox +port = PORT +socket = /tmp/PORT/mysql_sandboxPORT.sock + +[mysqld] +port = PORT +socket = /tmp/PORT/mysql_sandboxPORT.sock +pid-file = /tmp/PORT/data/mysql_sandboxPORT.pid +basedir = PERCONA_TOOLKIT_SANDBOX +datadir = /tmp/PORT/data +key_buffer_size = 16M +innodb_buffer_pool_size = 16M +innodb_data_home_dir = /tmp/PORT/data +innodb_log_group_home_dir = /tmp/PORT/data +innodb_data_file_path = ibdata1:10M:autoextend +innodb_log_file_size = 5M +log-bin = mysql-bin +relay_log = mysql-relay-bin +log_slave_updates +server-id = PORT +report-host = 127.0.0.1 +report-port = PORT +log-error = /tmp/PORT/data/mysqld.log +innodb_lock_wait_timeout = 3 +general_log +general_log_file = genlog +lower_case_table_names = 0 diff --git a/sandbox/servers/5.7/system_idb_tables.sql b/sandbox/servers/5.7/system_idb_tables.sql new file mode 100644 index 00000000..c97f5e6f --- /dev/null +++ b/sandbox/servers/5.7/system_idb_tables.sql @@ -0,0 +1,149 @@ +USE `mysql`; + +CREATE TABLE IF NOT EXISTS `innodb_index_stats` ( + `database_name` varchar(64) COLLATE utf8_bin NOT NULL, + `table_name` varchar(64) COLLATE utf8_bin NOT NULL, + `index_name` varchar(64) COLLATE utf8_bin NOT NULL, + `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `stat_name` varchar(64) COLLATE utf8_bin NOT NULL, + `stat_value` bigint(20) unsigned NOT NULL, + `sample_size` bigint(20) unsigned DEFAULT NULL, + `stat_description` varchar(1024) COLLATE utf8_bin NOT NULL, + PRIMARY KEY (`database_name`,`table_name`,`index_name`,`stat_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0; + +CREATE TABLE IF NOT EXISTS `innodb_table_stats` ( + `database_name` varchar(64) COLLATE utf8_bin NOT NULL, + `table_name` varchar(64) COLLATE utf8_bin NOT NULL, + `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `n_rows` bigint(20) unsigned NOT NULL, + `clustered_index_size` bigint(20) unsigned NOT NULL, + `sum_of_other_index_sizes` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`database_name`,`table_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0; + +CREATE TABLE IF NOT EXISTS `slave_master_info` ( + `Number_of_lines` int(10) unsigned NOT NULL COMMENT 'Number of lines in the file.', + `Master_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'The name of the master binary log currently being read from the master.', + `Master_log_pos` bigint(20) unsigned NOT NULL COMMENT 'The master log position of the last read event.', + `Host` char(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'The host name of the master.', + `User_name` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The user name used to connect to the master.', + `User_password` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The password used to connect to the master.', + `Port` int(10) unsigned NOT NULL COMMENT 'The network port used to connect to the master.', + `Connect_retry` int(10) unsigned NOT NULL COMMENT 'The period (in seconds) that the slave will wait before trying to reconnect to the master.', + `Enabled_ssl` tinyint(1) NOT NULL COMMENT 'Indicates whether the server supports SSL connections.', + `Ssl_ca` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The file used for the Certificate Authority (CA) certificate.', + `Ssl_capath` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The path to the Certificate Authority (CA) certificates.', + `Ssl_cert` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The name of the SSL certificate file.', + `Ssl_cipher` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The name of the cipher in use for the SSL connection.', + `Ssl_key` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The name of the SSL key file.', + `Ssl_verify_server_cert` tinyint(1) NOT NULL COMMENT 'Whether to verify the server certificate.', + `Heartbeat` float NOT NULL, + `Bind` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'Displays which interface is employed when connecting to the MySQL server', + `Ignored_server_ids` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The number of server IDs to be ignored, followed by the actual server IDs', + `Uuid` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The master server uuid.', + `Retry_count` bigint(20) unsigned NOT NULL COMMENT 'Number of reconnect attempts, to the master, before giving up.', + `Ssl_crl` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The file used for the Certificate Revocation List (CRL)', + `Ssl_crlpath` text CHARACTER SET utf8 COLLATE utf8_bin COMMENT 'The path used for Certificate Revocation List (CRL) files', + `Enabled_auto_position` tinyint(1) NOT NULL COMMENT 'Indicates whether GTIDs will be used to retrieve events from the master.', + PRIMARY KEY (`Host`,`Port`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Master Information'; + +CREATE TABLE IF NOT EXISTS `slave_relay_log_info` ( + `Number_of_lines` int(10) unsigned NOT NULL COMMENT 'Number of lines in the file or rows in the table. Used to version table definitions.', + `Relay_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'The name of the current relay log file.', + `Relay_log_pos` bigint(20) unsigned NOT NULL COMMENT 'The relay log position of the last executed event.', + `Master_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'The name of the master binary log file from which the events in the relay log file were read.', + `Master_log_pos` bigint(20) unsigned NOT NULL COMMENT 'The master log position of the last executed event.', + `Sql_delay` int(11) NOT NULL COMMENT 'The number of seconds that the slave must lag behind the master.', + `Number_of_workers` int(10) unsigned NOT NULL, + `Id` int(10) unsigned NOT NULL COMMENT 'Internal Id that uniquely identifies this record.', + PRIMARY KEY (`Id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Relay Log Information'; + + +CREATE TABLE IF NOT EXISTS `slave_worker_info` ( + `Id` int(10) unsigned NOT NULL, + `Relay_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `Relay_log_pos` bigint(20) unsigned NOT NULL, + `Master_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `Master_log_pos` bigint(20) unsigned NOT NULL, + `Checkpoint_relay_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `Checkpoint_relay_log_pos` bigint(20) unsigned NOT NULL, + `Checkpoint_master_log_name` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `Checkpoint_master_log_pos` bigint(20) unsigned NOT NULL, + `Checkpoint_seqno` int(10) unsigned NOT NULL, + `Checkpoint_group_size` int(10) unsigned NOT NULL, + `Checkpoint_group_bitmap` blob NOT NULL, + PRIMARY KEY (`Id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Worker Information'; + + +CREATE TABLE IF NOT EXISTS `help_category` ( + `help_category_id` smallint(5) unsigned NOT NULL, + `name` char(64) NOT NULL, + `parent_category_id` smallint(5) unsigned DEFAULT NULL, + `url` text NOT NULL, + PRIMARY KEY (`help_category_id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='help categories'; + +CREATE TABLE IF NOT EXISTS `help_keyword` ( + `help_keyword_id` int(10) unsigned NOT NULL, + `name` char(64) NOT NULL, + PRIMARY KEY (`help_keyword_id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='help keywords'; + +CREATE TABLE IF NOT EXISTS `help_relation` ( + `help_topic_id` int(10) unsigned NOT NULL, + `help_keyword_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`help_keyword_id`,`help_topic_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='keyword-topic relation'; + +CREATE TABLE IF NOT EXISTS `help_topic` ( + `help_topic_id` int(10) unsigned NOT NULL, + `name` char(64) NOT NULL, + `help_category_id` smallint(5) unsigned NOT NULL, + `description` text NOT NULL, + `example` text NOT NULL, + `url` text NOT NULL, + PRIMARY KEY (`help_topic_id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='help topics'; + +CREATE TABLE IF NOT EXISTS `time_zone` ( + `Time_zone_id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `Use_leap_seconds` enum('Y','N') NOT NULL DEFAULT 'N', + PRIMARY KEY (`Time_zone_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Time zones'; + +CREATE TABLE IF NOT EXISTS `time_zone_leap_second` ( + `Transition_time` bigint(20) NOT NULL, + `Correction` int(11) NOT NULL, + PRIMARY KEY (`Transition_time`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Leap seconds information for time zones'; + +CREATE TABLE IF NOT EXISTS `time_zone_name` ( + `Name` char(64) NOT NULL, + `Time_zone_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`Name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Time zone names'; + +CREATE TABLE IF NOT EXISTS `time_zone_transition` ( + `Time_zone_id` int(10) unsigned NOT NULL, + `Transition_time` bigint(20) NOT NULL, + `Transition_type_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`Time_zone_id`,`Transition_time`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Time zone transitions'; + +CREATE TABLE IF NOT EXISTS `time_zone_transition_type` ( + `Time_zone_id` int(10) unsigned NOT NULL, + `Transition_type_id` int(10) unsigned NOT NULL, + `Offset` int(11) NOT NULL DEFAULT '0', + `Is_DST` tinyint(3) unsigned NOT NULL DEFAULT '0', + `Abbreviation` char(8) NOT NULL DEFAULT '', + PRIMARY KEY (`Time_zone_id`,`Transition_type_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Time zone transition types'; + + diff --git a/sandbox/servers/pxc/5.6/my.sandbox.cnf b/sandbox/servers/pxc/5.6/my.sandbox.cnf new file mode 100644 index 00000000..ad2673d9 --- /dev/null +++ b/sandbox/servers/pxc/5.6/my.sandbox.cnf @@ -0,0 +1,42 @@ +[client] +user = msandbox +password = msandbox +port = PORT +socket = /tmp/PORT/mysql_sandboxPORT.sock + +[mysqld] +port = PORT +socket = /tmp/PORT/mysql_sandboxPORT.sock +pid-file = /tmp/PORT/data/mysql_sandboxPORT.pid +basedir = PERCONA_TOOLKIT_SANDBOX +datadir = /tmp/PORT/data +key_buffer_size = 16M +innodb_buffer_pool_size = 16M +innodb_data_home_dir = /tmp/PORT/data +innodb_log_group_home_dir = /tmp/PORT/data +innodb_data_file_path = ibdata1:10M:autoextend +innodb_log_file_size = 5M +log-bin = mysql-bin +relay_log = mysql-relay-bin +log_slave_updates +server-id = PORT +report-host = 127.0.0.1 +report-port = PORT +log-error = /tmp/PORT/data/mysqld.log +innodb_lock_wait_timeout = 3 +general_log +general_log_file = genlog + +binlog_format = ROW +wsrep_provider = LIBGALERA +wsrep_cluster_address = CLUSTER_AD +wsrep_sst_receive_address = ADDR:RECEIVE_PRT +wsrep_node_incoming_address= ADDR:PORT +wsrep_slave_threads = 2 +wsrep_cluster_name = CLUSTER_NAME +wsrep_provider_options = "gmcast.listen_addr=tcp://ADDR:LISTEN_PRT;" +wsrep_sst_method = rsync +wsrep_node_name = PORT +innodb_locks_unsafe_for_binlog = 1 +innodb_autoinc_lock_mode = 2 +wsrep-replicate-myisam diff --git a/t/lib/NibbleIterator.t b/t/lib/NibbleIterator.t index 701f2698..d9ada0e8 100644 --- a/t/lib/NibbleIterator.t +++ b/t/lib/NibbleIterator.t @@ -389,41 +389,59 @@ $ni = make_nibble_iter( select => $chunk_checksum, ); +# The following tests need a trick to make the timestamp column consistent +# across different test servers. +# Sakila uses '2006-02-15 11:44:00' for this column, which is converted to +# a different epoch vaule in different timezones, resulting in different +# checksum values according to the tz of the server where sakila was created. + +# save original value, just in case +my ($orig_datetime) = $dbh->selectrow_array("SELECT last_update FROM sakila.country LIMIT 1"); +# get locat datetime for UTC 2006-02-15 11:44:00 +my ($local_datetime_for_fixed_timestamp) = $dbh->selectrow_array("SELECT FROM_UNIXTIME(1140003840)"); +$dbh->do("UPDATE sakila.country SET last_update = '$local_datetime_for_fixed_timestamp'"); + +# now the following checksums are fixed, no matter the test server timezone +# where the sakila database was created my $row = $ni->next(); is_deeply( $row, - [25, 'd9c52498'], + [25, 'a947cb12'], "SELECT chunk checksum 1 FROM sakila.country" ) or diag(Dumper($row)); $row = $ni->next(); is_deeply( $row, - [25, 'ebdc982c'], + [25, 'c32790b9'], "SELECT chunk checksum 2 FROM sakila.country" ) or diag(Dumper($row)); $row = $ni->next(); is_deeply( $row, - [25, 'e8d9438d'], + [25, 'ea52549f'], "SELECT chunk checksum 3 FROM sakila.country" ) or diag(Dumper($row)); $row = $ni->next(); is_deeply( $row, - [25, '2e3b895d'], + [25, 'e78a7363'], "SELECT chunk checksum 4 FROM sakila.country" ) or diag(Dumper($row)); $row = $ni->next(); is_deeply( $row, - [9, 'bd08fd55'], + [9, 'd97ccb0d'], "SELECT chunk checksum 5 FROM sakila.country" ) or diag(Dumper($row)); +# revert timestamp to original value, in case other tests use it +$dbh->do("UPDATE sakila.country SET last_update = '$orig_datetime'"); + + # ######################################################################### # exec_nibble callback and explain_sth # ######################################################################### diff --git a/t/lib/RowChecksum.t b/t/lib/RowChecksum.t index 3ef193ea..88403d33 100644 --- a/t/lib/RowChecksum.t +++ b/t/lib/RowChecksum.t @@ -126,11 +126,11 @@ is( tbl => $tbl, func => 'SHA1', ), - q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, } + q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, UNIX_TIMESTAMP(`last_update`) AS `last_update`, } . q{SHA1(CONCAT_WS('#', } . q{`film_id`, `title`, `description`, `release_year`, `language_id`, } . q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, } - . q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0, } + . q{`replacement_cost`, `rating`, `special_features`, UNIX_TIMESTAMP(`last_update`), } . q{CONCAT(ISNULL(`description`), ISNULL(`release_year`), } . q{ISNULL(`original_language_id`), ISNULL(`length`), } . q{ISNULL(`rating`), ISNULL(`special_features`))))}, @@ -142,11 +142,11 @@ is( tbl => $tbl, func => 'FNV_64', ), - q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, } + q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, UNIX_TIMESTAMP(`last_update`) AS `last_update`, } . q{FNV_64(} . q{`film_id`, `title`, `description`, `release_year`, `language_id`, } . q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, } - . q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0)}, + . q{`replacement_cost`, `rating`, `special_features`, UNIX_TIMESTAMP(`last_update`))}, 'FNV_64 query for sakila.film', ); diff --git a/t/lib/bash/parse_options.sh b/t/lib/bash/parse_options.sh index ca6326c8..4beba5a0 100644 --- a/t/lib/bash/parse_options.sh +++ b/t/lib/bash/parse_options.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -plan 83 +plan 84 TMPFILE="$TEST_PT_TMPDIR/parse-opts-output" TOOL="pt-stalk" @@ -258,6 +258,19 @@ is "$OPT_NOTIFY_BY_EMAIL" "" "Bug 1038995: --notify-by-email is empty by default parse_options "$T_LIB_DIR/samples/bash/po005.sh" --notify-by-email foo@bar.com is "$OPT_NOTIFY_BY_EMAIL" "foo@bar.com" "Bug 1038995: ...but gets set without errors if specified" +# ############################################################################ +# Bug 1266869: fails when $HOME unset +# https://bugs.launchpad.net/percona-toolkit/+bug/1266869 +# ############################################################################ + +TMP_HOME="$HOME" +unset HOME +OUTPUT=`parse_options $T_LIB_DIR/samples/bash/po001.sh 2>&1` +echo "$OUTPUT" > "$TMPFILE" +cmd_ok "grep -q -v unbound $TMPFILE" "No error when \$HOME is not set" +HOME="$TMP_HOME" # just in case further tests below need it + + # ############################################################################ # Done # ############################################################################ diff --git a/t/pt-kill/match.t b/t/pt-kill/match.t index 9fbadf51..47be0382 100644 --- a/t/pt-kill/match.t +++ b/t/pt-kill/match.t @@ -9,7 +9,7 @@ BEGIN { use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); -use Test::More tests => 15; +use Test::More tests => 16; use PerconaTest; use Sandbox; @@ -137,6 +137,16 @@ like( "--match-all" ); +# --query-id option +$output = output( + sub { pt_kill::main(@args, "$trunk/t/lib/samples/pl/recset011.txt", qw(--match-all --print --query-id)); } +); +like( + $output, + qr/0x69962191E64980E6/, + '--query-id' +); + # ############################################################################# # Live tests. # ############################################################################# diff --git a/t/pt-slave-restart/gtid_parallelreplication.t b/t/pt-slave-restart/gtid_parallelreplication.t index 439870f4..de5276c7 100644 --- a/t/pt-slave-restart/gtid_parallelreplication.t +++ b/t/pt-slave-restart/gtid_parallelreplication.t @@ -49,7 +49,7 @@ my $output=`$trunk/bin/pt-slave-restart --run-time=1s -h 127.0.0.1 -P 12346 -u m like( $output, - qr/It is impossible to skip transactions properly./, + qr/Cannot skip transactions properly.*slave_parallel_workers/, "pt-slave-restart exits with multiple replication threads" ); diff --git a/t/pt-table-checksum/basics.t b/t/pt-table-checksum/basics.t index 056fbd1b..de2eb369 100644 --- a/t/pt-table-checksum/basics.t +++ b/t/pt-table-checksum/basics.t @@ -56,6 +56,7 @@ sub reset_repl_db { $master_dbh->do("use $repl_db"); } + # ############################################################################ # Default checksum and results. The tool does not technically require any # options on well-configured systems (which the test env cannot be). With @@ -508,6 +509,24 @@ is( "Bug 821675 (dot): 0 errors" ); +# ############################################################################# +# Bug 1019479: does not work with sql_mode ONLY_FULL_GROUP_BY +# ############################################################################# + +# add a couple more modes to test that commas don't affect setting +$master_dbh->do("SET sql_mode = 'NO_ZERO_DATE,ONLY_FULL_GROUP_BY,STRICT_ALL_TABLES'"); + +# force chunk-size because bug doesn't show up if table done in one chunk +$exit_status = pt_table_checksum::main(@args, + qw(--quiet --quiet -t sakila.actor --chunk-size=50)); + +is( + $exit_status, + 0, + "sql_mode ONLY_FULL_GROUP_BY is overidden" +); + + # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-table-checksum/bugs.t b/t/pt-table-checksum/bugs.t index cf67a889..2d9fb00d 100644 --- a/t/pt-table-checksum/bugs.t +++ b/t/pt-table-checksum/bugs.t @@ -294,6 +294,31 @@ like( "Bug 1210537: tool ran" ); +# ############################################################################# +# pt-table-checksum has errors when slaves have different system_time_zone +# https://bugs.launchpad.net/percona-toolkit/+bug/1388870 +# ############################################################################# + +# make slave set diferent system_time_zone by changing env var TZ. +diag(`/tmp/12346/stop >/dev/null`); +diag(`export TZ='HST';/tmp/12346/start >/dev/null`); + +$output = output( + sub { pt_table_checksum::main(@args, qw(-t sakila.payment)) }, +); + + +is( + PerconaTest::count_checksum_results($output, 'diffs'), + 0, + "Bug 1388870 - No false positive reported when system_tz differ on slave" +); + +# restore slave to original system_tz +diag(`/tmp/12346/stop >/dev/null`); +diag(`/tmp/12346/start >/dev/null`); + + # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-table-checksum/pxc.t b/t/pt-table-checksum/pxc.t index bfca0b3e..ea25ffb3 100644 --- a/t/pt-table-checksum/pxc.t +++ b/t/pt-table-checksum/pxc.t @@ -88,8 +88,8 @@ like( ); ok ( - $output =~ qr/WARNING/i && !$exit_status, - "Warns but doesn't die if --recursion-method=none - issue #1373937" + $output !~ qr/no other nodes or regular replicas were found/i && !$exit_status, + "checksums even if --recursion-method=none - issue 1373937" ); for my $args ( @@ -159,6 +159,7 @@ sub test_recursion_methods { my $same_ids = shift; my ($orig_id_1, $orig_id_2, $orig_id_3); + my ($orig_ia_1, $orig_ia_2, $orig_ia_3); if ($same_ids) { # save original values @@ -171,6 +172,19 @@ sub test_recursion_methods { $node1->do($sql); $node2->do($sql); $node3->do($sql); + + # since we're testing server id issues, set wsrep_node_incoming_address=AUTO ( https://bugs.launchpad.net/percona-toolkit/+bug/1399789 ) + # save original values + $sql = 'SELECT @@wsrep_node_incoming_address'; + ($orig_ia_1) = $node1->selectrow_array($sql); + ($orig_ia_2) = $node2->selectrow_array($sql); + ($orig_ia_3) = $node3->selectrow_array($sql); + # set wsrep_node_incoming_address value to AUTO on all nodes + $sql = 'SET GLOBAL wsrep_node_incoming_address = AUTO'; + $node1->do($sql); + $node2->do($sql); + $node3->do($sql); + } for my $args ( @@ -227,6 +241,10 @@ sub test_recursion_methods { $node1->do("SET GLOBAL server_id = $orig_id_1"); $node2->do("SET GLOBAL server_id = $orig_id_2"); $node3->do("SET GLOBAL server_id = $orig_id_3"); + # reset node wsrep_node_incoming_address to original values + $node1->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_1'"); + $node2->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_2'"); + $node3->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_3'"); } } diff --git a/t/pt-table-checksum/samples/chunkidx004.txt b/t/pt-table-checksum/samples/chunkidx004.txt index b3ceb684..31b89632 100644 --- a/t/pt-table-checksum/samples/chunkidx004.txt +++ b/t/pt-table-checksum/samples/chunkidx004.txt @@ -2,7 +2,7 @@ -- sakila.city -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `city_id`, `city`, `country_id`, `last_update` + 0)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`city` FORCE INDEX(`idx_fk_country_id`) WHERE ((`country_id` >= ?)) AND ((`country_id` <= ?)) AND (country_id > 100) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `city_id`, `city`, `country_id`, UNIX_TIMESTAMP(`last_update`))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`city` FORCE INDEX(`idx_fk_country_id`) WHERE ((`country_id` >= ?)) AND ((`country_id` <= ?)) AND (country_id > 100) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `sakila`.`city` FORCE INDEX(`idx_fk_country_id`) WHERE ((`country_id` < ?)) AND (country_id > 100) ORDER BY `country_id` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/chunkidx005.txt b/t/pt-table-checksum/samples/chunkidx005.txt index ae55d193..a6177f19 100644 --- a/t/pt-table-checksum/samples/chunkidx005.txt +++ b/t/pt-table-checksum/samples/chunkidx005.txt @@ -2,7 +2,7 @@ -- sakila.city -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `city_id`, `city`, `country_id`, `last_update` + 0)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`city` FORCE INDEX(`PRIMARY`) WHERE ((`city_id` >= ?)) AND ((`city_id` <= ?)) AND (country_id > 100) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `city_id`, `city`, `country_id`, UNIX_TIMESTAMP(`last_update`))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`city` FORCE INDEX(`PRIMARY`) WHERE ((`city_id` >= ?)) AND ((`city_id` <= ?)) AND (country_id > 100) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `sakila`.`city` FORCE INDEX(`PRIMARY`) WHERE ((`city_id` < ?)) AND (country_id > 100) ORDER BY `city_id` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/default-results-5.7.txt b/t/pt-table-checksum/samples/default-results-5.7.txt new file mode 100644 index 00000000..55c0bddd --- /dev/null +++ b/t/pt-table-checksum/samples/default-results-5.7.txt @@ -0,0 +1,41 @@ +ERRORS DIFFS ROWS SKIPPED TABLE +0 0 0 0 mysql.columns_priv +0 0 0 0 mysql.db +0 0 0 0 mysql.event +0 0 0 0 mysql.func +0 0 0 0 mysql.help_category +0 0 0 0 mysql.help_keyword +0 0 0 0 mysql.help_relation +0 0 0 0 mysql.help_topic +0 0 0 0 mysql.ndb_binlog_index +0 0 0 0 mysql.plugin +0 0 0 0 mysql.proc +0 0 0 0 mysql.procs_priv +0 0 1 0 mysql.proxies_priv +0 0 0 0 mysql.servers +0 0 0 0 mysql.tables_priv +0 0 0 0 mysql.time_zone +0 0 0 0 mysql.time_zone_leap_second +0 0 0 0 mysql.time_zone_name +0 0 0 0 mysql.time_zone_transition +0 0 0 0 mysql.time_zone_transition_type +0 0 8 0 mysql.user +0 0 18 0 percona_test.checksums +0 0 1 0 percona_test.load_data +0 0 1 0 percona_test.sentinel +0 0 200 0 sakila.actor +0 0 603 0 sakila.address +0 0 16 0 sakila.category +0 0 600 0 sakila.city +0 0 109 0 sakila.country +0 0 599 0 sakila.customer +0 0 1000 0 sakila.film +0 0 5462 0 sakila.film_actor +0 0 1000 0 sakila.film_category +0 0 1000 0 sakila.film_text +0 0 4581 0 sakila.inventory +0 0 6 0 sakila.language +0 0 16049 0 sakila.payment +0 0 16044 0 sakila.rental +0 0 2 0 sakila.staff +0 0 2 0 sakila.store diff --git a/t/pt-table-checksum/samples/n-chunk-index-cols.txt b/t/pt-table-checksum/samples/n-chunk-index-cols.txt index f08f80ea..b8d9e17e 100644 --- a/t/pt-table-checksum/samples/n-chunk-index-cols.txt +++ b/t/pt-table-checksum/samples/n-chunk-index-cols.txt @@ -2,7 +2,7 @@ -- sakila.rental -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `rental_id`, `rental_date`, `inventory_id`, `customer_id`, `return_date`, `staff_id`, `last_update` + 0, CONCAT(ISNULL(`return_date`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`rental` FORCE INDEX(`rental_date`) WHERE ((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` >= ?)) AND ((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` <= ?)) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `rental_id`, `rental_date`, `inventory_id`, `customer_id`, `return_date`, `staff_id`, UNIX_TIMESTAMP(`last_update`), CONCAT(ISNULL(`return_date`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `sakila`.`rental` FORCE INDEX(`rental_date`) WHERE ((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` >= ?)) AND ((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` <= ?)) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `sakila`.`rental` FORCE INDEX(`rental_date`) WHERE ((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` < ?)) ORDER BY `rental_date`, `inventory_id`, `customer_id` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt b/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt new file mode 100644 index 00000000..418b538f --- /dev/null +++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt @@ -0,0 +1,41 @@ +ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE +0 0 0 1 0 mysql.columns_priv +0 0 0 1 0 mysql.db +0 0 0 1 0 mysql.event +0 0 0 1 0 mysql.func +0 0 0 1 0 mysql.help_category +0 0 0 1 0 mysql.help_keyword +0 0 0 1 0 mysql.help_relation +0 0 0 1 0 mysql.help_topic +0 0 0 1 0 mysql.ndb_binlog_index +0 0 0 1 0 mysql.plugin +0 0 0 1 0 mysql.proc +0 0 0 1 0 mysql.procs_priv +0 0 1 1 0 mysql.proxies_priv +0 0 0 1 0 mysql.servers +0 0 0 1 0 mysql.tables_priv +0 0 0 1 0 mysql.time_zone +0 0 0 1 0 mysql.time_zone_leap_second +0 0 0 1 0 mysql.time_zone_name +0 0 0 1 0 mysql.time_zone_transition +0 0 0 1 0 mysql.time_zone_transition_type +0 0 8 1 0 mysql.user +0 0 18 1 0 percona_test.checksums +0 0 1 1 0 percona_test.load_data +0 0 1 1 0 percona_test.sentinel +0 0 200 1 0 sakila.actor +0 0 603 1 0 sakila.address +0 0 16 1 0 sakila.category +0 0 600 1 0 sakila.city +0 0 109 1 0 sakila.country +0 0 599 1 0 sakila.customer +0 0 1000 1 0 sakila.film +0 0 5462 8 0 sakila.film_actor +0 0 1000 1 0 sakila.film_category +0 0 1000 1 0 sakila.film_text +0 0 4581 7 0 sakila.inventory +0 0 6 1 0 sakila.language +0 0 16049 19 0 sakila.payment +0 0 16044 19 0 sakila.rental +0 0 2 1 0 sakila.staff +0 0 2 1 0 sakila.store