percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-19 02:05:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pt-dupe-key-fixes.

Browse Source
This commit is contained in:
Daniel Nichter
2013-12-11 19:49:47 -08:00
parent c726b8e624 e89ce13e97
commit 54e9522771
13 changed files with 806 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

431
bin/pt-duplicate-key-checker
View File

@@ -23,7 +23,7 @@ BEGIN {
Daemon
Schema
SchemaIterator
HTTPMicro
HTTP::Micro
VersionCheck
));
}
@@ -133,6 +133,8 @@ sub quote_val {
return $val if $val =~ m/^0x[0-9a-fA-F]+$/ # quote hex data
&& !$args{is_char}; # unless is_char is true
return $val if $args{is_float};
$val =~ s/(['\\])/\\$1/g;
return "'$val'";
}
@@ -2203,7 +2205,7 @@ sub get_key_size {
foreach my $col ( @cols ) {
push @where_cols, "$col=1";
}
if ( scalar @cols == 1 ) {
if ( scalar(@cols) == 1 && !$args{only_eq} ) {
push @where_cols, "$cols[0]<>1";
}
$sql .= join(' OR ', @where_cols);
@@ -2227,6 +2229,21 @@ sub get_key_size {
PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len,
'rows:', $rows);
if ( $chosen_key && $key_len eq '0' ) {
if ( $args{recurse} ) {
$self->{error} = "key_len = 0 in EXPLAIN:\n"
. _explain_to_text($explain);
return;
}
else {
return $self->get_key_size(
%args,
only_eq => 1,
recurse => 1,
);
}
}
my $key_size = 0;
if ( $key_len && $rows ) {
if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) {
@@ -2370,7 +2387,6 @@ sub get_duplicate_keys {
push @dupes,
$self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1);
my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}}
: undef;
PTDEBUG && _d('clustered key:',
@@ -2505,10 +2521,10 @@ sub remove_prefix_duplicates {
PTDEBUG && _d('Remove', $right_name);
my $reason;
if ( $right_keys->[$right_index]->{unconstrained} ) {
if ( my $type = $right_keys->[$right_index]->{unconstrained} ) {
$reason .= "Uniqueness of $right_name ignored because "
. $right_keys->[$right_index]->{constraining_key}->{name}
. " is a stronger constraint\n";
. " is a $type constraint\n";
}
my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1;
$reason .= $right_name
@@ -2608,11 +2624,18 @@ sub unconstrain_keys {
next unless $unique_key; # primary key may be undefined
my $cols = $unique_key->{cols};
if ( @$cols == 1 ) {
PTDEBUG && _d($unique_key->{name},'defines unique column:',$cols->[0]);
if ( !exists $unique_cols{$cols->[0]} ) {
PTDEBUG && _d($unique_key->{name}, 'defines unique column:',
$cols->[0]);
$unique_cols{$cols->[0]} = $unique_key;
$unique_key->{unique_col} = 1;
}
else {
PTDEBUG && _d($unique_key->{name},
'redundantly constrains unique column:', $cols->[0]);
$unique_key->{exact_dupe} = 1;
$unique_key->{constraining_key} = $unique_cols{$cols->[0]};
}
}
else {
local $LIST_SEPARATOR = '-';
@@ -2646,13 +2669,19 @@ sub unconstrain_keys {
for my $i ( 0..(scalar @$unique_keys-1) ) {
if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) {
PTDEBUG && _d('Unconstraining', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 1;
PTDEBUG && _d('Unconstraining weak', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'stronger';
$unique_keys->[$i]->{constraining_key}
= $unconstrain{$unique_keys->[$i]->{name}};
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
elsif ( $unique_keys->[$i]->{exact_dupe} ) {
PTDEBUG && _d('Unconstraining dupe', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'duplicate';
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
}
PTDEBUG && _d('No more keys');
@@ -2687,59 +2716,79 @@ package Daemon;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use POSIX qw(setsid);
use Fcntl qw(:DEFAULT);
sub new {
my ( $class, %args ) = @_;
foreach my $arg ( qw(o) ) {
die "I need a $arg argument" unless $args{$arg};
}
my $o = $args{o};
my ($class, %args) = @_;
my $self = {
o => $o,
log_file => $o->has('log') ? $o->get('log') : undef,
PID_file => $o->has('pid') ? $o->get('pid') : undef,
log_file => $args{log_file},
pid_file => $args{pid_file},
daemonize => $args{daemonize},
force_log_file => $args{force_log_file},
parent_exit => $args{parent_exit},
pid_file_owner => 0,
};
check_PID_file(undef, $self->{PID_file});
PTDEBUG && _d('Daemonized child will log to', $self->{log_file});
return bless $self, $class;
}
sub daemonize {
my ( $self ) = @_;
sub run {
my ($self) = @_;
PTDEBUG && _d('About to fork and daemonize');
defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR";
if ( $pid ) {
PTDEBUG && _d('Parent PID', $PID, 'exiting after forking child PID',$pid);
exit;
my $daemonize = $self->{daemonize};
my $pid_file = $self->{pid_file};
my $log_file = $self->{log_file};
my $force_log_file = $self->{force_log_file};
my $parent_exit = $self->{parent_exit};
PTDEBUG && _d('Starting daemon');
if ( $pid_file ) {
eval {
$self->_make_pid_file(
pid => $PID, # parent's pid
pid_file => $pid_file,
);
};
die "$EVAL_ERROR\n" if $EVAL_ERROR;
if ( !$daemonize ) {
$self->{pid_file_owner} = $PID; # parent's pid
}
}
PTDEBUG && _d('Daemonizing child PID', $PID);
$self->{PID_owner} = $PID;
$self->{child} = 1;
if ( $daemonize ) {
defined (my $child_pid = fork()) or die "Cannot fork: $OS_ERROR";
if ( $child_pid ) {
PTDEBUG && _d('Forked child', $child_pid);
$parent_exit->($child_pid) if $parent_exit;
exit 0;
}
POSIX::setsid() or die "Cannot start a new session: $OS_ERROR";
chdir '/' or die "Cannot chdir to /: $OS_ERROR";
$self->_make_PID_file();
$OUTPUT_AUTOFLUSH = 1;
if ( $pid_file ) {
$self->_update_pid_file(
pid => $PID, # child's pid
pid_file => $pid_file,
);
$self->{pid_file_owner} = $PID;
}
}
if ( $daemonize || $force_log_file ) {
PTDEBUG && _d('Redirecting STDIN to /dev/null');
close STDIN;
open STDIN, '/dev/null'
or die "Cannot reopen STDIN to /dev/null: $OS_ERROR";
if ( $self->{log_file} ) {
PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file});
if ( $log_file ) {
PTDEBUG && _d('Redirecting STDOUT and STDERR to', $log_file);
close STDOUT;
open STDOUT, '>>', $self->{log_file}
or die "Cannot open log file $self->{log_file}: $OS_ERROR";
open STDOUT, '>>', $log_file
or die "Cannot open log file $log_file: $OS_ERROR";
close STDERR;
open STDERR, ">&STDOUT"
@@ -2762,82 +2811,119 @@ sub daemonize {
}
}
$OUTPUT_AUTOFLUSH = 1;
}
PTDEBUG && _d('Daemon running');
return;
}
sub check_PID_file {
my ( $self, $file ) = @_;
my $PID_file = $self ? $self->{PID_file} : $file;
PTDEBUG && _d('Checking PID file', $PID_file);
if ( $PID_file && -f $PID_file ) {
my $pid;
eval {
chomp($pid = (slurp_file($PID_file) || ''));
sub _make_pid_file {
my ($self, %args) = @_;
my @required_args = qw(pid pid_file);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
if ( $EVAL_ERROR ) {
die "The PID file $PID_file already exists but it cannot be read: "
. $EVAL_ERROR;
my $pid = $args{pid};
my $pid_file = $args{pid_file};
eval {
sysopen(PID_FH, $pid_file, O_RDWR|O_CREAT|O_EXCL) or die $OS_ERROR;
print PID_FH $PID, "\n";
close PID_FH;
};
if ( my $e = $EVAL_ERROR ) {
if ( $e =~ m/file exists/i ) {
my $old_pid = $self->_check_pid_file(
pid_file => $pid_file,
pid => $PID,
);
if ( $old_pid ) {
warn "Overwriting PID file $pid_file because PID $old_pid "
. "is not running.\n";
}
PTDEBUG && _d('PID file exists; it contains PID', $pid);
if ( $pid ) {
my $pid_is_alive = kill 0, $pid;
$self->_update_pid_file(
pid => $PID,
pid_file => $pid_file
);
}
else {
die "Error creating PID file $pid_file: $e\n";
}
}
return;
}
sub _check_pid_file {
my ($self, %args) = @_;
my @required_args = qw(pid_file pid);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid_file = $args{pid_file};
my $pid = $args{pid};
PTDEBUG && _d('Checking if PID in', $pid_file, 'is running');
if ( ! -f $pid_file ) {
PTDEBUG && _d('PID file', $pid_file, 'does not exist');
return;
}
open my $fh, '<', $pid_file
or die "Error opening $pid_file: $OS_ERROR";
my $existing_pid = do { local $/; <$fh> };
chomp($existing_pid) if $existing_pid;
close $fh
or die "Error closing $pid_file: $OS_ERROR";
if ( $existing_pid ) {
if ( $existing_pid == $pid ) {
warn "The current PID $pid already holds the PID file $pid_file\n";
return;
}
else {
PTDEBUG && _d('Checking if PID', $existing_pid, 'is running');
my $pid_is_alive = kill 0, $existing_pid;
if ( $pid_is_alive ) {
die "The PID file $PID_file already exists "
. " and the PID that it contains, $pid, is running";
die "PID file $pid_file exists and PID $existing_pid is running\n";
}
else {
warn "Overwriting PID file $PID_file because the PID that it "
. "contains, $pid, is not running";
}
}
else {
die "The PID file $PID_file already exists but it does not "
. "contain a PID";
}
}
else {
PTDEBUG && _d('No PID file');
die "PID file $pid_file exists but it is empty. Remove the file "
. "if the process is no longer running.\n";
}
return $existing_pid;
}
sub _update_pid_file {
my ($self, %args) = @_;
my @required_args = qw(pid pid_file);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid = $args{pid};
my $pid_file = $args{pid_file};
open my $fh, '>', $pid_file
or die "Cannot open $pid_file: $OS_ERROR";
print { $fh } $pid, "\n"
or die "Cannot print to $pid_file: $OS_ERROR";
close $fh
or warn "Cannot close $pid_file: $OS_ERROR";
return;
}
sub make_PID_file {
my ( $self ) = @_;
if ( exists $self->{child} ) {
die "Do not call Daemon::make_PID_file() for daemonized scripts";
}
$self->_make_PID_file();
$self->{PID_owner} = $PID;
return;
}
sub _make_PID_file {
my ( $self ) = @_;
my $PID_file = $self->{PID_file};
if ( !$PID_file ) {
PTDEBUG && _d('No PID file to create');
return;
}
$self->check_PID_file();
open my $PID_FH, '>', $PID_file
or die "Cannot open PID file $PID_file: $OS_ERROR";
print $PID_FH $PID
or die "Cannot print to PID file $PID_file: $OS_ERROR";
close $PID_FH
or die "Cannot close PID file $PID_file: $OS_ERROR";
PTDEBUG && _d('Created PID file:', $self->{PID_file});
return;
}
sub _remove_PID_file {
my ( $self ) = @_;
if ( $self->{PID_file} && -f $self->{PID_file} ) {
unlink $self->{PID_file}
or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR";
sub remove_pid_file {
my ($self, $pid_file) = @_;
$pid_file ||= $self->{pid_file};
if ( $pid_file && -f $pid_file ) {
unlink $self->{pid_file}
or warn "Cannot remove PID file $pid_file: $OS_ERROR";
PTDEBUG && _d('Removed PID file');
}
else {
@@ -2847,20 +2933,15 @@ sub _remove_PID_file {
}
sub DESTROY {
my ( $self ) = @_;
my ($self) = @_;
$self->_remove_PID_file() if ($self->{PID_owner} || 0) == $PID;
if ( $self->{pid_file_owner} == $PID ) {
$self->remove_pid_file();
}
return;
}
sub slurp_file {
my ($file) = @_;
return unless $file;
open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
return do { local $/; <$fh> };
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
@@ -3543,25 +3624,23 @@ sub _d {
# ###########################################################################
# ###########################################################################
# HTTPMicro package
# HTTP::Micro package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/HTTPMicro.pm
# t/lib/HTTPMicro.t
# lib/HTTP/Micro.pm
# t/lib/HTTP/Micro.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
package HTTP::Micro;
our $VERSION = '0.01';
package HTTPMicro;
BEGIN {
$HTTPMicro::VERSION = '0.001';
}
use strict;
use warnings;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Carp ();
my @attributes;
BEGIN {
@attributes = qw(agent timeout);
@@ -3632,7 +3711,7 @@ sub _request {
headers => {},
};
my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout});
my $handle = HTTP::Micro::Handle->new(timeout => $self->{timeout});
$handle->connect($scheme, $host, $port);
@@ -3697,27 +3776,31 @@ sub _split_url {
return ($scheme, $host, $port, $path_query);
}
package
HTTPMicro::Handle; # hide from PAUSE/indexers
use strict;
use warnings;
} # HTTP::Micro
use Carp qw[croak];
use Errno qw[EINTR EPIPE];
use IO::Socket qw[SOCK_STREAM];
{
package HTTP::Micro::Handle;
sub BUFSIZE () { 32768 }
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
my $Printable = sub {
use Carp qw(croak);
use Errno qw(EINTR EPIPE);
use IO::Socket qw(SOCK_STREAM);
sub BUFSIZE () { 32768 }
my $Printable = sub {
local $_ = shift;
s/\r/\\r/g;
s/\n/\\n/g;
s/\t/\\t/g;
s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge;
$_;
};
};
sub new {
sub new {
my ($class, %args) = @_;
return bless {
rbuf => '',
@@ -3725,15 +3808,15 @@ sub new {
max_line_size => 16384,
%args
}, $class;
}
}
my $ssl_verify_args = {
my $ssl_verify_args = {
check_cn => "when_only",
wildcards_in_alt => "anywhere",
wildcards_in_cn => "anywhere"
};
};
sub connect {
sub connect {
@_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/);
my ($self, $scheme, $host, $port) = @_;
@@ -3747,7 +3830,7 @@ sub connect {
croak(qq/Unsupported URL scheme '$scheme'\n/);
}
$self->{fh} = 'IO::Socket::INET'->new(
$self->{fh} = IO::Socket::INET->new(
PeerHost => $host,
PeerPort => $port,
Proto => 'tcp',
@@ -3776,16 +3859,16 @@ sub connect {
$self->{port} = $port;
return $self;
}
}
sub close {
sub close {
@_ == 1 || croak(q/Usage: $handle->close()/);
my ($self) = @_;
CORE::close($self->{fh})
or croak(qq/Could not close socket: '$!'/);
}
}
sub write {
sub write {
@_ == 2 || croak(q/Usage: $handle->write(buf)/);
my ($self, $buf) = @_;
@@ -3811,9 +3894,9 @@ sub write {
}
}
return $off;
}
}
sub read {
sub read {
@_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/);
my ($self, $len) = @_;
@@ -3842,9 +3925,9 @@ sub read {
croak(q/Unexpected end of stream/);
}
return $buf;
}
}
sub readline {
sub readline {
@_ == 1 || croak(q/Usage: $handle->readline()/);
my ($self) = @_;
@@ -3863,9 +3946,9 @@ sub readline {
}
}
croak(q/Unexpected end of stream while looking for line/);
}
}
sub read_header_lines {
sub read_header_lines {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/);
my ($self, $headers) = @_;
$headers ||= {};
@@ -3894,9 +3977,9 @@ sub read_header_lines {
}
}
return $headers;
}
}
sub write_header_lines {
sub write_header_lines {
(@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/);
my($self, $headers) = @_;
@@ -3910,9 +3993,9 @@ sub write_header_lines {
}
$buf .= "\x0D\x0A";
return $self->write($buf);
}
}
sub read_content_body {
sub read_content_body {
@_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/);
my ($self, $cb, $response, $len) = @_;
$len ||= $response->{headers}{'content-length'};
@@ -3927,9 +4010,9 @@ sub read_content_body {
}
return;
}
}
sub write_content_body {
sub write_content_body {
@_ == 2 || croak(q/Usage: $handle->write_content_body(request)/);
my ($self, $request) = @_;
my ($len, $content_length) = (0, $request->{headers}{'content-length'});
@@ -3940,9 +4023,9 @@ sub write_content_body {
or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/);
return $len;
}
}
sub read_response_header {
sub read_response_header {
@_ == 1 || croak(q/Usage: $handle->read_response_header()/);
my ($self) = @_;
@@ -3959,17 +4042,17 @@ sub read_response_header {
headers => $self->read_header_lines,
protocol => $protocol,
};
}
}
sub write_request_header {
sub write_request_header {
@_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/);
my ($self, $method, $request_uri, $headers) = @_;
return $self->write("$method $request_uri HTTP/1.1\x0D\x0A")
+ $self->write_header_lines($headers);
}
}
sub _do_timeout {
sub _do_timeout {
my ($self, $type, $timeout) = @_;
$timeout = $self->{timeout}
unless defined $timeout && $timeout >= 0;
@@ -3998,19 +4081,20 @@ sub _do_timeout {
}
$! = 0;
return $nfound;
}
}
sub can_read {
sub can_read {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/);
my $self = shift;
return $self->_do_timeout('read', @_)
}
}
sub can_write {
sub can_write {
@_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/);
my $self = shift;
return $self->_do_timeout('write', @_)
}
}
} # HTTP::Micro::Handle
my $prog = <<'EOP';
BEGIN {
@@ -4031,6 +4115,7 @@ BEGIN {
}
}
{
use Carp qw(croak);
my %dispatcher = (
issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) },
subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) },
@@ -4186,9 +4271,8 @@ if ( $INC{"IO/Socket/SSL.pm"} ) {
}
1;
}
# ###########################################################################
# End HTTPMicro package
# End HTTP::Micro package
# ###########################################################################
# ###########################################################################
@@ -4222,7 +4306,7 @@ use FindBin qw();
eval {
require Percona::Toolkit;
require HTTPMicro;
require HTTP::Micro;
};
{
@@ -4453,7 +4537,7 @@ sub pingback {
my $url = $args{url};
my $instances = $args{instances};
my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
my $response = $ua->request('GET', $url);
PTDEBUG && _d('Server response:', Dumper($response));
@@ -4832,14 +4916,11 @@ sub main {
# ########################################################################
# If --pid, check it first since we'll die if it already exits.
# ########################################################################
my $daemon;
if ( $o->get('pid') ) {
# We're not daemoninzing, it just handles PID stuff. Keep $daemon
# in the the scope of main() because when it's destroyed it automatically
# removes the PID file.
$daemon = new Daemon(o=>$o);
$daemon->make_PID_file();
}
my $daemon = Daemon->new(
daemonize => 0,
pid_file => $o->get('pid'),
);
$daemon->run();
# #######################################################################
# Connect to MySQL.

38
lib/DuplicateKeyFinder.pm
View File

@@ -143,8 +143,6 @@ sub get_duplicate_keys {
push @dupes,
$self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1);
# TODO: other structs
# Remove clustered duplicates.
my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}}
: undef;
@@ -314,7 +312,7 @@ sub remove_prefix_duplicates {
if ( substr($left_cols, 0, $right_len_cols)
eq substr($right_cols, 0, $right_len_cols) ) {
# FULLTEXT keys, for example, are only duplicates if they
# UNIQUE and FULLTEXT indexes are only duplicates if they
# are exact duplicates.
if ( $args{exact_duplicates} && ($right_len_cols<$left_len_cols) ) {
PTDEBUG && _d($right_name, 'not exact duplicate of', $left_name);
@@ -333,10 +331,10 @@ sub remove_prefix_duplicates {
PTDEBUG && _d('Remove', $right_name);
my $reason;
if ( $right_keys->[$right_index]->{unconstrained} ) {
if ( my $type = $right_keys->[$right_index]->{unconstrained} ) {
$reason .= "Uniqueness of $right_name ignored because "
. $right_keys->[$right_index]->{constraining_key}->{name}
. " is a stronger constraint\n";
. " is a $type constraint\n";
}
my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1;
$reason .= $right_name
@@ -454,14 +452,23 @@ sub unconstrain_keys {
next unless $unique_key; # primary key may be undefined
my $cols = $unique_key->{cols};
if ( @$cols == 1 ) {
PTDEBUG && _d($unique_key->{name},'defines unique column:',$cols->[0]);
# Save only the first unique key for the unique col. If there
# are others, then they are exact duplicates and will be removed
# later when unique keys are compared to unique keys.
if ( !exists $unique_cols{$cols->[0]} ) {
PTDEBUG && _d($unique_key->{name}, 'defines unique column:',
$cols->[0]);
$unique_cols{$cols->[0]} = $unique_key;
$unique_key->{unique_col} = 1;
}
else {
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
# If two unique indexes are not exact, then they must be enforcing
# different uniqueness constraints. Else they're exact dupes
# so one can be treated as a non-unique and removed later
# when comparing unique to non-unique.
PTDEBUG && _d($unique_key->{name},
'redundantly constrains unique column:', $cols->[0]);
$unique_key->{exact_dupe} = 1;
$unique_key->{constraining_key} = $unique_cols{$cols->[0]};
}
}
else {
local $LIST_SEPARATOR = '-';
@@ -496,18 +503,25 @@ sub unconstrain_keys {
}
}
# And finally, unconstrain the redudantly unique sets found above by
# And finally, unconstrain the redundantly unique sets found above by
# removing them from the list of unique keys and adding them to the
# list of normal keys.
for my $i ( 0..(scalar @$unique_keys-1) ) {
if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) {
PTDEBUG && _d('Unconstraining', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 1;
PTDEBUG && _d('Unconstraining weak', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'stronger';
$unique_keys->[$i]->{constraining_key}
= $unconstrain{$unique_keys->[$i]->{name}};
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
elsif ( $unique_keys->[$i]->{exact_dupe} ) {
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
PTDEBUG && _d('Unconstraining dupe', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'duplicate';
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
}
PTDEBUG && _d('No more keys');

18
lib/KeySize.pm
View File

@@ -89,7 +89,7 @@ sub get_key_size {
# EXPLAIN rows will report only the rows that satisfy the query
# using the key, but this is not what we want. We want total table rows.
# In other words, we need an EXPLAIN type index, not ref or range.
if ( scalar @cols == 1 ) {
if ( scalar(@cols) == 1 && !$args{only_eq} ) {
push @where_cols, "$cols[0]<>1";
}
$sql .= join(' OR ', @where_cols);
@@ -113,6 +113,22 @@ sub get_key_size {
PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len,
'rows:', $rows);
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
if ( $chosen_key && $key_len eq '0' ) {
if ( $args{recurse} ) {
$self->{error} = "key_len = 0 in EXPLAIN:\n"
. _explain_to_text($explain);
return;
}
else {
return $self->get_key_size(
%args,
only_eq => 1,
recurse => 1,
);
}
}
my $key_size = 0;
if ( $key_len && $rows ) {
if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) {

63
t/lib/DuplicateKeyFinder.t
View File

@@ -10,6 +10,7 @@ use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
use VersionParser;
use DuplicateKeyFinder;
@@ -756,7 +757,7 @@ is_deeply(
duplicate_of_cols => [ 'row_id' ],
duplicate_of_ddl => 'PRIMARY KEY (`row_id`),',
key => 'row_id',
reason => 'row_id is a duplicate of PRIMARY',
reason => "Uniqueness of row_id ignored because PRIMARY is a duplicate constraint\nrow_id is a duplicate of PRIMARY",
},
{
cols => [ 'player_id' ],
@@ -772,6 +773,65 @@ is_deeply(
'Finds duplicates OK on uppercase columns',
);
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1214114
# #############################################################################
#$ddl = load_file('t/lib/samples/dupekeys/prefix_bug_1214114.sql');
#$dupes = [];
#($keys, $ck) = $tp->get_keys($ddl, $opt);
#$dk->get_duplicate_keys(
# $keys,
# clustered_key => $ck,
# clustered => 1,
# callback => $callback,
# tbl_info => { engine => 'InnoDB', ddl => $ddl },
#);
#
#is_deeply(
# $dupes,
# [{
# cols => ['b', 'id'],
# ddl => 'KEY `b` (`b`,`id`)',
# dupe_type => 'clustered',
# duplicate_of => 'PRIMARY',
# duplicate_of_cols => ['id'],
# duplicate_of_ddl => 'PRIMARY KEY (`id`),',
# key => 'b',
# reason => 'Key b ends with a prefix of the clustered index',
# short_key => '`b`',
# }],
# "Prefix bug 1214114"
#) or diag(Dumper($dupes));
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
#############################################################################
$ddl = load_file('t/lib/samples/dupekeys/simple_dupe_bug_1217013.sql');
$dupes = [];
($keys, $ck) = $tp->get_keys($ddl, $opt);
$dk->get_duplicate_keys(
$keys,
clustered_key => $ck,
clustered => 1,
callback => $callback,
tbl_info => { engine => 'InnoDB', ddl => $ddl },
);
is_deeply(
$dupes,
[{
cols => ['domain'],
ddl => 'UNIQUE KEY `domain` (`domain`),',
dupe_type => 'exact',
duplicate_of => 'unique_key_domain',
duplicate_of_cols => ['domain'],
duplicate_of_ddl => 'UNIQUE KEY `unique_key_domain` (`domain`)',
key => 'domain',
reason => "Uniqueness of domain ignored because unique_key_domain is a duplicate constraint\ndomain is a duplicate of unique_key_domain",
}],
"Exact dupe uniques (bug 1217013)"
) or diag(Dumper($dupes));
# #############################################################################
# Done.
# #############################################################################
@@ -787,4 +847,3 @@ like(
'_d() works'
);
done_testing;
exit;

106
t/lib/KeySize.t
View File

@@ -25,9 +25,6 @@ my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
plan skip_all => "Cannot connect to sandbox master";
}
else {
plan tests => 19;
}
my $q = new Quoter();
my $tp = new TableParser(Quoter => $q);
@@ -190,6 +187,107 @@ is(
'Query without FORCE INDEX (issue 364)'
);
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
# #############################################################################
$sb->load_file('master', "t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql");
($size, $chosen_key) = $ks->get_key_size(
name => 'child_ibfk_2',
cols => [qw(parent_id)],
tbl_name => 'fk_chosen_index_bug_1201443.child',
tbl_struct => {
charset => 'latin1',
clustered_key => undef,
col_posn => {
id => 0,
parent_id => 1
},
cols => [
'id',
'parent_id'
],
defs => {
id => ' `id` int(11) NOT NULL AUTO_INCREMENT',
parent_id => ' `parent_id` int(11) NOT NULL'
},
engine => 'InnoDB',
is_autoinc => {
id => 1,
parent_id => 0
},
is_col => {
id => 1,
parent_id => 1
},
is_nullable => {},
is_numeric => {
id => 1,
parent_id => 1
},
keys => {
id => {
col_prefixes => [
undef
],
colnames => '`id`',
cols => [
'id'
],
ddl => 'KEY `id` (`id`),',
is_col => {
id => 1
},
is_nullable => 0,
is_unique => 0,
name => 'id',
type => 'BTREE'
},
parent_id => {
col_prefixes => [
undef
],
colnames => '`parent_id`',
cols => [
'parent_id'
],
ddl => 'KEY `parent_id` (`parent_id`),',
is_col => {
parent_id => 1
},
is_nullable => 0,
is_unique => 0,
name => 'parent_id',
type => 'BTREE'
}
},
name => 'child',
null_cols => [],
numeric_cols => [
'id',
'parent_id'
],
type_for => {
id => 'int',
parent_id => 'int'
}
},
dbh => $dbh,
);
cmp_ok(
$size,
'>',
15_000, # estimages range from 15k to 30k
"Bug 1201443: size"
);
is(
$chosen_key,
'parent_id',
"Bug 1201443: chosen key"
);
# #############################################################################
# Done.
# #############################################################################
@@ -206,4 +304,4 @@ like(
);
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;
done_testing;

7
t/lib/samples/dupekeys/prefix_bug_1214114.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE `t` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`a` varchar(200) DEFAULT NULL,
`b` decimal(22,0) NOT NULL,
PRIMARY KEY (`id`),
KEY `b` (`b`,`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1

7
t/lib/samples/dupekeys/simple_dupe_bug_1217013.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE `domains` (
`id` bigint(20) NOT NULL,
`domain` varchar(175) COLLATE utf8_bin NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `domain` (`domain`),
UNIQUE KEY `unique_key_domain` (`domain`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin

19
t/pt-duplicate-key-checker/basics.t
View File

@@ -22,9 +22,6 @@ my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 13;
}
my $output;
my $sample = "t/pt-duplicate-key-checker/samples/";
@@ -136,9 +133,23 @@ ok(
'--key-types fk (explicit)'
);
# #############################################################################
# Exact unique dupes
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
# #############################################################################
$sb->load_file('master', 't/lib/samples/dupekeys/simple_dupe_bug_1217013.sql', 'test');
ok(
no_diff(
sub { pt_duplicate_key_checker::main(@args, qw(-t test.domains)) },
"$sample/simple_dupe_bug_1217013.txt"),
'Exact unique dupes (bug 1217013)'
) or diag($test_diff);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;
done_testing;

17
t/pt-duplicate-key-checker/clustered_keys.t
View File

@@ -41,7 +41,7 @@ ok(
: "$sample/issue_295.txt")
),
"Shorten, not remove, clustered dupes"
);
) or diag($test_diff);
# #############################################################################
# Error if InnoDB table has no PK or unique indexes
@@ -80,10 +80,23 @@ unlike(
'PTDEBUG doesn\'t auto-vivify cluster key hashref (bug 1036804)'
);
# #############################################################################
#
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
# #############################################################################
$sb->load_file('master', "t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql");
$output = `$trunk/bin/pt-duplicate-key-checker F=$cnf -d fk_chosen_index_bug_1201443 2>&1`;
unlike(
$output,
qr/Use of uninitialized value/,
'fk_chosen_index_bug_1201443'
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;
exit;

1
t/pt-duplicate-key-checker/samples/bug-894140.txt
View File

@@ -2,6 +2,7 @@
# test.bug_894140
# ########################################################################
# Uniqueness of row_id ignored because PRIMARY is a duplicate constraint
# row_id is a duplicate of PRIMARY
# Key definitions:
# UNIQUE KEY `row_id` (`row_id`),

21
t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql Normal file
View File

File diff suppressed because one or more lines are too long

21
t/pt-duplicate-key-checker/samples/simple_dupe_bug_1217013.txt Normal file
View File

@@ -0,0 +1,21 @@
# ########################################################################
# test.domains
# ########################################################################
# Uniqueness of domain ignored because unique_key_domain is a duplicate constraint
# domain is a duplicate of unique_key_domain
# Key definitions:
# UNIQUE KEY `domain` (`domain`),
# UNIQUE KEY `unique_key_domain` (`domain`)
# Column types:
# `domain` varchar(175) collate utf8_bin not null
# To remove this duplicate index, execute:
ALTER TABLE `test`.`domains` DROP INDEX `domain`;
# ########################################################################
# Summary of indexes
# ########################################################################
# Size Duplicate Indexes 527
# Total Duplicate Indexes 1
# Total Indexes 3

13
t/pt-duplicate-key-checker/standard_options.t
View File

@@ -18,20 +18,25 @@ require "$trunk/bin/pt-duplicate-key-checker";
my $output;
my $cnf = "/tmp/12345/my.sandbox.cnf";
my $cmd = "$trunk/bin/pt-duplicate-key-checker -F $cnf -h 127.1";
my $pid_file = "/tmp/pt-dupe-key-test.pid";
diag(`rm -f $pid_file >/dev/null`);
# #########################################################################
# Issue 391: Add --pid option to all scripts
# #########################################################################
`touch /tmp/mk-script.pid`;
$output = `$cmd -d issue_295 --pid /tmp/mk-script.pid 2>&1`;
diag(`touch $pid_file`);
$output = `$cmd -d issue_295 --pid $pid_file 2>&1`;
like(
$output,
qr{PID file /tmp/mk-script.pid already exists},
qr{PID file $pid_file exists},
'Dies if PID file already exists (issue 391)'
);
`rm -rf /tmp/mk-script.pid`;
# #############################################################################
# Done.
# #############################################################################
diag(`rm -f $pid_file >/dev/null`);
exit;