percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-19 02:05:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pt-dupe-key-fixes.

Browse Source
This commit is contained in:
Daniel Nichter
2013-12-11 19:49:47 -08:00
parent c726b8e624 e89ce13e97
commit 54e9522771
13 changed files with 806 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

353
bin/pt-duplicate-key-checker
View File

@@ -23,7 +23,7 @@ BEGIN {
Daemon Daemon
Schema Schema
SchemaIterator SchemaIterator
HTTPMicro HTTP::Micro
VersionCheck VersionCheck
)); ));
} }
@@ -133,6 +133,8 @@ sub quote_val {
return $val if $val =~ m/^0x[0-9a-fA-F]+$/ # quote hex data return $val if $val =~ m/^0x[0-9a-fA-F]+$/ # quote hex data
&& !$args{is_char}; # unless is_char is true && !$args{is_char}; # unless is_char is true
return $val if $args{is_float};
$val =~ s/(['\\])/\\$1/g; $val =~ s/(['\\])/\\$1/g;
return "'$val'"; return "'$val'";
} }
@@ -2203,7 +2205,7 @@ sub get_key_size {
foreach my $col ( @cols ) { foreach my $col ( @cols ) {
push @where_cols, "$col=1"; push @where_cols, "$col=1";
} }
if ( scalar @cols == 1 ) { if ( scalar(@cols) == 1 && !$args{only_eq} ) {
push @where_cols, "$cols[0]<>1"; push @where_cols, "$cols[0]<>1";
} }
$sql .= join(' OR ', @where_cols); $sql .= join(' OR ', @where_cols);
@@ -2227,6 +2229,21 @@ sub get_key_size {
PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len, PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len,
'rows:', $rows); 'rows:', $rows);
if ( $chosen_key && $key_len eq '0' ) {
if ( $args{recurse} ) {
$self->{error} = "key_len = 0 in EXPLAIN:\n"
. _explain_to_text($explain);
return;
}
else {
return $self->get_key_size(
%args,
only_eq => 1,
recurse => 1,
);
}
}
my $key_size = 0; my $key_size = 0;
if ( $key_len && $rows ) { if ( $key_len && $rows ) {
if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) { if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) {
@@ -2370,7 +2387,6 @@ sub get_duplicate_keys {
push @dupes, push @dupes,
$self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1); $self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1);
my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}} my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}}
: undef; : undef;
PTDEBUG && _d('clustered key:', PTDEBUG && _d('clustered key:',
@@ -2505,10 +2521,10 @@ sub remove_prefix_duplicates {
PTDEBUG && _d('Remove', $right_name); PTDEBUG && _d('Remove', $right_name);
my $reason; my $reason;
if ( $right_keys->[$right_index]->{unconstrained} ) { if ( my $type = $right_keys->[$right_index]->{unconstrained} ) {
$reason .= "Uniqueness of $right_name ignored because " $reason .= "Uniqueness of $right_name ignored because "
. $right_keys->[$right_index]->{constraining_key}->{name} . $right_keys->[$right_index]->{constraining_key}->{name}
. " is a stronger constraint\n"; . " is a $type constraint\n";
} }
my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1; my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1;
$reason .= $right_name $reason .= $right_name
@@ -2608,11 +2624,18 @@ sub unconstrain_keys {
next unless $unique_key; # primary key may be undefined next unless $unique_key; # primary key may be undefined
my $cols = $unique_key->{cols}; my $cols = $unique_key->{cols};
if ( @$cols == 1 ) { if ( @$cols == 1 ) {
PTDEBUG && _d($unique_key->{name},'defines unique column:',$cols->[0]);
if ( !exists $unique_cols{$cols->[0]} ) { if ( !exists $unique_cols{$cols->[0]} ) {
PTDEBUG && _d($unique_key->{name}, 'defines unique column:',
$cols->[0]);
$unique_cols{$cols->[0]} = $unique_key; $unique_cols{$cols->[0]} = $unique_key;
$unique_key->{unique_col} = 1; $unique_key->{unique_col} = 1;
} }
else {
PTDEBUG && _d($unique_key->{name},
'redundantly constrains unique column:', $cols->[0]);
$unique_key->{exact_dupe} = 1;
$unique_key->{constraining_key} = $unique_cols{$cols->[0]};
}
} }
else { else {
local $LIST_SEPARATOR = '-'; local $LIST_SEPARATOR = '-';
@@ -2646,13 +2669,19 @@ sub unconstrain_keys {
for my $i ( 0..(scalar @$unique_keys-1) ) { for my $i ( 0..(scalar @$unique_keys-1) ) {
if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) { if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) {
PTDEBUG && _d('Unconstraining', $unique_keys->[$i]->{name}); PTDEBUG && _d('Unconstraining weak', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 1; $unique_keys->[$i]->{unconstrained} = 'stronger';
$unique_keys->[$i]->{constraining_key} $unique_keys->[$i]->{constraining_key}
= $unconstrain{$unique_keys->[$i]->{name}}; = $unconstrain{$unique_keys->[$i]->{name}};
push @unconstrained_keys, $unique_keys->[$i]; push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i]; delete $unique_keys->[$i];
} }
elsif ( $unique_keys->[$i]->{exact_dupe} ) {
PTDEBUG && _d('Unconstraining dupe', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'duplicate';
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
} }
PTDEBUG && _d('No more keys'); PTDEBUG && _d('No more keys');
@@ -2687,59 +2716,79 @@ package Daemon;
use strict; use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use POSIX qw(setsid); use POSIX qw(setsid);
use Fcntl qw(:DEFAULT);
sub new { sub new {
my ($class, %args) = @_; my ($class, %args) = @_;
foreach my $arg ( qw(o) ) {
die "I need a $arg argument" unless $args{$arg};
}
my $o = $args{o};
my $self = { my $self = {
o => $o, log_file => $args{log_file},
log_file => $o->has('log') ? $o->get('log') : undef, pid_file => $args{pid_file},
PID_file => $o->has('pid') ? $o->get('pid') : undef, daemonize => $args{daemonize},
force_log_file => $args{force_log_file},
parent_exit => $args{parent_exit},
pid_file_owner => 0,
}; };
check_PID_file(undef, $self->{PID_file});
PTDEBUG && _d('Daemonized child will log to', $self->{log_file});
return bless $self, $class; return bless $self, $class;
} }
sub daemonize { sub run {
my ($self) = @_; my ($self) = @_;
PTDEBUG && _d('About to fork and daemonize'); my $daemonize = $self->{daemonize};
defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR"; my $pid_file = $self->{pid_file};
if ( $pid ) { my $log_file = $self->{log_file};
PTDEBUG && _d('Parent PID', $PID, 'exiting after forking child PID',$pid); my $force_log_file = $self->{force_log_file};
exit; my $parent_exit = $self->{parent_exit};
PTDEBUG && _d('Starting daemon');
if ( $pid_file ) {
eval {
$self->_make_pid_file(
pid => $PID, # parent's pid
pid_file => $pid_file,
);
};
die "$EVAL_ERROR\n" if $EVAL_ERROR;
if ( !$daemonize ) {
$self->{pid_file_owner} = $PID; # parent's pid
}
} }
PTDEBUG && _d('Daemonizing child PID', $PID); if ( $daemonize ) {
$self->{PID_owner} = $PID; defined (my $child_pid = fork()) or die "Cannot fork: $OS_ERROR";
$self->{child} = 1; if ( $child_pid ) {
PTDEBUG && _d('Forked child', $child_pid);
$parent_exit->($child_pid) if $parent_exit;
exit 0;
}
POSIX::setsid() or die "Cannot start a new session: $OS_ERROR"; POSIX::setsid() or die "Cannot start a new session: $OS_ERROR";
chdir '/' or die "Cannot chdir to /: $OS_ERROR"; chdir '/' or die "Cannot chdir to /: $OS_ERROR";
$self->_make_PID_file(); if ( $pid_file ) {
$self->_update_pid_file(
$OUTPUT_AUTOFLUSH = 1; pid => $PID, # child's pid
pid_file => $pid_file,
);
$self->{pid_file_owner} = $PID;
}
}
if ( $daemonize || $force_log_file ) {
PTDEBUG && _d('Redirecting STDIN to /dev/null'); PTDEBUG && _d('Redirecting STDIN to /dev/null');
close STDIN; close STDIN;
open STDIN, '/dev/null' open STDIN, '/dev/null'
or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; or die "Cannot reopen STDIN to /dev/null: $OS_ERROR";
if ( $log_file ) {
if ( $self->{log_file} ) { PTDEBUG && _d('Redirecting STDOUT and STDERR to', $log_file);
PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file});
close STDOUT; close STDOUT;
open STDOUT, '>>', $self->{log_file} open STDOUT, '>>', $log_file
or die "Cannot open log file $self->{log_file}: $OS_ERROR"; or die "Cannot open log file $log_file: $OS_ERROR";
close STDERR; close STDERR;
open STDERR, ">&STDOUT" open STDERR, ">&STDOUT"
@@ -2762,82 +2811,119 @@ sub daemonize {
} }
} }
$OUTPUT_AUTOFLUSH = 1;
}
PTDEBUG && _d('Daemon running');
return; return;
} }
sub check_PID_file { sub _make_pid_file {
my ( $self, $file ) = @_; my ($self, %args) = @_;
my $PID_file = $self ? $self->{PID_file} : $file; my @required_args = qw(pid pid_file);
PTDEBUG && _d('Checking PID file', $PID_file); foreach my $arg ( @required_args ) {
if ( $PID_file && -f $PID_file ) { die "I need a $arg argument" unless $args{$arg};
my $pid;
eval {
chomp($pid = (slurp_file($PID_file) || ''));
}; };
if ( $EVAL_ERROR ) { my $pid = $args{pid};
die "The PID file $PID_file already exists but it cannot be read: " my $pid_file = $args{pid_file};
. $EVAL_ERROR;
eval {
sysopen(PID_FH, $pid_file, O_RDWR|O_CREAT|O_EXCL) or die $OS_ERROR;
print PID_FH $PID, "\n";
close PID_FH;
};
if ( my $e = $EVAL_ERROR ) {
if ( $e =~ m/file exists/i ) {
my $old_pid = $self->_check_pid_file(
pid_file => $pid_file,
pid => $PID,
);
if ( $old_pid ) {
warn "Overwriting PID file $pid_file because PID $old_pid "
. "is not running.\n";
} }
PTDEBUG && _d('PID file exists; it contains PID', $pid); $self->_update_pid_file(
if ( $pid ) { pid => $PID,
my $pid_is_alive = kill 0, $pid; pid_file => $pid_file
);
}
else {
die "Error creating PID file $pid_file: $e\n";
}
}
return;
}
sub _check_pid_file {
my ($self, %args) = @_;
my @required_args = qw(pid_file pid);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid_file = $args{pid_file};
my $pid = $args{pid};
PTDEBUG && _d('Checking if PID in', $pid_file, 'is running');
if ( ! -f $pid_file ) {
PTDEBUG && _d('PID file', $pid_file, 'does not exist');
return;
}
open my $fh, '<', $pid_file
or die "Error opening $pid_file: $OS_ERROR";
my $existing_pid = do { local $/; <$fh> };
chomp($existing_pid) if $existing_pid;
close $fh
or die "Error closing $pid_file: $OS_ERROR";
if ( $existing_pid ) {
if ( $existing_pid == $pid ) {
warn "The current PID $pid already holds the PID file $pid_file\n";
return;
}
else {
PTDEBUG && _d('Checking if PID', $existing_pid, 'is running');
my $pid_is_alive = kill 0, $existing_pid;
if ( $pid_is_alive ) { if ( $pid_is_alive ) {
die "The PID file $PID_file already exists " die "PID file $pid_file exists and PID $existing_pid is running\n";
. " and the PID that it contains, $pid, is running";
} }
else {
warn "Overwriting PID file $PID_file because the PID that it "
. "contains, $pid, is not running";
} }
} }
else { else {
die "The PID file $PID_file already exists but it does not " die "PID file $pid_file exists but it is empty. Remove the file "
. "contain a PID"; . "if the process is no longer running.\n";
} }
return $existing_pid;
} }
else {
PTDEBUG && _d('No PID file'); sub _update_pid_file {
} my ($self, %args) = @_;
my @required_args = qw(pid pid_file);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid = $args{pid};
my $pid_file = $args{pid_file};
open my $fh, '>', $pid_file
or die "Cannot open $pid_file: $OS_ERROR";
print { $fh } $pid, "\n"
or die "Cannot print to $pid_file: $OS_ERROR";
close $fh
or warn "Cannot close $pid_file: $OS_ERROR";
return; return;
} }
sub make_PID_file { sub remove_pid_file {
my ( $self ) = @_; my ($self, $pid_file) = @_;
if ( exists $self->{child} ) { $pid_file ||= $self->{pid_file};
die "Do not call Daemon::make_PID_file() for daemonized scripts"; if ( $pid_file && -f $pid_file ) {
} unlink $self->{pid_file}
$self->_make_PID_file(); or warn "Cannot remove PID file $pid_file: $OS_ERROR";
$self->{PID_owner} = $PID;
return;
}
sub _make_PID_file {
my ( $self ) = @_;
my $PID_file = $self->{PID_file};
if ( !$PID_file ) {
PTDEBUG && _d('No PID file to create');
return;
}
$self->check_PID_file();
open my $PID_FH, '>', $PID_file
or die "Cannot open PID file $PID_file: $OS_ERROR";
print $PID_FH $PID
or die "Cannot print to PID file $PID_file: $OS_ERROR";
close $PID_FH
or die "Cannot close PID file $PID_file: $OS_ERROR";
PTDEBUG && _d('Created PID file:', $self->{PID_file});
return;
}
sub _remove_PID_file {
my ( $self ) = @_;
if ( $self->{PID_file} && -f $self->{PID_file} ) {
unlink $self->{PID_file}
or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR";
PTDEBUG && _d('Removed PID file'); PTDEBUG && _d('Removed PID file');
} }
else { else {
@@ -2849,16 +2935,11 @@ sub _remove_PID_file {
sub DESTROY { sub DESTROY {
my ($self) = @_; my ($self) = @_;
$self->_remove_PID_file() if ($self->{PID_owner} || 0) == $PID; if ( $self->{pid_file_owner} == $PID ) {
$self->remove_pid_file();
return;
} }
sub slurp_file { return;
my ($file) = @_;
return unless $file;
open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
return do { local $/; <$fh> };
} }
sub _d { sub _d {
@@ -3543,25 +3624,23 @@ sub _d {
# ########################################################################### # ###########################################################################
# ########################################################################### # ###########################################################################
# HTTPMicro package # HTTP::Micro package
# This package is a copy without comments from the original. The original # This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at, # with comments and its test file can be found in the Bazaar repository at,
# lib/HTTPMicro.pm # lib/HTTP/Micro.pm
# t/lib/HTTPMicro.t # t/lib/HTTP/Micro.t
# See https://launchpad.net/percona-toolkit for more information. # See https://launchpad.net/percona-toolkit for more information.
# ########################################################################### # ###########################################################################
{ {
package HTTP::Micro;
our $VERSION = '0.01';
package HTTPMicro;
BEGIN {
$HTTPMicro::VERSION = '0.001';
}
use strict; use strict;
use warnings; use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Carp (); use Carp ();
my @attributes; my @attributes;
BEGIN { BEGIN {
@attributes = qw(agent timeout); @attributes = qw(agent timeout);
@@ -3632,7 +3711,7 @@ sub _request {
headers => {}, headers => {},
}; };
my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout}); my $handle = HTTP::Micro::Handle->new(timeout => $self->{timeout});
$handle->connect($scheme, $host, $port); $handle->connect($scheme, $host, $port);
@@ -3697,14 +3776,18 @@ sub _split_url {
return ($scheme, $host, $port, $path_query); return ($scheme, $host, $port, $path_query);
} }
package } # HTTP::Micro
HTTPMicro::Handle; # hide from PAUSE/indexers
use strict;
use warnings;
use Carp qw[croak]; {
use Errno qw[EINTR EPIPE]; package HTTP::Micro::Handle;
use IO::Socket qw[SOCK_STREAM];
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Carp qw(croak);
use Errno qw(EINTR EPIPE);
use IO::Socket qw(SOCK_STREAM);
sub BUFSIZE () { 32768 } sub BUFSIZE () { 32768 }
@@ -3747,7 +3830,7 @@ sub connect {
croak(qq/Unsupported URL scheme '$scheme'\n/); croak(qq/Unsupported URL scheme '$scheme'\n/);
} }
$self->{fh} = 'IO::Socket::INET'->new( $self->{fh} = IO::Socket::INET->new(
PeerHost => $host, PeerHost => $host,
PeerPort => $port, PeerPort => $port,
Proto => 'tcp', Proto => 'tcp',
@@ -4011,6 +4094,7 @@ sub can_write {
my $self = shift; my $self = shift;
return $self->_do_timeout('write', @_) return $self->_do_timeout('write', @_)
} }
} # HTTP::Micro::Handle
my $prog = <<'EOP'; my $prog = <<'EOP';
BEGIN { BEGIN {
@@ -4031,6 +4115,7 @@ BEGIN {
} }
} }
{ {
use Carp qw(croak);
my %dispatcher = ( my %dispatcher = (
issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) }, issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) },
subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) }, subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) },
@@ -4186,9 +4271,8 @@ if ( $INC{"IO/Socket/SSL.pm"} ) {
} }
1; 1;
}
# ########################################################################### # ###########################################################################
# End HTTPMicro package # End HTTP::Micro package
# ########################################################################### # ###########################################################################
# ########################################################################### # ###########################################################################
@@ -4222,7 +4306,7 @@ use FindBin qw();
eval { eval {
require Percona::Toolkit; require Percona::Toolkit;
require HTTPMicro; require HTTP::Micro;
}; };
{ {
@@ -4453,7 +4537,7 @@ sub pingback {
my $url = $args{url}; my $url = $args{url};
my $instances = $args{instances}; my $instances = $args{instances};
my $ua = $args{ua} || HTTPMicro->new( timeout => 3 ); my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
my $response = $ua->request('GET', $url); my $response = $ua->request('GET', $url);
PTDEBUG && _d('Server response:', Dumper($response)); PTDEBUG && _d('Server response:', Dumper($response));
@@ -4832,14 +4916,11 @@ sub main {
# ######################################################################## # ########################################################################
# If --pid, check it first since we'll die if it already exits. # If --pid, check it first since we'll die if it already exits.
# ######################################################################## # ########################################################################
my $daemon; my $daemon = Daemon->new(
if ( $o->get('pid') ) { daemonize => 0,
# We're not daemoninzing, it just handles PID stuff. Keep $daemon pid_file => $o->get('pid'),
# in the the scope of main() because when it's destroyed it automatically );
# removes the PID file. $daemon->run();
$daemon = new Daemon(o=>$o);
$daemon->make_PID_file();
}
# ####################################################################### # #######################################################################
# Connect to MySQL. # Connect to MySQL.

38
lib/DuplicateKeyFinder.pm
View File

@@ -143,8 +143,6 @@ sub get_duplicate_keys {
push @dupes, push @dupes,
$self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1); $self->remove_prefix_duplicates(\@fulltext_keys, \@fulltext_keys, %args, exact_duplicates => 1);
# TODO: other structs
# Remove clustered duplicates. # Remove clustered duplicates.
my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}} my $clustered_key = $args{clustered_key} ? $keys{$args{clustered_key}}
: undef; : undef;
@@ -314,7 +312,7 @@ sub remove_prefix_duplicates {
if ( substr($left_cols, 0, $right_len_cols) if ( substr($left_cols, 0, $right_len_cols)
eq substr($right_cols, 0, $right_len_cols) ) { eq substr($right_cols, 0, $right_len_cols) ) {
# FULLTEXT keys, for example, are only duplicates if they # UNIQUE and FULLTEXT indexes are only duplicates if they
# are exact duplicates. # are exact duplicates.
if ( $args{exact_duplicates} && ($right_len_cols<$left_len_cols) ) { if ( $args{exact_duplicates} && ($right_len_cols<$left_len_cols) ) {
PTDEBUG && _d($right_name, 'not exact duplicate of', $left_name); PTDEBUG && _d($right_name, 'not exact duplicate of', $left_name);
@@ -333,10 +331,10 @@ sub remove_prefix_duplicates {
PTDEBUG && _d('Remove', $right_name); PTDEBUG && _d('Remove', $right_name);
my $reason; my $reason;
if ( $right_keys->[$right_index]->{unconstrained} ) { if ( my $type = $right_keys->[$right_index]->{unconstrained} ) {
$reason .= "Uniqueness of $right_name ignored because " $reason .= "Uniqueness of $right_name ignored because "
. $right_keys->[$right_index]->{constraining_key}->{name} . $right_keys->[$right_index]->{constraining_key}->{name}
. " is a stronger constraint\n"; . " is a $type constraint\n";
} }
my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1; my $exact_dupe = $right_len_cols < $left_len_cols ? 0 : 1;
$reason .= $right_name $reason .= $right_name
@@ -454,14 +452,23 @@ sub unconstrain_keys {
next unless $unique_key; # primary key may be undefined next unless $unique_key; # primary key may be undefined
my $cols = $unique_key->{cols}; my $cols = $unique_key->{cols};
if ( @$cols == 1 ) { if ( @$cols == 1 ) {
PTDEBUG && _d($unique_key->{name},'defines unique column:',$cols->[0]);
# Save only the first unique key for the unique col. If there
# are others, then they are exact duplicates and will be removed
# later when unique keys are compared to unique keys.
if ( !exists $unique_cols{$cols->[0]} ) { if ( !exists $unique_cols{$cols->[0]} ) {
PTDEBUG && _d($unique_key->{name}, 'defines unique column:',
$cols->[0]);
$unique_cols{$cols->[0]} = $unique_key; $unique_cols{$cols->[0]} = $unique_key;
$unique_key->{unique_col} = 1; $unique_key->{unique_col} = 1;
} }
else {
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
# If two unique indexes are not exact, then they must be enforcing
# different uniqueness constraints. Else they're exact dupes
# so one can be treated as a non-unique and removed later
# when comparing unique to non-unique.
PTDEBUG && _d($unique_key->{name},
'redundantly constrains unique column:', $cols->[0]);
$unique_key->{exact_dupe} = 1;
$unique_key->{constraining_key} = $unique_cols{$cols->[0]};
}
} }
else { else {
local $LIST_SEPARATOR = '-'; local $LIST_SEPARATOR = '-';
@@ -496,18 +503,25 @@ sub unconstrain_keys {
} }
} }
# And finally, unconstrain the redudantly unique sets found above by # And finally, unconstrain the redundantly unique sets found above by
# removing them from the list of unique keys and adding them to the # removing them from the list of unique keys and adding them to the
# list of normal keys. # list of normal keys.
for my $i ( 0..(scalar @$unique_keys-1) ) { for my $i ( 0..(scalar @$unique_keys-1) ) {
if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) { if ( exists $unconstrain{$unique_keys->[$i]->{name}} ) {
PTDEBUG && _d('Unconstraining', $unique_keys->[$i]->{name}); PTDEBUG && _d('Unconstraining weak', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 1; $unique_keys->[$i]->{unconstrained} = 'stronger';
$unique_keys->[$i]->{constraining_key} $unique_keys->[$i]->{constraining_key}
= $unconstrain{$unique_keys->[$i]->{name}}; = $unconstrain{$unique_keys->[$i]->{name}};
push @unconstrained_keys, $unique_keys->[$i]; push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i]; delete $unique_keys->[$i];
} }
elsif ( $unique_keys->[$i]->{exact_dupe} ) {
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
PTDEBUG && _d('Unconstraining dupe', $unique_keys->[$i]->{name});
$unique_keys->[$i]->{unconstrained} = 'duplicate';
push @unconstrained_keys, $unique_keys->[$i];
delete $unique_keys->[$i];
}
} }
PTDEBUG && _d('No more keys'); PTDEBUG && _d('No more keys');

18
lib/KeySize.pm
View File

@@ -89,7 +89,7 @@ sub get_key_size {
# EXPLAIN rows will report only the rows that satisfy the query # EXPLAIN rows will report only the rows that satisfy the query
# using the key, but this is not what we want. We want total table rows. # using the key, but this is not what we want. We want total table rows.
# In other words, we need an EXPLAIN type index, not ref or range. # In other words, we need an EXPLAIN type index, not ref or range.
if ( scalar @cols == 1 ) { if ( scalar(@cols) == 1 && !$args{only_eq} ) {
push @where_cols, "$cols[0]<>1"; push @where_cols, "$cols[0]<>1";
} }
$sql .= join(' OR ', @where_cols); $sql .= join(' OR ', @where_cols);
@@ -113,6 +113,22 @@ sub get_key_size {
PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len, PTDEBUG && _d('MySQL chose key:', $chosen_key, 'len:', $key_len,
'rows:', $rows); 'rows:', $rows);
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
if ( $chosen_key && $key_len eq '0' ) {
if ( $args{recurse} ) {
$self->{error} = "key_len = 0 in EXPLAIN:\n"
. _explain_to_text($explain);
return;
}
else {
return $self->get_key_size(
%args,
only_eq => 1,
recurse => 1,
);
}
}
my $key_size = 0; my $key_size = 0;
if ( $key_len && $rows ) { if ( $key_len && $rows ) {
if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) { if ( $chosen_key =~ m/,/ && $key_len =~ m/,/ ) {

63
t/lib/DuplicateKeyFinder.t
View File

@@ -10,6 +10,7 @@ use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use Test::More; use Test::More;
use Data::Dumper;
use VersionParser; use VersionParser;
use DuplicateKeyFinder; use DuplicateKeyFinder;
@@ -756,7 +757,7 @@ is_deeply(
duplicate_of_cols => [ 'row_id' ], duplicate_of_cols => [ 'row_id' ],
duplicate_of_ddl => 'PRIMARY KEY (`row_id`),', duplicate_of_ddl => 'PRIMARY KEY (`row_id`),',
key => 'row_id', key => 'row_id',
reason => 'row_id is a duplicate of PRIMARY', reason => "Uniqueness of row_id ignored because PRIMARY is a duplicate constraint\nrow_id is a duplicate of PRIMARY",
}, },
{ {
cols => [ 'player_id' ], cols => [ 'player_id' ],
@@ -772,6 +773,65 @@ is_deeply(
'Finds duplicates OK on uppercase columns', 'Finds duplicates OK on uppercase columns',
); );
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1214114
# #############################################################################
#$ddl = load_file('t/lib/samples/dupekeys/prefix_bug_1214114.sql');
#$dupes = [];
#($keys, $ck) = $tp->get_keys($ddl, $opt);
#$dk->get_duplicate_keys(
# $keys,
# clustered_key => $ck,
# clustered => 1,
# callback => $callback,
# tbl_info => { engine => 'InnoDB', ddl => $ddl },
#);
#
#is_deeply(
# $dupes,
# [{
# cols => ['b', 'id'],
# ddl => 'KEY `b` (`b`,`id`)',
# dupe_type => 'clustered',
# duplicate_of => 'PRIMARY',
# duplicate_of_cols => ['id'],
# duplicate_of_ddl => 'PRIMARY KEY (`id`),',
# key => 'b',
# reason => 'Key b ends with a prefix of the clustered index',
# short_key => '`b`',
# }],
# "Prefix bug 1214114"
#) or diag(Dumper($dupes));
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
#############################################################################
$ddl = load_file('t/lib/samples/dupekeys/simple_dupe_bug_1217013.sql');
$dupes = [];
($keys, $ck) = $tp->get_keys($ddl, $opt);
$dk->get_duplicate_keys(
$keys,
clustered_key => $ck,
clustered => 1,
callback => $callback,
tbl_info => { engine => 'InnoDB', ddl => $ddl },
);
is_deeply(
$dupes,
[{
cols => ['domain'],
ddl => 'UNIQUE KEY `domain` (`domain`),',
dupe_type => 'exact',
duplicate_of => 'unique_key_domain',
duplicate_of_cols => ['domain'],
duplicate_of_ddl => 'UNIQUE KEY `unique_key_domain` (`domain`)',
key => 'domain',
reason => "Uniqueness of domain ignored because unique_key_domain is a duplicate constraint\ndomain is a duplicate of unique_key_domain",
}],
"Exact dupe uniques (bug 1217013)"
) or diag(Dumper($dupes));
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
@@ -787,4 +847,3 @@ like(
'_d() works' '_d() works'
); );
done_testing; done_testing;
exit;

106
t/lib/KeySize.t
View File

@@ -25,9 +25,6 @@ my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) { if ( !$dbh ) {
plan skip_all => "Cannot connect to sandbox master"; plan skip_all => "Cannot connect to sandbox master";
} }
else {
plan tests => 19;
}
my $q = new Quoter(); my $q = new Quoter();
my $tp = new TableParser(Quoter => $q); my $tp = new TableParser(Quoter => $q);
@@ -190,6 +187,107 @@ is(
'Query without FORCE INDEX (issue 364)' 'Query without FORCE INDEX (issue 364)'
); );
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
# #############################################################################
$sb->load_file('master', "t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql");
($size, $chosen_key) = $ks->get_key_size(
name => 'child_ibfk_2',
cols => [qw(parent_id)],
tbl_name => 'fk_chosen_index_bug_1201443.child',
tbl_struct => {
charset => 'latin1',
clustered_key => undef,
col_posn => {
id => 0,
parent_id => 1
},
cols => [
'id',
'parent_id'
],
defs => {
id => ' `id` int(11) NOT NULL AUTO_INCREMENT',
parent_id => ' `parent_id` int(11) NOT NULL'
},
engine => 'InnoDB',
is_autoinc => {
id => 1,
parent_id => 0
},
is_col => {
id => 1,
parent_id => 1
},
is_nullable => {},
is_numeric => {
id => 1,
parent_id => 1
},
keys => {
id => {
col_prefixes => [
undef
],
colnames => '`id`',
cols => [
'id'
],
ddl => 'KEY `id` (`id`),',
is_col => {
id => 1
},
is_nullable => 0,
is_unique => 0,
name => 'id',
type => 'BTREE'
},
parent_id => {
col_prefixes => [
undef
],
colnames => '`parent_id`',
cols => [
'parent_id'
],
ddl => 'KEY `parent_id` (`parent_id`),',
is_col => {
parent_id => 1
},
is_nullable => 0,
is_unique => 0,
name => 'parent_id',
type => 'BTREE'
}
},
name => 'child',
null_cols => [],
numeric_cols => [
'id',
'parent_id'
],
type_for => {
id => 'int',
parent_id => 'int'
}
},
dbh => $dbh,
);
cmp_ok(
$size,
'>',
15_000, # estimages range from 15k to 30k
"Bug 1201443: size"
);
is(
$chosen_key,
'parent_id',
"Bug 1201443: chosen key"
);
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
@@ -206,4 +304,4 @@ like(
); );
$sb->wipe_clean($dbh); $sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit; done_testing;

7
t/lib/samples/dupekeys/prefix_bug_1214114.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE `t` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`a` varchar(200) DEFAULT NULL,
`b` decimal(22,0) NOT NULL,
PRIMARY KEY (`id`),
KEY `b` (`b`,`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1

7
t/lib/samples/dupekeys/simple_dupe_bug_1217013.sql Normal file
View File

@@ -0,0 +1,7 @@
CREATE TABLE `domains` (
`id` bigint(20) NOT NULL,
`domain` varchar(175) COLLATE utf8_bin NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `domain` (`domain`),
UNIQUE KEY `unique_key_domain` (`domain`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin

19
t/pt-duplicate-key-checker/basics.t
View File

@@ -22,9 +22,6 @@ my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) { if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master'; plan skip_all => 'Cannot connect to sandbox master';
} }
else {
plan tests => 13;
}
my $output; my $output;
my $sample = "t/pt-duplicate-key-checker/samples/"; my $sample = "t/pt-duplicate-key-checker/samples/";
@@ -136,9 +133,23 @@ ok(
'--key-types fk (explicit)' '--key-types fk (explicit)'
); );
# #############################################################################
# Exact unique dupes
# https://bugs.launchpad.net/percona-toolkit/+bug/1217013
# #############################################################################
$sb->load_file('master', 't/lib/samples/dupekeys/simple_dupe_bug_1217013.sql', 'test');
ok(
no_diff(
sub { pt_duplicate_key_checker::main(@args, qw(-t test.domains)) },
"$sample/simple_dupe_bug_1217013.txt"),
'Exact unique dupes (bug 1217013)'
) or diag($test_diff);
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
$sb->wipe_clean($dbh); $sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit; done_testing;

17
t/pt-duplicate-key-checker/clustered_keys.t
View File

@@ -41,7 +41,7 @@ ok(
: "$sample/issue_295.txt") : "$sample/issue_295.txt")
), ),
"Shorten, not remove, clustered dupes" "Shorten, not remove, clustered dupes"
); ) or diag($test_diff);
# ############################################################################# # #############################################################################
# Error if InnoDB table has no PK or unique indexes # Error if InnoDB table has no PK or unique indexes
@@ -80,10 +80,23 @@ unlike(
'PTDEBUG doesn\'t auto-vivify cluster key hashref (bug 1036804)' 'PTDEBUG doesn\'t auto-vivify cluster key hashref (bug 1036804)'
); );
# #############################################################################
#
# https://bugs.launchpad.net/percona-toolkit/+bug/1201443
# #############################################################################
$sb->load_file('master', "t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql");
$output = `$trunk/bin/pt-duplicate-key-checker F=$cnf -d fk_chosen_index_bug_1201443 2>&1`;
unlike(
$output,
qr/Use of uninitialized value/,
'fk_chosen_index_bug_1201443'
);
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
$sb->wipe_clean($dbh); $sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing; done_testing;
exit;

1
t/pt-duplicate-key-checker/samples/bug-894140.txt
View File

@@ -2,6 +2,7 @@
# test.bug_894140 # test.bug_894140
# ######################################################################## # ########################################################################
# Uniqueness of row_id ignored because PRIMARY is a duplicate constraint
# row_id is a duplicate of PRIMARY # row_id is a duplicate of PRIMARY
# Key definitions: # Key definitions:
# UNIQUE KEY `row_id` (`row_id`), # UNIQUE KEY `row_id` (`row_id`),

21
t/pt-duplicate-key-checker/samples/fk_chosen_index_bug_1201443.sql Normal file
View File

File diff suppressed because one or more lines are too long

21
t/pt-duplicate-key-checker/samples/simple_dupe_bug_1217013.txt Normal file
View File

@@ -0,0 +1,21 @@
# ########################################################################
# test.domains
# ########################################################################
# Uniqueness of domain ignored because unique_key_domain is a duplicate constraint
# domain is a duplicate of unique_key_domain
# Key definitions:
# UNIQUE KEY `domain` (`domain`),
# UNIQUE KEY `unique_key_domain` (`domain`)
# Column types:
# `domain` varchar(175) collate utf8_bin not null
# To remove this duplicate index, execute:
ALTER TABLE `test`.`domains` DROP INDEX `domain`;
# ########################################################################
# Summary of indexes
# ########################################################################
# Size Duplicate Indexes 527
# Total Duplicate Indexes 1
# Total Indexes 3

13
t/pt-duplicate-key-checker/standard_options.t
View File

@@ -18,20 +18,25 @@ require "$trunk/bin/pt-duplicate-key-checker";
my $output; my $output;
my $cnf = "/tmp/12345/my.sandbox.cnf"; my $cnf = "/tmp/12345/my.sandbox.cnf";
my $cmd = "$trunk/bin/pt-duplicate-key-checker -F $cnf -h 127.1"; my $cmd = "$trunk/bin/pt-duplicate-key-checker -F $cnf -h 127.1";
my $pid_file = "/tmp/pt-dupe-key-test.pid";
diag(`rm -f $pid_file >/dev/null`);
# ######################################################################### # #########################################################################
# Issue 391: Add --pid option to all scripts # Issue 391: Add --pid option to all scripts
# ######################################################################### # #########################################################################
`touch /tmp/mk-script.pid`;
$output = `$cmd -d issue_295 --pid /tmp/mk-script.pid 2>&1`; diag(`touch $pid_file`);
$output = `$cmd -d issue_295 --pid $pid_file 2>&1`;
like( like(
$output, $output,
qr{PID file /tmp/mk-script.pid already exists}, qr{PID file $pid_file exists},
'Dies if PID file already exists (issue 391)' 'Dies if PID file already exists (issue 391)'
); );
`rm -rf /tmp/mk-script.pid`;
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
diag(`rm -f $pid_file >/dev/null`);
exit; exit;