percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-11 13:40:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

PT-1757 Implemented fallback in NibbleIterator

Browse Source 
Since now NibbleIterator can fallback to nibble (from one chunk) pt-osc
can nibble tables even if due to unacurate stats, it choses one nible as
the initial nibbling method.
This commit is contained in:
Carlos Salguero
2019-10-13 21:58:56 -03:00
parent e58b9fbea8
commit 6a302e2b07
12 changed files with 549 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bin/pt-archiver
View File

@@ -3136,6 +3136,7 @@ sub generate_asc_stmt {
cols => \@cols, cols => \@cols,
quoter => $q, quoter => $q,
is_nullable => $tbl_struct->{is_nullable}, is_nullable => $tbl_struct->{is_nullable},
type_for => $tbl_struct->{type_for},
); );
$asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
} }
@@ -3156,6 +3157,7 @@ sub generate_cmp_where {
my @slice = @{$args{slice}}; my @slice = @{$args{slice}};
my @cols = @{$args{cols}}; my @cols = @{$args{cols}};
my $is_nullable = $args{is_nullable}; my $is_nullable = $args{is_nullable};
my $type_for = $args{type_for};
my $type = $args{type}; my $type = $args{type};
my $q = $self->{Quoter}; my $q = $self->{Quoter};
@@ -3172,13 +3174,14 @@ sub generate_cmp_where {
my $ord = $slice[$j]; my $ord = $slice[$j];
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))";
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
} }
else { else {
push @clause, "$quo = ?"; push @clause, "$quo = $val";
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
} }
@@ -3188,15 +3191,16 @@ sub generate_cmp_where {
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $end = $i == $#slice; # Last clause of the whole group. my $end = $i == $#slice; # Last clause of the whole group.
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
if ( $type =~ m/=/ && $end ) { if ( $type =~ m/=/ && $end ) {
push @clause, "(? IS NULL OR $quo $type ?)"; push @clause, "($val IS NULL OR $quo $type $val)";
} }
elsif ( $type =~ m/>/ ) { elsif ( $type =~ m/>/ ) {
push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)";
} }
else { # If $type =~ m/</ ) { else { # If $type =~ m/</ ) {
push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))"; push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))";
} }
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
@@ -3204,7 +3208,7 @@ sub generate_cmp_where {
else { else {
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?"); push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val");
} }
push @clauses, '(' . join(' AND ', @clause) . ')'; push @clauses, '(' . join(' AND ', @clause) . ')';

251
bin/pt-online-schema-change
View File

@@ -3057,15 +3057,13 @@ sub generate_cmp_where {
my $ord = $slice[$j]; my $ord = $slice[$j];
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $val = $type_for->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
#push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))";
push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))";
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
} }
else { else {
#push @clause, "$quo = ?";
push @clause, "$quo = $val"; push @clause, "$quo = $val";
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
@@ -3076,18 +3074,15 @@ sub generate_cmp_where {
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $end = $i == $#slice; # Last clause of the whole group. my $end = $i == $#slice; # Last clause of the whole group.
my $val = $type_for->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
if ( $type =~ m/=/ && $end ) { if ( $type =~ m/=/ && $end ) {
#push @clause, "(? IS NULL OR $quo $type ?)";
push @clause, "($val IS NULL OR $quo $type $val)"; push @clause, "($val IS NULL OR $quo $type $val)";
} }
elsif ( $type =~ m/>/ ) { elsif ( $type =~ m/>/ ) {
#push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))";
push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)";
} }
else { # If $type =~ m/</ ) { else { # If $type =~ m/</ ) {
#push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))";
push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))"; push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))";
} }
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
@@ -3096,7 +3091,6 @@ sub generate_cmp_where {
else { else {
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
#push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?");
push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val"); push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val");
} }
@@ -5457,7 +5451,6 @@ use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use IndexLength;
use Data::Dumper; use Data::Dumper;
$Data::Dumper::Indent = 1; $Data::Dumper::Indent = 1;
@@ -5495,82 +5488,131 @@ sub new {
my @cols = grep { !$ignore_col->{$_} } @$all_cols; my @cols = grep { !$ignore_col->{$_} } @$all_cols;
my $self; my $self;
if ( $nibble_params->{one_nibble} ) { if ( $nibble_params->{one_nibble} ) {
my $params = _one_nibble(\%args, \@cols, $where, $tbl, \%comments);
$self = {
%args,
one_nibble => 1,
limit => 0,
nibble_sql => $params->{nibble_sql},
explain_nibble_sql => $params->{explain_nibble_sql},
};
} else {
my $params = _nibble_params($nibble_params, $tbl, \%args, \@cols, $chunk_size, $where, \%comments, $q);
$self = {
%args,
index => $params->{index},
limit => $params->{limit},
first_lb_sql => $params->{first_lb_sql},
last_ub_sql => $params->{last_ub_sql},
ub_sql => $params->{ub_sql},
nibble_sql => $params->{nibble_sql},
explain_first_lb_sql => $params->{explain_first_lb_sql},
explain_ub_sql => $params->{explain_ub_sql},
explain_nibble_sql => $params->{explain_nibble_sql},
resume_lb_sql => $params->{resume_lb_sql},
sql => $params->{sql},
};
}
$self->{row_est} = $nibble_params->{row_est},
$self->{nibbleno} = 0;
$self->{have_rows} = 0;
$self->{rowno} = 0;
$self->{oktonibble} = 1;
$self->{pause_file} = $nibble_params->{pause_file};
$self->{sleep} = $args{sleep} || 60;
$self->{nibble_params} = $nibble_params;
$self->{tbl} = $tbl;
$self->{args} = \%args;
$self->{cols} = \@cols;
$self->{chunk_size} = $chunk_size;
$self->{where} = $where;
$self->{comments} = \%comments;
return bless $self, $class;
}
sub switch_to_nibble {
my $self = shift;
my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols},
$self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter});
$self->{one_nibble} = 0;
$self->{index} = $params->{index};
$self->{limit} = $params->{limit};
$self->{first_lb_sql} = $params->{first_lb_sql};
$self->{last_ub_sql} = $params->{last_ub_sql};
$self->{ub_sql} = $params->{ub_sql};
$self->{nibble_sql} = $params->{nibble_sql};
$self->{explain_first_lb_sql} = $params->{explain_first_lb_sql};
$self->{explain_ub_sql} = $params->{explain_ub_sql};
$self->{explain_nibble_sql} = $params->{explain_nibble_sql};
$self->{resume_lb_sql} = $params->{resume_lb_sql};
$self->{sql} = $params->{sql};
$self->_get_bounds();
$self->_prepare_sths();
}
sub _one_nibble {
my ($args, $cols, $where, $tbl, $comments) = @_;
my $q = new Quoter();
my $nibble_sql my $nibble_sql
= ($args{dml} ? "$args{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @cols)) : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ?
"CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*$comments{bite}*/"; . " /*$comments->{bite}*/";
PTDEBUG && _d('One nibble statement:', $nibble_sql); PTDEBUG && _d('One nibble statement:', $nibble_sql);
my $explain_nibble_sql my $explain_nibble_sql
= "EXPLAIN SELECT " = "EXPLAIN SELECT "
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @cols)) : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum'
? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*explain $comments{bite}*/"; . " /*explain $comments->{bite}*/";
PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql);
$self = { return {
%args,
one_nibble => 1, one_nibble => 1,
limit => 0, limit => 0,
nibble_sql => $nibble_sql, nibble_sql => $nibble_sql,
explain_nibble_sql => $explain_nibble_sql, explain_nibble_sql => $explain_nibble_sql,
}; };
} }
else {
sub _nibble_params {
my ($nibble_params, $tbl, $args, $cols, $chunk_size, $where, $comments, $q) = @_;
my $index = $nibble_params->{index}; # brevity my $index = $nibble_params->{index}; # brevity
my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols};
my $asc = $args{TableNibbler}->generate_asc_stmt( my $asc = $args->{TableNibbler}->generate_asc_stmt(
%args, %$args,
tbl_struct => $tbl->{tbl_struct}, tbl_struct => $tbl->{tbl_struct},
index => $index, index => $index,
n_index_cols => $args{n_chunk_index_cols}, n_index_cols => $args->{n_chunk_index_cols},
cols => \@cols, cols => $cols,
asc_only => 1, asc_only => 1,
); );
PTDEBUG && _d('Ascend params:', Dumper($asc)); PTDEBUG && _d('Ascend params:', Dumper($asc));
my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); my $force_concat_enums;
my $i=0;
for my $index (@{$index_cols}) {
last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols};
$i++;
if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') {
if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) {
my @items = split(/,\s*/, $1);
my $sorted = 1; # Asume the items list is sorted to later check if this is true
for (my $i=1; $i < scalar(@items); $i++) {
if ($items[$i-1] gt $items[$i]) {
$sorted = 0;
last;
}
}
if (!$force_concat_enums && !$sorted) {
die "The index " . $index . " in table " . $tbl->{name} .
" has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n";
}
}
}
}
my $from = "$tbl->{name} FORCE INDEX(`$index`)"; my $from = "$tbl->{name} FORCE INDEX(`$index`)";
my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols});
my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols});
my $first_lb_sql my $first_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY $order_by" . " ORDER BY $order_by"
@@ -5579,10 +5621,10 @@ sub new {
PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); PTDEBUG && _d('First lower boundary statement:', $first_lb_sql);
my $resume_lb_sql; my $resume_lb_sql;
if ( $args{resume} ) { if ( $args->{resume} ) {
$resume_lb_sql $resume_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>'} . " WHERE " . $asc->{boundaries}->{'>'}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -5594,7 +5636,7 @@ sub new {
my $last_ub_sql my $last_ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY " . " ORDER BY "
@@ -5605,7 +5647,7 @@ sub new {
my $ub_sql my $ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} . " WHERE " . $asc->{boundaries}->{'>='}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -5615,36 +5657,36 @@ sub new {
PTDEBUG && _d('Upper boundary statement:', $ub_sql); PTDEBUG && _d('Upper boundary statement:', $ub_sql);
my $nibble_sql my $nibble_sql
= ($args{dml} ? "$args{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
. " AND " . $asc->{boundaries}->{'<='} # upper boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
. ($args{order_by} ? " ORDER BY $order_by" : "") . ($args->{order_by} ? " ORDER BY $order_by" : "")
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*$comments{nibble}*/"; . " /*$comments->{nibble}*/";
PTDEBUG && _d('Nibble statement:', $nibble_sql); PTDEBUG && _d('Nibble statement:', $nibble_sql);
my $explain_nibble_sql my $explain_nibble_sql
= "EXPLAIN SELECT " = "EXPLAIN SELECT "
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $q->quote($_) } @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
. " AND " . $asc->{boundaries}->{'<='} # upper boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
. ($args{order_by} ? " ORDER BY $order_by" : "") . ($args->{order_by} ? " ORDER BY $order_by" : "")
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*explain $comments{nibble}*/"; . " /*explain $comments->{nibble}*/";
PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql);
my $limit = $chunk_size - 1; my $limit = $chunk_size - 1;
PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); PTDEBUG && _d('Initial chunk size (LIMIT):', $limit);
$self = { my $params = {
%args, one_nibble => 0,
index => $index, index => $index,
limit => $limit, limit => $limit,
first_lb_sql => $first_lb_sql, first_lb_sql => $first_lb_sql,
@@ -5663,17 +5705,7 @@ sub new {
order_by => $order_by, order_by => $order_by,
}, },
}; };
} return $params;
$self->{row_est} = $nibble_params->{row_est},
$self->{nibbleno} = 0;
$self->{have_rows} = 0;
$self->{rowno} = 0;
$self->{oktonibble} = 1;
$self->{pause_file} = $nibble_params->{pause_file};
$self->{sleep} = $args{sleep} || 60;
return bless $self, $class;
} }
sub next { sub next {
@@ -5710,7 +5742,6 @@ sub next {
NIBBLE: NIBBLE:
while ( $self->{have_rows} || $self->_next_boundaries() ) { while ( $self->{have_rows} || $self->_next_boundaries() ) {
if ($self->{pause_file}) { if ($self->{pause_file}) {
while(-f $self->{pause_file}) { while(-f $self->{pause_file}) {
print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n";
@@ -5822,12 +5853,8 @@ sub set_boundary {
} }
sub one_nibble { sub one_nibble {
my ($self, $value) = @_; my ($self) = @_;
if (@_ == 2) { return $self->{one_nibble};
$self->{one_nibble} = $value;
} else {
return $self->{one_nibble};
}
} }
sub limit { sub limit {
@@ -5861,11 +5888,11 @@ sub row_estimate {
sub can_nibble { sub can_nibble {
my (%args) = @_; my (%args) = @_;
my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser Quoter); my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser);
foreach my $arg ( @required_args ) { foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args};
my $where = $o->has('where') ? $o->get('where') : ''; my $where = $o->has('where') ? $o->get('where') : '';
@@ -5875,23 +5902,6 @@ sub can_nibble {
where => $where, where => $where,
); );
my $can_get_keys;
if ($mysql_index) {
my $idx_len = IndexLength->new(Quoter => $q);
my ($key_len, $key) = $idx_len->index_length(
Cxn => $args{Cxn},
tbl => $tbl,
index => $mysql_index,
n_index_cols => $o->get('chunk-index-columns'),
);
if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) {
$can_get_keys = 0;
} else {
$can_get_keys = 1;
}
}
if ( !$where ) { if ( !$where ) {
$mysql_index = undef; $mysql_index = undef;
} }
@@ -5900,10 +5910,6 @@ sub can_nibble {
my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
? $row_est <= $chunk_size * $chunk_size_limit ? $row_est <= $chunk_size * $chunk_size_limit
: 0; : 0;
if (!$can_get_keys) {
$one_nibble = 1;
}
PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
if ( $args{resume} if ( $args{resume}
@@ -6659,6 +6665,7 @@ use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use Data::Dumper; use Data::Dumper;
use Carp;
$Data::Dumper::Indent = 1; $Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1; $Data::Dumper::Sortkeys = 1;
$Data::Dumper::Quotekeys = 0; $Data::Dumper::Quotekeys = 0;
@@ -6671,7 +6678,7 @@ sub new {
} }
my $self = { my $self = {
Quoter => $args{Quoter}, Quoter => $args{Quoter},
}; };
return bless $self, $class; return bless $self, $class;
@@ -6728,8 +6735,16 @@ sub _get_first_values {
my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
my $index_cols = $index_struct->{cols}; my $index_cols = $index_struct->{cols};
my $index_columns = join (', ', my $index_columns;
eval {
$index_columns = join (', ',
map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]); map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]);
};
if ($EVAL_ERROR) {
confess "$EVAL_ERROR";
}
my @where; my @where;
foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) { foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) {
@@ -6763,8 +6778,7 @@ sub _make_range_query {
if ( $n_index_cols > 1 ) { if ( $n_index_cols > 1 ) {
foreach my $n ( 0..($n_index_cols - 2) ) { foreach my $n ( 0..($n_index_cols - 2) ) {
my $col = $index_cols->[$n]; my $col = $index_cols->[$n];
my $val = $vals->[$n]; my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
$val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
push @where, $q->quote($col) . " = " . $val; push @where, $q->quote($col) . " = " . $val;
} }
} }
@@ -9620,11 +9634,14 @@ sub main {
. " * chunk size limit=$limit).\n"; . " * chunk size limit=$limit).\n";
warn $msg; warn $msg;
$failed_to_nibble = 1; $failed_to_nibble = 1;
$nibble_iter->one_nibble(0); # disable nibbling for this table warn "Switching to nibble\n";
$nibble_iter->switch_to_nibble();
# die ts($msg); # die ts($msg);
} else {
return 0;
} }
} }
if (!$nibble_iter->one_nibble() || $failed_to_nibble) { # chunking the table if (!$nibble_iter->one_nibble()) { # chunking the table
if ( $o->get('check-plan') ) { if ( $o->get('check-plan') ) {
my $idx_len = new IndexLength(Quoter => $q); my $idx_len = new IndexLength(Quoter => $q);
my ($key_len, $key) = $idx_len->index_length( my ($key_len, $key) = $idx_len->index_length(

266
bin/pt-table-checksum
View File

@@ -4892,6 +4892,7 @@ sub generate_asc_stmt {
cols => \@cols, cols => \@cols,
quoter => $q, quoter => $q,
is_nullable => $tbl_struct->{is_nullable}, is_nullable => $tbl_struct->{is_nullable},
type_for => $tbl_struct->{type_for},
); );
$asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
} }
@@ -4912,6 +4913,7 @@ sub generate_cmp_where {
my @slice = @{$args{slice}}; my @slice = @{$args{slice}};
my @cols = @{$args{cols}}; my @cols = @{$args{cols}};
my $is_nullable = $args{is_nullable}; my $is_nullable = $args{is_nullable};
my $type_for = $args{type_for};
my $type = $args{type}; my $type = $args{type};
my $q = $self->{Quoter}; my $q = $self->{Quoter};
@@ -4928,13 +4930,14 @@ sub generate_cmp_where {
my $ord = $slice[$j]; my $ord = $slice[$j];
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))";
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
} }
else { else {
push @clause, "$quo = ?"; push @clause, "$quo = $val";
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
} }
@@ -4944,15 +4947,16 @@ sub generate_cmp_where {
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $end = $i == $#slice; # Last clause of the whole group. my $end = $i == $#slice; # Last clause of the whole group.
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
if ( $type =~ m/=/ && $end ) { if ( $type =~ m/=/ && $end ) {
push @clause, "(? IS NULL OR $quo $type ?)"; push @clause, "($val IS NULL OR $quo $type $val)";
} }
elsif ( $type =~ m/>/ ) { elsif ( $type =~ m/>/ ) {
push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)";
} }
else { # If $type =~ m/</ ) { else { # If $type =~ m/</ ) {
push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))"; push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))";
} }
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
@@ -4960,7 +4964,7 @@ sub generate_cmp_where {
else { else {
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?"); push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val");
} }
push @clauses, '(' . join(' AND ', @clause) . ')'; push @clauses, '(' . join(' AND ', @clause) . ')';
@@ -6293,7 +6297,6 @@ use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use IndexLength;
use Data::Dumper; use Data::Dumper;
$Data::Dumper::Indent = 1; $Data::Dumper::Indent = 1;
@@ -6331,82 +6334,131 @@ sub new {
my @cols = grep { !$ignore_col->{$_} } @$all_cols; my @cols = grep { !$ignore_col->{$_} } @$all_cols;
my $self; my $self;
if ( $nibble_params->{one_nibble} ) { if ( $nibble_params->{one_nibble} ) {
my $params = _one_nibble(\%args, \@cols, $where, $tbl, \%comments);
$self = {
%args,
one_nibble => 1,
limit => 0,
nibble_sql => $params->{nibble_sql},
explain_nibble_sql => $params->{explain_nibble_sql},
};
} else {
my $params = _nibble_params($nibble_params, $tbl, \%args, \@cols, $chunk_size, $where, \%comments, $q);
$self = {
%args,
index => $params->{index},
limit => $params->{limit},
first_lb_sql => $params->{first_lb_sql},
last_ub_sql => $params->{last_ub_sql},
ub_sql => $params->{ub_sql},
nibble_sql => $params->{nibble_sql},
explain_first_lb_sql => $params->{explain_first_lb_sql},
explain_ub_sql => $params->{explain_ub_sql},
explain_nibble_sql => $params->{explain_nibble_sql},
resume_lb_sql => $params->{resume_lb_sql},
sql => $params->{sql},
};
}
$self->{row_est} = $nibble_params->{row_est},
$self->{nibbleno} = 0;
$self->{have_rows} = 0;
$self->{rowno} = 0;
$self->{oktonibble} = 1;
$self->{pause_file} = $nibble_params->{pause_file};
$self->{sleep} = $args{sleep} || 60;
$self->{nibble_params} = $nibble_params;
$self->{tbl} = $tbl;
$self->{args} = \%args;
$self->{cols} = \@cols;
$self->{chunk_size} = $chunk_size;
$self->{where} = $where;
$self->{comments} = \%comments;
return bless $self, $class;
}
sub switch_to_nibble {
my $self = shift;
my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols},
$self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter});
$self->{one_nibble} = 0;
$self->{index} = $params->{index};
$self->{limit} = $params->{limit};
$self->{first_lb_sql} = $params->{first_lb_sql};
$self->{last_ub_sql} = $params->{last_ub_sql};
$self->{ub_sql} = $params->{ub_sql};
$self->{nibble_sql} = $params->{nibble_sql};
$self->{explain_first_lb_sql} = $params->{explain_first_lb_sql};
$self->{explain_ub_sql} = $params->{explain_ub_sql};
$self->{explain_nibble_sql} = $params->{explain_nibble_sql};
$self->{resume_lb_sql} = $params->{resume_lb_sql};
$self->{sql} = $params->{sql};
$self->_get_bounds();
$self->_prepare_sths();
}
sub _one_nibble {
my ($args, $cols, $where, $tbl, $comments) = @_;
my $q = new Quoter();
my $nibble_sql my $nibble_sql
= ($args{dml} ? "$args{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @cols)) : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ?
"CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*$comments{bite}*/"; . " /*$comments->{bite}*/";
PTDEBUG && _d('One nibble statement:', $nibble_sql); PTDEBUG && _d('One nibble statement:', $nibble_sql);
my $explain_nibble_sql my $explain_nibble_sql
= "EXPLAIN SELECT " = "EXPLAIN SELECT "
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @cols)) : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum'
? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*explain $comments{bite}*/"; . " /*explain $comments->{bite}*/";
PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql);
$self = { return {
%args,
one_nibble => 1, one_nibble => 1,
limit => 0, limit => 0,
nibble_sql => $nibble_sql, nibble_sql => $nibble_sql,
explain_nibble_sql => $explain_nibble_sql, explain_nibble_sql => $explain_nibble_sql,
}; };
} }
else {
sub _nibble_params {
my ($nibble_params, $tbl, $args, $cols, $chunk_size, $where, $comments, $q) = @_;
my $index = $nibble_params->{index}; # brevity my $index = $nibble_params->{index}; # brevity
my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols};
my $asc = $args{TableNibbler}->generate_asc_stmt( my $asc = $args->{TableNibbler}->generate_asc_stmt(
%args, %$args,
tbl_struct => $tbl->{tbl_struct}, tbl_struct => $tbl->{tbl_struct},
index => $index, index => $index,
n_index_cols => $args{n_chunk_index_cols}, n_index_cols => $args->{n_chunk_index_cols},
cols => \@cols, cols => $cols,
asc_only => 1, asc_only => 1,
); );
PTDEBUG && _d('Ascend params:', Dumper($asc)); PTDEBUG && _d('Ascend params:', Dumper($asc));
my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); my $force_concat_enums;
my $i=0;
for my $index (@{$index_cols}) {
last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols};
$i++;
if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') {
if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) {
my @items = split(/,\s*/, $1);
my $sorted = 1; # Asume the items list is sorted to later check if this is true
for (my $i=1; $i < scalar(@items); $i++) {
if ($items[$i-1] gt $items[$i]) {
$sorted = 0;
last;
}
}
if (!$force_concat_enums && !$sorted) {
die "The index " . $index . " in table " . $tbl->{name} .
" has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n";
}
}
}
}
my $from = "$tbl->{name} FORCE INDEX(`$index`)"; my $from = "$tbl->{name} FORCE INDEX(`$index`)";
my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols});
my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols});
my $first_lb_sql my $first_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY $order_by" . " ORDER BY $order_by"
@@ -6415,10 +6467,10 @@ sub new {
PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); PTDEBUG && _d('First lower boundary statement:', $first_lb_sql);
my $resume_lb_sql; my $resume_lb_sql;
if ( $args{resume} ) { if ( $args->{resume} ) {
$resume_lb_sql $resume_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>'} . " WHERE " . $asc->{boundaries}->{'>'}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -6430,7 +6482,7 @@ sub new {
my $last_ub_sql my $last_ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY " . " ORDER BY "
@@ -6441,7 +6493,7 @@ sub new {
my $ub_sql my $ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} . " WHERE " . $asc->{boundaries}->{'>='}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -6451,36 +6503,36 @@ sub new {
PTDEBUG && _d('Upper boundary statement:', $ub_sql); PTDEBUG && _d('Upper boundary statement:', $ub_sql);
my $nibble_sql my $nibble_sql
= ($args{dml} ? "$args{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
. " AND " . $asc->{boundaries}->{'<='} # upper boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
. ($args{order_by} ? " ORDER BY $order_by" : "") . ($args->{order_by} ? " ORDER BY $order_by" : "")
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*$comments{nibble}*/"; . " /*$comments->{nibble}*/";
PTDEBUG && _d('Nibble statement:', $nibble_sql); PTDEBUG && _d('Nibble statement:', $nibble_sql);
my $explain_nibble_sql my $explain_nibble_sql
= "EXPLAIN SELECT " = "EXPLAIN SELECT "
. ($args{select} ? $args{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $q->quote($_) } @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
. " AND " . $asc->{boundaries}->{'<='} # upper boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
. ($args{order_by} ? " ORDER BY $order_by" : "") . ($args->{order_by} ? " ORDER BY $order_by" : "")
. ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
. " /*explain $comments{nibble}*/"; . " /*explain $comments->{nibble}*/";
PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql);
my $limit = $chunk_size - 1; my $limit = $chunk_size - 1;
PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); PTDEBUG && _d('Initial chunk size (LIMIT):', $limit);
$self = { my $params = {
%args, one_nibble => 0,
index => $index, index => $index,
limit => $limit, limit => $limit,
first_lb_sql => $first_lb_sql, first_lb_sql => $first_lb_sql,
@@ -6499,17 +6551,7 @@ sub new {
order_by => $order_by, order_by => $order_by,
}, },
}; };
} return $params;
$self->{row_est} = $nibble_params->{row_est},
$self->{nibbleno} = 0;
$self->{have_rows} = 0;
$self->{rowno} = 0;
$self->{oktonibble} = 1;
$self->{pause_file} = $nibble_params->{pause_file};
$self->{sleep} = $args{sleep} || 60;
return bless $self, $class;
} }
sub next { sub next {
@@ -6546,7 +6588,6 @@ sub next {
NIBBLE: NIBBLE:
while ( $self->{have_rows} || $self->_next_boundaries() ) { while ( $self->{have_rows} || $self->_next_boundaries() ) {
if ($self->{pause_file}) { if ($self->{pause_file}) {
while(-f $self->{pause_file}) { while(-f $self->{pause_file}) {
print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n";
@@ -6658,12 +6699,8 @@ sub set_boundary {
} }
sub one_nibble { sub one_nibble {
my ($self, $value) = @_; my ($self) = @_;
if (@_ == 2) { return $self->{one_nibble};
$self->{one_nibble} = $value;
} else {
return $self->{one_nibble};
}
} }
sub limit { sub limit {
@@ -6697,11 +6734,11 @@ sub row_estimate {
sub can_nibble { sub can_nibble {
my (%args) = @_; my (%args) = @_;
my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser Quoter); my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser);
foreach my $arg ( @required_args ) { foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args};
my $where = $o->has('where') ? $o->get('where') : ''; my $where = $o->has('where') ? $o->get('where') : '';
@@ -6711,23 +6748,6 @@ sub can_nibble {
where => $where, where => $where,
); );
my $can_get_keys;
if ($mysql_index) {
my $idx_len = IndexLength->new(Quoter => $q);
my ($key_len, $key) = $idx_len->index_length(
Cxn => $args{Cxn},
tbl => $tbl,
index => $mysql_index,
n_index_cols => $o->get('chunk-index-columns'),
);
if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) {
$can_get_keys = 0;
} else {
$can_get_keys = 1;
}
}
if ( !$where ) { if ( !$where ) {
$mysql_index = undef; $mysql_index = undef;
} }
@@ -6736,10 +6756,6 @@ sub can_nibble {
my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
? $row_est <= $chunk_size * $chunk_size_limit ? $row_est <= $chunk_size * $chunk_size_limit
: 0; : 0;
if (!$can_get_keys) {
$one_nibble = 1;
}
PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
if ( $args{resume} if ( $args{resume}
@@ -9606,6 +9622,7 @@ use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use Data::Dumper; use Data::Dumper;
use Carp;
$Data::Dumper::Indent = 1; $Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1; $Data::Dumper::Sortkeys = 1;
$Data::Dumper::Quotekeys = 0; $Data::Dumper::Quotekeys = 0;
@@ -9618,7 +9635,7 @@ sub new {
} }
my $self = { my $self = {
Quoter => $args{Quoter}, Quoter => $args{Quoter},
}; };
return bless $self, $class; return bless $self, $class;
@@ -9670,11 +9687,21 @@ sub _get_first_values {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args}; my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args};
my $q = new Quoter();
my $q = $self->{Quoter};
my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
my $index_cols = $index_struct->{cols}; my $index_cols = $index_struct->{cols};
my $index_columns = join (', ', my $index_columns;
eval {
$index_columns = join (', ',
map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]); map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]);
};
if ($EVAL_ERROR) {
confess "$EVAL_ERROR";
}
my @where; my @where;
foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) { foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) {
@@ -9686,19 +9713,21 @@ sub _get_first_values {
. "WHERE " . join(' AND ', @where) . "WHERE " . join(' AND ', @where)
. " ORDER BY $index_columns " . " ORDER BY $index_columns "
. "LIMIT 1 /*key_len*/"; # only need 1 row . "LIMIT 1 /*key_len*/"; # only need 1 row
PTDEBUG && _d("_get_first_values: $sql"); PTDEBUG && _d($sql);
my $vals = $cxn->dbh()->selectrow_arrayref($sql); my $vals = $cxn->dbh()->selectrow_arrayref($sql);
return $vals; return $vals;
} }
sub _make_range_query { sub _make_range_query {
my ($self, %args) = @_; my ($self, %args) = @_;
my @required_args = qw(tbl index n_index_cols); my @required_args = qw(tbl index n_index_cols vals);
foreach my $arg ( @required_args ) { foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args}; my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args};
my $q = new Quoter();
my $q = $self->{Quoter};
my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
my $index_cols = $index_struct->{cols}; my $index_cols = $index_struct->{cols};
@@ -9706,12 +9735,15 @@ sub _make_range_query {
if ( $n_index_cols > 1 ) { if ( $n_index_cols > 1 ) {
foreach my $n ( 0..($n_index_cols - 2) ) { foreach my $n ( 0..($n_index_cols - 2) ) {
my $col = $index_cols->[$n]; my $col = $index_cols->[$n];
push @where, $q->quote($col) . " = ?"; my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
push @where, $q->quote($col) . " = " . $val;
} }
} }
my $col = $index_cols->[$n_index_cols - 1]; my $col = $index_cols->[$n_index_cols - 1];
push @where, $q->quote($col) . " >= ?"; my $val = $vals->[-1]; # should only be as many vals as cols
my $condition = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
push @where, $q->quote($col) . " >= " . $condition;
my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * " my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * "
. "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") " . "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") "

16
bin/pt-table-sync
View File

@@ -6437,6 +6437,7 @@ sub generate_asc_stmt {
cols => \@cols, cols => \@cols,
quoter => $q, quoter => $q,
is_nullable => $tbl_struct->{is_nullable}, is_nullable => $tbl_struct->{is_nullable},
type_for => $tbl_struct->{type_for},
); );
$asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
} }
@@ -6457,6 +6458,7 @@ sub generate_cmp_where {
my @slice = @{$args{slice}}; my @slice = @{$args{slice}};
my @cols = @{$args{cols}}; my @cols = @{$args{cols}};
my $is_nullable = $args{is_nullable}; my $is_nullable = $args{is_nullable};
my $type_for = $args{type_for};
my $type = $args{type}; my $type = $args{type};
my $q = $self->{Quoter}; my $q = $self->{Quoter};
@@ -6473,13 +6475,14 @@ sub generate_cmp_where {
my $ord = $slice[$j]; my $ord = $slice[$j];
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))";
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
} }
else { else {
push @clause, "$quo = ?"; push @clause, "$quo = $val";
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
} }
@@ -6489,15 +6492,16 @@ sub generate_cmp_where {
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $end = $i == $#slice; # Last clause of the whole group. my $end = $i == $#slice; # Last clause of the whole group.
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
if ( $type =~ m/=/ && $end ) { if ( $type =~ m/=/ && $end ) {
push @clause, "(? IS NULL OR $quo $type ?)"; push @clause, "($val IS NULL OR $quo $type $val)";
} }
elsif ( $type =~ m/>/ ) { elsif ( $type =~ m/>/ ) {
push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)";
} }
else { # If $type =~ m/</ ) { else { # If $type =~ m/</ ) {
push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))"; push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))";
} }
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
@@ -6505,7 +6509,7 @@ sub generate_cmp_where {
else { else {
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?"); push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val");
} }
push @clauses, '(' . join(' AND ', @clause) . ')'; push @clauses, '(' . join(' AND ', @clause) . ')';

7
lib/IndexLength.pm
View File

@@ -151,8 +151,8 @@ sub _make_range_query {
# we don't want the last column; that's added below. # we don't want the last column; that's added below.
foreach my $n ( 0..($n_index_cols - 2) ) { foreach my $n ( 0..($n_index_cols - 2) ) {
my $col = $index_cols->[$n]; my $col = $index_cols->[$n];
my $val = $vals->[$n]; my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
push @where, $q->quote($col) . " = ?"; push @where, $q->quote($col) . " = " . $val;
} }
} }
@@ -160,7 +160,8 @@ sub _make_range_query {
# the N left-most index columns. # the N left-most index columns.
my $col = $index_cols->[$n_index_cols - 1]; my $col = $index_cols->[$n_index_cols - 1];
my $val = $vals->[-1]; # should only be as many vals as cols my $val = $vals->[-1]; # should only be as many vals as cols
push @where, $q->quote($col) . " >= ?"; my $condition = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
push @where, $q->quote($col) . " >= " . $condition;
my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * " my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * "
. "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") " . "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") "

28
lib/NibbleIterator.pm
View File

@@ -139,6 +139,7 @@ sub switch_to_nibble {
my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols},
$self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter}); $self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter});
$self->{one_nibble} = 0;
$self->{index} = $params->{index}; $self->{index} = $params->{index};
$self->{limit} = $params->{limit}; $self->{limit} = $params->{limit};
$self->{first_lb_sql} = $params->{first_lb_sql}; $self->{first_lb_sql} = $params->{first_lb_sql};
@@ -150,6 +151,8 @@ sub switch_to_nibble {
$self->{explain_nibble_sql} = $params->{explain_nibble_sql}; $self->{explain_nibble_sql} = $params->{explain_nibble_sql};
$self->{resume_lb_sql} = $params->{resume_lb_sql}; $self->{resume_lb_sql} = $params->{resume_lb_sql};
$self->{sql} = $params->{sql}; $self->{sql} = $params->{sql};
$self->_get_bounds();
$self->_prepare_sths();
} }
sub _one_nibble { sub _one_nibble {
@@ -161,7 +164,9 @@ sub _one_nibble {
my $nibble_sql my $nibble_sql
= ($args->{dml} ? "$args->{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args->{select} ? $args->{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @$cols)) # : join(', ', map { $q->quote($_) } @$cols))
: join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ?
"CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
@@ -171,7 +176,8 @@ sub _one_nibble {
my $explain_nibble_sql my $explain_nibble_sql
= "EXPLAIN SELECT " = "EXPLAIN SELECT "
. ($args->{select} ? $args->{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @$cols)) : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum'
? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols))
. " FROM $tbl->{name}" . " FROM $tbl->{name}"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "")
@@ -211,17 +217,14 @@ sub _nibble_params {
# are needed to ensure deterministic nibbling. # are needed to ensure deterministic nibbling.
my $from = "$tbl->{name} FORCE INDEX(`$index`)"; my $from = "$tbl->{name} FORCE INDEX(`$index`)";
my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols});
my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum'
? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols});
# The real first row in the table. Usually we start nibbling from # The real first row in the table. Usually we start nibbling from
# this row. Called once in _get_bounds(). # this row. Called once in _get_bounds().
my $first_lb_sql my $first_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY $order_by" . " ORDER BY $order_by"
@@ -235,7 +238,7 @@ sub _nibble_params {
if ( $args->{resume} ) { if ( $args->{resume} ) {
$resume_lb_sql $resume_lb_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>'} . " WHERE " . $asc->{boundaries}->{'>'}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -250,7 +253,7 @@ sub _nibble_params {
# upper in some cases. Called once in _get_bounds(). # upper in some cases. Called once in _get_bounds().
my $last_ub_sql my $last_ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. ($where ? " WHERE $where" : '') . ($where ? " WHERE $where" : '')
. " ORDER BY " . " ORDER BY "
@@ -269,7 +272,7 @@ sub _nibble_params {
# for the next nibble. See _next_boundaries(). # for the next nibble. See _next_boundaries().
my $ub_sql my $ub_sql
= "SELECT /*!40001 SQL_NO_CACHE */ " = "SELECT /*!40001 SQL_NO_CACHE */ "
. join(', ', map { $q->quote($_) } @{$asc->{scols}}) . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}})
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} . " WHERE " . $asc->{boundaries}->{'>='}
. ($where ? " AND ($where)" : '') . ($where ? " AND ($where)" : '')
@@ -283,7 +286,7 @@ sub _nibble_params {
my $nibble_sql my $nibble_sql
= ($args->{dml} ? "$args->{dml} " : "SELECT ") = ($args->{dml} ? "$args->{dml} " : "SELECT ")
. ($args->{select} ? $args->{select} . ($args->{select} ? $args->{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
. " AND " . $asc->{boundaries}->{'<='} # upper boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary
@@ -369,7 +372,6 @@ sub next {
# If there's another nibble, fetch the rows within it. # If there's another nibble, fetch the rows within it.
NIBBLE: NIBBLE:
while ( $self->{have_rows} || $self->_next_boundaries() ) { while ( $self->{have_rows} || $self->_next_boundaries() ) {
if ($self->{pause_file}) { if ($self->{pause_file}) {
while(-f $self->{pause_file}) { while(-f $self->{pause_file}) {
print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n";

16
lib/TableNibbler.pm
View File

@@ -125,6 +125,7 @@ sub generate_asc_stmt {
cols => \@cols, cols => \@cols,
quoter => $q, quoter => $q,
is_nullable => $tbl_struct->{is_nullable}, is_nullable => $tbl_struct->{is_nullable},
type_for => $tbl_struct->{type_for},
); );
$asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
} }
@@ -152,6 +153,7 @@ sub generate_cmp_where {
my @slice = @{$args{slice}}; my @slice = @{$args{slice}};
my @cols = @{$args{cols}}; my @cols = @{$args{cols}};
my $is_nullable = $args{is_nullable}; my $is_nullable = $args{is_nullable};
my $type_for = $args{type_for};
my $type = $args{type}; my $type = $args{type};
my $q = $self->{Quoter}; my $q = $self->{Quoter};
@@ -169,13 +171,14 @@ sub generate_cmp_where {
my $ord = $slice[$j]; my $ord = $slice[$j];
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))";
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
} }
else { else {
push @clause, "$quo = ?"; push @clause, "$quo = $val";
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
} }
@@ -188,15 +191,16 @@ sub generate_cmp_where {
my $col = $cols[$ord]; my $col = $cols[$ord];
my $quo = $q->quote($col); my $quo = $q->quote($col);
my $end = $i == $#slice; # Last clause of the whole group. my $end = $i == $#slice; # Last clause of the whole group.
my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?";
if ( $is_nullable->{$col} ) { if ( $is_nullable->{$col} ) {
if ( $type =~ m/=/ && $end ) { if ( $type =~ m/=/ && $end ) {
push @clause, "(? IS NULL OR $quo $type ?)"; push @clause, "($val IS NULL OR $quo $type $val)";
} }
elsif ( $type =~ m/>/ ) { elsif ( $type =~ m/>/ ) {
push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)";
} }
else { # If $type =~ m/</ ) { else { # If $type =~ m/</ ) {
push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))"; push @clauses, "(($val IS NOT NULL AND $quo IS NULL) OR ($quo $cmp $val))";
} }
push @r_slice, $ord, $ord; push @r_slice, $ord, $ord;
push @r_scols, $col, $col; push @r_scols, $col, $col;
@@ -204,7 +208,7 @@ sub generate_cmp_where {
else { else {
push @r_slice, $ord; push @r_slice, $ord;
push @r_scols, $col; push @r_scols, $col;
push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?"); push @clause, ($type =~ m/=/ && $end ? "$quo $type $val" : "$quo $cmp $val");
} }
# Add the clause to the larger WHERE clause. # Add the clause to the larger WHERE clause.

72
t/lib/NibbleIterator.t
View File

@@ -39,7 +39,7 @@ my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) { if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master'; plan skip_all => 'Cannot connect to sandbox master';
} else { } else {
plan tests => 62; plan tests => 56;
} }
my $q = new Quoter(); my $q = new Quoter();
@@ -896,76 +896,6 @@ is(
"Use non-unique index with highest cardinality (bug 1199591)" "Use non-unique index with highest cardinality (bug 1199591)"
); );
$sb->load_file('master', "t/lib/samples/NibbleIterator/enum_keys.sql");
$ni = undef;
eval {
$ni = make_nibble_iter(
db => 'test',
tbl => 't1',
argv => [qw(--databases test --chunk-size 3)],
);
};
like(
$EVAL_ERROR,
qr/The index f3 in table `test`.`t1` has unsorted enum items/,
"PT-1572 Die on unsorted enum items in index",
);
eval {
$ni = make_nibble_iter(
db => 'test',
tbl => 't1',
argv => [qw(--databases test --force-concat-enums --chunk-size 3)],
);
};
like(
$ni->{explain_first_lb_sql},
qr/ORDER BY `f1`, `f2`, CONCAT\(`f3`\)/,
"PT-1572 Use of CONCAT for unsorted ENUM field items without --",
);
eval {
$ni = make_nibble_iter(
db => 'test',
tbl => 't2',
argv => [qw(--databases test --chunk-size 3)],
);
};
is(
$EVAL_ERROR,
'',
"PT-1572 No errors on sorted enum items in index",
);
like(
$ni->{explain_first_lb_sql},
qr/ORDER BY `f1`, `f2`, `f3`/,
"PT-1572 Don't use CONCAT for sorted ENUM field items without --force-concat-enums",
);
eval {
$ni = make_nibble_iter(
db => 'test',
tbl => 't1',
argv => [qw(--databases test --chunk-size 3 --chunk-index-columns 2)],
);
};
is(
$EVAL_ERROR,
'',
"PT-1572 No errors on unsorted enum items in index and --chunk-index-columns",
);
like(
$ni->{explain_first_lb_sql},
qr/ORDER BY `f1`, `f2`, `f3`/,
"PT-1572 Don't use CONCAT for sorted ENUM field items without --force-concat-enums & --chunk-index-columns",
);
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################

164
t/lib/TableNibbler.t
View File

@@ -417,17 +417,24 @@ is_deeply(
'>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND '
. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` '
. '= ? AND (? IS NULL OR `customer_id` >= ?)))', . '= ? AND (? IS NULL OR `customer_id` >= ?)))',
'>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR '
. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) '
. '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' . 'OR (`customer_id` > ?)))',
. 'OR (`customer_id` > ?))))', # '((`rental_date` > ?) OR (`rental_date` = ? AND '
#. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` '
# . '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) '
# . 'OR (`customer_id` > ?))))',
'<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND '
. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` '
. '= ? AND (? IS NULL OR `customer_id` <= ?)))', . '= ? AND (? IS NULL OR `customer_id` <= ?)))',
'<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` < ?) OR '
. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '((? IS NOT NULL AND `customer_id` IS NULL) OR (`customer_id` < ?)) OR '
. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' . '(`rental_date` = ? AND `inventory_id` = ?))',
. 'OR (`customer_id` < ?))))',
# '((`rental_date` < ?) OR (`rental_date` = ? AND '
#. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` '
#. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) '
#. 'OR (`customer_id` < ?))))',
}, },
}, },
'Alternate index on sakila.rental with nullable customer_id', 'Alternate index on sakila.rental with nullable customer_id',
@@ -460,26 +467,35 @@ is_deeply(
cols => [qw(rental_id rental_date inventory_id customer_id cols => [qw(rental_id rental_date inventory_id customer_id
return_date staff_id last_update)], return_date staff_id last_update)],
index => 'rental_date', index => 'rental_date',
where => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?)' where => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR '
. ' OR (`rental_date` = ? AND `inventory_id` = ? AND ' . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) '
. '((? IS NULL AND `customer_id` IS NOT NULL) OR (`customer_id` > ?))))', . 'OR (`customer_id` > ?)))',
# '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?)'
#. ' OR (`rental_date` = ? AND `inventory_id` = ? AND '
#. '((? IS NULL AND `customer_id` IS NOT NULL) OR (`customer_id` > ?))))',
slice => [1, 1, 2, 1, 2, 3, 3], slice => [1, 1, 2, 1, 2, 3, 3],
scols => [qw(rental_date rental_date inventory_id rental_date inventory_id customer_id customer_id)], scols => [qw(rental_date rental_date inventory_id rental_date inventory_id customer_id customer_id)],
boundaries => { boundaries => {
'>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND '
. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` '
. '= ? AND (? IS NULL OR `customer_id` >= ?)))', . '= ? AND (? IS NULL OR `customer_id` >= ?)))',
'>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR '
. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) OR '
. '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' . '(`customer_id` > ?)))',
. 'OR (`customer_id` > ?))))', # '((`rental_date` > ?) OR (`rental_date` = ? AND '
#. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` '
#. '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) '
#. 'OR (`customer_id` > ?))))',
'<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND '
. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` '
. '= ? AND (? IS NULL OR `customer_id` <= ?)))', . '= ? AND (? IS NULL OR `customer_id` <= ?)))',
'<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` < ?) OR '
. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '((? IS NOT NULL AND `customer_id` IS NULL) OR (`customer_id` < ?)) OR '
. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' . '(`rental_date` = ? AND `inventory_id` = ?))',
. 'OR (`customer_id` < ?))))', # '((`rental_date` < ?) OR (`rental_date` = ? AND '
#. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` '
#. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) '
#. 'OR (`customer_id` < ?))))',
}, },
}, },
'Alternate index on sakila.rental with nullable customer_id and strict ascending', 'Alternate index on sakila.rental with nullable customer_id and strict ascending',
@@ -500,30 +516,45 @@ is_deeply(
cols => [qw(rental_id rental_date inventory_id customer_id cols => [qw(rental_id rental_date inventory_id customer_id
return_date staff_id last_update)], return_date staff_id last_update)],
index => 'rental_date', index => 'rental_date',
where => '((`rental_date` > ?) OR ' where => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL)'
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' . ' OR (`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' . '(`inventory_id` = ?)) AND `customer_id` >= ?))',
. 'OR (`inventory_id` = ?)) AND `customer_id` >= ?))', # '((`rental_date` > ?) OR '
#. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))'
#. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
#. 'OR (`inventory_id` = ?)) AND `customer_id` >= ?))',
slice => [1, 1, 2, 2, 1, 2, 2, 3], slice => [1, 1, 2, 2, 1, 2, 2, 3],
scols => [qw(rental_date rental_date inventory_id inventory_id scols => [qw(rental_date rental_date inventory_id inventory_id
rental_date inventory_id inventory_id customer_id)], rental_date inventory_id inventory_id customer_id)],
boundaries => { boundaries => {
'>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR '
. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' . '(`inventory_id` = ?)) AND `customer_id` >= ?))',
. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', # '((`rental_date` > ?) OR (`rental_date` = ? AND '
'>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' #. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` '
. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' #. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` '
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' #. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))',
. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR '
'<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' . '(`inventory_id` = ?)) AND `customer_id` > ?))',
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' # '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL '
. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', #. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR '
'<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' #. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))',
. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' '<=' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) '
. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', . 'OR (`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '(`inventory_id` = ?)) AND `customer_id` <= ?))',
# '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT '
#. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR '
#. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
#. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))',
'<' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) '
. 'OR (`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '(`inventory_id` = ?)) AND `customer_id` < ?))',
# '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT '
#. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) '
#. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS '
#. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))',
}, },
}, },
'Alternate index on sakila.rental with nullable inventory_id', 'Alternate index on sakila.rental with nullable inventory_id',
@@ -540,30 +571,45 @@ is_deeply(
cols => [qw(rental_id rental_date inventory_id customer_id cols => [qw(rental_id rental_date inventory_id customer_id
return_date staff_id last_update)], return_date staff_id last_update)],
index => 'rental_date', index => 'rental_date',
where => '((`rental_date` > ?) OR ' where => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR '
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' . '(`inventory_id` = ?)) AND `customer_id` > ?))',
. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', # '((`rental_date` > ?) OR '
#. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))'
#. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
#. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))',
slice => [1, 1, 2, 2, 1, 2, 2, 3], slice => [1, 1, 2, 2, 1, 2, 2, 3],
scols => [qw(rental_date rental_date inventory_id inventory_id scols => [qw(rental_date rental_date inventory_id inventory_id
rental_date inventory_id inventory_id customer_id)], rental_date inventory_id inventory_id customer_id)],
boundaries => { boundaries => {
'>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR '
. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' . '(`inventory_id` = ?)) AND `customer_id` >= ?))',
. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', # '((`rental_date` > ?) OR (`rental_date` = ? AND '
'>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' #. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` '
. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' #. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` '
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' #. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))',
. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR '
'<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' . '(`inventory_id` = ?)) AND `customer_id` > ?))',
. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' # '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL '
. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', #. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR '
'<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' #. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))',
. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' '<=' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) OR '
. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', . '(`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '(`inventory_id` = ?)) AND `customer_id` <= ?))',
# '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT '
#. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR '
#. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) '
#. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))',
'<' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) OR '
. '(`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR '
. '(`inventory_id` = ?)) AND `customer_id` < ?))',
#'((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT '
#. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) '
#. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS '
#. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))',
}, },
}, },
'Alternate index on sakila.rental with nullable inventory_id and strict ascending', 'Alternate index on sakila.rental with nullable inventory_id and strict ascending',

96
t/pt-online-schema-change/pt_1757.t Normal file
View File

@@ -0,0 +1,96 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use threads;
use threads::shared;
use Thread::Semaphore;
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
use PerconaTest;
use Sandbox;
use SqlModes;
use File::Temp qw/ tempdir /;
if ($sandbox_version lt '5.7') {
plan skip_all => 'This test needs MySQL 5.7+';
} else {
plan tests => 3;
}
require "$trunk/bin/pt-online-schema-change";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
my $dsn = $sb->dsn_for("master");
my $slave_dbh = $sb->get_dbh_for('slave1');
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the
# tool will die.
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $master_port = $sb->port_for('master');
my $num_rows = 5000;
$sb->load_file('master', "t/pt-online-schema-change/samples/pt-1757.sql");
diag(`util/mysql_random_data_load --host=127.0.0.1 --port=$master_port --user=msandbox --password=msandbox test t1 $num_rows`);
# Let's alter the stats to force this scenario:
# 1) On master, we are going to put 100 as the number of rows in the table. This will make osc to try to run in one chunk
# 2) On the slave, we are going to put the real number of rows. This will cause a fallback to nibble and pt-osc should call
# NibbleIterator->switch_to_nibble()
$dbh->do('SET @@SESSION.sql_log_bin=0');
$dbh->do('update mysql.innodb_table_stats set n_rows=100 where table_name="t1"');
$dbh->do('update mysql.innodb_index_stats set stat_value=100 where stat_description in("id") and table_name="t1"');
$dbh->do('SET @@SESSION.sql_log_bin=1');
$slave_dbh->do('SET @@SESSION.sql_log_bin=0');
$slave_dbh->do("update mysql.innodb_table_stats set n_rows=$num_rows where table_name='t1'");
$slave_dbh->do("update mysql.innodb_index_stats set stat_value=$num_rows where stat_description in('id') and table_name='t1'");
$slave_dbh->do('SET @@SESSION.sql_log_bin=1');
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=test,t=t1",
'--execute', '--alter', "ADD COLUMN new_col INT NOT NULL DEFAULT 1",
'--chunk-size', '25',
),
},
stderr => 1,
);
is(
$exit_status,
0,
"Altered OK status",
);
# The WHERE clause here is important as a double check that the table was altered and new_col exists
my $rows = $dbh->selectrow_arrayref("SELECT COUNT(*) FROM test.t1 WHERE new_col = 1");
is(
$rows->[0],
$num_rows,
"Correct rows count"
) or diag(Dumper($rows));
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;

15
t/pt-online-schema-change/samples/pt-1757.sql Normal file
View File

@@ -0,0 +1,15 @@
DROP DATABASE IF EXISTS `test`;
CREATE DATABASE `test`;
CREATE TABLE `test`.`t1` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`f2` char(32) DEFAULT NULL,
`f3` ENUM('red','green','blue'),
PRIMARY KEY (`id`),
UNIQUE KEY `c` (`f2`)
) ENGINE=InnoDB
AUTO_INCREMENT=0
DEFAULT CHARSET=utf8
STATS_AUTO_RECALC = 0
STATS_PERSISTENT = 1;

BIN
util/mysql_random_data_load
View File

Binary file not shown.