diff --git a/bin/pt-archiver b/bin/pt-archiver index e4ed893c..0cf1f490 100755 --- a/bin/pt-archiver +++ b/bin/pt-archiver @@ -3136,6 +3136,7 @@ sub generate_asc_stmt { cols => \@cols, quoter => $q, is_nullable => $tbl_struct->{is_nullable}, + type_for => $tbl_struct->{type_for}, ); $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; } @@ -3156,6 +3157,7 @@ sub generate_cmp_where { my @slice = @{$args{slice}}; my @cols = @{$args{cols}}; my $is_nullable = $args{is_nullable}; + my $type_for = $args{type_for}; my $type = $args{type}; my $q = $self->{Quoter}; @@ -3172,13 +3174,14 @@ sub generate_cmp_where { my $ord = $slice[$j]; my $col = $cols[$ord]; my $quo = $q->quote($col); + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { - push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; + push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @r_slice, $ord, $ord; push @r_scols, $col, $col; } else { - push @clause, "$quo = ?"; + push @clause, "$quo = $val"; push @r_slice, $ord; push @r_scols, $col; } @@ -3188,15 +3191,16 @@ sub generate_cmp_where { my $col = $cols[$ord]; my $quo = $q->quote($col); my $end = $i == $#slice; # Last clause of the whole group. + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { if ( $type =~ m/=/ && $end ) { - push @clause, "(? IS NULL OR $quo $type ?)"; + push @clause, "($val IS NULL OR $quo $type $val)"; } elsif ( $type =~ m/>/ ) { - push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; + push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; } else { # If $type =~ m/quote($col); - my $val = $type_for->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { - #push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @r_slice, $ord, $ord; push @r_scols, $col, $col; } else { - #push @clause, "$quo = ?"; push @clause, "$quo = $val"; push @r_slice, $ord; push @r_scols, $col; @@ -3076,18 +3074,15 @@ sub generate_cmp_where { my $col = $cols[$ord]; my $quo = $q->quote($col); my $end = $i == $#slice; # Last clause of the whole group. - my $val = $type_for->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { if ( $type =~ m/=/ && $end ) { - #push @clause, "(? IS NULL OR $quo $type ?)"; push @clause, "($val IS NULL OR $quo $type $val)"; } elsif ( $type =~ m/>/ ) { - #push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; } else { # If $type =~ m/ 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -use IndexLength; use Data::Dumper; $Data::Dumper::Indent = 1; @@ -5495,82 +5488,131 @@ sub new { my @cols = grep { !$ignore_col->{$_} } @$all_cols; my $self; if ( $nibble_params->{one_nibble} ) { + my $params = _one_nibble(\%args, \@cols, $where, $tbl, \%comments); + $self = { + %args, + one_nibble => 1, + limit => 0, + nibble_sql => $params->{nibble_sql}, + explain_nibble_sql => $params->{explain_nibble_sql}, + }; + } else { + my $params = _nibble_params($nibble_params, $tbl, \%args, \@cols, $chunk_size, $where, \%comments, $q); + $self = { + %args, + index => $params->{index}, + limit => $params->{limit}, + first_lb_sql => $params->{first_lb_sql}, + last_ub_sql => $params->{last_ub_sql}, + ub_sql => $params->{ub_sql}, + nibble_sql => $params->{nibble_sql}, + explain_first_lb_sql => $params->{explain_first_lb_sql}, + explain_ub_sql => $params->{explain_ub_sql}, + explain_nibble_sql => $params->{explain_nibble_sql}, + resume_lb_sql => $params->{resume_lb_sql}, + sql => $params->{sql}, + }; + } + + $self->{row_est} = $nibble_params->{row_est}, + $self->{nibbleno} = 0; + $self->{have_rows} = 0; + $self->{rowno} = 0; + $self->{oktonibble} = 1; + $self->{pause_file} = $nibble_params->{pause_file}; + $self->{sleep} = $args{sleep} || 60; + + $self->{nibble_params} = $nibble_params; + $self->{tbl} = $tbl; + $self->{args} = \%args; + $self->{cols} = \@cols; + $self->{chunk_size} = $chunk_size; + $self->{where} = $where; + $self->{comments} = \%comments; + + return bless $self, $class; +} + +sub switch_to_nibble { + my $self = shift; + my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, + $self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter}); + + $self->{one_nibble} = 0; + $self->{index} = $params->{index}; + $self->{limit} = $params->{limit}; + $self->{first_lb_sql} = $params->{first_lb_sql}; + $self->{last_ub_sql} = $params->{last_ub_sql}; + $self->{ub_sql} = $params->{ub_sql}; + $self->{nibble_sql} = $params->{nibble_sql}; + $self->{explain_first_lb_sql} = $params->{explain_first_lb_sql}; + $self->{explain_ub_sql} = $params->{explain_ub_sql}; + $self->{explain_nibble_sql} = $params->{explain_nibble_sql}; + $self->{resume_lb_sql} = $params->{resume_lb_sql}; + $self->{sql} = $params->{sql}; + $self->_get_bounds(); + $self->_prepare_sths(); +} + +sub _one_nibble { + my ($args, $cols, $where, $tbl, $comments) = @_; + my $q = new Quoter(); + my $nibble_sql - = ($args{dml} ? "$args{dml} " : "SELECT ") - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @cols)) + = ($args->{dml} ? "$args->{dml} " : "SELECT ") + . ($args->{select} ? $args->{select} + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? + "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*$comments{bite}*/"; + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments->{bite}*/"; PTDEBUG && _d('One nibble statement:', $nibble_sql); my $explain_nibble_sql = "EXPLAIN SELECT " - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @cols)) + . ($args->{select} ? $args->{select} + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*explain $comments{bite}*/"; + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments->{bite}*/"; PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); - $self = { - %args, + return { one_nibble => 1, limit => 0, nibble_sql => $nibble_sql, explain_nibble_sql => $explain_nibble_sql, }; - } - else { +} + +sub _nibble_params { + my ($nibble_params, $tbl, $args, $cols, $chunk_size, $where, $comments, $q) = @_; my $index = $nibble_params->{index}; # brevity my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; - my $asc = $args{TableNibbler}->generate_asc_stmt( - %args, + my $asc = $args->{TableNibbler}->generate_asc_stmt( + %$args, tbl_struct => $tbl->{tbl_struct}, index => $index, - n_index_cols => $args{n_chunk_index_cols}, - cols => \@cols, + n_index_cols => $args->{n_chunk_index_cols}, + cols => $cols, asc_only => 1, ); PTDEBUG && _d('Ascend params:', Dumper($asc)); - my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); - my $i=0; - for my $index (@{$index_cols}) { - last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols}; - $i++; - if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { - if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { - my @items = split(/,\s*/, $1); - my $sorted = 1; # Asume the items list is sorted to later check if this is true - for (my $i=1; $i < scalar(@items); $i++) { - if ($items[$i-1] gt $items[$i]) { - $sorted = 0; - last; - } - } - if (!$force_concat_enums && !$sorted) { - die "The index " . $index . " in table " . $tbl->{name} . - " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; - } - } - } - } + my $force_concat_enums; my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - - my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); + my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); + my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols}); my $first_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY $order_by" @@ -5579,10 +5621,10 @@ sub new { PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); my $resume_lb_sql; - if ( $args{resume} ) { + if ( $args->{resume} ) { $resume_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>'} . ($where ? " AND ($where)" : '') @@ -5594,7 +5636,7 @@ sub new { my $last_ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY " @@ -5605,7 +5647,7 @@ sub new { my $ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} . ($where ? " AND ($where)" : '') @@ -5615,36 +5657,36 @@ sub new { PTDEBUG && _d('Upper boundary statement:', $ub_sql); my $nibble_sql - = ($args{dml} ? "$args{dml} " : "SELECT ") - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @{$asc->{cols}})) + = ($args->{dml} ? "$args->{dml} " : "SELECT ") + . ($args->{select} ? $args->{select} + : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($where ? " AND ($where)" : '') - . ($args{order_by} ? " ORDER BY $order_by" : "") - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*$comments{nibble}*/"; + . ($args->{order_by} ? " ORDER BY $order_by" : "") + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments->{nibble}*/"; PTDEBUG && _d('Nibble statement:', $nibble_sql); my $explain_nibble_sql = "EXPLAIN SELECT " - . ($args{select} ? $args{select} + . ($args->{select} ? $args->{select} : join(', ', map { $q->quote($_) } @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($where ? " AND ($where)" : '') - . ($args{order_by} ? " ORDER BY $order_by" : "") - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*explain $comments{nibble}*/"; + . ($args->{order_by} ? " ORDER BY $order_by" : "") + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments->{nibble}*/"; PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); my $limit = $chunk_size - 1; PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); - $self = { - %args, + my $params = { + one_nibble => 0, index => $index, limit => $limit, first_lb_sql => $first_lb_sql, @@ -5663,17 +5705,7 @@ sub new { order_by => $order_by, }, }; - } - - $self->{row_est} = $nibble_params->{row_est}, - $self->{nibbleno} = 0; - $self->{have_rows} = 0; - $self->{rowno} = 0; - $self->{oktonibble} = 1; - $self->{pause_file} = $nibble_params->{pause_file}; - $self->{sleep} = $args{sleep} || 60; - - return bless $self, $class; + return $params; } sub next { @@ -5710,7 +5742,6 @@ sub next { NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { - if ($self->{pause_file}) { while(-f $self->{pause_file}) { print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; @@ -5822,12 +5853,8 @@ sub set_boundary { } sub one_nibble { - my ($self, $value) = @_; - if (@_ == 2) { - $self->{one_nibble} = $value; - } else { - return $self->{one_nibble}; - } + my ($self) = @_; + return $self->{one_nibble}; } sub limit { @@ -5861,11 +5888,11 @@ sub row_estimate { sub can_nibble { my (%args) = @_; - my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser Quoter); + my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; + my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args}; my $where = $o->has('where') ? $o->get('where') : ''; @@ -5875,23 +5902,6 @@ sub can_nibble { where => $where, ); - my $can_get_keys; - if ($mysql_index) { - my $idx_len = IndexLength->new(Quoter => $q); - my ($key_len, $key) = $idx_len->index_length( - Cxn => $args{Cxn}, - tbl => $tbl, - index => $mysql_index, - n_index_cols => $o->get('chunk-index-columns'), - ); - - if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) { - $can_get_keys = 0; - } else { - $can_get_keys = 1; - } - } - if ( !$where ) { $mysql_index = undef; } @@ -5900,10 +5910,6 @@ sub can_nibble { my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} ? $row_est <= $chunk_size * $chunk_size_limit : 0; - if (!$can_get_keys) { - $one_nibble = 1; - } - PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); if ( $args{resume} @@ -6659,6 +6665,7 @@ use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; use Data::Dumper; +use Carp; $Data::Dumper::Indent = 1; $Data::Dumper::Sortkeys = 1; $Data::Dumper::Quotekeys = 0; @@ -6671,7 +6678,7 @@ sub new { } my $self = { - Quoter => $args{Quoter}, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -6728,8 +6735,16 @@ sub _get_first_values { my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_cols = $index_struct->{cols}; - my $index_columns = join (', ', + my $index_columns; + eval { + $index_columns = join (', ', map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]); + }; + if ($EVAL_ERROR) { + confess "$EVAL_ERROR"; + } + + my @where; foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) { @@ -6763,8 +6778,7 @@ sub _make_range_query { if ( $n_index_cols > 1 ) { foreach my $n ( 0..($n_index_cols - 2) ) { my $col = $index_cols->[$n]; - my $val = $vals->[$n]; - $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; push @where, $q->quote($col) . " = " . $val; } } @@ -9620,11 +9634,14 @@ sub main { . " * chunk size limit=$limit).\n"; warn $msg; $failed_to_nibble = 1; - $nibble_iter->one_nibble(0); # disable nibbling for this table + warn "Switching to nibble\n"; + $nibble_iter->switch_to_nibble(); # die ts($msg); + } else { + return 0; } } - if (!$nibble_iter->one_nibble() || $failed_to_nibble) { # chunking the table + if (!$nibble_iter->one_nibble()) { # chunking the table if ( $o->get('check-plan') ) { my $idx_len = new IndexLength(Quoter => $q); my ($key_len, $key) = $idx_len->index_length( diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index cc15207b..5e2bc0e6 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -4892,6 +4892,7 @@ sub generate_asc_stmt { cols => \@cols, quoter => $q, is_nullable => $tbl_struct->{is_nullable}, + type_for => $tbl_struct->{type_for}, ); $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; } @@ -4912,6 +4913,7 @@ sub generate_cmp_where { my @slice = @{$args{slice}}; my @cols = @{$args{cols}}; my $is_nullable = $args{is_nullable}; + my $type_for = $args{type_for}; my $type = $args{type}; my $q = $self->{Quoter}; @@ -4928,13 +4930,14 @@ sub generate_cmp_where { my $ord = $slice[$j]; my $col = $cols[$ord]; my $quo = $q->quote($col); + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { - push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; + push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @r_slice, $ord, $ord; push @r_scols, $col, $col; } else { - push @clause, "$quo = ?"; + push @clause, "$quo = $val"; push @r_slice, $ord; push @r_scols, $col; } @@ -4944,15 +4947,16 @@ sub generate_cmp_where { my $col = $cols[$ord]; my $quo = $q->quote($col); my $end = $i == $#slice; # Last clause of the whole group. + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { if ( $type =~ m/=/ && $end ) { - push @clause, "(? IS NULL OR $quo $type ?)"; + push @clause, "($val IS NULL OR $quo $type $val)"; } elsif ( $type =~ m/>/ ) { - push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; + push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; } else { # If $type =~ m/ 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -use IndexLength; use Data::Dumper; $Data::Dumper::Indent = 1; @@ -6331,82 +6334,131 @@ sub new { my @cols = grep { !$ignore_col->{$_} } @$all_cols; my $self; if ( $nibble_params->{one_nibble} ) { + my $params = _one_nibble(\%args, \@cols, $where, $tbl, \%comments); + $self = { + %args, + one_nibble => 1, + limit => 0, + nibble_sql => $params->{nibble_sql}, + explain_nibble_sql => $params->{explain_nibble_sql}, + }; + } else { + my $params = _nibble_params($nibble_params, $tbl, \%args, \@cols, $chunk_size, $where, \%comments, $q); + $self = { + %args, + index => $params->{index}, + limit => $params->{limit}, + first_lb_sql => $params->{first_lb_sql}, + last_ub_sql => $params->{last_ub_sql}, + ub_sql => $params->{ub_sql}, + nibble_sql => $params->{nibble_sql}, + explain_first_lb_sql => $params->{explain_first_lb_sql}, + explain_ub_sql => $params->{explain_ub_sql}, + explain_nibble_sql => $params->{explain_nibble_sql}, + resume_lb_sql => $params->{resume_lb_sql}, + sql => $params->{sql}, + }; + } + + $self->{row_est} = $nibble_params->{row_est}, + $self->{nibbleno} = 0; + $self->{have_rows} = 0; + $self->{rowno} = 0; + $self->{oktonibble} = 1; + $self->{pause_file} = $nibble_params->{pause_file}; + $self->{sleep} = $args{sleep} || 60; + + $self->{nibble_params} = $nibble_params; + $self->{tbl} = $tbl; + $self->{args} = \%args; + $self->{cols} = \@cols; + $self->{chunk_size} = $chunk_size; + $self->{where} = $where; + $self->{comments} = \%comments; + + return bless $self, $class; +} + +sub switch_to_nibble { + my $self = shift; + my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, + $self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter}); + + $self->{one_nibble} = 0; + $self->{index} = $params->{index}; + $self->{limit} = $params->{limit}; + $self->{first_lb_sql} = $params->{first_lb_sql}; + $self->{last_ub_sql} = $params->{last_ub_sql}; + $self->{ub_sql} = $params->{ub_sql}; + $self->{nibble_sql} = $params->{nibble_sql}; + $self->{explain_first_lb_sql} = $params->{explain_first_lb_sql}; + $self->{explain_ub_sql} = $params->{explain_ub_sql}; + $self->{explain_nibble_sql} = $params->{explain_nibble_sql}; + $self->{resume_lb_sql} = $params->{resume_lb_sql}; + $self->{sql} = $params->{sql}; + $self->_get_bounds(); + $self->_prepare_sths(); +} + +sub _one_nibble { + my ($args, $cols, $where, $tbl, $comments) = @_; + my $q = new Quoter(); + my $nibble_sql - = ($args{dml} ? "$args{dml} " : "SELECT ") - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @cols)) + = ($args->{dml} ? "$args->{dml} " : "SELECT ") + . ($args->{select} ? $args->{select} + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? + "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*$comments{bite}*/"; + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments->{bite}*/"; PTDEBUG && _d('One nibble statement:', $nibble_sql); my $explain_nibble_sql = "EXPLAIN SELECT " - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @cols)) + . ($args->{select} ? $args->{select} + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*explain $comments{bite}*/"; + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments->{bite}*/"; PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); - $self = { - %args, + return { one_nibble => 1, limit => 0, nibble_sql => $nibble_sql, explain_nibble_sql => $explain_nibble_sql, }; - } - else { +} + +sub _nibble_params { + my ($nibble_params, $tbl, $args, $cols, $chunk_size, $where, $comments, $q) = @_; my $index = $nibble_params->{index}; # brevity my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; - my $asc = $args{TableNibbler}->generate_asc_stmt( - %args, + my $asc = $args->{TableNibbler}->generate_asc_stmt( + %$args, tbl_struct => $tbl->{tbl_struct}, index => $index, - n_index_cols => $args{n_chunk_index_cols}, - cols => \@cols, + n_index_cols => $args->{n_chunk_index_cols}, + cols => $cols, asc_only => 1, ); PTDEBUG && _d('Ascend params:', Dumper($asc)); - my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); - my $i=0; - for my $index (@{$index_cols}) { - last if $args{n_chunk_index_cols} && $i >= $args{n_chunk_index_cols}; - $i++; - if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { - if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { - my @items = split(/,\s*/, $1); - my $sorted = 1; # Asume the items list is sorted to later check if this is true - for (my $i=1; $i < scalar(@items); $i++) { - if ($items[$i-1] gt $items[$i]) { - $sorted = 0; - last; - } - } - if (!$force_concat_enums && !$sorted) { - die "The index " . $index . " in table " . $tbl->{name} . - " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; - } - } - } - } + my $force_concat_enums; my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - - my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); + my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); + my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols}); my $first_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY $order_by" @@ -6415,10 +6467,10 @@ sub new { PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); my $resume_lb_sql; - if ( $args{resume} ) { + if ( $args->{resume} ) { $resume_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>'} . ($where ? " AND ($where)" : '') @@ -6430,7 +6482,7 @@ sub new { my $last_ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY " @@ -6441,7 +6493,7 @@ sub new { my $ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} . ($where ? " AND ($where)" : '') @@ -6451,36 +6503,36 @@ sub new { PTDEBUG && _d('Upper boundary statement:', $ub_sql); my $nibble_sql - = ($args{dml} ? "$args{dml} " : "SELECT ") - . ($args{select} ? $args{select} - : join(', ', map { $q->quote($_) } @{$asc->{cols}})) + = ($args->{dml} ? "$args->{dml} " : "SELECT ") + . ($args->{select} ? $args->{select} + : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($where ? " AND ($where)" : '') - . ($args{order_by} ? " ORDER BY $order_by" : "") - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*$comments{nibble}*/"; + . ($args->{order_by} ? " ORDER BY $order_by" : "") + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*$comments->{nibble}*/"; PTDEBUG && _d('Nibble statement:', $nibble_sql); my $explain_nibble_sql = "EXPLAIN SELECT " - . ($args{select} ? $args{select} + . ($args->{select} ? $args->{select} : join(', ', map { $q->quote($_) } @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($where ? " AND ($where)" : '') - . ($args{order_by} ? " ORDER BY $order_by" : "") - . ($args{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") - . " /*explain $comments{nibble}*/"; + . ($args->{order_by} ? " ORDER BY $order_by" : "") + . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") + . " /*explain $comments->{nibble}*/"; PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); my $limit = $chunk_size - 1; PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); - $self = { - %args, + my $params = { + one_nibble => 0, index => $index, limit => $limit, first_lb_sql => $first_lb_sql, @@ -6499,17 +6551,7 @@ sub new { order_by => $order_by, }, }; - } - - $self->{row_est} = $nibble_params->{row_est}, - $self->{nibbleno} = 0; - $self->{have_rows} = 0; - $self->{rowno} = 0; - $self->{oktonibble} = 1; - $self->{pause_file} = $nibble_params->{pause_file}; - $self->{sleep} = $args{sleep} || 60; - - return bless $self, $class; + return $params; } sub next { @@ -6546,7 +6588,6 @@ sub next { NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { - if ($self->{pause_file}) { while(-f $self->{pause_file}) { print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; @@ -6658,12 +6699,8 @@ sub set_boundary { } sub one_nibble { - my ($self, $value) = @_; - if (@_ == 2) { - $self->{one_nibble} = $value; - } else { - return $self->{one_nibble}; - } + my ($self) = @_; + return $self->{one_nibble}; } sub limit { @@ -6697,11 +6734,11 @@ sub row_estimate { sub can_nibble { my (%args) = @_; - my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser Quoter); + my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } - my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; + my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args}; my $where = $o->has('where') ? $o->get('where') : ''; @@ -6711,23 +6748,6 @@ sub can_nibble { where => $where, ); - my $can_get_keys; - if ($mysql_index) { - my $idx_len = IndexLength->new(Quoter => $q); - my ($key_len, $key) = $idx_len->index_length( - Cxn => $args{Cxn}, - tbl => $tbl, - index => $mysql_index, - n_index_cols => $o->get('chunk-index-columns'), - ); - - if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) { - $can_get_keys = 0; - } else { - $can_get_keys = 1; - } - } - if ( !$where ) { $mysql_index = undef; } @@ -6736,10 +6756,6 @@ sub can_nibble { my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} ? $row_est <= $chunk_size * $chunk_size_limit : 0; - if (!$can_get_keys) { - $one_nibble = 1; - } - PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); if ( $args{resume} @@ -9606,6 +9622,7 @@ use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; use Data::Dumper; +use Carp; $Data::Dumper::Indent = 1; $Data::Dumper::Sortkeys = 1; $Data::Dumper::Quotekeys = 0; @@ -9618,7 +9635,7 @@ sub new { } my $self = { - Quoter => $args{Quoter}, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -9670,11 +9687,21 @@ sub _get_first_values { die "I need a $arg argument" unless $args{$arg}; } my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args}; - my $q = new Quoter(); + + my $q = $self->{Quoter}; + my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_cols = $index_struct->{cols}; - my $index_columns = join (', ', + my $index_columns; + eval { + $index_columns = join (', ', map { $q->quote($_) } @{$index_cols}[0..($n_index_cols - 1)]); + }; + if ($EVAL_ERROR) { + confess "$EVAL_ERROR"; + } + + my @where; foreach my $col ( @{$index_cols}[0..($n_index_cols - 1)] ) { @@ -9686,19 +9713,21 @@ sub _get_first_values { . "WHERE " . join(' AND ', @where) . " ORDER BY $index_columns " . "LIMIT 1 /*key_len*/"; # only need 1 row - PTDEBUG && _d("_get_first_values: $sql"); + PTDEBUG && _d($sql); my $vals = $cxn->dbh()->selectrow_arrayref($sql); return $vals; } sub _make_range_query { my ($self, %args) = @_; - my @required_args = qw(tbl index n_index_cols); + my @required_args = qw(tbl index n_index_cols vals); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args}; - my $q = new Quoter(); + + my $q = $self->{Quoter}; + my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_cols = $index_struct->{cols}; @@ -9706,12 +9735,15 @@ sub _make_range_query { if ( $n_index_cols > 1 ) { foreach my $n ( 0..($n_index_cols - 2) ) { my $col = $index_cols->[$n]; - push @where, $q->quote($col) . " = ?"; + my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + push @where, $q->quote($col) . " = " . $val; } } my $col = $index_cols->[$n_index_cols - 1]; - push @where, $q->quote($col) . " >= ?"; + my $val = $vals->[-1]; # should only be as many vals as cols + my $condition = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + push @where, $q->quote($col) . " >= " . $condition; my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * " . "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") " diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 5ddba648..9ec71806 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -6437,6 +6437,7 @@ sub generate_asc_stmt { cols => \@cols, quoter => $q, is_nullable => $tbl_struct->{is_nullable}, + type_for => $tbl_struct->{type_for}, ); $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; } @@ -6457,6 +6458,7 @@ sub generate_cmp_where { my @slice = @{$args{slice}}; my @cols = @{$args{cols}}; my $is_nullable = $args{is_nullable}; + my $type_for = $args{type_for}; my $type = $args{type}; my $q = $self->{Quoter}; @@ -6473,13 +6475,14 @@ sub generate_cmp_where { my $ord = $slice[$j]; my $col = $cols[$ord]; my $quo = $q->quote($col); + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { - push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; + push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @r_slice, $ord, $ord; push @r_scols, $col, $col; } else { - push @clause, "$quo = ?"; + push @clause, "$quo = $val"; push @r_slice, $ord; push @r_scols, $col; } @@ -6489,15 +6492,16 @@ sub generate_cmp_where { my $col = $cols[$ord]; my $quo = $q->quote($col); my $end = $i == $#slice; # Last clause of the whole group. + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { if ( $type =~ m/=/ && $end ) { - push @clause, "(? IS NULL OR $quo $type ?)"; + push @clause, "($val IS NULL OR $quo $type $val)"; } elsif ( $type =~ m/>/ ) { - push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; + push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; } else { # If $type =~ m/[$n]; - my $val = $vals->[$n]; - push @where, $q->quote($col) . " = ?"; + my $val = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + push @where, $q->quote($col) . " = " . $val; } } @@ -160,7 +160,8 @@ sub _make_range_query { # the N left-most index columns. my $col = $index_cols->[$n_index_cols - 1]; my $val = $vals->[-1]; # should only be as many vals as cols - push @where, $q->quote($col) . " >= ?"; + my $condition = $tbl->{tbl_struct}->{type_for}->{$col} eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; + push @where, $q->quote($col) . " >= " . $condition; my $sql = "EXPLAIN SELECT /*!40001 SQL_NO_CACHE */ * " . "FROM $tbl->{name} FORCE INDEX (" . $q->quote($index) . ") " diff --git a/lib/NibbleIterator.pm b/lib/NibbleIterator.pm index 8376a831..8662b252 100644 --- a/lib/NibbleIterator.pm +++ b/lib/NibbleIterator.pm @@ -139,6 +139,7 @@ sub switch_to_nibble { my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, $self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter}); + $self->{one_nibble} = 0; $self->{index} = $params->{index}; $self->{limit} = $params->{limit}; $self->{first_lb_sql} = $params->{first_lb_sql}; @@ -150,6 +151,8 @@ sub switch_to_nibble { $self->{explain_nibble_sql} = $params->{explain_nibble_sql}; $self->{resume_lb_sql} = $params->{resume_lb_sql}; $self->{sql} = $params->{sql}; + $self->_get_bounds(); + $self->_prepare_sths(); } sub _one_nibble { @@ -161,7 +164,9 @@ sub _one_nibble { my $nibble_sql = ($args->{dml} ? "$args->{dml} " : "SELECT ") . ($args->{select} ? $args->{select} - : join(', ', map { $q->quote($_) } @$cols)) + # : join(', ', map { $q->quote($_) } @$cols)) + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? + "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") @@ -171,7 +176,8 @@ sub _one_nibble { my $explain_nibble_sql = "EXPLAIN SELECT " . ($args->{select} ? $args->{select} - : join(', ', map { $q->quote($_) } @$cols)) + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') . ($args->{lock_in_share_mode} ? " LOCK IN SHARE MODE" : "") @@ -211,17 +217,14 @@ sub _nibble_params { # are needed to ensure deterministic nibbling. my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - - my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' - ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); + my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); + my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols}); # The real first row in the table. Usually we start nibbling from # this row. Called once in _get_bounds(). my $first_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY $order_by" @@ -235,7 +238,7 @@ sub _nibble_params { if ( $args->{resume} ) { $resume_lb_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>'} . ($where ? " AND ($where)" : '') @@ -250,7 +253,7 @@ sub _nibble_params { # upper in some cases. Called once in _get_bounds(). my $last_ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . ($where ? " WHERE $where" : '') . " ORDER BY " @@ -269,7 +272,7 @@ sub _nibble_params { # for the next nibble. See _next_boundaries(). my $ub_sql = "SELECT /*!40001 SQL_NO_CACHE */ " - . join(', ', map { $q->quote($_) } @{$asc->{scols}}) + . join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{scols}}) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} . ($where ? " AND ($where)" : '') @@ -283,7 +286,7 @@ sub _nibble_params { my $nibble_sql = ($args->{dml} ? "$args->{dml} " : "SELECT ") . ($args->{select} ? $args->{select} - : join(', ', map { $q->quote($_) } @{$asc->{cols}})) + : join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_)} @{$asc->{cols}})) . " FROM $from" . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary . " AND " . $asc->{boundaries}->{'<='} # upper boundary @@ -369,7 +372,6 @@ sub next { # If there's another nibble, fetch the rows within it. NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { - if ($self->{pause_file}) { while(-f $self->{pause_file}) { print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; diff --git a/lib/TableNibbler.pm b/lib/TableNibbler.pm index 945da85f..122e4e7b 100644 --- a/lib/TableNibbler.pm +++ b/lib/TableNibbler.pm @@ -125,6 +125,7 @@ sub generate_asc_stmt { cols => \@cols, quoter => $q, is_nullable => $tbl_struct->{is_nullable}, + type_for => $tbl_struct->{type_for}, ); $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; } @@ -152,6 +153,7 @@ sub generate_cmp_where { my @slice = @{$args{slice}}; my @cols = @{$args{cols}}; my $is_nullable = $args{is_nullable}; + my $type_for = $args{type_for}; my $type = $args{type}; my $q = $self->{Quoter}; @@ -169,13 +171,14 @@ sub generate_cmp_where { my $ord = $slice[$j]; my $col = $cols[$ord]; my $quo = $q->quote($col); + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { - push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; + push @clause, "(($val IS NULL AND $quo IS NULL) OR ($quo = $val))"; push @r_slice, $ord, $ord; push @r_scols, $col, $col; } else { - push @clause, "$quo = ?"; + push @clause, "$quo = $val"; push @r_slice, $ord; push @r_scols, $col; } @@ -188,15 +191,16 @@ sub generate_cmp_where { my $col = $cols[$ord]; my $quo = $q->quote($col); my $end = $i == $#slice; # Last clause of the whole group. + my $val = ($col && ($type_for->{$col} || '')) eq 'enum' ? "CAST(? AS UNSIGNED)" : "?"; if ( $is_nullable->{$col} ) { if ( $type =~ m/=/ && $end ) { - push @clause, "(? IS NULL OR $quo $type ?)"; + push @clause, "($val IS NULL OR $quo $type $val)"; } elsif ( $type =~ m/>/ ) { - push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; + push @clause, "(($val IS NULL AND $quo IS NOT NULL) OR ($quo $cmp $val)"; } else { # If $type =~ m/get_dbh_for('master'); if ( !$dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 62; + plan tests => 56; } my $q = new Quoter(); @@ -896,76 +896,6 @@ is( "Use non-unique index with highest cardinality (bug 1199591)" ); -$sb->load_file('master', "t/lib/samples/NibbleIterator/enum_keys.sql"); -$ni = undef; -eval { - $ni = make_nibble_iter( - db => 'test', - tbl => 't1', - argv => [qw(--databases test --chunk-size 3)], - ); -}; - -like( - $EVAL_ERROR, - qr/The index f3 in table `test`.`t1` has unsorted enum items/, - "PT-1572 Die on unsorted enum items in index", -); - -eval { - $ni = make_nibble_iter( - db => 'test', - tbl => 't1', - argv => [qw(--databases test --force-concat-enums --chunk-size 3)], - ); -}; - -like( - $ni->{explain_first_lb_sql}, - qr/ORDER BY `f1`, `f2`, CONCAT\(`f3`\)/, - "PT-1572 Use of CONCAT for unsorted ENUM field items without --", -); - -eval { - $ni = make_nibble_iter( - db => 'test', - tbl => 't2', - argv => [qw(--databases test --chunk-size 3)], - ); -}; - -is( - $EVAL_ERROR, - '', - "PT-1572 No errors on sorted enum items in index", -); - -like( - $ni->{explain_first_lb_sql}, - qr/ORDER BY `f1`, `f2`, `f3`/, - "PT-1572 Don't use CONCAT for sorted ENUM field items without --force-concat-enums", -); - -eval { - $ni = make_nibble_iter( - db => 'test', - tbl => 't1', - argv => [qw(--databases test --chunk-size 3 --chunk-index-columns 2)], - ); -}; - -is( - $EVAL_ERROR, - '', - "PT-1572 No errors on unsorted enum items in index and --chunk-index-columns", -); - -like( - $ni->{explain_first_lb_sql}, - qr/ORDER BY `f1`, `f2`, `f3`/, - "PT-1572 Don't use CONCAT for sorted ENUM field items without --force-concat-enums & --chunk-index-columns", -); - # ############################################################################# # Done. # ############################################################################# diff --git a/t/lib/TableNibbler.t b/t/lib/TableNibbler.t index 4df9a4a5..6457bca3 100644 --- a/t/lib/TableNibbler.t +++ b/t/lib/TableNibbler.t @@ -417,17 +417,24 @@ is_deeply( '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '= ? AND (? IS NULL OR `customer_id` >= ?)))', - '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' - . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' - . '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' - . 'OR (`customer_id` > ?))))', + '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR ' + . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' + . 'OR (`customer_id` > ?)))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ' + #. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' + # . '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' + # . 'OR (`customer_id` > ?))))', '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '= ? AND (? IS NULL OR `customer_id` <= ?)))', - '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' - . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' - . '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' - . 'OR (`customer_id` < ?))))', + '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` < ?) OR ' + . '((? IS NOT NULL AND `customer_id` IS NULL) OR (`customer_id` < ?)) OR ' + . '(`rental_date` = ? AND `inventory_id` = ?))', + + # '((`rental_date` < ?) OR (`rental_date` = ? AND ' + #. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' + #. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' + #. 'OR (`customer_id` < ?))))', }, }, 'Alternate index on sakila.rental with nullable customer_id', @@ -460,26 +467,35 @@ is_deeply( cols => [qw(rental_id rental_date inventory_id customer_id return_date staff_id last_update)], index => 'rental_date', - where => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?)' - . ' OR (`rental_date` = ? AND `inventory_id` = ? AND ' - . '((? IS NULL AND `customer_id` IS NOT NULL) OR (`customer_id` > ?))))', + where => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR ' + . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' + . 'OR (`customer_id` > ?)))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?)' + #. ' OR (`rental_date` = ? AND `inventory_id` = ? AND ' + #. '((? IS NULL AND `customer_id` IS NOT NULL) OR (`customer_id` > ?))))', slice => [1, 1, 2, 1, 2, 3, 3], scols => [qw(rental_date rental_date inventory_id rental_date inventory_id customer_id customer_id)], boundaries => { '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' . '= ? AND (? IS NULL OR `customer_id` >= ?)))', - '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' - . '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' - . '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' - . 'OR (`customer_id` > ?))))', + '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND `inventory_id` > ?) OR ' + . '(`rental_date` = ? AND `inventory_id` = ? AND ((? IS NULL AND `customer_id` IS NOT NULL) OR ' + . '(`customer_id` > ?)))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ' + #. '`inventory_id` > ?) OR (`rental_date` = ? AND `inventory_id` ' + #. '= ? AND ((? IS NULL AND `customer_id` IS NOT NULL) ' + #. 'OR (`customer_id` > ?))))', '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' . '= ? AND (? IS NULL OR `customer_id` <= ?)))', - '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ' - . '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' - . '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' - . 'OR (`customer_id` < ?))))', + '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND `inventory_id` < ?) OR ' + . '((? IS NOT NULL AND `customer_id` IS NULL) OR (`customer_id` < ?)) OR ' + . '(`rental_date` = ? AND `inventory_id` = ?))', + # '((`rental_date` < ?) OR (`rental_date` = ? AND ' + #. '`inventory_id` < ?) OR (`rental_date` = ? AND `inventory_id` ' + #. '= ? AND ((? IS NOT NULL AND `customer_id` IS NULL) ' + #. 'OR (`customer_id` < ?))))', }, }, 'Alternate index on sakila.rental with nullable customer_id and strict ascending', @@ -500,30 +516,45 @@ is_deeply( cols => [qw(rental_id rental_date inventory_id customer_id return_date staff_id last_update)], index => 'rental_date', - where => '((`rental_date` > ?) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' - . ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` >= ?))', + where => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL)' + . ' OR (`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` >= ?))', + # '((`rental_date` > ?) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' + #. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` >= ?))', slice => [1, 1, 2, 2, 1, 2, 2, 3], scols => [qw(rental_date rental_date inventory_id inventory_id rental_date inventory_id inventory_id customer_id)], boundaries => { - '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' - . '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' - . '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' - . 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', - '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' - . 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', - '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' - . 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', - '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' - . 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' - . 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' - . 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', + '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR ' + . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` >= ?))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ' + #. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' + #. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' + #. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', + '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR ' + . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` > ?))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' + #. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', + '<=' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) ' + . 'OR (`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` <= ?))', + # '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' + #. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', + '<' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) ' + . 'OR (`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` < ?))', + # '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' + #. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' + #. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' + #. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', }, }, 'Alternate index on sakila.rental with nullable inventory_id', @@ -540,30 +571,45 @@ is_deeply( cols => [qw(rental_id rental_date inventory_id customer_id return_date staff_id last_update)], index => 'rental_date', - where => '((`rental_date` > ?) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' - . ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', + where => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR ' + . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` > ?))', + # '((`rental_date` > ?) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?)))' + #. ' OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', slice => [1, 1, 2, 2, 1, 2, 2, 3], scols => [qw(rental_date rental_date inventory_id inventory_id rental_date inventory_id inventory_id customer_id)], boundaries => { - '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ' - . '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' - . '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' - . 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', - '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' - . 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', - '<=' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' - . 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' - . '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' - . 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', - '<' => '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' - . 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' - . 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' - . 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', + '>=' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR ' + . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` >= ?))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ' + #. '((? IS NULL AND `inventory_id` IS NOT NULL) OR (`inventory_id` ' + #. '> ?))) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` ' + #. 'IS NULL) OR (`inventory_id` = ?)) AND `customer_id` >= ?))', + '>' => '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NOT NULL) OR ' + . '(`inventory_id` > ?)) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` > ?))', + # '((`rental_date` > ?) OR (`rental_date` = ? AND ((? IS NULL ' + #. 'AND `inventory_id` IS NOT NULL) OR (`inventory_id` > ?))) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` > ?))', + '<=' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) OR ' + . '(`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` <= ?))', + # '((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' + #. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) OR ' + #. '(`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) ' + #. 'OR (`inventory_id` = ?)) AND `customer_id` <= ?))', + '<' => '((`rental_date` < ?) OR ((? IS NOT NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?)) OR ' + . '(`rental_date` = ?) OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS NULL) OR ' + . '(`inventory_id` = ?)) AND `customer_id` < ?))', + #'((`rental_date` < ?) OR (`rental_date` = ? AND ((? IS NOT ' + #. 'NULL AND `inventory_id` IS NULL) OR (`inventory_id` < ?))) ' + #. 'OR (`rental_date` = ? AND ((? IS NULL AND `inventory_id` IS ' + #. 'NULL) OR (`inventory_id` = ?)) AND `customer_id` < ?))', }, }, 'Alternate index on sakila.rental with nullable inventory_id and strict ascending', diff --git a/t/pt-online-schema-change/pt_1757.t b/t/pt-online-schema-change/pt_1757.t new file mode 100644 index 00000000..d5840de5 --- /dev/null +++ b/t/pt-online-schema-change/pt_1757.t @@ -0,0 +1,96 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use threads; +use threads::shared; +use Thread::Semaphore; + +use English qw(-no_match_vars); +use Test::More; + +use Data::Dumper; +use PerconaTest; +use Sandbox; +use SqlModes; +use File::Temp qw/ tempdir /; + +if ($sandbox_version lt '5.7') { + plan skip_all => 'This test needs MySQL 5.7+'; +} else { + plan tests => 3; +} + +require "$trunk/bin/pt-online-schema-change"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); + +my $dbh = $sb->get_dbh_for('master'); +my $dsn = $sb->dsn_for("master"); + +my $slave_dbh = $sb->get_dbh_for('slave1'); + +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic +# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the +# tool will die. +my @args = (qw(--set-vars innodb_lock_wait_timeout=3)); +my $output; +my $exit_status; +my $master_port = $sb->port_for('master'); +my $num_rows = 5000; + +$sb->load_file('master', "t/pt-online-schema-change/samples/pt-1757.sql"); +diag(`util/mysql_random_data_load --host=127.0.0.1 --port=$master_port --user=msandbox --password=msandbox test t1 $num_rows`); + + +# Let's alter the stats to force this scenario: +# 1) On master, we are going to put 100 as the number of rows in the table. This will make osc to try to run in one chunk +# 2) On the slave, we are going to put the real number of rows. This will cause a fallback to nibble and pt-osc should call +# NibbleIterator->switch_to_nibble() + +$dbh->do('SET @@SESSION.sql_log_bin=0'); +$dbh->do('update mysql.innodb_table_stats set n_rows=100 where table_name="t1"'); +$dbh->do('update mysql.innodb_index_stats set stat_value=100 where stat_description in("id") and table_name="t1"'); +$dbh->do('SET @@SESSION.sql_log_bin=1'); + +$slave_dbh->do('SET @@SESSION.sql_log_bin=0'); +$slave_dbh->do("update mysql.innodb_table_stats set n_rows=$num_rows where table_name='t1'"); +$slave_dbh->do("update mysql.innodb_index_stats set stat_value=$num_rows where stat_description in('id') and table_name='t1'"); +$slave_dbh->do('SET @@SESSION.sql_log_bin=1'); + +($output, $exit_status) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=test,t=t1", + '--execute', '--alter', "ADD COLUMN new_col INT NOT NULL DEFAULT 1", + '--chunk-size', '25', + ), + }, + stderr => 1, +); + +is( + $exit_status, + 0, + "Altered OK status", +); + +# The WHERE clause here is important as a double check that the table was altered and new_col exists +my $rows = $dbh->selectrow_arrayref("SELECT COUNT(*) FROM test.t1 WHERE new_col = 1"); +is( + $rows->[0], + $num_rows, + "Correct rows count" +) or diag(Dumper($rows)); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +done_testing; diff --git a/t/pt-online-schema-change/samples/pt-1757.sql b/t/pt-online-schema-change/samples/pt-1757.sql new file mode 100644 index 00000000..ad1852f5 --- /dev/null +++ b/t/pt-online-schema-change/samples/pt-1757.sql @@ -0,0 +1,15 @@ +DROP DATABASE IF EXISTS `test`; + +CREATE DATABASE `test`; + +CREATE TABLE `test`.`t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `f2` char(32) DEFAULT NULL, + `f3` ENUM('red','green','blue'), + PRIMARY KEY (`id`), + UNIQUE KEY `c` (`f2`) +) ENGINE=InnoDB +AUTO_INCREMENT=0 +DEFAULT CHARSET=utf8 +STATS_AUTO_RECALC = 0 +STATS_PERSISTENT = 1; diff --git a/util/mysql_random_data_load b/util/mysql_random_data_load index c8df0b33..eb1be653 100755 Binary files a/util/mysql_random_data_load and b/util/mysql_random_data_load differ