percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-12 14:18:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix --ignore-columns in NibbleIterator. Increase test coverage to 93%.

Browse Source
This commit is contained in:
Daniel Nichter
2011-10-18 09:32:48 -06:00
parent e08719ac4a
commit 8d2259e5b3
3 changed files with 283 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
bin/pt-table-checksum
View File

@@ -3477,14 +3477,13 @@ sub new {
die "There is no good index and the table is oversized."; die "There is no good index and the table is oversized.";
} }
my $where = $o->get('where'); my $tbl_struct = $tbl->{tbl_struct};
my $ignore_col = $o->get('ignore-columns') || {};
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
my @cols = grep { !$ignore_col->{$_} } @$all_cols;
my $where = $o->get('where');
my $self; my $self;
if ( $one_nibble ) { if ( $one_nibble ) {
my $tbl_struct = $tbl->{tbl_struct};
my $ignore_col = $o->get('ignore-columns') || {};
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
my @cols = grep { !$ignore_col->{$_} } @$all_cols;
my $nibble_sql my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ") = ($args{dms} ? "$args{dms} " : "SELECT ")
. ($args{select} ? $args{select} . ($args{select} ? $args{select}
@@ -3518,6 +3517,7 @@ sub new {
%args, %args,
tbl_struct => $tbl->{tbl_struct}, tbl_struct => $tbl->{tbl_struct},
index => $index, index => $index,
cols => \@cols,
asc_only => 1, asc_only => 1,
); );
MKDEBUG && _d('Ascend params:', Dumper($asc)); MKDEBUG && _d('Ascend params:', Dumper($asc));
@@ -3604,10 +3604,11 @@ sub new {
}; };
} }
$self->{row_est} = $row_est; $self->{row_est} = $row_est;
$self->{nibbleno} = 0; $self->{nibbleno} = 0;
$self->{have_rows} = 0; $self->{have_rows} = 0;
$self->{rowno} = 0; $self->{rowno} = 0;
$self->{oktonibble} = 1;
return bless $self, $class; return bless $self, $class;
} }
@@ -3615,6 +3616,11 @@ sub new {
sub next { sub next {
my ($self) = @_; my ($self) = @_;
if ( !$self->{oktonibble} ) {
MKDEBUG && _d('Not ok to nibble');
return;
}
my %callback_args = ( my %callback_args = (
Cxn => $self->{Cxn}, Cxn => $self->{Cxn},
tbl => $self->{tbl}, tbl => $self->{tbl},
@@ -3625,9 +3631,9 @@ sub next {
$self->_prepare_sths(); $self->_prepare_sths();
$self->_get_bounds(); $self->_get_bounds();
if ( my $callback = $self->{callbacks}->{init} ) { if ( my $callback = $self->{callbacks}->{init} ) {
my $oktonibble = $callback->(%callback_args); $self->{oktonibble} = $callback->(%callback_args);
MKDEBUG && _d('init callback returned', $oktonibble); MKDEBUG && _d('init callback returned', $self->{oktonibble});
if ( !$oktonibble ) { if ( !$self->{oktonibble} ) {
$self->{no_more_boundaries} = 1; $self->{no_more_boundaries} = 1;
return; return;
} }
@@ -3733,7 +3739,7 @@ sub one_nibble {
sub chunk_size { sub chunk_size {
my ($self) = @_; my ($self) = @_;
return $self->{limit}; return $self->{limit} + 1;
} }
sub set_chunk_size { sub set_chunk_size {

34
lib/NibbleIterator.pm
View File

@@ -70,14 +70,13 @@ sub new {
die "There is no good index and the table is oversized."; die "There is no good index and the table is oversized.";
} }
my $where = $o->get('where'); my $tbl_struct = $tbl->{tbl_struct};
my $ignore_col = $o->get('ignore-columns') || {};
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
my @cols = grep { !$ignore_col->{$_} } @$all_cols;
my $where = $o->get('where');
my $self; my $self;
if ( $one_nibble ) { if ( $one_nibble ) {
my $tbl_struct = $tbl->{tbl_struct};
my $ignore_col = $o->get('ignore-columns') || {};
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
my @cols = grep { !$ignore_col->{$_} } @$all_cols;
# If the chunk size is >= number of rows in table, then we don't # If the chunk size is >= number of rows in table, then we don't
# need to chunk; we can just select all rows, in order, at once. # need to chunk; we can just select all rows, in order, at once.
my $nibble_sql my $nibble_sql
@@ -114,6 +113,7 @@ sub new {
%args, %args,
tbl_struct => $tbl->{tbl_struct}, tbl_struct => $tbl->{tbl_struct},
index => $index, index => $index,
cols => \@cols,
asc_only => 1, asc_only => 1,
); );
MKDEBUG && _d('Ascend params:', Dumper($asc)); MKDEBUG && _d('Ascend params:', Dumper($asc));
@@ -214,10 +214,11 @@ sub new {
}; };
} }
$self->{row_est} = $row_est; $self->{row_est} = $row_est;
$self->{nibbleno} = 0; $self->{nibbleno} = 0;
$self->{have_rows} = 0; $self->{have_rows} = 0;
$self->{rowno} = 0; $self->{rowno} = 0;
$self->{oktonibble} = 1;
return bless $self, $class; return bless $self, $class;
} }
@@ -225,6 +226,11 @@ sub new {
sub next { sub next {
my ($self) = @_; my ($self) = @_;
if ( !$self->{oktonibble} ) {
MKDEBUG && _d('Not ok to nibble');
return;
}
my %callback_args = ( my %callback_args = (
Cxn => $self->{Cxn}, Cxn => $self->{Cxn},
tbl => $self->{tbl}, tbl => $self->{tbl},
@@ -237,9 +243,9 @@ sub next {
$self->_prepare_sths(); $self->_prepare_sths();
$self->_get_bounds(); $self->_get_bounds();
if ( my $callback = $self->{callbacks}->{init} ) { if ( my $callback = $self->{callbacks}->{init} ) {
my $oktonibble = $callback->(%callback_args); $self->{oktonibble} = $callback->(%callback_args);
MKDEBUG && _d('init callback returned', $oktonibble); MKDEBUG && _d('init callback returned', $self->{oktonibble});
if ( !$oktonibble ) { if ( !$self->{oktonibble} ) {
$self->{no_more_boundaries} = 1; $self->{no_more_boundaries} = 1;
return; return;
} }
@@ -352,7 +358,7 @@ sub one_nibble {
sub chunk_size { sub chunk_size {
my ($self) = @_; my ($self) = @_;
return $self->{limit}; return $self->{limit} + 1;
} }
sub set_chunk_size { sub set_chunk_size {

374
t/lib/NibbleIterator.t
View File

@@ -39,7 +39,7 @@ if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master'; plan skip_all => 'Cannot connect to sandbox master';
} }
else { else {
plan tests => 30; plan tests => 42;
} }
my $q = new Quoter(); my $q = new Quoter();
@@ -108,6 +108,17 @@ my $ni = make_nibble_iter(
argv => [qw(--databases test --chunk-size 5)], argv => [qw(--databases test --chunk-size 5)],
); );
ok(
!$ni->one_nibble(),
"one_nibble() false"
);
is(
$ni->nibble_number(),
0,
"nibble_number() 0"
);
my @rows = (); my @rows = ();
for (1..5) { for (1..5) {
push @rows, $ni->next(); push @rows, $ni->next();
@@ -118,6 +129,12 @@ is_deeply(
'a-z nibble 1' 'a-z nibble 1'
) or print Dumper(\@rows); ) or print Dumper(\@rows);
is(
$ni->nibble_number(),
1,
"nibble_number() 1"
);
@rows = (); @rows = ();
for (1..5) { for (1..5) {
push @rows, $ni->next(); push @rows, $ni->next();
@@ -128,6 +145,12 @@ is_deeply(
'a-z nibble 2' 'a-z nibble 2'
) or print Dumper(\@rows); ) or print Dumper(\@rows);
is(
$ni->nibble_number(),
2,
"nibble_number() 2"
);
@rows = (); @rows = ();
for (1..5) { for (1..5) {
push @rows, $ni->next(); push @rows, $ni->next();
@@ -158,6 +181,11 @@ is_deeply(
'a-z nibble 5' 'a-z nibble 5'
) or print Dumper(\@rows); ) or print Dumper(\@rows);
ok(
$ni->more_boundaries(),
"more_boundaries() true"
);
# There's only 1 row left but extra calls shouldn't return anything or crash. # There's only 1 row left but extra calls shouldn't return anything or crash.
@rows = (); @rows = ();
for (1..5) { for (1..5) {
@@ -169,6 +197,17 @@ is_deeply(
'a-z nibble 6' 'a-z nibble 6'
) or print Dumper(\@rows); ) or print Dumper(\@rows);
ok(
!$ni->more_boundaries(),
"more_boundaries() false"
);
is(
$ni->chunk_size(),
5,
"chunk_size()"
);
# ############################################################################ # ############################################################################
# a-y w/ chunk-size 5, even nibbles # a-y w/ chunk-size 5, even nibbles
# ############################################################################ # ############################################################################
@@ -219,6 +258,11 @@ $ni = make_nibble_iter(
argv => [qw(--databases test --chunk-size 100)], argv => [qw(--databases test --chunk-size 100)],
); );
ok(
$ni->one_nibble(),
"one_nibble() true"
);
@rows = (); @rows = ();
for (1..3) { for (1..3) {
push @rows, $ni->next(); push @rows, $ni->next();
@@ -284,153 +328,206 @@ done
"callbacks" "callbacks"
); );
# Test that init callback can stop nibbling.
$ni = make_nibble_iter(
db => 'test',
tbl => 't',
argv => [qw(--databases test --chunk-size 2)],
callbacks => {
init => sub { print "init\n"; return 0; },
after_nibble => sub { print "after nibble\n"; },
done => sub { print "done\n"; },
}
);
$dbh->do('delete from test.t limit 20'); # 6 rows left
$output = output(
sub {
for (1..8) { $ni->next() }
},
);
is(
$output,
"init
",
"init callbacks can stop nibbling"
);
# ############################################################################ # ############################################################################
# Nibble a larger table by numeric pk id # Nibble a larger table by numeric pk id
# ############################################################################ # ############################################################################
SKIP: { $ni = make_nibble_iter(
skip "Sakila database is not loaded", 8 db => 'sakila',
unless @{ $dbh->selectall_arrayref('show databases like "sakila"') }; tbl => 'payment',
argv => [qw(--databases sakila --tables payment --chunk-size 100)],
);
$ni = make_nibble_iter( my $n_nibbles = 0;
db => 'sakila', $n_nibbles++ while $ni->next();
tbl => 'payment', is(
argv => [qw(--databases sakila --tables payment --chunk-size 100)], $n_nibbles,
); 16049,
"Nibble sakila.payment (16049 rows)"
);
my $n_nibbles = 0; my $tbl = {
$n_nibbles++ while $ni->next(); db => 'sakila',
is( tbl => 'country',
$n_nibbles, tbl_struct => $tp->parse(
16049, $tp->get_create_table($dbh, 'sakila', 'country')),
"Nibble sakila.payment (16049 rows)" };
); my $chunk_checksum = $rc->make_chunk_checksum(
dbh => $dbh,
tbl => $tbl,
);
$ni = make_nibble_iter(
db => 'sakila',
tbl => 'country',
argv => [qw(--databases sakila --tables country --chunk-size 25)],
select => $chunk_checksum,
);
my $tbl = { my $row = $ni->next();
db => 'sakila', is_deeply(
tbl => 'country', $row,
tbl_struct => $tp->parse( [25, 'da79784d'],
$tp->get_create_table($dbh, 'sakila', 'country')), "SELECT chunk checksum 1 FROM sakila.country"
}; ) or print STDERR Dumper($row);
my $chunk_checksum = $rc->make_chunk_checksum(
dbh => $dbh,
tbl => $tbl,
);
$ni = make_nibble_iter(
db => 'sakila',
tbl => 'country',
argv => [qw(--databases sakila --tables country --chunk-size 25)],
select => $chunk_checksum,
);
my $row = $ni->next(); $row = $ni->next();
is_deeply( is_deeply(
$row, $row,
[25, 'da79784d'], [25, 'e860c4f9'],
"SELECT chunk checksum 1 FROM sakila.country" "SELECT chunk checksum 2 FROM sakila.country"
) or print STDERR Dumper($row); ) or print STDERR Dumper($row);
$row = $ni->next(); $row = $ni->next();
is_deeply( is_deeply(
$row, $row,
[25, 'e860c4f9'], [25, 'eb651f58'],
"SELECT chunk checksum 2 FROM sakila.country" "SELECT chunk checksum 3 FROM sakila.country"
) or print STDERR Dumper($row); ) or print STDERR Dumper($row);
$row = $ni->next(); $row = $ni->next();
is_deeply( is_deeply(
$row, $row,
[25, 'eb651f58'], [25, '2d87d588'],
"SELECT chunk checksum 3 FROM sakila.country" "SELECT chunk checksum 4 FROM sakila.country"
) or print STDERR Dumper($row); ) or print STDERR Dumper($row);
$row = $ni->next(); $row = $ni->next();
is_deeply( is_deeply(
$row, $row,
[25, '2d87d588'], [9, 'beb4a180'],
"SELECT chunk checksum 4 FROM sakila.country" "SELECT chunk checksum 5 FROM sakila.country"
) or print STDERR Dumper($row); ) or print STDERR Dumper($row);
$row = $ni->next();
is_deeply(
$row,
[9, 'beb4a180'],
"SELECT chunk checksum 5 FROM sakila.country"
) or print STDERR Dumper($row);
# ######################################################################### # #########################################################################
# exec_nibble callback and explain_sth # exec_nibble callback and explain_sth
# ######################################################################### # #########################################################################
my @expl; my @expl;
$ni = make_nibble_iter( $ni = make_nibble_iter(
db => 'sakila', db => 'sakila',
tbl => 'country', tbl => 'country',
argv => [qw(--databases sakila --tables country --chunk-size 60)], argv => [qw(--databases sakila --tables country --chunk-size 60)],
select => $chunk_checksum, select => $chunk_checksum,
callbacks => { callbacks => {
exec_nibble => sub { exec_nibble => sub {
my (%args) = @_; my (%args) = @_;
my $nibble_iter = $args{NibbleIterator}; my $nibble_iter = $args{NibbleIterator};
my $sth = $nibble_iter->statements(); my $sth = $nibble_iter->statements();
my $boundary = $nibble_iter->boundaries(); my $boundary = $nibble_iter->boundaries();
$sth->{explain_nibble}->execute( $sth->{explain_nibble}->execute(
@{$boundary->{lower}}, @{$boundary->{upper}}); @{$boundary->{lower}}, @{$boundary->{upper}});
push @expl, $sth->{explain_nibble}->fetchrow_hashref(); push @expl, $sth->{explain_nibble}->fetchrow_hashref();
return 0; return 0;
},
}, },
one_nibble => 0, },
); one_nibble => 0,
$ni->next(); );
$ni->next(); $ni->next();
is_deeply( $ni->next();
\@expl, is_deeply(
[ \@expl,
{ [
id => '1', {
key => 'PRIMARY', id => '1',
key_len => '2', key => 'PRIMARY',
possible_keys => 'PRIMARY', key_len => '2',
ref => undef, possible_keys => 'PRIMARY',
rows => '54', ref => undef,
select_type => 'SIMPLE', rows => '54',
table => 'country', select_type => 'SIMPLE',
type => 'range', table => 'country',
extra => 'Using where', type => 'range',
}, extra => 'Using where',
{ },
id => '1', {
key => 'PRIMARY', id => '1',
key_len => '2', key => 'PRIMARY',
possible_keys => 'PRIMARY', key_len => '2',
ref => undef, possible_keys => 'PRIMARY',
rows => '49', ref => undef,
select_type => 'SIMPLE', rows => '49',
table => 'country', select_type => 'SIMPLE',
type => 'range', table => 'country',
extra => 'Using where', type => 'range',
}, extra => 'Using where',
], },
'exec_nibble callbackup and explain_sth' ],
) or print STDERR Dumper(\@expl); 'exec_nibble callbackup and explain_sth'
) or print STDERR Dumper(\@expl);
# ######################################################################### # #########################################################################
# film_actor, multi-column pk # film_actor, multi-column pk
# ######################################################################### # #########################################################################
$ni = make_nibble_iter( $ni = make_nibble_iter(
db => 'sakila', db => 'sakila',
tbl => 'film_actor', tbl => 'film_actor',
argv => [qw(--tables sakila.film_actor --chunk-size 1000)], argv => [qw(--tables sakila.film_actor --chunk-size 1000)],
); );
$n_nibbles = 0; $n_nibbles = 0;
$n_nibbles++ while $ni->next(); $n_nibbles++ while $ni->next();
is( is(
$n_nibbles, $n_nibbles,
5462, 5462,
"Nibble sakila.film_actor (multi-column pk)" "Nibble sakila.film_actor (multi-column pk)"
); );
}
is_deeply(
$ni->sql(),
{
boundaries => {
'<' => '((`actor_id` < ?) OR (`actor_id` = ? AND `film_id` < ?))',
'<=' => '((`actor_id` < ?) OR (`actor_id` = ? AND `film_id` <= ?))',
'>' => '((`actor_id` > ?) OR (`actor_id` = ? AND `film_id` > ?))',
'>=' => '((`actor_id` > ?) OR (`actor_id` = ? AND `film_id` >= ?))'
},
columns => [qw(actor_id actor_id film_id)],
from => '`sakila`.`film_actor` FORCE INDEX(`PRIMARY`)',
where => undef,
order_by => '`actor_id`, `film_id`',
},
"sql()"
);
$ni = make_nibble_iter(
db => 'sakila',
tbl => 'address',
argv => [qw(--tables sakila.address --chunk-size 10),
'--ignore-columns', 'phone,last_update'],
);
$ni->next();
is(
$ni->statements()->{nibble}->{Statement},
"SELECT `address_id`, `address`, `address2`, `district`, `city_id`, `postal_code` FROM `sakila`.`address` FORCE INDEX(`PRIMARY`) WHERE ((`address_id` >= ?)) AND ((`address_id` <= ?)) ORDER BY `address_id` /*checksum chunk*/",
"--ignore-columns"
);
# ############################################################################ # ############################################################################
# Reset chunk size on-the-fly. # Reset chunk size on-the-fly.
@@ -623,9 +720,18 @@ is_deeply(
"--chunk-size-limit 0 on empty table" "--chunk-size-limit 0 on empty table"
); );
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################
{
local *STDERR;
open STDERR, '>', \$output;
$ni->_d('Complete test coverage');
}
like(
$output,
qr/Complete test coverage/,
'_d() works'
);
$sb->wipe_clean($dbh); $sb->wipe_clean($dbh);
exit; exit;