percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-14 07:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete --resume tests and fixes.

Browse Source
This commit is contained in:
Daniel Nichter
2011-11-17 11:26:59 -07:00
parent ff78656859
commit 41be2d9e2c
3 changed files with 196 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
bin/pt-table-checksum
View File

@@ -3490,12 +3490,11 @@ sub new {
: 0; : 0;
MKDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); MKDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
if ( my $nibble = $args{resume} ) { if ( $args{resume}
if ( !defined $nibble->{lower_boundary} && !defined $args{resume}->{lower_boundary}
&& !defined $nibble->{upper_boundary} ) { && !defined $args{resume}->{upper_boundary} ) {
MKDEBUG && _d('Resuming from one nibble table'); MKDEBUG && _d('Resuming from one nibble table');
$one_nibble = 1; $one_nibble = 1;
}
} }
my $index = _find_best_index(%args, mysql_index => $mysql_index); my $index = _find_best_index(%args, mysql_index => $mysql_index);
@@ -3511,7 +3510,7 @@ sub new {
my $self; my $self;
if ( $one_nibble ) { if ( $one_nibble ) {
my $nibble_sql my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ") = ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @cols)) : join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
@@ -3534,7 +3533,6 @@ sub new {
limit => 0, limit => 0,
nibble_sql => $nibble_sql, nibble_sql => $nibble_sql,
explain_nibble_sql => $explain_nibble_sql, explain_nibble_sql => $explain_nibble_sql,
no_more_boundaries => $args{resume} ? 1 : 0,
}; };
} }
else { else {
@@ -3599,7 +3597,7 @@ sub new {
MKDEBUG && _d('Upper boundary statement:', $ub_sql); MKDEBUG && _d('Upper boundary statement:', $ub_sql);
my $nibble_sql my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ") = ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select} . ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}})) : join(', ', map { $q->quote($_) } @{$asc->{cols}}))
. " FROM $from" . " FROM $from"
@@ -3635,7 +3633,6 @@ sub new {
nibble_sql => $nibble_sql, nibble_sql => $nibble_sql,
explain_ub_sql => "EXPLAIN $ub_sql", explain_ub_sql => "EXPLAIN $ub_sql",
explain_nibble_sql => $explain_nibble_sql, explain_nibble_sql => $explain_nibble_sql,
resume => $args{resume},
resume_lb_sql => $resume_lb_sql, resume_lb_sql => $resume_lb_sql,
sql => { sql => {
columns => $asc->{scols}, columns => $asc->{scols},
@@ -3937,7 +3934,13 @@ sub _prepare_sths {
sub _get_bounds { sub _get_bounds {
my ($self) = @_; my ($self) = @_;
return if $self->{one_nibble};
if ( $self->{one_nibble} ) {
if ( $self->{resume} ) {
$self->{no_more_boundaries} = 1;
}
return;
}
my $dbh = $self->{Cxn}->dbh(); my $dbh = $self->{Cxn}->dbh();
@@ -3945,7 +3948,8 @@ sub _get_bounds {
MKDEBUG && _d('First lower boundary:', Dumper($self->{first_lower})); MKDEBUG && _d('First lower boundary:', Dumper($self->{first_lower}));
if ( my $nibble = $self->{resume} ) { if ( my $nibble = $self->{resume} ) {
if ( defined $nibble->{upper_boundary} ) { if ( defined $nibble->{lower_boundary}
&& defined $nibble->{upper_boundary} ) {
my $sth = $dbh->prepare($self->{resume_lb_sql}); my $sth = $dbh->prepare($self->{resume_lb_sql});
my @ub = split ',', $nibble->{upper_boundary}; my @ub = split ',', $nibble->{upper_boundary};
MKDEBUG && _d($sth->{Statement}, 'params:', @ub); MKDEBUG && _d($sth->{Statement}, 'params:', @ub);
@@ -3953,16 +3957,17 @@ sub _get_bounds {
$self->{next_lower} = $sth->fetchrow_arrayref(); $self->{next_lower} = $sth->fetchrow_arrayref();
$sth->finish(); $sth->finish();
} }
else {
MKDEBUG && _d('No more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
} }
else { else {
$self->{next_lower} = $self->{first_lower}; $self->{next_lower} = $self->{first_lower};
} }
MKDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower})); MKDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower}));
if ( !$self->{next_lower} ) {
MKDEBUG && _d('At end of table, or no more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
$self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql}); $self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql});
MKDEBUG && _d('Last upper boundary:', Dumper($self->{last_upper})); MKDEBUG && _d('Last upper boundary:', Dumper($self->{last_upper}));
@@ -3983,12 +3988,6 @@ sub _next_boundaries {
return 1; # continue nibbling return 1; # continue nibbling
} }
if ( !$self->{next_lower} ) {
MKDEBUG && _d('At end of table');
$self->{no_more_boundaries} = 1; # for next call
return; # stop nibbling
}
if ( $self->identical_boundaries($self->{lower}, $self->{next_lower}) ) { if ( $self->identical_boundaries($self->{lower}, $self->{next_lower}) ) {
MKDEBUG && _d('Infinite loop detected'); MKDEBUG && _d('Infinite loop detected');
my $tbl = $self->{tbl}; my $tbl = $self->{tbl};
@@ -4124,7 +4123,7 @@ sub new {
if ( !$self->one_nibble() ) { if ( !$self->one_nibble() ) {
my $head_sql my $head_sql
= ($args{past_dms} || "SELECT ") = ($args{past_dml} || "SELECT ")
. ($args{past_select} . ($args{past_select}
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}})) || join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
. " FROM " . $self->{sql}->{from}; . " FROM " . $self->{sql}->{from};
@@ -4172,18 +4171,18 @@ sub new {
$self->{explain_past_lower_sql} = $explain_past_lower_sql; $self->{explain_past_lower_sql} = $explain_past_lower_sql;
$self->{explain_past_upper_sql} = $explain_past_upper_sql; $self->{explain_past_upper_sql} = $explain_past_upper_sql;
my @past_nibbles = qw(lower upper); $self->{past_nibbles} = [qw(lower upper)];
if ( my $nibble = $args{resume} ) { if ( my $nibble = $args{resume} ) {
if ( !defined $nibble->{lower_boundary} ) { if ( !defined $nibble->{lower_boundary}
$self->{past_nibbles} = [qw(upper)]; || !defined $nibble->{upper_boundary} ) {
} $self->{past_nibbles} = !defined $nibble->{lower_boundary}
elsif ( !defined $nibble->{upper_boundary} ) { ? ['upper']
$self->{past_nibbles} = []; : [];
} }
} }
MKDEBUG && _d('Nibble past', @past_nibbles); MKDEBUG && _d('Nibble past', @{$self->{past_nibbles}});
$self->{past_nibbles} = \@past_nibbles,
} } # not one nibble
return bless $self, $class; return bless $self, $class;
} }
@@ -6208,6 +6207,26 @@ sub main {
my $tbl = $args{tbl}; my $tbl = $args{tbl};
my $nibble_iter = $args{NibbleIterator}; my $nibble_iter = $args{NibbleIterator};
my $oktonibble = 1; my $oktonibble = 1;
if ( $last_chunk ) { # resuming
if ( have_more_chunks(%args, last_chunk => $last_chunk) ) {
$nibble_iter->set_nibble_number($last_chunk->{chunk});
MKDEBUG && _d('Have more chunks; resuming from',
$last_chunk->{chunk}, 'at', $last_chunk->{ts});
if ( !$o->get('quiet') ) {
print "Resuming from $tbl->{db}.$tbl->{tbl} chunk "
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n";
}
}
else {
# Problem resuming or no next lower boundary.
MKDEBUG && _d('No more chunks; resuming from next table');
$oktonibble = 0; # don't nibble table; next table
}
# Just need to call us once to kick-start the resume process.
$last_chunk = undef;
}
if ( $o->get('explain') ) { if ( $o->get('explain') ) {
# --explain level 1: print the checksum and next boundary # --explain level 1: print the checksum and next boundary
@@ -6227,48 +6246,32 @@ sub main {
$oktonibble = 0; # don't nibble table; next table $oktonibble = 0; # don't nibble table; next table
} }
} }
elsif ( $last_chunk ) { # resuming else {
if ( have_more_chunks(%args, last_chunk => $last_chunk) ) { if ( $o->get('empty-replicate-table') ) {
$nibble_iter->set_nibble_number($last_chunk->{chunk}); use_repl_db(
MKDEBUG && _d('Have more chunks; resuming from', dbh => $master_cxn->dbh(),
$last_chunk->{chunk}, 'at', $last_chunk->{ts}); repl_table => $repl_table,
if ( !$o->get('quiet') ) { OptionParser => $o,
print "Resuming from $tbl->{db}.$tbl->{tbl} chunk " Quoter => $q,
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n"; );
} MKDEBUG && _d($delete_sth->{Statement});
} $delete_sth->execute($tbl->{db}, $tbl->{tbl});
else {
# Problem resuming or no next lower boundary.
$oktonibble = 0; # don't nibble table; next table
} }
# Just need to call us once to kick-start the resume process. # USE the correct db while checksumming this table. The "correct"
$last_chunk = undef; # db is a complicated subject; see sub for comments.
}
elsif ( $o->get('empty-replicate-table') ) {
use_repl_db( use_repl_db(
dbh => $master_cxn->dbh(), dbh => $master_cxn->dbh(),
tbl => $tbl, # XXX working on this table
repl_table => $repl_table, repl_table => $repl_table,
OptionParser => $o, OptionParser => $o,
Quoter => $q, Quoter => $q,
); );
MKDEBUG && _d($delete_sth->{Statement}); # #########################################################
$delete_sth->execute($tbl->{db}, $tbl->{tbl}); # XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX
# #########################################################
} }
# USE the correct db while checksumming this table. The "correct"
# db is a complicated subject; see sub for comments.
use_repl_db(
dbh => $master_cxn->dbh(),
tbl => $tbl, # XXX working on this table
repl_table => $repl_table,
OptionParser => $o,
Quoter => $q,
);
# #########################################################
# XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX
# #########################################################
return $oktonibble; # continue nibbling table? return $oktonibble; # continue nibbling table?
}, },
next_boundaries => sub { next_boundaries => sub {
@@ -6603,9 +6606,9 @@ sub main {
tbl => $tbl, tbl => $tbl,
chunk_size => $tbl->{chunk_size}, chunk_size => $tbl->{chunk_size},
chunk_index => $o->get('chunk-index'), chunk_index => $o->get('chunk-index'),
dms => $checksum_dml, dml => $checksum_dml,
select => $checksum_cols, select => $checksum_cols,
past_dms => $checksum_dml, past_dml => $checksum_dml,
past_select => $past_cols, past_select => $past_cols,
callbacks => $callbacks, callbacks => $callbacks,
resume => $last_chunk, resume => $last_chunk,
@@ -7121,6 +7124,7 @@ sub last_chunk {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($dbh, $repl_table, $q) = @args{@required_args}; my ($dbh, $repl_table, $q) = @args{@required_args};
MKDEBUG && _d('Getting last chunk for --resume');
my $sql = "SELECT MAX(ts) FROM $repl_table WHERE master_crc IS NOT NULL"; my $sql = "SELECT MAX(ts) FROM $repl_table WHERE master_crc IS NOT NULL";
MKDEBUG && _d($sql); MKDEBUG && _d($sql);
@@ -7166,11 +7170,12 @@ sub have_more_chunks {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($tbl, $last_chunk, $nibble_iter) = @args{@required_args}; my ($tbl, $last_chunk, $nibble_iter) = @args{@required_args};
MKDEBUG && _d('Checking for more chunks beyond last chunk');
# If there's no next lower boundary, then this is the last # If there's no next lower boundary, then this is the last
# chunk of the table. # chunk of the table.
if ( !$nibble_iter->boundaries()->{next_lower} ) { if ( !$nibble_iter->more_boundaries() ) {
MKDEBUG && _d('No more rows in table; resuming from next table'); MKDEBUG && _d('No more boundaries');
return 0; return 0;
} }

121
t/pt-table-checksum/resume.t
View File

@@ -29,7 +29,7 @@ elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave'; plan skip_all => 'Cannot connect to sandbox slave';
} }
else { else {
plan tests => 33; plan tests => 45;
} }
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic # The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
@@ -483,7 +483,7 @@ is_deeply(
[qw( test t2 2 ), '2011-11-08 00:00:10'], [qw( test t2 2 ), '2011-11-08 00:00:10'],
[qw( test t2 3 ), '2011-11-08 00:00:11'], [qw( test t2 3 ), '2011-11-08 00:00:11'],
], ],
"Checksum results through tbl2 chunk 3" "Checksum results through t2 chunk 3"
); );
$output = output( $output = output(
@@ -534,6 +534,123 @@ is_deeply(
"t1 and t2 checksums not updated" "t1 and t2 checksums not updated"
); );
# ############################################################################
# Resume from table that finished bounded chunks but not the 2 oob chunks.
# ############################################################################
$sb->load_file('master', "t/pt-table-checksum/samples/3tbl-resume.sql");
load_data_infile("3tbl-resume", "ts='2011-11-08 00:00:24'");
# This will truncate the checksum results after t1 chunk 6 where chunk 7
# is the lower oob and chunk 8 is the upper oob.
$master_dbh->do("delete from percona.checksums where ts > '2011-11-08 00:00:06'");
is_deeply(
$master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums order by db, tbl"),
[
[qw(test t1 1), '2011-11-08 00:00:01'],
[qw(test t1 2), '2011-11-08 00:00:02'],
[qw(test t1 3), '2011-11-08 00:00:03'],
[qw(test t1 4), '2011-11-08 00:00:04'],
[qw(test t1 5), '2011-11-08 00:00:05'],
[qw(test t1 6), '2011-11-08 00:00:06'],
],
"Checksum results through bounded t1 chunks"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(-d test --resume),
qw(--chunk-size 5)) },
);
(undef, undef, $first_tbl) = split /\n/, $output;
like(
$first_tbl,
qr/test.t1$/,
"Resumed from test.t1"
);
like(
$output,
qr/Resuming from test.t1 chunk 6, timestamp 2011-11-08 00:00:06/,
"Resumed from test.t1 chunk 6"
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"Resumed 0 errors"
);
is(
PerconaTest::count_checksum_results($output, 'rows'),
52,
"Resumed 52 rows"
);
is(
PerconaTest::count_checksum_results($output, 'chunks'),
18,
"Resumed 18 chunks"
);
# ############################################################################
# Resume from table that finished bounded chunks and the lower oob chunk
# but not the upper oob chunk.
# ############################################################################
$sb->load_file('master', "t/pt-table-checksum/samples/3tbl-resume.sql");
load_data_infile("3tbl-resume", "ts='2011-11-08 00:00:24'");
$master_dbh->do("delete from percona.checksums where ts > '2011-11-08 00:00:07'");
is_deeply(
$master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums order by db, tbl"),
[
[qw(test t1 1), '2011-11-08 00:00:01'],
[qw(test t1 2), '2011-11-08 00:00:02'],
[qw(test t1 3), '2011-11-08 00:00:03'],
[qw(test t1 4), '2011-11-08 00:00:04'],
[qw(test t1 5), '2011-11-08 00:00:05'],
[qw(test t1 6), '2011-11-08 00:00:06'],
[qw(test t1 7), '2011-11-08 00:00:07'], # lower oob
],
"Checksum results through t1 lower oob chunk"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(-d test --resume),
qw(--chunk-size 5)) },
);
(undef, undef, $first_tbl) = split /\n/, $output;
like(
$first_tbl,
qr/test.t1$/,
"Resumed from test.t1"
);
like(
$output,
qr/Resuming from test.t1 chunk 7, timestamp 2011-11-08 00:00:07/,
"Resumed from test.t1 chunk 7"
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"Resumed 0 errors"
);
is(
PerconaTest::count_checksum_results($output, 'rows'),
52,
"Resumed 52 rows"
);
is(
PerconaTest::count_checksum_results($output, 'chunks'),
17,
"Resumed 17 chunks"
);
# ############################################################################# # #############################################################################
# Done. # Done.
# ############################################################################# # #############################################################################

8
t/pt-table-checksum/samples/checksum_results/sakila-done-1k-chunks
View File

@@ -59,7 +59,7 @@ sakila rental 14 0.008716 PRIMARY 13005 14004 ea862225 1000 ea862225 1000 2011-1
sakila rental 15 0.006528 PRIMARY 14005 15004 dc7ca09f 1000 dc7ca09f 1000 2011-10-15 13:00:53 sakila rental 15 0.006528 PRIMARY 14005 15004 dc7ca09f 1000 dc7ca09f 1000 2011-10-15 13:00:53
sakila rental 16 0.0096 PRIMARY 15005 16005 113818d1 1000 113818d1 1000 2011-10-15 13:00:54 sakila rental 16 0.0096 PRIMARY 15005 16005 113818d1 1000 113818d1 1000 2011-10-15 13:00:54
sakila rental 17 0.001746 PRIMARY 16006 16049 dc02888c 44 dc02888c 44 2011-10-15 13:00:55 sakila rental 17 0.001746 PRIMARY 16006 16049 dc02888c 44 dc02888c 44 2011-10-15 13:00:55
sakila rental 18 0.001069 PRIMARY \N 1 0 0 0 0 2011-10-15 13:00:55 sakila rental 18 0.001069 PRIMARY \N 1 0 0 0 0 2011-10-15 13:00:56
sakila rental 19 0.000857 PRIMARY 16049 \N 0 0 0 0 2011-10-15 13:00:55 sakila rental 19 0.000857 PRIMARY 16049 \N 0 0 0 0 2011-10-15 13:00:57
sakila staff 1 0.001279 \N \N \N 233668ae 2 233668ae 2 2011-10-15 13:00:56 sakila staff 1 0.001279 \N \N \N 233668ae 2 233668ae 2 2011-10-15 13:00:58
sakila store 1 0.000905 \N \N \N 6ce7245a 2 6ce7245a 2 2011-10-15 13:00:57 sakila store 1 0.000905 \N \N \N 6ce7245a 2 6ce7245a 2 2011-10-15 13:00:59