percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-14 15:33:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete --resume tests and fixes.

Browse Source
This commit is contained in:
Daniel Nichter
2011-11-17 11:26:59 -07:00
parent ff78656859
commit 41be2d9e2c
3 changed files with 196 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
bin/pt-table-checksum
View File

@@ -3490,13 +3490,12 @@ sub new {
: 0;
MKDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
if ( my $nibble = $args{resume} ) {
if ( !defined $nibble->{lower_boundary}
&& !defined $nibble->{upper_boundary} ) {
if ( $args{resume}
&& !defined $args{resume}->{lower_boundary}
&& !defined $args{resume}->{upper_boundary} ) {
MKDEBUG && _d('Resuming from one nibble table');
$one_nibble = 1;
}
}
my $index = _find_best_index(%args, mysql_index => $mysql_index);
if ( !$index && !$one_nibble ) {
@@ -3511,7 +3510,7 @@ sub new {
my $self;
if ( $one_nibble ) {
my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ")
= ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
@@ -3534,7 +3533,6 @@ sub new {
limit => 0,
nibble_sql => $nibble_sql,
explain_nibble_sql => $explain_nibble_sql,
no_more_boundaries => $args{resume} ? 1 : 0,
};
}
else {
@@ -3599,7 +3597,7 @@ sub new {
MKDEBUG && _d('Upper boundary statement:', $ub_sql);
my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ")
= ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}}))
. " FROM $from"
@@ -3635,7 +3633,6 @@ sub new {
nibble_sql => $nibble_sql,
explain_ub_sql => "EXPLAIN $ub_sql",
explain_nibble_sql => $explain_nibble_sql,
resume => $args{resume},
resume_lb_sql => $resume_lb_sql,
sql => {
columns => $asc->{scols},
@@ -3937,7 +3934,13 @@ sub _prepare_sths {
sub _get_bounds {
my ($self) = @_;
return if $self->{one_nibble};
if ( $self->{one_nibble} ) {
if ( $self->{resume} ) {
$self->{no_more_boundaries} = 1;
}
return;
}
my $dbh = $self->{Cxn}->dbh();
@@ -3945,7 +3948,8 @@ sub _get_bounds {
MKDEBUG && _d('First lower boundary:', Dumper($self->{first_lower}));
if ( my $nibble = $self->{resume} ) {
if ( defined $nibble->{upper_boundary} ) {
if ( defined $nibble->{lower_boundary}
&& defined $nibble->{upper_boundary} ) {
my $sth = $dbh->prepare($self->{resume_lb_sql});
my @ub = split ',', $nibble->{upper_boundary};
MKDEBUG && _d($sth->{Statement}, 'params:', @ub);
@@ -3953,16 +3957,17 @@ sub _get_bounds {
$self->{next_lower} = $sth->fetchrow_arrayref();
$sth->finish();
}
else {
MKDEBUG && _d('No more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
}
else {
$self->{next_lower} = $self->{first_lower};
}
MKDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower}));
if ( !$self->{next_lower} ) {
MKDEBUG && _d('At end of table, or no more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
$self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql});
MKDEBUG && _d('Last upper boundary:', Dumper($self->{last_upper}));
@@ -3983,12 +3988,6 @@ sub _next_boundaries {
return 1; # continue nibbling
}
if ( !$self->{next_lower} ) {
MKDEBUG && _d('At end of table');
$self->{no_more_boundaries} = 1; # for next call
return; # stop nibbling
}
if ( $self->identical_boundaries($self->{lower}, $self->{next_lower}) ) {
MKDEBUG && _d('Infinite loop detected');
my $tbl = $self->{tbl};
@@ -4124,7 +4123,7 @@ sub new {
if ( !$self->one_nibble() ) {
my $head_sql
= ($args{past_dms} || "SELECT ")
= ($args{past_dml} || "SELECT ")
. ($args{past_select}
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
. " FROM " . $self->{sql}->{from};
@@ -4172,18 +4171,18 @@ sub new {
$self->{explain_past_lower_sql} = $explain_past_lower_sql;
$self->{explain_past_upper_sql} = $explain_past_upper_sql;
my @past_nibbles = qw(lower upper);
$self->{past_nibbles} = [qw(lower upper)];
if ( my $nibble = $args{resume} ) {
if ( !defined $nibble->{lower_boundary} ) {
$self->{past_nibbles} = [qw(upper)];
}
elsif ( !defined $nibble->{upper_boundary} ) {
$self->{past_nibbles} = [];
if ( !defined $nibble->{lower_boundary}
|| !defined $nibble->{upper_boundary} ) {
$self->{past_nibbles} = !defined $nibble->{lower_boundary}
? ['upper']
: [];
}
}
MKDEBUG && _d('Nibble past', @past_nibbles);
$self->{past_nibbles} = \@past_nibbles,
}
MKDEBUG && _d('Nibble past', @{$self->{past_nibbles}});
} # not one nibble
return bless $self, $class;
}
@@ -6209,6 +6208,26 @@ sub main {
my $nibble_iter = $args{NibbleIterator};
my $oktonibble = 1;
if ( $last_chunk ) { # resuming
if ( have_more_chunks(%args, last_chunk => $last_chunk) ) {
$nibble_iter->set_nibble_number($last_chunk->{chunk});
MKDEBUG && _d('Have more chunks; resuming from',
$last_chunk->{chunk}, 'at', $last_chunk->{ts});
if ( !$o->get('quiet') ) {
print "Resuming from $tbl->{db}.$tbl->{tbl} chunk "
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n";
}
}
else {
# Problem resuming or no next lower boundary.
MKDEBUG && _d('No more chunks; resuming from next table');
$oktonibble = 0; # don't nibble table; next table
}
# Just need to call us once to kick-start the resume process.
$last_chunk = undef;
}
if ( $o->get('explain') ) {
# --explain level 1: print the checksum and next boundary
# statements.
@@ -6227,25 +6246,8 @@ sub main {
$oktonibble = 0; # don't nibble table; next table
}
}
elsif ( $last_chunk ) { # resuming
if ( have_more_chunks(%args, last_chunk => $last_chunk) ) {
$nibble_iter->set_nibble_number($last_chunk->{chunk});
MKDEBUG && _d('Have more chunks; resuming from',
$last_chunk->{chunk}, 'at', $last_chunk->{ts});
if ( !$o->get('quiet') ) {
print "Resuming from $tbl->{db}.$tbl->{tbl} chunk "
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n";
}
}
else {
# Problem resuming or no next lower boundary.
$oktonibble = 0; # don't nibble table; next table
}
# Just need to call us once to kick-start the resume process.
$last_chunk = undef;
}
elsif ( $o->get('empty-replicate-table') ) {
if ( $o->get('empty-replicate-table') ) {
use_repl_db(
dbh => $master_cxn->dbh(),
repl_table => $repl_table,
@@ -6268,6 +6270,7 @@ sub main {
# #########################################################
# XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX
# #########################################################
}
return $oktonibble; # continue nibbling table?
},
@@ -6603,9 +6606,9 @@ sub main {
tbl => $tbl,
chunk_size => $tbl->{chunk_size},
chunk_index => $o->get('chunk-index'),
dms => $checksum_dml,
dml => $checksum_dml,
select => $checksum_cols,
past_dms => $checksum_dml,
past_dml => $checksum_dml,
past_select => $past_cols,
callbacks => $callbacks,
resume => $last_chunk,
@@ -7121,6 +7124,7 @@ sub last_chunk {
die "I need a $arg argument" unless $args{$arg};
}
my ($dbh, $repl_table, $q) = @args{@required_args};
MKDEBUG && _d('Getting last chunk for --resume');
my $sql = "SELECT MAX(ts) FROM $repl_table WHERE master_crc IS NOT NULL";
MKDEBUG && _d($sql);
@@ -7166,11 +7170,12 @@ sub have_more_chunks {
die "I need a $arg argument" unless $args{$arg};
}
my ($tbl, $last_chunk, $nibble_iter) = @args{@required_args};
MKDEBUG && _d('Checking for more chunks beyond last chunk');
# If there's no next lower boundary, then this is the last
# chunk of the table.
if ( !$nibble_iter->boundaries()->{next_lower} ) {
MKDEBUG && _d('No more rows in table; resuming from next table');
if ( !$nibble_iter->more_boundaries() ) {
MKDEBUG && _d('No more boundaries');
return 0;
}

121
t/pt-table-checksum/resume.t
View File

@@ -29,7 +29,7 @@ elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
else {
plan tests => 33;
plan tests => 45;
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
@@ -483,7 +483,7 @@ is_deeply(
[qw( test t2 2 ), '2011-11-08 00:00:10'],
[qw( test t2 3 ), '2011-11-08 00:00:11'],
],
"Checksum results through tbl2 chunk 3"
"Checksum results through t2 chunk 3"
);
$output = output(
@@ -534,6 +534,123 @@ is_deeply(
"t1 and t2 checksums not updated"
);
# ############################################################################
# Resume from table that finished bounded chunks but not the 2 oob chunks.
# ############################################################################
$sb->load_file('master', "t/pt-table-checksum/samples/3tbl-resume.sql");
load_data_infile("3tbl-resume", "ts='2011-11-08 00:00:24'");
# This will truncate the checksum results after t1 chunk 6 where chunk 7
# is the lower oob and chunk 8 is the upper oob.
$master_dbh->do("delete from percona.checksums where ts > '2011-11-08 00:00:06'");
is_deeply(
$master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums order by db, tbl"),
[
[qw(test t1 1), '2011-11-08 00:00:01'],
[qw(test t1 2), '2011-11-08 00:00:02'],
[qw(test t1 3), '2011-11-08 00:00:03'],
[qw(test t1 4), '2011-11-08 00:00:04'],
[qw(test t1 5), '2011-11-08 00:00:05'],
[qw(test t1 6), '2011-11-08 00:00:06'],
],
"Checksum results through bounded t1 chunks"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(-d test --resume),
qw(--chunk-size 5)) },
);
(undef, undef, $first_tbl) = split /\n/, $output;
like(
$first_tbl,
qr/test.t1$/,
"Resumed from test.t1"
);
like(
$output,
qr/Resuming from test.t1 chunk 6, timestamp 2011-11-08 00:00:06/,
"Resumed from test.t1 chunk 6"
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"Resumed 0 errors"
);
is(
PerconaTest::count_checksum_results($output, 'rows'),
52,
"Resumed 52 rows"
);
is(
PerconaTest::count_checksum_results($output, 'chunks'),
18,
"Resumed 18 chunks"
);
# ############################################################################
# Resume from table that finished bounded chunks and the lower oob chunk
# but not the upper oob chunk.
# ############################################################################
$sb->load_file('master', "t/pt-table-checksum/samples/3tbl-resume.sql");
load_data_infile("3tbl-resume", "ts='2011-11-08 00:00:24'");
$master_dbh->do("delete from percona.checksums where ts > '2011-11-08 00:00:07'");
is_deeply(
$master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums order by db, tbl"),
[
[qw(test t1 1), '2011-11-08 00:00:01'],
[qw(test t1 2), '2011-11-08 00:00:02'],
[qw(test t1 3), '2011-11-08 00:00:03'],
[qw(test t1 4), '2011-11-08 00:00:04'],
[qw(test t1 5), '2011-11-08 00:00:05'],
[qw(test t1 6), '2011-11-08 00:00:06'],
[qw(test t1 7), '2011-11-08 00:00:07'], # lower oob
],
"Checksum results through t1 lower oob chunk"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(-d test --resume),
qw(--chunk-size 5)) },
);
(undef, undef, $first_tbl) = split /\n/, $output;
like(
$first_tbl,
qr/test.t1$/,
"Resumed from test.t1"
);
like(
$output,
qr/Resuming from test.t1 chunk 7, timestamp 2011-11-08 00:00:07/,
"Resumed from test.t1 chunk 7"
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"Resumed 0 errors"
);
is(
PerconaTest::count_checksum_results($output, 'rows'),
52,
"Resumed 52 rows"
);
is(
PerconaTest::count_checksum_results($output, 'chunks'),
17,
"Resumed 17 chunks"
);
# #############################################################################
# Done.
# #############################################################################

8
t/pt-table-checksum/samples/checksum_results/sakila-done-1k-chunks
View File

@@ -59,7 +59,7 @@ sakila rental 14 0.008716 PRIMARY 13005 14004 ea862225 1000 ea862225 1000 2011-1
sakila rental 15 0.006528 PRIMARY 14005 15004 dc7ca09f 1000 dc7ca09f 1000 2011-10-15 13:00:53
sakila rental 16 0.0096 PRIMARY 15005 16005 113818d1 1000 113818d1 1000 2011-10-15 13:00:54
sakila rental 17 0.001746 PRIMARY 16006 16049 dc02888c 44 dc02888c 44 2011-10-15 13:00:55
sakila rental 18 0.001069 PRIMARY \N 1 0 0 0 0 2011-10-15 13:00:55
sakila rental 19 0.000857 PRIMARY 16049 \N 0 0 0 0 2011-10-15 13:00:55
sakila staff 1 0.001279 \N \N \N 233668ae 2 233668ae 2 2011-10-15 13:00:56
sakila store 1 0.000905 \N \N \N 6ce7245a 2 6ce7245a 2 2011-10-15 13:00:57
sakila rental 18 0.001069 PRIMARY \N 1 0 0 0 0 2011-10-15 13:00:56
sakila rental 19 0.000857 PRIMARY 16049 \N 0 0 0 0 2011-10-15 13:00:57
sakila staff 1 0.001279 \N \N \N 233668ae 2 233668ae 2 2011-10-15 13:00:58
sakila store 1 0.000905 \N \N \N 6ce7245a 2 6ce7245a 2 2011-10-15 13:00:59