Fix --resume --ignore-tables.

This commit is contained in:
Daniel Nichter
2011-11-08 10:20:59 -07:00
parent f45003731c
commit 5465634885
7 changed files with 236 additions and 128 deletions

View File

@@ -4265,8 +4265,10 @@ sub new {
my %resume;
if ( my $table = $args{resume} ) {
MKDEBUG && _d('Will resume from', $table);
MKDEBUG && _d('Will resume from or after', $table);
my ($db, $tbl) = $args{Quoter}->split_unquote($table);
die "Resume table must be database-qualified: $table"
unless $db && $tbl;
$resume{db} = $db;
$resume{tbl} = $tbl;
}
@@ -4337,6 +4339,16 @@ sub _make_filters {
sub next {
my ( $self ) = @_;
if ( !$self->{initialized} ) {
$self->{initialized} = 1;
if ( $self->{resume}->{tbl}
&& !$self->table_is_allowed(@{$self->{resume}}{qw(db tbl)}) ) {
MKDEBUG && _d('Will resume after',
join('.', @{$self->{resume}}{qw(db tbl)}));
$self->{resume}->{after} = 1;
}
}
my $schema_obj;
if ( $self->{file_itr} ) {
$schema_obj= $self->_iterate_files();
@@ -4399,8 +4411,8 @@ sub _iterate_files {
my ($tbl) = $chunk =~ m/$tbl_name/;
$tbl =~ s/^\s*`//;
$tbl =~ s/`\s*$//;
if ( $self->table_is_allowed($self->{db}, $tbl)
&& $self->_resume_from_table($tbl) ) {
if ( $self->_resume_from_table($tbl)
&& $self->table_is_allowed($self->{db}, $tbl) ) {
my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;
if ( !$ddl ) {
warn "Failed to parse CREATE TABLE from\n" . $chunk;
@@ -4459,8 +4471,9 @@ sub _iterate_dbh {
}
grep {
my ($tbl, $type) = @$_;
$self->table_is_allowed($self->{db}, $tbl)
&& (!$type || ($type ne 'VIEW'));
(!$type || ($type ne 'VIEW'))
&& $self->_resume_from_table($tbl)
&& $self->table_is_allowed($self->{db}, $tbl);
}
@{$dbh->selectall_arrayref($sql)};
MKDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
@@ -4468,8 +4481,6 @@ sub _iterate_dbh {
}
while ( my $tbl = shift @{$self->{tbls}} ) {
next unless $self->_resume_from_table($tbl);
my $tbl_status;
if ( $self->{filters}->{'engines'}
|| $self->{filters}->{'ignore-engines'}
@@ -4637,9 +4648,15 @@ sub _resume_from_table {
return 1 unless $self->{resume}->{tbl};
if ( $tbl eq $self->{resume}->{tbl} ) {
MKDEBUG && _d('At resume table', $tbl);
delete $self->{resume}->{tbl};
return 1;
if ( !$self->{resume}->{after} ) {
MKDEBUG && _d('Resuming from table', $tbl);
delete $self->{resume}->{tbl};
return 1;
}
else {
MKDEBUG && _d('Resuming after table', $tbl);
delete $self->{resume}->{tbl};
}
}
return 0;
@@ -5909,6 +5926,23 @@ sub main {
);
}
my $schema_iter = new SchemaIterator(
dbh => $master_dbh,
resume => $last_chunk ? $q->quote(@{$last_chunk}{qw(db tbl)})
: "",
keep_tbl_status => 1,
OptionParser => $o,
TableParser => $tp,
Quoter => $q,
);
if ( $last_chunk &&
!$schema_iter->table_is_allowed(@{$last_chunk}{qw(db tbl)}) ) {
MKDEBUG && _d('Ignoring last table', @{$last_chunk}{qw(db tbl)},
'and resuming from next table');
$last_chunk = undef;
}
# ########################################################################
# Various variables and modules for checksumming the tables.
# ########################################################################
@@ -6297,15 +6331,6 @@ sub main {
# ########################################################################
# Checksum each table.
# ########################################################################
my $schema_iter = new SchemaIterator(
dbh => $master_dbh,
resume => $last_chunk ? $q->quote(@{$last_chunk}{qw(db tbl)})
: "",
keep_tbl_status => 1,
OptionParser => $o,
TableParser => $tp,
Quoter => $q,
);
TABLE:
while ( $oktorun && (my $tbl = $schema_iter->next()) ) {

View File

@@ -80,8 +80,10 @@ sub new {
my %resume;
if ( my $table = $args{resume} ) {
MKDEBUG && _d('Will resume from', $table);
MKDEBUG && _d('Will resume from or after', $table);
my ($db, $tbl) = $args{Quoter}->split_unquote($table);
die "Resume table must be database-qualified: $table"
unless $db && $tbl;
$resume{db} = $db;
$resume{tbl} = $tbl;
}
@@ -201,6 +203,16 @@ sub _make_filters {
sub next {
my ( $self ) = @_;
if ( !$self->{initialized} ) {
$self->{initialized} = 1;
if ( $self->{resume}->{tbl}
&& !$self->table_is_allowed(@{$self->{resume}}{qw(db tbl)}) ) {
MKDEBUG && _d('Will resume after',
join('.', @{$self->{resume}}{qw(db tbl)}));
$self->{resume}->{after} = 1;
}
}
my $schema_obj;
if ( $self->{file_itr} ) {
$schema_obj= $self->_iterate_files();
@@ -275,8 +287,8 @@ sub _iterate_files {
my ($tbl) = $chunk =~ m/$tbl_name/;
$tbl =~ s/^\s*`//;
$tbl =~ s/`\s*$//;
if ( $self->table_is_allowed($self->{db}, $tbl)
&& $self->_resume_from_table($tbl) ) {
if ( $self->_resume_from_table($tbl)
&& $self->table_is_allowed($self->{db}, $tbl) ) {
my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;
if ( !$ddl ) {
warn "Failed to parse CREATE TABLE from\n" . $chunk;
@@ -338,8 +350,9 @@ sub _iterate_dbh {
}
grep {
my ($tbl, $type) = @$_;
$self->table_is_allowed($self->{db}, $tbl)
&& (!$type || ($type ne 'VIEW'));
(!$type || ($type ne 'VIEW'))
&& $self->_resume_from_table($tbl)
&& $self->table_is_allowed($self->{db}, $tbl);
}
@{$dbh->selectall_arrayref($sql)};
MKDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
@@ -347,8 +360,6 @@ sub _iterate_dbh {
}
while ( my $tbl = shift @{$self->{tbls}} ) {
next unless $self->_resume_from_table($tbl);
# If there are engine filters, we have to get the table status.
# Else, get it if the user wants to keep it since they'll expect
# it to be available.
@@ -533,9 +544,15 @@ sub _resume_from_table {
return 1 unless $self->{resume}->{tbl};
if ( $tbl eq $self->{resume}->{tbl} ) {
MKDEBUG && _d('At resume table', $tbl);
delete $self->{resume}->{tbl};
return 1;
if ( !$self->{resume}->{after} ) {
MKDEBUG && _d('Resuming from table', $tbl);
delete $self->{resume}->{tbl};
return 1;
}
else {
MKDEBUG && _d('Resuming after table', $tbl);
delete $self->{resume}->{tbl};
}
}
return 0;

View File

@@ -9,7 +9,7 @@ BEGIN {
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More tests => 30;
use Test::More tests => 31;
use SchemaIterator;
use FileIterator;
@@ -416,17 +416,20 @@ is(
# ############################################################################
# Resume
# ############################################################################
SKIP: {
skip 'Sandbox master does not have the sakila database', 1
unless @{$dbh->selectcol_arrayref('SHOW DATABASES LIKE "sakila"')};
test_so(
filters => [qw(-d sakila)],
result => "$out/resume-from-sakila-payment.txt",
resume => 'sakila.payment',
test_name => "Resume"
);
test_so(
filters => [qw(-d sakila)],
result => "$out/resume-from-sakila-payment.txt",
resume => 'sakila.payment',
test_name => "Resume"
);
};
# Ignore the table being resumed from; resume from next table.
test_so(
filters => [qw(-d sakila --ignore-tables sakila.payment)],
result => "$out/resume-from-ignored-sakila-payment.txt",
resume => 'sakila.payment',
test_name => "Resume from ignored table"
);
# #############################################################################
# Done.

View File

@@ -0,0 +1,3 @@
sakila.rental
sakila.staff
sakila.store

View File

@@ -27,7 +27,7 @@ elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
else {
plan tests => 17;
plan tests => 21;
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
@@ -197,6 +197,37 @@ my $first_half = [
[qw(sakila payment 6 1000 )],
[qw(sakila payment 7 1000 )],
];
my $second_half = [
[qw(sakila payment 8 1000 )],
[qw(sakila payment 9 1000 )],
[qw(sakila payment 10 1000 )],
[qw(sakila payment 11 1000 )],
[qw(sakila payment 12 1000 )],
[qw(sakila payment 13 1000 )],
[qw(sakila payment 14 1000 )],
[qw(sakila payment 15 1000 )],
[qw(sakila payment 16 1000 )],
[qw(sakila payment 17 49 )],
[qw(sakila rental 1 1000 )],
[qw(sakila rental 2 1000 )],
[qw(sakila rental 3 1000 )],
[qw(sakila rental 4 1000 )],
[qw(sakila rental 5 1000 )],
[qw(sakila rental 6 1000 )],
[qw(sakila rental 7 1000 )],
[qw(sakila rental 8 1000 )],
[qw(sakila rental 9 1000 )],
[qw(sakila rental 10 1000 )],
[qw(sakila rental 11 1000 )],
[qw(sakila rental 12 1000 )],
[qw(sakila rental 13 1000 )],
[qw(sakila rental 14 1000 )],
[qw(sakila rental 15 1000 )],
[qw(sakila rental 16 1000 )],
[qw(sakila rental 17 44 )],
[qw(sakila staff 1 2 )],
[qw(sakila store 1 2 )],
];
$row = $master_dbh->selectall_arrayref('select db, tbl, chunk, master_cnt from percona.checksums order by db, tbl');
is_deeply(
@@ -216,35 +247,7 @@ is_deeply(
$row,
[
@$first_half,
[qw(sakila payment 8 1000 )],
[qw(sakila payment 9 1000 )],
[qw(sakila payment 10 1000 )],
[qw(sakila payment 11 1000 )],
[qw(sakila payment 12 1000 )],
[qw(sakila payment 13 1000 )],
[qw(sakila payment 14 1000 )],
[qw(sakila payment 15 1000 )],
[qw(sakila payment 16 1000 )],
[qw(sakila payment 17 49 )],
[qw(sakila rental 1 1000 )],
[qw(sakila rental 2 1000 )],
[qw(sakila rental 3 1000 )],
[qw(sakila rental 4 1000 )],
[qw(sakila rental 5 1000 )],
[qw(sakila rental 6 1000 )],
[qw(sakila rental 7 1000 )],
[qw(sakila rental 8 1000 )],
[qw(sakila rental 9 1000 )],
[qw(sakila rental 10 1000 )],
[qw(sakila rental 11 1000 )],
[qw(sakila rental 12 1000 )],
[qw(sakila rental 13 1000 )],
[qw(sakila rental 14 1000 )],
[qw(sakila rental 15 1000 )],
[qw(sakila rental 16 1000 )],
[qw(sakila rental 17 44 )],
[qw(sakila staff 1 2 )],
[qw(sakila store 1 2 )],
@$second_half,
],
"Resume finished sakila"
);
@@ -297,35 +300,7 @@ is_deeply(
$row,
[
@$first_half,
[qw(sakila payment 8 1000 )],
[qw(sakila payment 9 1000 )],
[qw(sakila payment 10 1000 )],
[qw(sakila payment 11 1000 )],
[qw(sakila payment 12 1000 )],
[qw(sakila payment 13 1000 )],
[qw(sakila payment 14 1000 )],
[qw(sakila payment 15 1000 )],
[qw(sakila payment 16 1000 )],
[qw(sakila payment 17 49 )],
[qw(sakila rental 1 1000 )],
[qw(sakila rental 2 1000 )],
[qw(sakila rental 3 1000 )],
[qw(sakila rental 4 1000 )],
[qw(sakila rental 5 1000 )],
[qw(sakila rental 6 1000 )],
[qw(sakila rental 7 1000 )],
[qw(sakila rental 8 1000 )],
[qw(sakila rental 9 1000 )],
[qw(sakila rental 10 1000 )],
[qw(sakila rental 11 1000 )],
[qw(sakila rental 12 1000 )],
[qw(sakila rental 13 1000 )],
[qw(sakila rental 14 1000 )],
[qw(sakila rental 15 1000 )],
[qw(sakila rental 16 1000 )],
[qw(sakila rental 17 44 )],
[qw(sakila staff 1 2 )],
[qw(sakila store 1 2 )],
@$second_half,
],
"Resume finished sakila"
);
@@ -398,35 +373,7 @@ is_deeply(
$row,
[
@$first_half,
[qw(sakila payment 8 1000 )],
[qw(sakila payment 9 1000 )],
[qw(sakila payment 10 1000 )],
[qw(sakila payment 11 1000 )],
[qw(sakila payment 12 1000 )],
[qw(sakila payment 13 1000 )],
[qw(sakila payment 14 1000 )],
[qw(sakila payment 15 1000 )],
[qw(sakila payment 16 1000 )],
[qw(sakila payment 17 49 )],
[qw(sakila rental 1 1000 )],
[qw(sakila rental 2 1000 )],
[qw(sakila rental 3 1000 )],
[qw(sakila rental 4 1000 )],
[qw(sakila rental 5 1000 )],
[qw(sakila rental 6 1000 )],
[qw(sakila rental 7 1000 )],
[qw(sakila rental 8 1000 )],
[qw(sakila rental 9 1000 )],
[qw(sakila rental 10 1000 )],
[qw(sakila rental 11 1000 )],
[qw(sakila rental 12 1000 )],
[qw(sakila rental 13 1000 )],
[qw(sakila rental 14 1000 )],
[qw(sakila rental 15 1000 )],
[qw(sakila rental 16 1000 )],
[qw(sakila rental 17 44 )],
[qw(sakila staff 1 2 )],
[qw(sakila store 1 2 )],
@$second_half,
],
"Resume finished sakila"
);
@@ -454,6 +401,74 @@ ok(
"Chunk 12 master_crc updated"
);
# ############################################################################
# Resume with --ignore-table.
# ############################################################################
$sb->load_file('master', "t/pt-table-checksum/samples/3tbl-resume.sql");
load_data_infile("3tbl-resume", "ts='2011-11-08 00:00:18'");
$master_dbh->do("delete from percona.checksums where ts > '2011-11-08 00:00:09'");
my $before = $master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums where tbl='t1' or tbl='t2' order by db, tbl");
is_deeply(
$before,
[
[qw( test t1 1 ), '2011-11-08 00:00:01'],
[qw( test t1 2 ), '2011-11-08 00:00:02'],
[qw( test t1 3 ), '2011-11-08 00:00:03'],
[qw( test t1 4 ), '2011-11-08 00:00:04'],
[qw( test t1 5 ), '2011-11-08 00:00:05'],
[qw( test t1 6 ), '2011-11-08 00:00:06'],
[qw( test t2 1 ), '2011-11-08 00:00:07'],
[qw( test t2 2 ), '2011-11-08 00:00:08'],
[qw( test t2 3 ), '2011-11-08 00:00:09'],
],
"Checksum results through tbl2 chunk 3"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(-d test --resume),
qw(--ignore-tables test.t2 --chunk-size 5 --chunk-time 0)) },
trf => sub { return PerconaTest::normalize_checksum_results(@_) },
);
is(
$output,
"ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 26 6 0 test.t3
",
"Resumed from t3"
);
$row = $master_dbh->selectall_arrayref('select db, tbl, chunk from percona.checksums order by db, tbl');
is_deeply(
$row,
[
[qw( test t1 1 )],
[qw( test t1 2 )],
[qw( test t1 3 )],
[qw( test t1 4 )],
[qw( test t1 5 )],
[qw( test t1 6 )],
[qw( test t2 1 )],
[qw( test t2 2 )],
[qw( test t2 3 )],
# t2 not resumed
[qw( test t3 1 )],
[qw( test t3 2 )],
[qw( test t3 3 )],
[qw( test t3 4 )],
[qw( test t3 5 )],
[qw( test t3 6 )],
],
"--resume and --ignore-table"
);
is_deeply(
$master_dbh->selectall_arrayref("select db, tbl, chunk, ts from percona.checksums where tbl='t1' or tbl='t2' order by db, tbl"),
$before,
"t1 and t2 checksums not updated"
);
# #############################################################################
# Done.
# #############################################################################

View File

@@ -0,0 +1,27 @@
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
USE test;
CREATE TABLE `t1` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`c` varchar(26) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM;
INSERT INTO `t1` VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),(6,'f'),(7,'g'),(8,'h'),(9,'i'),(10,'j'),(11,'k'),(12,'l'),(13,'m'),(14,'n'),(15,'o'),(16,'p'),(17,'q'),(18,'r'),(19,'s'),(20,'t'),(21,'u'),(22,'v'),(23,'w'),(24,'x'),(25,'y'),(26,'z');
CREATE TABLE `t2` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`c` varchar(26) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM;
INSERT INTO `t2` VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),(6,'f'),(7,'g'),(8,'h'),(9,'i'),(10,'j'),(11,'k'),(12,'l'),(13,'m'),(14,'n'),(15,'o'),(16,'p'),(17,'q'),(18,'r'),(19,'s'),(20,'t'),(21,'u'),(22,'v'),(23,'w'),(24,'x'),(25,'y'),(26,'z');
CREATE TABLE `t3` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
`c` varchar(26) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM;
INSERT INTO `t3` VALUES (1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'),(6,'f'),(7,'g'),(8,'h'),(9,'i'),(10,'j'),(11,'k'),(12,'l'),(13,'m'),(14,'n'),(15,'o'),(16,'p'),(17,'q'),(18,'r'),(19,'s'),(20,'t'),(21,'u'),(22,'v'),(23,'w'),(24,'x'),(25,'y'),(26,'z');

View File

@@ -0,0 +1,18 @@
test t1 1 0.001062 PRIMARY 1 5 5ef4701a 5 5ef4701a 5 2011-11-08 00:00:01
test t1 2 0.000908 PRIMARY 6 10 5e765a6f 5 5e765a6f 5 2011-11-08 00:00:02
test t1 3 0.000890 PRIMARY 11 15 28b300ce 5 28b300ce 5 2011-11-08 00:00:03
test t1 4 0.000874 PRIMARY 16 20 b6ccc8e2 5 b6ccc8e2 5 2011-11-08 00:00:04
test t1 5 0.000915 PRIMARY 21 25 c0099243 5 c0099243 5 2011-11-08 00:00:05
test t1 6 0.001000 PRIMARY 26 26 55f99957 1 55f99957 1 2011-11-08 00:00:06
test t2 1 0.000983 PRIMARY 1 5 5ef4701a 5 5ef4701a 5 2011-11-08 00:00:07
test t2 2 0.000734 PRIMARY 6 10 5e765a6f 5 5e765a6f 5 2011-11-08 00:00:08
test t2 3 0.000938 PRIMARY 11 15 28b300ce 5 28b300ce 5 2011-11-08 00:00:09
test t2 4 0.000889 PRIMARY 16 20 b6ccc8e2 5 b6ccc8e2 5 2011-11-08 00:00:10
test t2 5 0.000916 PRIMARY 21 25 c0099243 5 c0099243 5 2011-11-08 00:00:11
test t2 6 0.000908 PRIMARY 26 26 55f99957 1 55f99957 1 2011-11-08 00:00:12
test t3 1 0.000920 PRIMARY 1 5 5ef4701a 5 5ef4701a 5 2011-11-08 00:00:13
test t3 2 0.000880 PRIMARY 6 10 5e765a6f 5 5e765a6f 5 2011-11-08 00:00:14
test t3 3 0.002164 PRIMARY 11 15 28b300ce 5 28b300ce 5 2011-11-08 00:00:15
test t3 4 0.000957 PRIMARY 16 20 b6ccc8e2 5 b6ccc8e2 5 2011-11-08 00:00:16
test t3 5 0.000708 PRIMARY 21 25 c0099243 5 c0099243 5 2011-11-08 00:00:17
test t3 6 0.000834 PRIMARY 26 26 55f99957 1 55f99957 1 2011-11-08 00:00:18