Re-implement --replicate-check. Add --quiet. Update find_replication_differences().

This commit is contained in:
Daniel Nichter
2011-09-29 10:46:22 -06:00
parent 206ac93214
commit c4092295a5
2 changed files with 73 additions and 48 deletions

View File

@@ -3286,24 +3286,31 @@ sub _make_xor_slices {
} }
sub find_replication_differences { sub find_replication_differences {
my ( $self, $dbh, $table ) = @_; my ($self, %args) = @_;
my @required_args = qw(dbh repl_table);
foreach my $arg( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($dbh, $repl_table) = @args{@required_args};
(my $sql = <<" EOF") =~ s/\s+/ /gm; (my $sql = <<" EOF") =~ s/\s+/ /gm;
SELECT db, tbl, chunk, boundaries, SELECT
CONCAT(db, '.', tbl) AS `table`,
chunk, chunk_index, lower_boundary, upper_boundary,
COALESCE(this_cnt-master_cnt, 0) AS cnt_diff, COALESCE(this_cnt-master_cnt, 0) AS cnt_diff,
COALESCE( COALESCE(
this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc), this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc),
0 0
) AS crc_diff, ) AS crc_diff,
this_cnt, master_cnt, this_crc, master_crc this_cnt, master_cnt, this_crc, master_crc
FROM $table FROM $repl_table
WHERE master_cnt <> this_cnt OR master_crc <> this_crc WHERE master_cnt <> this_cnt OR master_crc <> this_crc
OR ISNULL(master_crc) <> ISNULL(this_crc) OR ISNULL(master_crc) <> ISNULL(this_crc)
EOF EOF
MKDEBUG && _d($sql); MKDEBUG && _d($sql);
my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} }); my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} });
return @$diffs; return $diffs;
} }
sub _d { sub _d {
@@ -5345,26 +5352,31 @@ sub main {
my $retry = new Retry(); my $retry = new Retry();
# ######################################################################## # ########################################################################
# Check replication slaves if desired. If only --replicate-check is given, # Check replication slaves.
# then we will exit here. If --recheck is also given, then we'll continue
# through the entire script but checksum only the inconsistent tables found
# here.
# ######################################################################## # ########################################################################
my $repl_table = $q->quote($q->split_unquote($o->get('replicate'))); my $repl_table = $q->quote($q->split_unquote($o->get('replicate')));
if ( defined $o->get('replicate-check') ) { if ( $o->get('replicate-check') && !$o->get('recheck') ) {
foreach my $host ( @$slaves ) { MKDEBUG && _d('Will --replicate-check and exit');
my @tbls = $rc->find_replication_differences($host->{dbh},$repl_table);
next unless @tbls; foreach my $slave ( @$slaves ) {
my $diffs = $rc->find_replication_differences(
dbh => $slave->{dbh},
repl_table => $repl_table,
);
MKDEBUG && _d(scalar @$diffs, 'checksum diffs on', $slave->{dsn}->{n});
if ( @$diffs ) {
$exit_status |= 1; $exit_status |= 1;
print_inconsistent_tbls( next if $o->get('quiet');
dsn => $host->{dsn}, print_checksum_diffs(
tbls => \@tbls, cxn => $slave,
OptionParser => $o, diffs => $diffs,
DSNParser => $dp,
); );
} }
return $exit_status unless $o->get('recheck'); }
MKDEBUG && _d('Exit status', $exit_status, 'oktorun', $oktorun);
return $exit_status;
} }
# ######################################################################## # ########################################################################
@@ -5475,7 +5487,7 @@ sub main {
my $limit = $o->get('chunk-size-limit'); my $limit = $o->get('chunk-size-limit');
# ######################################################################## # ########################################################################
# Resume # Get last chunk for --resume.
# ######################################################################## # ########################################################################
my $last_chunk; my $last_chunk;
if ( $o->get('resume') ) { if ( $o->get('resume') ) {
@@ -5508,14 +5520,17 @@ sub main {
Quoter => $q, Quoter => $q,
); );
if ( !$next_lb ) { if ( !$next_lb ) {
# This can happen if the tool stops after the last checksum
# of a table. So we just start with the next table.
MKDEBUG && _d('Resuming from last chunk in table;', MKDEBUG && _d('Resuming from last chunk in table;',
'getting next table'); 'getting next table');
$oktonibble = 0; $oktonibble = 0;
} }
else { else {
MKDEBUG && _d('Resuming from chunk', $last_chunk->{chunk});
$nibble_iter->set_nibble_number($last_chunk->{chunk}); $nibble_iter->set_nibble_number($last_chunk->{chunk});
$nibble_iter->set_boundary('next_lower', $next_lb); $nibble_iter->set_boundary('next_lower', $next_lb);
print "Resuming from $tbl->{db}.$tbl->{tbl} at chunk "
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n";
} }
# Just need to call us once to kick-start the resume process. # Just need to call us once to kick-start the resume process.
@@ -5944,6 +5959,28 @@ sub print_checksum_results {
} }
} }
{
my @headers = qw(table chunk cnt_diff crc_diff chunk_index lower_boundary upper_boundary);
sub print_checksum_diffs {
my ( %args ) = @_;
my @required_args = qw(cxn diffs);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($cxn, $diffs) = @args{@required_args};
print "Differences on $cxn->{dsn}->{n}\n";
print join(' ', map { uc $_ } @headers), "\n";
foreach my $diff ( @$diffs ) {
print join(' ', map { defined $_ ? $_ : '' } @{$diff}{@headers}), "\n";
}
print "\n";
return;
}
}
# Check for existence and privileges on the replication table before # Check for existence and privileges on the replication table before
# starting, and prepare the statements that will be used to update it. # starting, and prepare the statements that will be used to update it.
# Also clean out the checksum table. And create it if needed. # Also clean out the checksum table. And create it if needed.
@@ -6142,27 +6179,6 @@ sub explain_statement {
return $expl; return $expl;
} }
sub print_inconsistent_tbls {
my ( %args ) = @_;
my @required_args = qw(dsn tbls OptionParser DSNParser);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($dsn, $tbls, $o, $dp) = @args{@required_args};
my @headers = qw(db tbl chunk cnt_diff crc_diff boundaries);
print "Differences on " . $dp->as_string($dsn, [qw(h P F)]) . "\n";
my $max_db = max(5, map { length($_->{db}) } @$tbls);
my $max_tbl = max(5, map { length($_->{tbl}) } @$tbls);
my $fmt = "%-${max_db}s %-${max_tbl}s %5s %8s %8s %s\n";
printf($fmt, map { uc } @headers) or die "Cannot print: $OS_ERROR";
foreach my $tbl ( @$tbls ) {
printf($fmt, @{$tbl}{@headers}) or die "Cannot print: $OS_ERROR";
}
print "\n" or die "Cannot print: $OS_ERROR";
return;
}
sub table_progress { sub table_progress {
my (%args) = @_; my (%args) = @_;
my @required_args = qw(dbh tbl OptionParser Quoter); my @required_args = qw(dbh tbl OptionParser Quoter);
@@ -6841,6 +6857,10 @@ The value is a comma-separated list with two parts. The first part can be
percentage, time, or iterations; the second part specifies how often an update percentage, time, or iterations; the second part specifies how often an update
should be printed, in percentage, seconds, or number of iterations. should be printed, in percentage, seconds, or number of iterations.
=item --quiet
Don't print anything to STDOUT.
=item --recheck =item --recheck
Re-checksum chunks that L<"--replicate-check"> found to be different. Re-checksum chunks that L<"--replicate-check"> found to be different.
@@ -6971,8 +6991,6 @@ The table specified by L<"--replicate"> will never be checksummed itself.
=item --replicate-check =item --replicate-check
type: int
Check results in L<"--replicate"> table, to the specified depth. You must use Check results in L<"--replicate"> table, to the specified depth. You must use
this after you run the tool normally; it skips the checksum step and only checks this after you run the tool normally; it skips the checksum step and only checks
results. results.

View File

@@ -446,24 +446,31 @@ sub _make_xor_slices {
# Queries the replication table for chunks that differ from the master's data. # Queries the replication table for chunks that differ from the master's data.
sub find_replication_differences { sub find_replication_differences {
my ( $self, $dbh, $table ) = @_; my ($self, %args) = @_;
my @required_args = qw(dbh repl_table);
foreach my $arg( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($dbh, $repl_table) = @args{@required_args};
(my $sql = <<" EOF") =~ s/\s+/ /gm; (my $sql = <<" EOF") =~ s/\s+/ /gm;
SELECT db, tbl, chunk, boundaries, SELECT
CONCAT(db, '.', tbl) AS `table`,
chunk, chunk_index, lower_boundary, upper_boundary,
COALESCE(this_cnt-master_cnt, 0) AS cnt_diff, COALESCE(this_cnt-master_cnt, 0) AS cnt_diff,
COALESCE( COALESCE(
this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc), this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc),
0 0
) AS crc_diff, ) AS crc_diff,
this_cnt, master_cnt, this_crc, master_crc this_cnt, master_cnt, this_crc, master_crc
FROM $table FROM $repl_table
WHERE master_cnt <> this_cnt OR master_crc <> this_crc WHERE master_cnt <> this_cnt OR master_crc <> this_crc
OR ISNULL(master_crc) <> ISNULL(this_crc) OR ISNULL(master_crc) <> ISNULL(this_crc)
EOF EOF
MKDEBUG && _d($sql); MKDEBUG && _d($sql);
my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} }); my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} });
return @$diffs; return $diffs;
} }
sub _d { sub _d {