percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-11 05:29:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

More repl table columns: chunk_time, chunk_index, lower_boundary, and upper_boundary (those two replace boundaries).

Browse Source
This commit is contained in:
Daniel Nichter
2011-09-23 10:13:57 -06:00
parent b783470aaa
commit 07cb6010a2
2 changed files with 68 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
bin/pt-table-checksum
View File

@@ -3525,6 +3525,11 @@ sub nibble_number {
return $self->{nibbleno};
}
sub nibble_index {
my ($self) = @_;
return $self->{index};
}
sub set_chunk_size {
my ($self, $limit) = @_;
MKDEBUG && _d('Setting new chunk size (LIMIT):', $limit);
@@ -4326,10 +4331,13 @@ sub _d {
# ###########################################################################
# Transformers package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/Transformers.pm
# t/lib/Transformers.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
# Package: Transformers
# Transformers exports subroutines that convert and beautify values.
package Transformers;
use strict;
@@ -4361,7 +4369,6 @@ our @EXPORT_OK = qw(
our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/;
our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/;
our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks
# like a MySQL YYMMDD without hh:mm:ss.
sub micro_t {
my ( $t, %args ) = @_;
@@ -4371,12 +4378,8 @@ sub micro_t {
$t = 0 if $t < 0;
# "Remove" scientific notation so the regex below does not make
# 6.123456e+18 into 6.123456.
$t = sprintf('%.17f', $t) if $t =~ /e/;
# Truncate after 6 decimal places to avoid 0.9999997 becoming 1
# because sprintf() rounds.
$t =~ s/\.(\d{1,6})\d*/\.$1/;
if ($t > 0 && $t <= 0.000999) {
@@ -4397,7 +4400,6 @@ sub micro_t {
return $f;
}
# Returns what percentage $is of $of.
sub percentage_of {
my ( $is, $of, %args ) = @_;
my $p = $args{p} || 0; # float precision
@@ -4410,7 +4412,6 @@ sub secs_to_time {
$secs ||= 0;
return '00:00' unless $secs;
# Decide what format to use, if not given
$fmt ||= $secs >= 86_400 ? 'd'
: $secs >= 3_600 ? 'h'
: 'm';
@@ -4433,8 +4434,6 @@ sub secs_to_time {
$secs % 60);
}
# Convert time values to number of seconds:
# 1s = 1, 1m = 60, 1h = 3600, 1d = 86400.
sub time_to_secs {
my ( $val, $default_suffix ) = @_;
die "I need a val argument" unless defined $val;
@@ -4472,8 +4471,6 @@ sub shorten {
$num, $units[$n]);
}
# Turns a unix timestamp into an ISO8601 formatted date and time. $gmt makes
# this relative to GMT, for test determinism.
sub ts {
my ( $time, $gmt ) = @_;
my ( $sec, $min, $hour, $mday, $mon, $year )
@@ -4490,8 +4487,6 @@ sub ts {
return $val;
}
# Turns MySQL's 071015 21:43:52 into a properly formatted timestamp. Also
# handles a timestamp with fractions after it.
sub parse_timestamp {
my ( $val ) = @_;
if ( my($y, $m, $d, $h, $i, $s, $f)
@@ -4504,9 +4499,6 @@ sub parse_timestamp {
return $val;
}
# Turns a properly formatted timestamp like 2007-10-15 01:43:52
# into an int (seconds since epoch). Optional microseconds are printed. $gmt
# makes it use GMT time instead of local time (to make tests deterministic).
sub unix_timestamp {
my ( $val, $gmt ) = @_;
if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) {
@@ -4522,15 +4514,6 @@ sub unix_timestamp {
return $val;
}
# Turns several different types of timestamps into a unix timestamp.
# Each type is auto-detected. Supported types are:
# * N[shdm] Now - N[shdm]
# * 071015 21:43:52 MySQL slow log timestamp
# * 2009-07-01 [3:43:01] Proper timestamp with options HH:MM:SS
# * NOW() A MySQL time express
# For the last type, the callback arg is required. It is passed the
# given value/expression and is expected to return a single value
# (the result of the expression).
sub any_unix_timestamp {
my ( $val, $callback ) = @_;
@@ -4544,9 +4527,6 @@ sub any_unix_timestamp {
return time - $n;
}
elsif ( $val =~ m/^\d{9,}/ ) {
# unix timestamp 100000000 is roughly March, 1973, so older
# dates won't be caught here; they'll probably be mistaken
# for a MySQL slow log timestamp.
MKDEBUG && _d('ts is already a unix timestamp');
return $val;
}
@@ -4569,7 +4549,6 @@ sub any_unix_timestamp {
return;
}
# Returns the rightmost 64 bits of an MD5 checksum of the value.
sub make_checksum {
my ( $val ) = @_;
my $checksum = uc substr(md5_hex($val), -16);
@@ -4577,9 +4556,6 @@ sub make_checksum {
return $checksum;
}
# Perl implementation of CRC32, ripped off from Digest::Crc32. The results
# ought to match what you get from any standard CRC32 implementation, such as
# that inside MySQL.
sub crc32 {
my ( $string ) = @_;
return unless $string;
@@ -5225,13 +5201,14 @@ sub main {
# ########################################################################
my %crc_args = $rc->get_crc_args(dbh => $dbh);
my $checksum_dms = "REPLACE INTO $repl_table "
. "(db, tbl, chunk, boundaries, this_cnt, this_crc) "
. "SELECT ?, ?, ?, ?,";
. "(db, tbl, chunk, chunk_index,"
. " lower_boundary, upper_boundary, this_cnt, this_crc) "
. "SELECT ?, ?, ?, ?, ?, ?,";
my $fetch_sth = $dbh->prepare(
"SELECT this_crc, this_cnt FROM $repl_table "
. "WHERE db = ? AND tbl = ? AND chunk = ?");
my $update_sth = $dbh->prepare(
"UPDATE $repl_table SET master_crc = ?, master_cnt = ? "
"UPDATE $repl_table SET chunk_time = ?, master_crc = ?, master_cnt = ? "
. "WHERE db = ? AND tbl = ? AND chunk = ?");
# ########################################################################
@@ -5288,7 +5265,7 @@ sub main {
# Exec and time the chunk checksum query. If it fails, retry.
# Should return 0 rows which will fetch the next boundary.
my $t_start = time;
my $rows = exec_nibble(%args, Retry => $retry);
my $rows = exec_nibble(%args, Quoter => $q, Retry => $retry);
$tbl->{nibble_time} = time - $t_start;
return $rows;
},
@@ -5303,11 +5280,20 @@ sub main {
$fetch_sth->execute(@{$tbl}{qw(db tbl)}, $args{nibbleno});
my ($crc, $cnt) = $fetch_sth->fetchrow_array();
$tbl->{checksum_results}->{n_rows} += $cnt || 0;
# We're working on the master, so update the checksum's master_cnt
# and master_crc.
$tbl->{checksum_results}->{n_rows} += $cnt || 0;
$update_sth->execute(
$crc, $cnt, @{$tbl}{qw(db tbl)}, $args{nibbleno});
# UPDATE repl_table SET
sprintf('%.3f', $tbl->{nibble_time}), # chunk_time
$crc, # master_crc
$cnt, # master_cnt
# WHERE
$tbl->{db}, # db
$tbl->{tbl}, # tbl
$args{nibbleno}, # chunk
);
# Should be don't automatically, but I like to be explicit.
$fetch_sth->finish();
@@ -5343,6 +5329,7 @@ sub main {
$args{NibbleIterator}->set_chunk_size($tbl->{chunk_size});
}
# Every table should have a Progress obj; update it.
if ( my $tbl_pr = $tbl->{progress} ) {
$tbl_pr->update(sub {return $tbl->{checksum_results}->{n_rows}});
}
@@ -5480,11 +5467,12 @@ sub get_cxn {
sub exec_nibble {
my (%args) = @_;
my @required_args = qw(dbh tbl sth lb ub Retry);
my @required_args = qw(dbh tbl sth lb ub NibbleIterator Retry Quoter);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($dbh, $tbl, $sth, $lb, $ub, $retry) = @args{@required_args};
my ($dbh, $tbl, $sth, $lb, $ub, $nibble_iter, $retry, $q)
= @args{@required_args};
return $retry->retry(
tries => 2,
@@ -5497,22 +5485,23 @@ sub exec_nibble {
MKDEBUG && _d($sql);
$dbh->do($sql);
my $boundaries = @$lb || @$ub ? join(',', @$lb, @$ub) : '1=1';
my $lb_quoted = join(',', map { $q->quote_val($_) } @$lb);
my $ub_quoted = join(',', map { $q->quote_val($_) } @$ub);
# Execute the REPLACE...SELECT checksum query.
MKDEBUG && _d($sth->{Statement}, 'params:',
@{$tbl}{qw(db tbl)},
$args{nibbleno},
$boundaries,
@$lb,
@$ub,
);
# MKDEBUG && _d($sth->{Statement}, 'params:',
# );
$sth->execute(
@{$tbl}{qw(db tbl)},
$args{nibbleno},
$boundaries,
@$lb,
@$ub,
# REPLACE INTO repl_table SELECT
$tbl->{db}, # db
$tbl->{tbl}, # tbl
$args{nibbleno}, # chunk
$nibble_iter->nibble_index(), # chunk_index
$lb_quoted, # lower_boundary
$ub_quoted, # upper_boundary
# this_cnt, this_crc WHERE
@$lb, # upper boundary values
@$ub, # lower boundary values
);
# Check if checksum query caused any warnings.
@@ -5533,14 +5522,11 @@ sub exec_nibble {
MKDEBUG && _d('Ignoring warning:', $warning->{message});
}
else {
# die doesn't permit extra line breaks so warn then die.
warn "\nChecksum query caused a warning:\n"
. join("\n",
map { "\t$_: " . $warning->{$_} || '' }
qw(level code message)
)
. "\n\tquery: " . $sth->{Statement} . "\n\n";
die;
die "Checksum query caused a warning:\n"
. " Level: " . ($warning->{level} || '') . "\n"
. " Code: " . ($warning->{code} || '') . "\n"
. " Message: " . ($warning->{message} || '') . "\n"
. " Query: " . $sth->{Statement} . "\n";
}
}
@@ -6501,14 +6487,17 @@ L<"--create-replicate-table"> (MAGIC_create_replicate):
db char(64) NOT NULL,
tbl char(64) NOT NULL,
chunk int NOT NULL,
boundaries char(100) NOT NULL,
chunk_time float NULL,
chunk_index varchar(200) NOT NULL,
lower_boundary text NOT NULL,
upper_boundary text NOT NULL,
this_crc char(40) NOT NULL,
this_cnt int NOT NULL,
master_crc char(40) NULL,
master_cnt int NULL,
ts timestamp NOT NULL,
PRIMARY KEY (db, tbl, chunk)
);
) ENGINE=InnoDB;
Be sure to choose an appropriate storage engine for the checksum table. If you
are checksumming InnoDB tables, for instance, a deadlock will break replication

7
lib/NibbleIterator.pm
View File

@@ -44,7 +44,7 @@ $Data::Dumper::Quotekeys = 0;
# Quoter - <Quoter> object
#
# Optional Arguments:
# chunk_indexd - Index to use for nibbling
# chunk_index - Index to use for nibbling
#
# Returns:
# NibbleIterator object
@@ -278,6 +278,11 @@ sub nibble_number {
return $self->{nibbleno};
}
sub nibble_index {
my ($self) = @_;
return $self->{index};
}
sub set_chunk_size {
my ($self, $limit) = @_;
MKDEBUG && _d('Setting new chunk size (LIMIT):', $limit);