mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-10-18 16:40:23 +00:00
Add OobNibbleIterator.pm.
This commit is contained in:
Daniel Nichter
206
lib/OobNibbleIterator.pm
Normal file
206
lib/OobNibbleIterator.pm
Normal file
@@ -0,0 +1,206 @@
|
||||
# This program is copyright 2011 Percona Inc.
|
||||
# Feedback and improvements are welcome.
|
||||
#
|
||||
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
|
||||
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it under
|
||||
# the terms of the GNU General Public License as published by the Free Software
|
||||
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
|
||||
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
|
||||
# licenses.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
||||
# Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
# ###########################################################################
|
||||
# OobNibbleIterator package
|
||||
# ###########################################################################
|
||||
{
|
||||
# Package: OobNibbleIterator
|
||||
# OobNibbleIterator is a <NibbleIterator> that nibbles values which are
|
||||
# out-of-bounds: beyond the lower and upper boundaries. NibbleIterator
|
||||
# nibbles a table from its lowest to its highest value, but sometimes
|
||||
# another server's copy of the table might have more values below or above
|
||||
# the first table's boundaires. When the parent NibbleIterator is done,
|
||||
# this class executes two more nibbles for values past the lower boundary
|
||||
# and past the upper boundary.
|
||||
package OobNibbleIterator;
|
||||
use base 'NibbleIterator';
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
|
||||
|
||||
use Data::Dumper;
|
||||
$Data::Dumper::Indent = 1;
|
||||
$Data::Dumper::Sortkeys = 1;
|
||||
$Data::Dumper::Quotekeys = 0;
|
||||
|
||||
sub new {
|
||||
my ( $class, %args ) = @_;
|
||||
my @required_args = qw();
|
||||
foreach my $arg ( @required_args ) {
|
||||
die "I need a $arg argument" unless $args{$arg};
|
||||
}
|
||||
|
||||
# Let the parent do all the init work.
|
||||
my $self = $class->SUPER::new(%args);
|
||||
|
||||
my $q = $self->{Quoter};
|
||||
my $o = $self->{OptionParser};
|
||||
my $where = $o->get('where');
|
||||
|
||||
# If it's not a single nibble table, init our special statements.
|
||||
if ( !$self->one_nibble() ) {
|
||||
# Make statements for the lower and upper ranges.
|
||||
my $head_sql
|
||||
= ($args{past_dms} || "SELECT ")
|
||||
. ($args{past_select}
|
||||
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
|
||||
. " FROM " . $self->{sql}->{from};
|
||||
|
||||
my $tail_sql
|
||||
= ($where ? " AND ($where)" : '')
|
||||
. " ORDER BY " . $self->{sql}->{order_by};
|
||||
|
||||
my $past_lower_sql
|
||||
= $head_sql
|
||||
. " WHERE " . $self->{sql}->{boundaries}->{'<'}
|
||||
. $tail_sql
|
||||
. " /*past lower chunk*/";
|
||||
MKDEBUG && _d('Past lower statement:', $past_lower_sql);
|
||||
|
||||
my $explain_past_lower_sql
|
||||
= "EXPLAIN SELECT "
|
||||
. ($args{past_select}
|
||||
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
|
||||
. " FROM " . $self->{sql}->{from}
|
||||
. " WHERE " . $self->{sql}->{boundaries}->{'<'}
|
||||
. $tail_sql
|
||||
. " /*explain past lower chunk*/";
|
||||
MKDEBUG && _d('Past lower statement:', $explain_past_lower_sql);
|
||||
|
||||
my $past_upper_sql
|
||||
= $head_sql
|
||||
. " WHERE " . $self->{sql}->{boundaries}->{'>'}
|
||||
. $tail_sql
|
||||
. " /*past upper chunk*/";
|
||||
MKDEBUG && _d('Past upper statement:', $past_upper_sql);
|
||||
|
||||
my $explain_past_upper_sql
|
||||
= "EXPLAIN SELECT "
|
||||
. ($args{past_select}
|
||||
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
|
||||
. " FROM " . $self->{sql}->{from}
|
||||
. " WHERE " . $self->{sql}->{boundaries}->{'>'}
|
||||
. $tail_sql
|
||||
. " /*explain past upper chunk*/";
|
||||
MKDEBUG && _d('Past upper statement:', $explain_past_upper_sql);
|
||||
|
||||
$self->{past_lower_sql} = $past_lower_sql;
|
||||
$self->{past_upper_sql} = $past_upper_sql;
|
||||
$self->{explain_past_lower_sql} = $explain_past_lower_sql;
|
||||
$self->{explain_past_upper_sql} = $explain_past_upper_sql;
|
||||
$self->{past_nibbles} = [qw(lower upper)]; # what we nibble
|
||||
}
|
||||
|
||||
return bless $self, $class;
|
||||
}
|
||||
|
||||
sub more_boundaries {
|
||||
my ($self) = @_;
|
||||
return $self->SUPER::more_boundaries() if $self->{one_nibble};
|
||||
return scalar @{$self->{past_nibbles}} ? 1 : 0;
|
||||
}
|
||||
|
||||
sub statements {
|
||||
my ($self) = @_;
|
||||
|
||||
# Get the parent's statements.
|
||||
my $sths = $self->SUPER::statements();
|
||||
|
||||
# Add our special statements.
|
||||
$sths->{past_lower_boundary} = $self->{past_lower_sth};
|
||||
$sths->{past_upper_boundary} = $self->{past_upper_sth};
|
||||
|
||||
return $sths;
|
||||
}
|
||||
|
||||
sub _prepare_sths {
|
||||
my ($self) = @_;
|
||||
MKDEBUG && _d('Preparing boundless statement handles');
|
||||
|
||||
# Prepare our statements for nibbles past the lower and upper boundaries.
|
||||
if ( !$self->{one_nibble} ) {
|
||||
my $dbh = $self->{Cxn}->dbh();
|
||||
$self->{past_lower_sth} = $dbh->prepare($self->{past_lower_sql});
|
||||
$self->{past_upper_sth} = $dbh->prepare($self->{past_upper_sql});
|
||||
$self->{explain_past_lower_sth} = $dbh->prepare($self->{explain_past_lower_sql});
|
||||
$self->{explain_past_upper_sth} = $dbh->prepare($self->{explain_past_upper_sql});
|
||||
}
|
||||
|
||||
# Let the parent prepare its statements.
|
||||
return $self->SUPER::_prepare_sths();
|
||||
}
|
||||
|
||||
sub _next_boundaries {
|
||||
my ($self) = @_;
|
||||
|
||||
# Use the parent's boundaries.
|
||||
return $self->SUPER::_next_boundaries() unless $self->{no_more_boundaries};
|
||||
|
||||
# Parent has no more boundaries. Use our past boundaries.
|
||||
if ( my $past = shift @{$self->{past_nibbles}} ) {
|
||||
if ( $past eq 'lower' ) {
|
||||
MKDEBUG && _d('Nibbling values below lower boundary');
|
||||
$self->{nibble_sth} = $self->{past_lower_sth};
|
||||
$self->{explain_nibble_sth} = $self->{explain_past_lower_sth};
|
||||
$self->{lower} = [];
|
||||
$self->{upper} = $self->boundaries()->{first_lower};
|
||||
$self->{next_lower} = undef;
|
||||
}
|
||||
elsif ( $past eq 'upper' ) {
|
||||
MKDEBUG && _d('Nibbling values above upper boundary');
|
||||
$self->{nibble_sth} = $self->{past_upper_sth};
|
||||
$self->{explain_nibble_sth} = $self->{explain_past_upper_sth};
|
||||
$self->{lower} = $self->boundaries()->{last_upper};
|
||||
$self->{upper} = [];
|
||||
$self->{next_lower} = undef;
|
||||
}
|
||||
else {
|
||||
die "Invalid past nibble: $past";
|
||||
}
|
||||
return 1; # continue nibbling
|
||||
}
|
||||
|
||||
MKDEBUG && _d('Done nibbling past boundaries');
|
||||
return; # stop nibbling
|
||||
}
|
||||
|
||||
sub DESTROY {
|
||||
my ( $self ) = @_;
|
||||
foreach my $key ( keys %$self ) {
|
||||
if ( $key =~ m/_sth$/ ) {
|
||||
MKDEBUG && _d('Finish', $key);
|
||||
$self->{$key}->finish();
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
sub _d {
|
||||
my ($package, undef, $line) = caller 0;
|
||||
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
|
||||
map { defined $_ ? $_ : 'undef' }
|
||||
@_;
|
||||
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
# ###########################################################################
|
||||
# End OobNibbleIterator package
|
||||
# ###########################################################################
|
||||
264
t/lib/OobNibbleIterator.t
Normal file
264
t/lib/OobNibbleIterator.t
Normal file
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More;
|
||||
|
||||
use Schema;
|
||||
use SchemaIterator;
|
||||
use Quoter;
|
||||
use DSNParser;
|
||||
use Sandbox;
|
||||
use OptionParser;
|
||||
use TableParser;
|
||||
use TableNibbler;
|
||||
use RowChecksum;
|
||||
use NibbleIterator;
|
||||
use OobNibbleIterator;
|
||||
use Cxn;
|
||||
use PerconaTest;
|
||||
|
||||
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
|
||||
|
||||
use Data::Dumper;
|
||||
$Data::Dumper::Indent = 1;
|
||||
$Data::Dumper::Sortkeys = 1;
|
||||
$Data::Dumper::Quotekeys = 0;
|
||||
|
||||
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
my $dbh = $sb->get_dbh_for('master');
|
||||
my $output;
|
||||
|
||||
if ( !$dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox master';
|
||||
}
|
||||
else {
|
||||
plan tests => 15;
|
||||
}
|
||||
|
||||
my $q = new Quoter();
|
||||
my $tp = new TableParser(Quoter=>$q);
|
||||
my $nb = new TableNibbler(TableParser=>$tp, Quoter=>$q);
|
||||
my $o = new OptionParser(description => 'OobNibbleIterator');
|
||||
my $rc = new RowChecksum(OptionParser => $o, Quoter=>$q);
|
||||
my $cxn = new Cxn(
|
||||
dbh => $dbh,
|
||||
dsn => { h=>'127.1', P=>'12345', n=>'h=127.1,P=12345' },
|
||||
DSNParser => $dp,
|
||||
OptionParser => $o,
|
||||
);
|
||||
|
||||
$o->get_specs("$trunk/bin/pt-table-checksum");
|
||||
|
||||
my %common_modules = (
|
||||
Quoter => $q,
|
||||
TableParser => $tp,
|
||||
TableNibbler => $nb,
|
||||
OptionParser => $o,
|
||||
);
|
||||
my $in = "/t/lib/samples/NibbleIterator/";
|
||||
|
||||
sub make_nibble_iter {
|
||||
my (%args) = @_;
|
||||
|
||||
if (my $file = $args{sql_file}) {
|
||||
$sb->load_file('master', "$in/$file");
|
||||
}
|
||||
|
||||
@ARGV = $args{argv} ? @{$args{argv}} : ();
|
||||
$o->get_opts();
|
||||
|
||||
my $schema = new Schema();
|
||||
my $si = new SchemaIterator(
|
||||
dbh => $dbh,
|
||||
keep_ddl => 1,
|
||||
keep_tbl_status => 1,
|
||||
Schema => $schema,
|
||||
%common_modules,
|
||||
);
|
||||
1 while $si->next();
|
||||
|
||||
my $ni = new OobNibbleIterator(
|
||||
Cxn => $cxn,
|
||||
tbl => $schema->get_table($args{db}, $args{tbl}),
|
||||
chunk_size => $o->get('chunk-size'),
|
||||
chunk_index => $o->get('chunk-index'),
|
||||
callbacks => $args{callbacks},
|
||||
select => $args{select},
|
||||
one_nibble => $args{one_nibble},
|
||||
%common_modules,
|
||||
);
|
||||
|
||||
return $ni;
|
||||
}
|
||||
|
||||
# ############################################################################
|
||||
# a-z w/ chunk-size 5, z is final boundary and single value
|
||||
# ############################################################################
|
||||
my $ni = make_nibble_iter(
|
||||
sql_file => "a-z.sql",
|
||||
db => 'test',
|
||||
tbl => 't',
|
||||
argv => [qw(--databases test --chunk-size 5)],
|
||||
);
|
||||
|
||||
my @rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['a'],['b'],['c'],['d'],['e']],
|
||||
'a-z nibble 1'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['f'],['g'],['h'],['i'],['j']],
|
||||
'a-z nibble 2'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['k'],['l'],['m'],['n'],['o']],
|
||||
'a-z nibble 3'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['p'],['q'],['r'],['s'],['t']],
|
||||
'a-z nibble 4'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['u'],['v'],['w'],['x'],['y']],
|
||||
'a-z nibble 5'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..5) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['z']],
|
||||
'a-z nibble 6'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
is(
|
||||
$ni->nibble_number(),
|
||||
8,
|
||||
"8 nibbles"
|
||||
);
|
||||
|
||||
ok(
|
||||
!$ni->more_boundaries(),
|
||||
"No more boundaries"
|
||||
);
|
||||
|
||||
# ############################################################################
|
||||
# Get lower and upper oob values.
|
||||
# ############################################################################
|
||||
$ni = make_nibble_iter(
|
||||
sql_file => "a-z.sql",
|
||||
db => 'test',
|
||||
tbl => 't',
|
||||
argv => [qw(--databases test --chunk-size 8)],
|
||||
);
|
||||
|
||||
|
||||
$dbh->do("delete from test.t where c='a' or c='z'");
|
||||
|
||||
@rows = ();
|
||||
for (1..8) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['b'],['c'],['d'],['e'],['f'],['g'],['h'],['i']],
|
||||
'a-z nibble 1 with oob'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
$dbh->do("insert into test.t values ('a'), ('z')");
|
||||
|
||||
@rows = ();
|
||||
for (1..8) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['j'],['k'],['l'],['m'],['n'],['o'],['p'],['q']],
|
||||
'a-z nibble 2 with oob'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
for (1..8) {
|
||||
push @rows, $ni->next();
|
||||
}
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['r'],['s'],['t'],['u'],['v'],['w'],['x'],['y']],
|
||||
'a-z nibble 3 with oob'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
push @rows, $ni->next();
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['a']],
|
||||
'a-z nibble 4 lower oob'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
@rows = ();
|
||||
push @rows, $ni->next();
|
||||
is_deeply(
|
||||
\@rows,
|
||||
[['z']],
|
||||
'a-z nibble 4 upper oob'
|
||||
) or print Dumper(\@rows);
|
||||
|
||||
ok(
|
||||
!$ni->more_boundaries(),
|
||||
"No more boundaries"
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
{
|
||||
local *STDERR;
|
||||
open STDERR, '>', \$output;
|
||||
$ni->_d('Complete test coverage');
|
||||
}
|
||||
like(
|
||||
$output,
|
||||
qr/Complete test coverage/,
|
||||
'_d() works'
|
||||
);
|
||||
$sb->wipe_clean($dbh);
|
||||
exit;
|
||||
Reference in New Issue
Block a user