percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-07 04:49:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix various resume issues.

Browse Source
This commit is contained in:
Daniel Nichter
2011-11-17 11:20:00 -07:00
parent 14dc2366f6
commit ff78656859
3 changed files with 57 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
lib/NibbleIterator.pm
View File

@@ -35,18 +35,22 @@ $Data::Dumper::Quotekeys = 0;
# Sub: new
#
# Required Arguments:
# cxn - <Cxn> object
# Cxn - <Cxn> object
# tbl - Standard tbl ref
# chunk_size - Number of rows to nibble per chunk
# OptionParser - <OptionParser> object
# Quoter - <Quoter> object
# TableNibbler - <TableNibbler> object
# TableParser - <TableParser> object
# Quoter - <Quoter> object
#
# Optional Arguments:
# dml - Data manipulation statment to precede the SELECT statement
# select - Arrayref of table columns to select
# chunk_index - Index to use for nibbling
# one_nibble - Allow one-chunk tables (default yes)
# where - WHERE clause
# resume - Hashref with lower_boundary and upper_boundary values
# to resume nibble from
#
# Returns:
# NibbleIterator object
@@ -64,12 +68,11 @@ sub new {
: 0;
MKDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
if ( my $nibble = $args{resume} ) {
if ( !defined $nibble->{lower_boundary}
&& !defined $nibble->{upper_boundary} ) {
MKDEBUG && _d('Resuming from one nibble table');
$one_nibble = 1;
}
if ( $args{resume}
&& !defined $args{resume}->{lower_boundary}
&& !defined $args{resume}->{upper_boundary} ) {
MKDEBUG && _d('Resuming from one nibble table');
$one_nibble = 1;
}
# Get an index to nibble by. We'll order rows by the index's columns.
@@ -88,7 +91,7 @@ sub new {
# If the chunk size is >= number of rows in table, then we don't
# need to chunk; we can just select all rows, in order, at once.
my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ")
= ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
@@ -111,7 +114,6 @@ sub new {
limit => 0,
nibble_sql => $nibble_sql,
explain_nibble_sql => $explain_nibble_sql,
no_more_boundaries => $args{resume} ? 1 : 0,
};
}
else {
@@ -197,7 +199,7 @@ sub new {
# This statement does the actual nibbling work; its rows are returned
# to the caller via next().
my $nibble_sql
= ($args{dms} ? "$args{dms} " : "SELECT ")
= ($args{dml} ? "$args{dml} " : "SELECT ")
. ($args{select} ? $args{select}
: join(', ', map { $q->quote($_) } @{$asc->{cols}}))
. " FROM $from"
@@ -233,7 +235,6 @@ sub new {
nibble_sql => $nibble_sql,
explain_ub_sql => "EXPLAIN $ub_sql",
explain_nibble_sql => $explain_nibble_sql,
resume => $args{resume},
resume_lb_sql => $resume_lb_sql,
sql => {
columns => $asc->{scols},
@@ -547,7 +548,13 @@ sub _prepare_sths {
sub _get_bounds {
my ($self) = @_;
return if $self->{one_nibble};
if ( $self->{one_nibble} ) {
if ( $self->{resume} ) {
$self->{no_more_boundaries} = 1;
}
return;
}
my $dbh = $self->{Cxn}->dbh();
@@ -556,9 +563,11 @@ sub _get_bounds {
MKDEBUG && _d('First lower boundary:', Dumper($self->{first_lower}));
# The next boundary is the first lower boundary. If resuming,
# this should be something > the real first lower boundary.
# this should be something > the real first lower boundary and
# bounded (else it's not one of our chunks).
if ( my $nibble = $self->{resume} ) {
if ( defined $nibble->{upper_boundary} ) {
if ( defined $nibble->{lower_boundary}
&& defined $nibble->{upper_boundary} ) {
my $sth = $dbh->prepare($self->{resume_lb_sql});
my @ub = split ',', $nibble->{upper_boundary};
MKDEBUG && _d($sth->{Statement}, 'params:', @ub);
@@ -566,16 +575,19 @@ sub _get_bounds {
$self->{next_lower} = $sth->fetchrow_arrayref();
$sth->finish();
}
else {
MKDEBUG && _d('No more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
}
else {
$self->{next_lower} = $self->{first_lower};
}
MKDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower}));
if ( !$self->{next_lower} ) {
# This happens if we resume from the end of the table, or if the
# last chunk for resuming isn't bounded.
MKDEBUG && _d('At end of table, or no more boundaries to resume');
$self->{no_more_boundaries} = 1;
}
# Get the real last upper boundary, i.e. the last row of the table
# at this moment. If rows are inserted after, we won't see them.
$self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql});
@@ -598,13 +610,6 @@ sub _next_boundaries {
return 1; # continue nibbling
}
if ( !$self->{next_lower} ) {
# This happens if we resume from the end of the table.
MKDEBUG && _d('At end of table');
$self->{no_more_boundaries} = 1; # for next call
return; # stop nibbling
}
# Detect infinite loops. If the lower boundary we just nibbled from
# is identical to the next lower boundary, then this next nibble won't
# go anywhere, so to speak, unless perhaps the chunk size has changed

36
lib/OobNibbleIterator.pm
View File

@@ -39,6 +39,13 @@ $Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Quotekeys = 0;
# Sub: new
#
# Required Arguments:
# See <NibbleIterator::new()>
#
# Returns:
# OobNibbleIterator object
sub new {
my ( $class, %args ) = @_;
my @required_args = qw();
@@ -57,7 +64,7 @@ sub new {
if ( !$self->one_nibble() ) {
# Make statements for the lower and upper ranges.
my $head_sql
= ($args{past_dms} || "SELECT ")
= ($args{past_dml} || "SELECT ")
. ($args{past_select}
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
. " FROM " . $self->{sql}->{from};
@@ -105,21 +112,24 @@ sub new {
$self->{explain_past_lower_sql} = $explain_past_lower_sql;
$self->{explain_past_upper_sql} = $explain_past_upper_sql;
my @past_nibbles = qw(lower upper);
$self->{past_nibbles} = [qw(lower upper)];
if ( my $nibble = $args{resume} ) {
if ( !defined $nibble->{lower_boundary} ) {
# Have past lower nibble, do only past upper nibble.
$self->{past_nibbles} = [qw(upper)];
}
elsif ( !defined $nibble->{upper_boundary} ) {
# Have past upper nibble, so the past lower nibble
# should have alredy been done. Nothing for us to do.
$self->{past_nibbles} = [];
if ( !defined $nibble->{lower_boundary}
|| !defined $nibble->{upper_boundary} ) {
# One or the other boundary isn't defined, so the last chunk
# we're resuming from is one our oob chunks. The parent doesn't
# have any more bounded boundaries, and if the lower boundary
# isn't defined then it's the lower oob chunk, so only do the
# upper oob chunk, or if the upper boundary isn't defined, then
# we're resuming from the upper oob chunk so we're already done.
$self->{past_nibbles} = !defined $nibble->{lower_boundary}
? ['upper']
: [];
}
}
MKDEBUG && _d('Nibble past', @past_nibbles);
$self->{past_nibbles} = \@past_nibbles,
}
MKDEBUG && _d('Nibble past', @{$self->{past_nibbles}});
} # not one nibble
return bless $self, $class;
}

6
t/lib/NibbleIterator.t
View File

@@ -729,7 +729,7 @@ $ni = make_nibble_iter(
db => 'test',
tbl => 't',
argv => [qw(--databases test --chunk-size 5)],
resume => { upper_boundary => 'j' },
resume => { lower_boundary => 'a', upper_boundary => 'e' },
);
@rows = ();
@@ -739,7 +739,7 @@ while (my $row = $ni->next()) {
is_deeply(
\@rows,
[ ('k'..'z') ],
[ ('f'..'z') ],
"Resume from middle"
);
@@ -748,7 +748,7 @@ $ni = make_nibble_iter(
db => 'test',
tbl => 't',
argv => [qw(--databases test --chunk-size 5)],
resume => { upper_boundary => 'z' },
resume => { lower_boundary => 'z', upper_boundary => 'z' },
);
@rows = ();