diff --git a/lib/NibbleIterator.pm b/lib/NibbleIterator.pm index 8b600664..9956e145 100644 --- a/lib/NibbleIterator.pm +++ b/lib/NibbleIterator.pm @@ -35,18 +35,22 @@ $Data::Dumper::Quotekeys = 0; # Sub: new # # Required Arguments: -# cxn - object +# Cxn - object # tbl - Standard tbl ref # chunk_size - Number of rows to nibble per chunk # OptionParser - object +# Quoter - object # TableNibbler - object # TableParser - object -# Quoter - object # # Optional Arguments: +# dml - Data manipulation statment to precede the SELECT statement +# select - Arrayref of table columns to select # chunk_index - Index to use for nibbling # one_nibble - Allow one-chunk tables (default yes) # where - WHERE clause +# resume - Hashref with lower_boundary and upper_boundary values +# to resume nibble from # # Returns: # NibbleIterator object @@ -64,12 +68,11 @@ sub new { : 0; MKDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); - if ( my $nibble = $args{resume} ) { - if ( !defined $nibble->{lower_boundary} - && !defined $nibble->{upper_boundary} ) { - MKDEBUG && _d('Resuming from one nibble table'); - $one_nibble = 1; - } + if ( $args{resume} + && !defined $args{resume}->{lower_boundary} + && !defined $args{resume}->{upper_boundary} ) { + MKDEBUG && _d('Resuming from one nibble table'); + $one_nibble = 1; } # Get an index to nibble by. We'll order rows by the index's columns. @@ -88,7 +91,7 @@ sub new { # If the chunk size is >= number of rows in table, then we don't # need to chunk; we can just select all rows, in order, at once. my $nibble_sql - = ($args{dms} ? "$args{dms} " : "SELECT ") + = ($args{dml} ? "$args{dml} " : "SELECT ") . ($args{select} ? $args{select} : join(', ', map { $q->quote($_) } @cols)) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) @@ -111,7 +114,6 @@ sub new { limit => 0, nibble_sql => $nibble_sql, explain_nibble_sql => $explain_nibble_sql, - no_more_boundaries => $args{resume} ? 1 : 0, }; } else { @@ -197,7 +199,7 @@ sub new { # This statement does the actual nibbling work; its rows are returned # to the caller via next(). my $nibble_sql - = ($args{dms} ? "$args{dms} " : "SELECT ") + = ($args{dml} ? "$args{dml} " : "SELECT ") . ($args{select} ? $args{select} : join(', ', map { $q->quote($_) } @{$asc->{cols}})) . " FROM $from" @@ -233,7 +235,6 @@ sub new { nibble_sql => $nibble_sql, explain_ub_sql => "EXPLAIN $ub_sql", explain_nibble_sql => $explain_nibble_sql, - resume => $args{resume}, resume_lb_sql => $resume_lb_sql, sql => { columns => $asc->{scols}, @@ -547,7 +548,13 @@ sub _prepare_sths { sub _get_bounds { my ($self) = @_; - return if $self->{one_nibble}; + + if ( $self->{one_nibble} ) { + if ( $self->{resume} ) { + $self->{no_more_boundaries} = 1; + } + return; + } my $dbh = $self->{Cxn}->dbh(); @@ -556,9 +563,11 @@ sub _get_bounds { MKDEBUG && _d('First lower boundary:', Dumper($self->{first_lower})); # The next boundary is the first lower boundary. If resuming, - # this should be something > the real first lower boundary. + # this should be something > the real first lower boundary and + # bounded (else it's not one of our chunks). if ( my $nibble = $self->{resume} ) { - if ( defined $nibble->{upper_boundary} ) { + if ( defined $nibble->{lower_boundary} + && defined $nibble->{upper_boundary} ) { my $sth = $dbh->prepare($self->{resume_lb_sql}); my @ub = split ',', $nibble->{upper_boundary}; MKDEBUG && _d($sth->{Statement}, 'params:', @ub); @@ -566,16 +575,19 @@ sub _get_bounds { $self->{next_lower} = $sth->fetchrow_arrayref(); $sth->finish(); } - else { - MKDEBUG && _d('No more boundaries to resume'); - $self->{no_more_boundaries} = 1; - } } else { $self->{next_lower} = $self->{first_lower}; } MKDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower})); + if ( !$self->{next_lower} ) { + # This happens if we resume from the end of the table, or if the + # last chunk for resuming isn't bounded. + MKDEBUG && _d('At end of table, or no more boundaries to resume'); + $self->{no_more_boundaries} = 1; + } + # Get the real last upper boundary, i.e. the last row of the table # at this moment. If rows are inserted after, we won't see them. $self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql}); @@ -598,13 +610,6 @@ sub _next_boundaries { return 1; # continue nibbling } - if ( !$self->{next_lower} ) { - # This happens if we resume from the end of the table. - MKDEBUG && _d('At end of table'); - $self->{no_more_boundaries} = 1; # for next call - return; # stop nibbling - } - # Detect infinite loops. If the lower boundary we just nibbled from # is identical to the next lower boundary, then this next nibble won't # go anywhere, so to speak, unless perhaps the chunk size has changed diff --git a/lib/OobNibbleIterator.pm b/lib/OobNibbleIterator.pm index fb1f2754..6e701384 100644 --- a/lib/OobNibbleIterator.pm +++ b/lib/OobNibbleIterator.pm @@ -39,6 +39,13 @@ $Data::Dumper::Indent = 1; $Data::Dumper::Sortkeys = 1; $Data::Dumper::Quotekeys = 0; +# Sub: new +# +# Required Arguments: +# See +# +# Returns: +# OobNibbleIterator object sub new { my ( $class, %args ) = @_; my @required_args = qw(); @@ -57,7 +64,7 @@ sub new { if ( !$self->one_nibble() ) { # Make statements for the lower and upper ranges. my $head_sql - = ($args{past_dms} || "SELECT ") + = ($args{past_dml} || "SELECT ") . ($args{past_select} || join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}})) . " FROM " . $self->{sql}->{from}; @@ -105,21 +112,24 @@ sub new { $self->{explain_past_lower_sql} = $explain_past_lower_sql; $self->{explain_past_upper_sql} = $explain_past_upper_sql; - my @past_nibbles = qw(lower upper); + $self->{past_nibbles} = [qw(lower upper)]; if ( my $nibble = $args{resume} ) { - if ( !defined $nibble->{lower_boundary} ) { - # Have past lower nibble, do only past upper nibble. - $self->{past_nibbles} = [qw(upper)]; - } - elsif ( !defined $nibble->{upper_boundary} ) { - # Have past upper nibble, so the past lower nibble - # should have alredy been done. Nothing for us to do. - $self->{past_nibbles} = []; + if ( !defined $nibble->{lower_boundary} + || !defined $nibble->{upper_boundary} ) { + # One or the other boundary isn't defined, so the last chunk + # we're resuming from is one our oob chunks. The parent doesn't + # have any more bounded boundaries, and if the lower boundary + # isn't defined then it's the lower oob chunk, so only do the + # upper oob chunk, or if the upper boundary isn't defined, then + # we're resuming from the upper oob chunk so we're already done. + $self->{past_nibbles} = !defined $nibble->{lower_boundary} + ? ['upper'] + : []; } } - MKDEBUG && _d('Nibble past', @past_nibbles); - $self->{past_nibbles} = \@past_nibbles, - } + MKDEBUG && _d('Nibble past', @{$self->{past_nibbles}}); + + } # not one nibble return bless $self, $class; } diff --git a/t/lib/NibbleIterator.t b/t/lib/NibbleIterator.t index 4d310747..c48ac2d4 100644 --- a/t/lib/NibbleIterator.t +++ b/t/lib/NibbleIterator.t @@ -729,7 +729,7 @@ $ni = make_nibble_iter( db => 'test', tbl => 't', argv => [qw(--databases test --chunk-size 5)], - resume => { upper_boundary => 'j' }, + resume => { lower_boundary => 'a', upper_boundary => 'e' }, ); @rows = (); @@ -739,7 +739,7 @@ while (my $row = $ni->next()) { is_deeply( \@rows, - [ ('k'..'z') ], + [ ('f'..'z') ], "Resume from middle" ); @@ -748,7 +748,7 @@ $ni = make_nibble_iter( db => 'test', tbl => 't', argv => [qw(--databases test --chunk-size 5)], - resume => { upper_boundary => 'z' }, + resume => { lower_boundary => 'z', upper_boundary => 'z' }, ); @rows = ();