From c91350f2eb5118bcedcdfff0bb1d5899198ca079 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Tue, 1 Nov 2016 19:05:49 -0300 Subject: [PATCH] Added pause to NibbleIterator --- bin/pt-online-schema-change | 30 ++++++++++++++++++++++++++++++ bin/pt-table-checksum | 30 ++++++++++++++++++++++++++++++ lib/NibbleIterator.pm | 24 ++++++++++++++++++++++++ t/lib/NibbleIterator.t | 26 ++++++++++++++++++++++++++ 4 files changed, 110 insertions(+) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index e5ad2a68..b532f54b 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -5529,6 +5529,8 @@ sub new { $self->{have_rows} = 0; $self->{rowno} = 0; $self->{oktonibble} = 1; + $self->{pause_file} = $nibble_params->{pause_file}; + $self->{sleep} = $args{sleep} || 60; return bless $self, $class; } @@ -5567,6 +5569,24 @@ sub next { NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { + + if ($self->{pause_file}) { + while(-f $self->{pause_file}) { + print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; + my $dbh = $self->{Cxn}->dbh(); + if ( !$dbh || !$dbh->ping() ) { + eval { $dbh = $self->{Cxn}->connect() }; # connect or die trying + if ( $EVAL_ERROR ) { + chomp $EVAL_ERROR; + die "Lost connection to " . $self->{Cxn}->name() . " while waiting for " + . "replica lag ($EVAL_ERROR)\n"; + } + } + $dbh->do("SELECT 'nibble iterator keepalive'"); + sleep($self->{sleep}); + } + } + if ( !$self->{have_rows} ) { $self->{nibbleno}++; PTDEBUG && _d('Nibble:', $self->{nibble_sth}->{Statement}, 'params:', @@ -5596,6 +5616,7 @@ sub next { } $self->{rowno} = 0; $self->{have_rows} = 0; + } PTDEBUG && _d('Done nibbling'); @@ -5731,10 +5752,13 @@ sub can_nibble { die "There is no good index and the table is oversized."; } + my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef; + return { row_est => $row_est, # nibble about this many rows index => $index, # using this index one_nibble => $one_nibble, # if the table fits in one nibble/chunk + pause_file => $pause_file, }; } @@ -11723,6 +11747,12 @@ short form: -p; type: string Password to use when connecting. If password contains commas they must be escaped with a backslash: "exam\,ple" +=item --pause-file + +type: string + +Execution will be paused while the file specified by this param exists. + =item --pid type: string diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 9183e55d..09c3e6b1 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -6336,6 +6336,8 @@ sub new { $self->{have_rows} = 0; $self->{rowno} = 0; $self->{oktonibble} = 1; + $self->{pause_file} = $nibble_params->{pause_file}; + $self->{sleep} = $args{sleep} || 60; return bless $self, $class; } @@ -6374,6 +6376,24 @@ sub next { NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { + + if ($self->{pause_file}) { + while(-f $self->{pause_file}) { + print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; + my $dbh = $self->{Cxn}->dbh(); + if ( !$dbh || !$dbh->ping() ) { + eval { $dbh = $self->{Cxn}->connect() }; # connect or die trying + if ( $EVAL_ERROR ) { + chomp $EVAL_ERROR; + die "Lost connection to " . $self->{Cxn}->name() . " while waiting for " + . "replica lag ($EVAL_ERROR)\n"; + } + } + $dbh->do("SELECT 'nibble iterator keepalive'"); + sleep($self->{sleep}); + } + } + if ( !$self->{have_rows} ) { $self->{nibbleno}++; PTDEBUG && _d('Nibble:', $self->{nibble_sth}->{Statement}, 'params:', @@ -6403,6 +6423,7 @@ sub next { } $self->{rowno} = 0; $self->{have_rows} = 0; + } PTDEBUG && _d('Done nibbling'); @@ -6538,10 +6559,13 @@ sub can_nibble { die "There is no good index and the table is oversized."; } + my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef; + return { row_est => $row_est, # nibble about this many rows index => $index, # using this index one_nibble => $one_nibble, # if the table fits in one nibble/chunk + pause_file => $pause_file, }; } @@ -12397,6 +12421,12 @@ short form: -p; type: string; group: Connection Password to use when connecting. If password contains commas they must be escaped with a backslash: "exam\,ple" +=item --pause-file + +type: string + +Execution will be paused while the file specified by this param exists. + =item --pid type: string diff --git a/lib/NibbleIterator.pm b/lib/NibbleIterator.pm index b363abc4..18f6f849 100644 --- a/lib/NibbleIterator.pm +++ b/lib/NibbleIterator.pm @@ -263,6 +263,8 @@ sub new { $self->{have_rows} = 0; $self->{rowno} = 0; $self->{oktonibble} = 1; + $self->{pause_file} = $nibble_params->{pause_file}; + $self->{sleep} = $args{sleep} || 60; return bless $self, $class; } @@ -304,6 +306,24 @@ sub next { # If there's another nibble, fetch the rows within it. NIBBLE: while ( $self->{have_rows} || $self->_next_boundaries() ) { + + if ($self->{pause_file}) { + while(-f $self->{pause_file}) { + print "Sleeping $self->{sleep} seconds because $self->{pause_file} exists\n"; + my $dbh = $self->{Cxn}->dbh(); + if ( !$dbh || !$dbh->ping() ) { + eval { $dbh = $self->{Cxn}->connect() }; # connect or die trying + if ( $EVAL_ERROR ) { + chomp $EVAL_ERROR; + die "Lost connection to " . $self->{Cxn}->name() . " while waiting for " + . "replica lag ($EVAL_ERROR)\n"; + } + } + $dbh->do("SELECT 'nibble iterator keepalive'"); + sleep($self->{sleep}); + } + } + # If no rows, then we just got the next boundaries, which start # the next nibble. if ( !$self->{have_rows} ) { @@ -341,6 +361,7 @@ sub next { } $self->{rowno} = 0; $self->{have_rows} = 0; + } PTDEBUG && _d('Done nibbling'); @@ -493,10 +514,13 @@ sub can_nibble { # The table can be nibbled if this point is reached, else we would have # died earlier. Return some values about nibbling the table. + my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef; + return { row_est => $row_est, # nibble about this many rows index => $index, # using this index one_nibble => $one_nibble, # if the table fits in one nibble/chunk + pause_file => $pause_file, }; } diff --git a/t/lib/NibbleIterator.t b/t/lib/NibbleIterator.t index d9ada0e8..6c0f68c2 100644 --- a/t/lib/NibbleIterator.t +++ b/t/lib/NibbleIterator.t @@ -23,6 +23,7 @@ use RowChecksum; use NibbleIterator; use Cxn; use PerconaTest; +use File::Temp qw/ tempfile /; use constant PTDEBUG => $ENV{PTDEBUG} || 0; @@ -92,6 +93,8 @@ sub make_nibble_iter { resume => $args{resume}, order_by => $args{order_by}, comments => $args{comments}, + pause_file => $o->get('pause-file'), + sleep => $args{sleep} || 60, %common_modules, ); @@ -355,6 +358,29 @@ is( "init callbacks can stop nibbling" ); +my ($fh, $filename) = tempfile(); +system ("sleep 3 && rm $filename &"); + +$ni = make_nibble_iter( + db => 'sakila', + tbl => 'payment', + sleep => 3, + argv => [qw(--databases sakila --tables payment --chunk-size 100 --pause-file ), $filename], +); + +$output = output( + sub { + for (1..8) { $ni->next() } + }, +); + +like( + $output, + qr/Sleeping 3 seconds because/, + "nibbles paused" +); + + # ############################################################################ # Nibble a larger table by numeric pk id # ############################################################################