percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-25 21:54:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement retry_on_error for Pipeline processes. Retry iteration proc twice, then fail completely. Fix mirror.t.

Browse Source
This commit is contained in:
Daniel Nichter
2012-05-23 16:07:05 -06:00
parent 74b1caa241
commit 491c076e75
5 changed files with 105 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
bin/pt-query-digest
View File

@@ -11745,6 +11745,9 @@ sub add {
push @{$self->{procs}}, $process;
push @{$self->{names}}, $name;
if ( my $n = $args{retry_on_error} ) {
$self->{retries}->{$name} = $n;
}
if ( $self->{instrument} ) {
$self->{instrumentation}->{$name} = { time => 0, calls => 0 };
}
@@ -11809,10 +11812,28 @@ sub execute {
}
};
if ( $EVAL_ERROR ) {
warn "Pipeline process $procno ("
. ($self->{names}->[$procno] || "")
. ") caused an error: $EVAL_ERROR";
die $EVAL_ERROR unless $self->{continue_on_error};
my $name = $self->{names}->[$procno] || "";
my $msg = "Pipeline process " . ($procno + 1)
. " ($name) caused an error: "
. $EVAL_ERROR;
if ( defined $self->{retries}->{$name} ) {
my $n = $self->{retries}->{$name};
if ( $n ) {
warn $msg . "Will retry pipeline process $procno ($name) "
. "$n more " . ($n > 1 ? "times" : "time") . ".\n";
$self->{retries}->{$name}--;
}
else {
die $msg . "Terminating pipeline because process $procno "
. "($name) caused too many errors.\n";
}
}
elsif ( !$self->{continue_on_error} ) {
die $msg;
}
else {
warn $msg;
}
}
}
@@ -12562,6 +12583,12 @@ sub main {
{ # iteration
$pipeline->add(
# This is a critical proc: if we die here, we probably need
# to stop, else an infinite loop can develop:
# https://bugs.launchpad.net/percona-toolkit/+bug/888114
# We'll retry twice in case the problem is just one bad
# query class, or something like that.
retry_on_error => 2,
name => 'iteration',
process => sub {
my ( $args ) = @_;

29
lib/Pipeline.pm
View File

@@ -71,6 +71,9 @@ sub add {
push @{$self->{procs}}, $process;
push @{$self->{names}}, $name;
if ( my $n = $args{retry_on_error} ) {
$self->{retries}->{$name} = $n;
}
if ( $self->{instrument} ) {
$self->{instrumentation}->{$name} = { time => 0, calls => 0 };
}
@@ -156,10 +159,28 @@ sub execute {
}
};
if ( $EVAL_ERROR ) {
warn "Pipeline process $procno ("
. ($self->{names}->[$procno] || "")
. ") caused an error: $EVAL_ERROR";
die $EVAL_ERROR unless $self->{continue_on_error};
my $name = $self->{names}->[$procno] || "";
my $msg = "Pipeline process " . ($procno + 1)
. " ($name) caused an error: "
. $EVAL_ERROR;
if ( defined $self->{retries}->{$name} ) {
my $n = $self->{retries}->{$name};
if ( $n ) {
warn $msg . "Will retry pipeline process $procno ($name) "
. "$n more " . ($n > 1 ? "times" : "time") . ".\n";
$self->{retries}->{$name}--;
}
else {
die $msg . "Terminating pipeline because process $procno "
. "($name) caused too many errors.\n";
}
}
elsif ( !$self->{continue_on_error} ) {
die $msg;
}
else {
warn $msg;
}
}
}

48
t/lib/Pipeline.t
View File

@@ -9,7 +9,7 @@ BEGIN {
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More tests => 10;
use Test::More tests => 11;
use Time::HiRes qw(usleep);
@@ -206,11 +206,13 @@ is(
$oktorun = 1;
$pipeline = new Pipeline(continue_on_error=>1);
my @called = ();
my @die = qw(1 0);
$pipeline->add(
name => 'proc1',
process => sub {
my ( $args ) = @_;
push @called, 'proc1';
die "I'm an error" if shift @die;
return $args;
},
@@ -218,7 +220,7 @@ $pipeline->add(
$pipeline->add(
name => 'proc2',
process => sub {
print "proc2";
push @called, 'proc2';
$oktorun = 0;
return;
},
@@ -229,12 +231,48 @@ $output = output(
stderr => 1,
);
like(
$output,
qr/0 \(proc1\) caused an error.+proc2/s,
is_deeply(
\@called,
[qw(proc1 proc1 proc2)],
"Continues on error"
);
# Override global for critical procs that must kill the pipeline if they die.
$oktorun = 1;
$pipeline = new Pipeline(continue_on_error=>1);
@called = ();
$pipeline->add(
name => 'proc1',
process => sub {
my ( $args ) = @_;
push @called, 'proc1';
$oktorun = 0 if @called > 8;
return $args;
},
);
$pipeline->add(
retry_on_error => 2,
name => 'proc2',
process => sub {
push @called, 'proc2';
die;
},
);
$output = output(
sub { $pipeline->execute(%args) },
stderr => 1,
);
is_deeply(
\@called,
[qw(proc1 proc2), # first attempt
qw(proc1 proc2), # retry 1/2
qw(proc1 proc2)], # retry 2/2
"Retry proc"
) or diag($output);
# #############################################################################
# Done.
# #############################################################################

4
t/pt-query-digest/continue_on_error.t
View File

@@ -49,8 +49,8 @@ $output = output(
like(
$output,
qr/Query\s+1/,
"No infinite loop in report crashes (bug 888114)"
qr/Argument \S+ isn't numeric/,
"Report crashed, but no infinite loop (bug 888114)"
);
# #############################################################################

3
t/pt-query-digest/mirror.t
View File

@@ -36,9 +36,6 @@ my $cmd;
my $pid_file = "/tmp/pt-query-digest-mirror-test.pid";
diag(`rm $pid_file 2>/dev/null`);
my $pid_file = '/tmp/pt-query-digest.test.pid';
`rm -rf $pid_file >/dev/null`;
# ##########################################################################
# Tests for swapping --processlist and --execute
# ##########################################################################