Merge pull request #302 from dolphin-ops/read-only-interval

PT-1508 Adding --read-only-interval flag, and read-only check on wake-up

bin/pt-heartbeat

@@ -5923,8 +5923,9 @@ sub main {
       my $start_time         = time;
       my $run_time           = $o->get('run-time');
       my $interval           = $o->get('interval') || 5;
+      my $read_only_interval = $o->get('read-only-interval') || $interval;
       while (server_is_readonly($dbh)) {
-         sleep($interval); 
+         sleep($read_only_interval);
          if (
             ($run_time && $run_time < time - $start_time)
             || -f $sentinel
@@ -6322,6 +6323,8 @@ sub main {
    PTDEBUG && _d($end ? ('Will exit at', ts($end)) : 'Running forever');
 
    my $get_next_interval = make_interval_iter($interval, $skew);
+   my $max_successive_errors = $o->get('fail-successive-errors') || 0;
+   my $num_successive_errors = 0;
 
    while (                         # Stop if...
       (!$end || int(time) < $end)  # runtime exceeded, or
@@ -6341,6 +6344,18 @@ sub main {
          sleep $next_interval - $time;
          PTDEBUG && _d('Woke up at', ts(time));
 
+         if ( $o->get('check-read-only') && $o->get('update') ) {
+           my $read_only_interval = $o->get('read-only-interval') || $interval;
+           while (server_is_readonly($dbh)) {
+              sleep($read_only_interval);
+              if (
+                 -f $sentinel
+              ) {
+                 return 0;
+              }
+           }
+         }
+
          # Connect or reconnect if necessary.
          if ( !$dbh->ping() ) {
             $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 });
@@ -6383,6 +6398,7 @@ sub main {
          }
       };
       if ( $EVAL_ERROR ) {
+         $num_successive_errors = $num_successive_errors + 1;
          my ( $err ) = $EVAL_ERROR =~ m/^(?:DBI|DBD).*failed: (.*?)\s*at \S+ line .*/;
          if ( $err ) {
             warn "$err\n";
@@ -6390,6 +6406,12 @@ sub main {
          else {
             die $EVAL_ERROR;
          }
+         if ($max_successive_errors > 0 && $num_successive_errors >= $max_successive_errors) {
+            die $EVAL_ERROR;
+         }
+      }
+      else {
+        $num_successive_errors = 0;
       }
    }
 
@@ -6737,7 +6759,7 @@ before its delay.  L<"--recurse"> only works with MySQL.
 =item --check-read-only
 
 Check if the server has read_only enabled; If it does, the tool skips doing
-any inserts.
+any inserts. See also L<"--read-only-interval">
 
 =item --config
 
@@ -6924,6 +6946,13 @@ L<"--frames">.  For example,
 
  5s [  0.25s,  0.05s,  0.02s ]
 
+=item --fail-successive-errors
+
+type: int
+
+If specified, pt-heartbeat will fail after given number of successive DBI errors
+(failure to connect to server or issue a query).
+
 =item --password
 
 short form: -p; type: string
@@ -6953,6 +6982,13 @@ Print the auto-detected or given L<"--master-server-id">.  If L<"--check">
 or L<"--monitor"> is specified, specifying this option will print the
 auto-detected or given L<"--master-server-id"> at the end of each line.
 
+=item --read-only-interval
+
+type: int
+
+When L<"--check-read-only"> is specified, the interval to sleep while the
+server is found to be read-only. If unspecified, L<"--interval"> is used.
+
 =item --recurse
 
 type: int