Handle exit status 200 and 254.

bin/pt-agent

@@ -6936,12 +6936,13 @@ sub run_service {
 
    # Run the tasks, spool any data.
    my @output_files;
-   my $data_file      = $prefix . '.' . $service->name . '.data';
+   my $recursive_service = '--run-service ' . $service->name;
-   my $tmp_data_file  = "$tmp_dir/$data_file";
+   my $data_file         = $prefix . '.' . $service->name . '.data';
-   my $taskno         = 0;
+   my $tmp_data_file     = "$tmp_dir/$data_file";
-   my $metadata       = { data_ts => $prefix };
+   my $taskno            = 0;
-   my $store          = {};
+   my $metadata          = { data_ts => $prefix };
-   my $env_vars       = env_vars();
+   my $store             = {};
+   my $env_vars          = env_vars();
 
    TASK:
    foreach my $task ( @$tasks ) {
@@ -7085,6 +7086,11 @@ sub run_service {
          );
          $logger->info("Task $taskno command: $cmd");
 
+         if ( $cmd =~ m/$recursive_service/ ) {
+            $logger->fatal("Recursive service detected: $cmd");
+            die;  # fatal() should die, but just in case
+         }
+
          # Execute this run.
          my $t0 = time;
          system($cmd);
@@ -7105,8 +7111,24 @@ sub run_service {
             exit_status => $cmd_exit_status,
          };
 
-         if ( $cmd_exit_status != 0 ) {
+         if ( $cmd_exit_status == 200 ) {
-            $logger->info($task->name . ' exit status not zero, stopping');
+            $logger->error($task->name . ' exit status not zero, '
+               . 'stopping ' . $service->name . ' service');
+            stop_service(
+               service => $service->name,
+               lib_dir => $lib_dir,
+            );
+         }
+         elsif ( $cmd_exit_status == 254 ) {
+            $logger->error($task->name . ' exit status not zero, '
+               . 'stopping all services');
+            stop_all_services(
+               lib_dir => $lib_dir
+            );
+         }
+         elsif ( $cmd_exit_status != 0 ) {
+            $logger->info($task->name . ' exit status not zero, '
+               . 'stopping tasks');
             last TASK;
          }
       }
@@ -7860,6 +7882,54 @@ sub stop_all_services {
    return;
 }
 
+sub stop_service {
+   my (%args) = @_;
+
+   have_required_args(\%args, qw(
+      service
+      lib_dir
+   )) or die;
+   my $service = $args{service};
+   my $lib_dir = $args{lib_dir};
+   
+   # Optional args
+   my $bin_dir = defined $args{bin_dir} ? $args{bin_dir}
+               :                          "$FindBin::Bin/";
+
+   if ( -d "$lib_dir/services" ) {
+      my $stop_service = "$lib_dir/services/stop-$service";
+      if ( -f $stop_service ) {
+         my $env_vars = env_vars();
+         my $stop_log = "$lib_dir/logs/$service.stop";
+         my $run_log  = "$lib_dir/logs/$service.run";
+         my $cmd      = ($env_vars ? "$env_vars " : '')
+                      . "${bin_dir}pt-agent --run-service $service"
+                      . " </dev/null"
+                      . " >$stop_log 2>&1";
+         $logger->info("Stopping $service...");
+         PTDEBUG && _d($cmd);
+         system($cmd);
+         my $cmd_exit_status = $CHILD_ERROR >> 8;
+         if ( $cmd_exit_status != 0 ) {
+            my $err = -f $run_log ? slurp($run_log) : '';
+            $logger->error("Error stopping $service.  Check $stop_log and the "
+               . "online logs for details.  The service may still be running.");
+            next SERVICE;
+         }
+         unlink $stop_log
+            or $logger->warning("Cannot remove $stop_log: $OS_ERROR");
+      }
+      else {
+         $logger->warning("$stop_service does not exist, cannot stop $service");
+      }
+   }
+   else {
+      $logger->warning("$lib_dir/services does not exist, cannot stop $service");
+   }
+
+   return;
+}
+
 sub reset_agent {
    my (%args) = @_;