Implement and test pt-agent --run-service.

bin/pt-agent

@@ -4679,6 +4679,7 @@ sub run_agent {
          # write the new one to disk, then apply it.
          if ( resource_diff($config, $new_config) ) {
             _info('New config');
+
             write_config(
                config => $new_config,
                file   => $config_file,
@@ -4694,9 +4695,10 @@ sub run_agent {
 
             # Apply new config, i.e. update the current, running config.
             $config = $new_config;
+            _info('Config updated successfully');
          }
          else {
-            _info("Config has not changed");
+            _info('Config has not changed');
          }
       };
       if ( $EVAL_ERROR ) {
@@ -4718,18 +4720,24 @@ sub run_agent {
             # write the new ones to disk, then schedule them.
             if ( resource_diff($services, $new_services) ) {
                _info('New services');
+
                write_services(
                   services => $new_services,
                   lib_dir  => $lib_dir,
                );
+
                schedule_services(
                   services => $new_services,
                   lib_dir  => $lib_dir,
                );
+
                $services = $new_services;
+               _info('Services updated successfully: '
+                  . join(', ', map { $_->alias . ' (' . $_->name . ')' }
+                               @$services));
             }
             else {
-               _info("Services have not changed")
+               _info('Services have not changed');
             }
          };
          if ( $EVAL_ERROR ) {
@@ -4830,11 +4838,14 @@ sub write_services {
          _info("Removed $file");
       }
    }
-   closedir $dh;
+   closedir $dh
+      or die "Error closing $lib_dir: $OS_ERROR";
 
    return;
 }
 
+# Write Service->schedule lines to crontab, along with any other
+# non-pt-agent lines, and load.
 sub schedule_services {
    my (%args) = @_;
 
@@ -4868,6 +4879,8 @@ sub schedule_services {
    return;
 }
 
+# Combine Service->schedule lines with non-pt-agent lines,
+# i.e. don't clobber the user's other crontab lines.
 sub make_new_crontab {
    my (%args) = @_;
 
@@ -4900,6 +4913,140 @@ sub make_new_crontab {
 # #################### #
 
 sub run_service {
+   my (%args) = @_;
+
+   have_required_args(\%args, qw(
+      service
+      spool_dir
+      lib_dir
+   )) or die;
+   my $service   = $args{service};
+   my $spool_dir = $args{spool_dir};
+   my $lib_dir   = $args{lib_dir};
+
+   # TODO: where should this output go?
+   _info("Running $service service");
+
+   $service = load_service(
+      service => $service,
+      lib_dir => $lib_dir,
+   );
+
+   my @output_files;
+   my $final_exit_status = 0;
+   my $spool_file        = "$spool_dir/" . $service->name;
+   my $runs              = $service->runs;
+   my $runno             = 0;
+   foreach my $run ( @$runs ) {
+
+      # Set up the output file, i.e. where this run puts its results.
+      # Runs can access each other's output files.  E.g. run0 may
+      # write to fileX, then subsequent runs can access that file
+      # with the special var __RUN_N_OUTPUT__ where N=0.
+      my $output_file;
+      my $output = $run->output;
+      if ( $output eq 'spool' ) {
+         $output_file = $spool_file; 
+         push @output_files, $spool_file;
+      }
+      elsif ( $output eq 'tmp' ) {
+         my ($fh, $file) = tempfile();
+         close $fh;
+         $output_file = $file;
+         push @output_files, $file;
+      }
+      elsif ( $output eq 'none' ) {
+         $output_file = '/dev/null';
+         push @output_files, undef;
+      }
+      else {
+         die "Invalid output: $output\n";
+      }
+
+      # Create the full command line to execute, replacing any
+      # special vars like __RUN_N_OUTPUT__, __TMPDIR__, etc.
+      my $cmd = join(' ',
+         $run->program,
+         $run->options,
+         '>',
+         $output_file,
+      );
+      $cmd = replace_special_vars(
+         cmd          => $cmd,
+         service      => $service,
+         output_files => \@output_files,
+      );
+      _info("Run $runno: $cmd");
+
+      # Execute this run.
+      system($cmd);
+      my $exit_status = $CHILD_ERROR >> 8;
+      _info("Run $runno: exit $exit_status");
+
+      $final_exit_status |= $exit_status;
+      $runno++;
+   }
+
+   # Remove temp output files.
+   foreach my $file ( @output_files ) {
+      next if $file eq $spool_file;
+      unlink $file
+         or _warn("Error removing $file: $OS_ERROR");
+   }
+
+   _info("Done running " . $service->name);
+
+   return $final_exit_status;
+}
+
+sub load_service {
+   my (%args) = @_;
+
+   have_required_args(\%args, qw(
+      service
+      lib_dir
+   )) or die;
+   my $service   = $args{service};
+   my $lib_dir   = $args{lib_dir};
+
+   my $service_file = "$lib_dir/services/$service";
+   if ( ! -f $service_file ) {
+      die "$service_file does not exist.\n";
+   }
+
+   my $service_hash = decode_json(slurp($service_file));
+   my $service_obj  = Percona::WebAPI::Resource::Service->new(%$service_hash);
+
+   return $service_obj;
+}
+
+sub replace_special_vars {
+   my (%args) = @_;
+
+   have_required_args(\%args, qw(
+      cmd
+      output_files
+   )) or die;
+   my $cmd          = $args{cmd};
+   my $output_files = $args{output_files};
+
+   my $new_cmd = join(' ',
+      map {
+         my $word = $_;
+         if ( my ($runno) = $word =~ m/__RUN_(\d)_OUTPUT__/ ) {
+            if ( $output_files->[$runno] ) {
+               $word = $output_files->[$runno];
+            }
+            else {
+               die "Run$runno has no output for $word to access.\n";
+            }
+         }
+         $word;
+      }
+      split(/\s+/, $cmd)
+   );
+
+   return $new_cmd;
 }
 
 # ################## #
@@ -4940,6 +5087,19 @@ sub init_config_file {
    return;
 }
 
+sub slurp {
+   my ($file) = @_;
+   return unless -f $file;
+   open my $fh, '<', $file
+      or die "Error opening $file: $OS_ERROR";
+   my $data = do {
+      local $INPUT_RECORD_SEPARATOR = undef;
+      <$fh>;
+   };
+   close $fh;
+   return $data;
+}
+
 sub _log {
    my ($level, $msg) = @_;
    $msg .= "\n" if $msg !~ m/\n$/;

t/pt-agent/replace_special_vars.t

@@ -0,0 +1,53 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+use JSON;
+use File::Temp qw(tempfile);
+
+use Percona::Test;
+require "$trunk/bin/pt-agent";
+
+Percona::Toolkit->import(qw(have_required_args Dumper));
+
+my @output_files = ();
+
+sub test_replace {
+   my (%args) = @_;
+   have_required_args(\%args, qw(
+      cmd
+      expect
+   )) or die;
+   my $cmd    = $args{cmd};
+   my $expect = $args{expect};
+
+   my $new_cmd = pt_agent::replace_special_vars(
+      cmd          => $cmd,
+      output_files => \@output_files,
+   );
+
+   is(
+      $new_cmd,
+      $expect,
+      $cmd,
+   );
+};
+
+@output_files = qw(zero one two);
+test_replace(
+   cmd    => "pt-query-digest __RUN_0_OUTPUT__",
+   expect => "pt-query-digest zero",
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+done_testing;

t/pt-agent/run_service.t

@@ -0,0 +1,188 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+use JSON;
+use File::Temp qw(tempdir);
+
+use Percona::Test;
+use Percona::Test::Mock::UserAgent;
+require "$trunk/bin/pt-agent";
+
+Percona::Toolkit->import(qw(Dumper have_required_args));
+Percona::WebAPI::Representation->import(qw(as_hashref));
+
+my $sample = "t/pt-agent/samples";
+
+# Create fake spool and lib dirs.  Service-related subs in pt-agent
+# automatically add "/services" to the lib dir, but the spool dir is
+# used as-is.
+my $tmpdir = tempdir("/tmp/pt-agent.$PID.XXXXXX", CLEANUP => 1);
+mkdir "$tmpdir/spool"    or die "Error making $tmpdir/spool: $OS_ERROR";
+mkdir "$tmpdir/services" or die "Error making $tmpdir/services: $OS_ERROR";
+my $spool_dir = "$tmpdir/spool";
+
+sub write_svc_files {
+   my (%args) = @_;
+   have_required_args(\%args, qw(
+      services
+   )) or die;
+   my $services = $args{services};
+
+   my $output = output(
+      sub {
+         pt_agent::write_services(
+            services => $services,
+            lib_dir  => $tmpdir,
+         );
+      },
+      stderr => 1,
+      die    => 1,
+   );
+}
+
+# #############################################################################
+# Simple single run service
+# #############################################################################
+
+my $run0 = Percona::WebAPI::Resource::Run->new(
+   number  => '0',
+   program => "$trunk/bin/pt-query-digest",
+   options => "--report-format profile $trunk/t/lib/samples/slowlogs/slow008.txt",
+   output  => 'spool',
+);
+
+my $svc0 = Percona::WebAPI::Resource::Service->new(
+   name     => 'query-monitor',
+   alias    => 'Query Monitor',
+   schedule => '* * * * *',
+   runs     => [ $run0 ],
+);
+
+write_svc_files(
+   services => [ $svc0 ],
+);
+
+my $exit_status;
+my $output = output(
+   sub {
+      $exit_status = pt_agent::run_service(
+         service   => 'query-monitor',
+         spool_dir => $spool_dir,
+         lib_dir   => $tmpdir,
+      );
+   },
+   stderr => 1,
+);
+
+ok(
+   no_diff(
+      "cat $tmpdir/spool/query-monitor",
+      "$sample/spool001.txt",
+   ),
+   "1 run: spool data (spool001.txt)"
+);
+
+chomp(my $n_files = `ls -1 $spool_dir | wc -l | awk '{print \$1}'`);
+is(
+   $n_files,
+   1,
+   "1 run: only wrote spool data (spool001.txt)"
+) or diag(`ls -l $spool_dir`);
+
+is(
+   $exit_status,
+   0,
+   "1 run: exit 0"
+);
+
+# #############################################################################
+# Service with two runs
+# #############################################################################
+
+diag(`rm -rf $tmpdir/spool/* $tmpdir/services/*`);
+
+# The result is the same as the previous single-run test, but instead of
+# having pqd read the slowlog directly, we have the first run cat the
+# log to a tmp file which pt-agent should auto-create.  Then pqd in run1
+# references this tmp file.
+
+$run0 = Percona::WebAPI::Resource::Run->new(
+   number  => '0',
+   program => "cat",
+   options => "$trunk/t/lib/samples/slowlogs/slow008.txt",
+   output  => 'tmp',
+);
+
+my $run1 = Percona::WebAPI::Resource::Run->new(
+   number  => '1',
+   program => "$trunk/bin/pt-query-digest",
+   options => "--report-format profile __RUN_0_OUTPUT__",
+   output  => 'spool',
+);
+
+$svc0 = Percona::WebAPI::Resource::Service->new(
+   name     => 'query-monitor',
+   alias    => 'Query Monitor',
+   schedule => '* * * * *',
+   runs     => [ $run0, $run1 ],
+);
+
+write_svc_files(
+   services => [ $svc0 ],
+);
+
+$output = output(
+   sub {
+      $exit_status = pt_agent::run_service(
+         service   => 'query-monitor',
+         spool_dir => $spool_dir,
+         lib_dir   => $tmpdir,
+      );
+   },
+   stderr => 1,
+);
+
+ok(
+   no_diff(
+      "cat $tmpdir/spool/query-monitor",
+      "$sample/spool001.txt",
+   ),
+   "2 runs: spool data"
+);
+
+chomp($n_files = `ls -1 $spool_dir | wc -l | awk '{print \$1}'`);
+is(
+   $n_files,
+   1,
+   "2 runs: only wrote spool data"
+) or diag(`ls -l $spool_dir`);
+
+is(
+   $exit_status,
+   0,
+   "2 runs: exit 0"
+);
+
+# Get the temp file created by pt-agent by matching it from
+# the output line like:
+#   2013-01-08T13:14:23.627040 INFO Run 0: cat /Users/daniel/p/pt-agent/t/lib/samples/slowlogs/slow008.txt > /var/folders/To/ToaPSttnFbqvgRqcHPY7qk+++TI/-Tmp-/q1EnzzlDoL
+my ($tmpfile) = $output =~ m/cat \S+ > (\S+)/;
+
+ok(
+   ! -f $tmpfile,
+   "2 runs: temp file removed"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+done_testing;

t/pt-agent/samples/spool001.txt

@@ -0,0 +1,6 @@
+
+# Profile
+# Rank Query ID           Response time Calls R/Call Apdx V/M   Item
+# ==== ================== ============= ===== ====== ==== ===== ==========
+#    1 0xC72BF45D68E35A6E  0.0188 95.4%     1 0.0188 1.00  0.00 SELECT tbl
+# MISC 0xMISC              0.0009  4.6%     2 0.0005   NS   0.0 <2 ITEMS>