Implement and test running queries for tasks.

This commit is contained in:
Daniel Nichter
2013-03-19 18:17:24 -06:00
parent 846daa6c31
commit cf0342c69e
3 changed files with 235 additions and 27 deletions

View File

@@ -3600,7 +3600,7 @@ sub new {
set => $args{set},
NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
dbh_set => 0,
ask_pass => $o->get('ask-pass'),
ask_pass => $args{ask_pass},
DSNParser => $dp,
is_cluster_node => undef,
parent => $args{parent},
@@ -4703,10 +4703,16 @@ sub main {
# This runs locally and offline, doesn't need a web API connection.
# ########################################################################
if ( my $service = $o->get('run-service') ) {
my $cxn = Cxn->new(
dsn_string => '',
OptionParser => $o,
DSNParser => $dp,
);
$exit_status = run_service(
service => $service,
spool_dir => $o->get('spool'),
lib_dir => $o->get('lib'),
Cxn => $cxn,
);
_info("Done running $service, exit $exit_status");
exit $exit_status;
@@ -5366,10 +5372,12 @@ sub run_service {
service
spool_dir
lib_dir
Cxn
)) or die;
my $service = $args{service};
my $spool_dir = $args{spool_dir};
my $lib_dir = $args{lib_dir};
my $cxn = $args{Cxn};
# TODO: where should this output go?
_info("Running $service service");
@@ -5378,12 +5386,25 @@ sub run_service {
service => $service,
lib_dir => $lib_dir,
);
my $tasks = $service->tasks;
# Take a quick look through all the tasks to see if any
# will require a MySQL connection. If so, connect now.
if ( grep { $_->query } @$tasks ) {
eval {
$cxn->connect();
};
if ( $EVAL_ERROR ) {
_warn("Cannot connect to MySQL: $EVAL_ERROR");
return 1;
}
}
my @output_files;
my $final_exit_status = 0;
my $spool_file = "$spool_dir/" . $service->name;
my $tasks = $service->tasks;
my $taskno = 0;
TASK:
foreach my $task ( @$tasks ) {
PTDEBUG && _d("Task $taskno:", $task->name);
@@ -5410,33 +5431,51 @@ sub run_service {
}
PTDEBUG && _d("Task $taskno output:", Dumper(\@output_files));
# Create the full command line to execute, replacing any
# special vars like __RUN_N_OUTPUT__, __TMPDIR__, etc.
# TODO: handle query tasks
my $cmd = join(' ',
$task->program,
$task->options,
'>',
$output_file,
);
$cmd = replace_special_vars(
cmd => $cmd,
service => $service,
output_files => \@output_files,
);
_info("Task $taskno command: $cmd");
if ( my $query = $task->query ) {
_info("Task $taskno query: $query");
eval {
$cxn->dbh->do($query);
};
if ( $EVAL_ERROR ) {
_warn("Error executing $query: $EVAL_ERROR");
$final_exit_status |= 1;
last TASK;
}
}
elsif ( my $program = $task->program ) {
# Create the full command line to execute, replacing any
# special vars like __RUN_N_OUTPUT__, __TMPDIR__, etc.
my $cmd = join(' ',
$task->program,
$task->options,
'>',
$output_file,
);
$cmd = replace_special_vars(
cmd => $cmd,
service => $service,
output_files => \@output_files,
);
_info("Task $taskno command: $cmd");
# Execute this run.
system($cmd);
my $exit_status = $CHILD_ERROR >> 8;
_info("Run $taskno: exit $exit_status");
# Execute this run.
system($cmd);
my $exit_status = $CHILD_ERROR >> 8;
$final_exit_status |= $exit_status;
_info("Run $taskno: exit $exit_status");
}
else {
_warn('Invalid Task resource:', Dumper($task));
$final_exit_status |= 1;
last TASK;
}
$final_exit_status |= $exit_status;
$taskno++;
}
# Remove temp output files.
foreach my $file ( @output_files ) {
next unless defined $file;
next if $file eq $spool_file;
unlink $file
or _warn("Error removing $file: $OS_ERROR");

View File

@@ -49,7 +49,6 @@ use constant {
#
# Required Arguments:
# DSNParser - <DSNParser> object
# OptionParser - <OptionParser> object
# dsn - DSN hashref, or...
# dsn_string - ... DSN string like "h=127.1,P=12345"
#
@@ -109,7 +108,7 @@ sub new {
set => $args{set},
NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
dbh_set => 0,
ask_pass => $o->get('ask-pass'),
ask_pass => $args{ask_pass},
DSNParser => $dp,
is_cluster_node => undef,
parent => $args{parent},

View File

@@ -16,9 +16,18 @@ use File::Temp qw(tempdir);
$ENV{PTTEST_PRETTY_JSON} = 1;
use Percona::Test;
use Sandbox;
use Percona::Test::Mock::UserAgent;
require "$trunk/bin/pt-agent";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
my $dsn = $sb->dsn_for('master');
my $o = new OptionParser();
$o->get_specs("$trunk/bin/pt-agent");
$o->get_opts();
Percona::Toolkit->import(qw(Dumper have_required_args));
Percona::WebAPI::Representation->import(qw(as_hashref));
@@ -27,7 +36,7 @@ my $sample = "t/pt-agent/samples";
# Create fake spool and lib dirs. Service-related subs in pt-agent
# automatically add "/services" to the lib dir, but the spool dir is
# used as-is.
my $tmpdir = tempdir("/tmp/pt-agent.$PID.XXXXXX", CLEANUP => 1);
my $tmpdir = tempdir("/tmp/pt-agent.$PID.XXXXXX", CLEANUP => 0);
mkdir "$tmpdir/spool" or die "Error making $tmpdir/spool: $OS_ERROR";
mkdir "$tmpdir/services" or die "Error making $tmpdir/services: $OS_ERROR";
my $spool_dir = "$tmpdir/spool";
@@ -52,7 +61,7 @@ sub write_svc_files {
}
# #############################################################################
# Simple single run service
# Simple single task service using a program.
# #############################################################################
my $run0 = Percona::WebAPI::Resource::Task->new(
@@ -81,6 +90,7 @@ my $output = output(
service => 'query-history',
spool_dir => $spool_dir,
lib_dir => $tmpdir,
Cxn => '',
);
},
stderr => 1,
@@ -108,7 +118,7 @@ is(
);
# #############################################################################
# Service with two runs
# Service with two task, both using a program.
# #############################################################################
diag(`rm -rf $tmpdir/spool/* $tmpdir/services/*`);
@@ -151,6 +161,7 @@ $output = output(
service => 'query-history',
spool_dir => $spool_dir,
lib_dir => $tmpdir,
Cxn => '',
);
},
stderr => 1,
@@ -187,6 +198,165 @@ ok(
"2 runs: temp file removed"
);
# #############################################################################
# More realistc: 3 services, multiple tasks, using programs and queries.
# #############################################################################
SKIP: {
skip 'Cannot connect to sandbox master', 5 unless $dbh;
skip 'No HOME environment variable', 5 unless $ENV{HOME};
diag(`rm -rf $tmpdir/spool/* $tmpdir/services/*`);
my (undef, $old_genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
my $new_genlog = "$tmpdir/genlog";
# First service: set up
my $task00 = Percona::WebAPI::Resource::Task->new(
name => 'disable-gen-log',
number => '0',
query => "SET GLOBAL general_log=OFF",
);
my $task01 = Percona::WebAPI::Resource::Task->new(
name => 'set-gen-log-file',
number => '1',
query => "SET GLOBAL general_log_file='$new_genlog'",
);
my $task02 = Percona::WebAPI::Resource::Task->new(
name => 'enable-gen-log',
number => '2',
query => "SET GLOBAL general_log=ON",
);
my $svc0 = Percona::WebAPI::Resource::Service->new(
name => 'enable-gen-log',
run_schedule => '1 * * * *',
spool_schedule => '2 * * * *',
tasks => [ $task00, $task01, $task02 ],
);
# Second service: the actual service
my $task10 = Percona::WebAPI::Resource::Task->new(
name => 'query-history',
number => '1',
program => "$trunk/bin/pt-query-digest",
options => "--output json --type genlog $new_genlog",
output => 'spool',
);
my $svc1 = Percona::WebAPI::Resource::Service->new(
name => 'query-history',
run_schedule => '3 * * * *',
spool_schedule => '4 * * * *',
tasks => [ $task10 ],
);
# Third service: tear down
my $task20 = Percona::WebAPI::Resource::Task->new(
name => 'disable-gen-log',
number => '0',
query => "SET GLOBAL general_log=OFF",
);
my $task21 = Percona::WebAPI::Resource::Task->new(
name => 'set-gen-log-file',
number => '1',
query => "SET GLOBAL general_log_file='$old_genlog'",
);
my $task22 = Percona::WebAPI::Resource::Task->new(
name => 'enable-gen-log',
number => '2',
query => "SET GLOBAL general_log=ON",
);
my $svc2 = Percona::WebAPI::Resource::Service->new(
name => 'disable-gen-log',
run_schedule => '5 * * * *',
spool_schedule => '6 * * * *',
tasks => [ $task20, $task21, $task22 ],
);
write_svc_files(
services => [ $svc0, $svc1, $svc2 ],
);
my $cxn = Cxn->new(
dsn_string => $dsn,
OptionParser => $o,
DSNParser => $dp,
);
# Run the first service.
$output = output(
sub {
$exit_status = pt_agent::run_service(
service => 'enable-gen-log',
spool_dir => $spool_dir,
lib_dir => $tmpdir,
Cxn => $cxn,
);
},
stderr => 1,
);
my (undef, $genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
is(
$genlog,
$new_genlog,
"Task set MySQL var"
) or diag($output);
# Pretend some time passes...
# The next service doesn't need MySQL, so it shouldn't connect to it.
# To check this, the genlog before running and after running should
# be identical.
`cp $new_genlog $tmpdir/genlog-before`;
# Run the second service.
$output = output(
sub {
$exit_status = pt_agent::run_service(
service => 'query-history',
spool_dir => $spool_dir,
lib_dir => $tmpdir,
Cxn => $cxn,
);
},
stderr => 1,
);
`cp $new_genlog $tmpdir/genlog-after`;
my $diff = `diff $tmpdir/genlog-before $tmpdir/genlog-after`;
is(
$diff,
'',
"Tasks didn't need MySQL, didn't connect to MySQL"
) or diag($output);
# Pretend more time passes...
# Run the third service.
$output = output(
sub {
$exit_status = pt_agent::run_service(
service => 'disable-gen-log',
spool_dir => $spool_dir,
lib_dir => $tmpdir,
Cxn => $cxn,
);
},
stderr => 1,
);
(undef, $genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
is(
$genlog,
$old_genlog,
"Task restored MySQL var"
) or diag($output);
$dbh->do("SET GLOBAL general_log=ON");
$dbh->do("SET GLOBAL general_log_file='$old_genlog'");
}
# #############################################################################
# Done.
# #############################################################################