mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-10-22 19:04:30 +00:00
Implement and test running queries for tasks.
This commit is contained in:
Daniel Nichter
83
bin/pt-agent
83
bin/pt-agent
@@ -3600,7 +3600,7 @@ sub new {
|
||||
set => $args{set},
|
||||
NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
|
||||
dbh_set => 0,
|
||||
ask_pass => $o->get('ask-pass'),
|
||||
ask_pass => $args{ask_pass},
|
||||
DSNParser => $dp,
|
||||
is_cluster_node => undef,
|
||||
parent => $args{parent},
|
||||
@@ -4703,10 +4703,16 @@ sub main {
|
||||
# This runs locally and offline, doesn't need a web API connection.
|
||||
# ########################################################################
|
||||
if ( my $service = $o->get('run-service') ) {
|
||||
my $cxn = Cxn->new(
|
||||
dsn_string => '',
|
||||
OptionParser => $o,
|
||||
DSNParser => $dp,
|
||||
);
|
||||
$exit_status = run_service(
|
||||
service => $service,
|
||||
spool_dir => $o->get('spool'),
|
||||
lib_dir => $o->get('lib'),
|
||||
Cxn => $cxn,
|
||||
);
|
||||
_info("Done running $service, exit $exit_status");
|
||||
exit $exit_status;
|
||||
@@ -5366,10 +5372,12 @@ sub run_service {
|
||||
service
|
||||
spool_dir
|
||||
lib_dir
|
||||
Cxn
|
||||
)) or die;
|
||||
my $service = $args{service};
|
||||
my $spool_dir = $args{spool_dir};
|
||||
my $lib_dir = $args{lib_dir};
|
||||
my $cxn = $args{Cxn};
|
||||
|
||||
# TODO: where should this output go?
|
||||
_info("Running $service service");
|
||||
@@ -5378,12 +5386,25 @@ sub run_service {
|
||||
service => $service,
|
||||
lib_dir => $lib_dir,
|
||||
);
|
||||
my $tasks = $service->tasks;
|
||||
|
||||
# Take a quick look through all the tasks to see if any
|
||||
# will require a MySQL connection. If so, connect now.
|
||||
if ( grep { $_->query } @$tasks ) {
|
||||
eval {
|
||||
$cxn->connect();
|
||||
};
|
||||
if ( $EVAL_ERROR ) {
|
||||
_warn("Cannot connect to MySQL: $EVAL_ERROR");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
my @output_files;
|
||||
my $final_exit_status = 0;
|
||||
my $spool_file = "$spool_dir/" . $service->name;
|
||||
my $tasks = $service->tasks;
|
||||
my $taskno = 0;
|
||||
TASK:
|
||||
foreach my $task ( @$tasks ) {
|
||||
PTDEBUG && _d("Task $taskno:", $task->name);
|
||||
|
||||
@@ -5410,33 +5431,51 @@ sub run_service {
|
||||
}
|
||||
PTDEBUG && _d("Task $taskno output:", Dumper(\@output_files));
|
||||
|
||||
# Create the full command line to execute, replacing any
|
||||
# special vars like __RUN_N_OUTPUT__, __TMPDIR__, etc.
|
||||
# TODO: handle query tasks
|
||||
my $cmd = join(' ',
|
||||
$task->program,
|
||||
$task->options,
|
||||
'>',
|
||||
$output_file,
|
||||
);
|
||||
$cmd = replace_special_vars(
|
||||
cmd => $cmd,
|
||||
service => $service,
|
||||
output_files => \@output_files,
|
||||
);
|
||||
_info("Task $taskno command: $cmd");
|
||||
if ( my $query = $task->query ) {
|
||||
_info("Task $taskno query: $query");
|
||||
eval {
|
||||
$cxn->dbh->do($query);
|
||||
};
|
||||
if ( $EVAL_ERROR ) {
|
||||
_warn("Error executing $query: $EVAL_ERROR");
|
||||
$final_exit_status |= 1;
|
||||
last TASK;
|
||||
}
|
||||
}
|
||||
elsif ( my $program = $task->program ) {
|
||||
# Create the full command line to execute, replacing any
|
||||
# special vars like __RUN_N_OUTPUT__, __TMPDIR__, etc.
|
||||
my $cmd = join(' ',
|
||||
$task->program,
|
||||
$task->options,
|
||||
'>',
|
||||
$output_file,
|
||||
);
|
||||
$cmd = replace_special_vars(
|
||||
cmd => $cmd,
|
||||
service => $service,
|
||||
output_files => \@output_files,
|
||||
);
|
||||
_info("Task $taskno command: $cmd");
|
||||
|
||||
# Execute this run.
|
||||
system($cmd);
|
||||
my $exit_status = $CHILD_ERROR >> 8;
|
||||
_info("Run $taskno: exit $exit_status");
|
||||
# Execute this run.
|
||||
system($cmd);
|
||||
my $exit_status = $CHILD_ERROR >> 8;
|
||||
$final_exit_status |= $exit_status;
|
||||
_info("Run $taskno: exit $exit_status");
|
||||
}
|
||||
else {
|
||||
_warn('Invalid Task resource:', Dumper($task));
|
||||
$final_exit_status |= 1;
|
||||
last TASK;
|
||||
}
|
||||
|
||||
$final_exit_status |= $exit_status;
|
||||
$taskno++;
|
||||
}
|
||||
|
||||
# Remove temp output files.
|
||||
foreach my $file ( @output_files ) {
|
||||
next unless defined $file;
|
||||
next if $file eq $spool_file;
|
||||
unlink $file
|
||||
or _warn("Error removing $file: $OS_ERROR");
|
||||
|
||||
@@ -49,7 +49,6 @@ use constant {
|
||||
#
|
||||
# Required Arguments:
|
||||
# DSNParser - <DSNParser> object
|
||||
# OptionParser - <OptionParser> object
|
||||
# dsn - DSN hashref, or...
|
||||
# dsn_string - ... DSN string like "h=127.1,P=12345"
|
||||
#
|
||||
@@ -109,7 +108,7 @@ sub new {
|
||||
set => $args{set},
|
||||
NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
|
||||
dbh_set => 0,
|
||||
ask_pass => $o->get('ask-pass'),
|
||||
ask_pass => $args{ask_pass},
|
||||
DSNParser => $dp,
|
||||
is_cluster_node => undef,
|
||||
parent => $args{parent},
|
||||
|
||||
@@ -16,9 +16,18 @@ use File::Temp qw(tempdir);
|
||||
$ENV{PTTEST_PRETTY_JSON} = 1;
|
||||
|
||||
use Percona::Test;
|
||||
use Sandbox;
|
||||
use Percona::Test::Mock::UserAgent;
|
||||
require "$trunk/bin/pt-agent";
|
||||
|
||||
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
my $dbh = $sb->get_dbh_for('master');
|
||||
my $dsn = $sb->dsn_for('master');
|
||||
my $o = new OptionParser();
|
||||
$o->get_specs("$trunk/bin/pt-agent");
|
||||
$o->get_opts();
|
||||
|
||||
Percona::Toolkit->import(qw(Dumper have_required_args));
|
||||
Percona::WebAPI::Representation->import(qw(as_hashref));
|
||||
|
||||
@@ -27,7 +36,7 @@ my $sample = "t/pt-agent/samples";
|
||||
# Create fake spool and lib dirs. Service-related subs in pt-agent
|
||||
# automatically add "/services" to the lib dir, but the spool dir is
|
||||
# used as-is.
|
||||
my $tmpdir = tempdir("/tmp/pt-agent.$PID.XXXXXX", CLEANUP => 1);
|
||||
my $tmpdir = tempdir("/tmp/pt-agent.$PID.XXXXXX", CLEANUP => 0);
|
||||
mkdir "$tmpdir/spool" or die "Error making $tmpdir/spool: $OS_ERROR";
|
||||
mkdir "$tmpdir/services" or die "Error making $tmpdir/services: $OS_ERROR";
|
||||
my $spool_dir = "$tmpdir/spool";
|
||||
@@ -52,7 +61,7 @@ sub write_svc_files {
|
||||
}
|
||||
|
||||
# #############################################################################
|
||||
# Simple single run service
|
||||
# Simple single task service using a program.
|
||||
# #############################################################################
|
||||
|
||||
my $run0 = Percona::WebAPI::Resource::Task->new(
|
||||
@@ -81,6 +90,7 @@ my $output = output(
|
||||
service => 'query-history',
|
||||
spool_dir => $spool_dir,
|
||||
lib_dir => $tmpdir,
|
||||
Cxn => '',
|
||||
);
|
||||
},
|
||||
stderr => 1,
|
||||
@@ -108,7 +118,7 @@ is(
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Service with two runs
|
||||
# Service with two task, both using a program.
|
||||
# #############################################################################
|
||||
|
||||
diag(`rm -rf $tmpdir/spool/* $tmpdir/services/*`);
|
||||
@@ -151,6 +161,7 @@ $output = output(
|
||||
service => 'query-history',
|
||||
spool_dir => $spool_dir,
|
||||
lib_dir => $tmpdir,
|
||||
Cxn => '',
|
||||
);
|
||||
},
|
||||
stderr => 1,
|
||||
@@ -187,6 +198,165 @@ ok(
|
||||
"2 runs: temp file removed"
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# More realistc: 3 services, multiple tasks, using programs and queries.
|
||||
# #############################################################################
|
||||
|
||||
SKIP: {
|
||||
skip 'Cannot connect to sandbox master', 5 unless $dbh;
|
||||
skip 'No HOME environment variable', 5 unless $ENV{HOME};
|
||||
|
||||
diag(`rm -rf $tmpdir/spool/* $tmpdir/services/*`);
|
||||
|
||||
my (undef, $old_genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
|
||||
|
||||
my $new_genlog = "$tmpdir/genlog";
|
||||
|
||||
# First service: set up
|
||||
my $task00 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'disable-gen-log',
|
||||
number => '0',
|
||||
query => "SET GLOBAL general_log=OFF",
|
||||
);
|
||||
my $task01 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'set-gen-log-file',
|
||||
number => '1',
|
||||
query => "SET GLOBAL general_log_file='$new_genlog'",
|
||||
);
|
||||
my $task02 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'enable-gen-log',
|
||||
number => '2',
|
||||
query => "SET GLOBAL general_log=ON",
|
||||
);
|
||||
my $svc0 = Percona::WebAPI::Resource::Service->new(
|
||||
name => 'enable-gen-log',
|
||||
run_schedule => '1 * * * *',
|
||||
spool_schedule => '2 * * * *',
|
||||
tasks => [ $task00, $task01, $task02 ],
|
||||
);
|
||||
|
||||
# Second service: the actual service
|
||||
my $task10 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'query-history',
|
||||
number => '1',
|
||||
program => "$trunk/bin/pt-query-digest",
|
||||
options => "--output json --type genlog $new_genlog",
|
||||
output => 'spool',
|
||||
);
|
||||
my $svc1 = Percona::WebAPI::Resource::Service->new(
|
||||
name => 'query-history',
|
||||
run_schedule => '3 * * * *',
|
||||
spool_schedule => '4 * * * *',
|
||||
tasks => [ $task10 ],
|
||||
);
|
||||
|
||||
# Third service: tear down
|
||||
my $task20 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'disable-gen-log',
|
||||
number => '0',
|
||||
query => "SET GLOBAL general_log=OFF",
|
||||
);
|
||||
my $task21 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'set-gen-log-file',
|
||||
number => '1',
|
||||
query => "SET GLOBAL general_log_file='$old_genlog'",
|
||||
);
|
||||
my $task22 = Percona::WebAPI::Resource::Task->new(
|
||||
name => 'enable-gen-log',
|
||||
number => '2',
|
||||
query => "SET GLOBAL general_log=ON",
|
||||
);
|
||||
my $svc2 = Percona::WebAPI::Resource::Service->new(
|
||||
name => 'disable-gen-log',
|
||||
run_schedule => '5 * * * *',
|
||||
spool_schedule => '6 * * * *',
|
||||
tasks => [ $task20, $task21, $task22 ],
|
||||
);
|
||||
|
||||
write_svc_files(
|
||||
services => [ $svc0, $svc1, $svc2 ],
|
||||
);
|
||||
|
||||
my $cxn = Cxn->new(
|
||||
dsn_string => $dsn,
|
||||
OptionParser => $o,
|
||||
DSNParser => $dp,
|
||||
);
|
||||
|
||||
# Run the first service.
|
||||
$output = output(
|
||||
sub {
|
||||
$exit_status = pt_agent::run_service(
|
||||
service => 'enable-gen-log',
|
||||
spool_dir => $spool_dir,
|
||||
lib_dir => $tmpdir,
|
||||
Cxn => $cxn,
|
||||
);
|
||||
},
|
||||
stderr => 1,
|
||||
);
|
||||
|
||||
my (undef, $genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
|
||||
is(
|
||||
$genlog,
|
||||
$new_genlog,
|
||||
"Task set MySQL var"
|
||||
) or diag($output);
|
||||
|
||||
# Pretend some time passes...
|
||||
|
||||
# The next service doesn't need MySQL, so it shouldn't connect to it.
|
||||
# To check this, the genlog before running and after running should
|
||||
# be identical.
|
||||
`cp $new_genlog $tmpdir/genlog-before`;
|
||||
|
||||
# Run the second service.
|
||||
$output = output(
|
||||
sub {
|
||||
$exit_status = pt_agent::run_service(
|
||||
service => 'query-history',
|
||||
spool_dir => $spool_dir,
|
||||
lib_dir => $tmpdir,
|
||||
Cxn => $cxn,
|
||||
);
|
||||
},
|
||||
stderr => 1,
|
||||
);
|
||||
|
||||
`cp $new_genlog $tmpdir/genlog-after`;
|
||||
my $diff = `diff $tmpdir/genlog-before $tmpdir/genlog-after`;
|
||||
is(
|
||||
$diff,
|
||||
'',
|
||||
"Tasks didn't need MySQL, didn't connect to MySQL"
|
||||
) or diag($output);
|
||||
|
||||
# Pretend more time passes...
|
||||
|
||||
# Run the third service.
|
||||
$output = output(
|
||||
sub {
|
||||
$exit_status = pt_agent::run_service(
|
||||
service => 'disable-gen-log',
|
||||
spool_dir => $spool_dir,
|
||||
lib_dir => $tmpdir,
|
||||
Cxn => $cxn,
|
||||
);
|
||||
},
|
||||
stderr => 1,
|
||||
);
|
||||
|
||||
(undef, $genlog) = $dbh->selectrow_array("SHOW VARIABLES LIKE 'general_log_file'");
|
||||
is(
|
||||
$genlog,
|
||||
$old_genlog,
|
||||
"Task restored MySQL var"
|
||||
) or diag($output);
|
||||
|
||||
$dbh->do("SET GLOBAL general_log=ON");
|
||||
$dbh->do("SET GLOBAL general_log_file='$old_genlog'");
|
||||
}
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
|
||||
Reference in New Issue
Block a user