percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-11 13:40:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update modules in pt-query-digest to fix bug 821692 and bug 984053. Replace HTTPMicro with HTTP::Micro.

Browse Source
This commit is contained in:
Daniel Nichter
2013-08-03 11:50:15 -07:00
parent e66415764d
commit 1ed23c7845
4 changed files with 598 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
bin/pt-query-digest
View File

@@ -48,7 +48,7 @@ BEGIN {
FileIterator FileIterator
Runtime Runtime
Pipeline Pipeline
HTTPMicro HTTP::Micro
VersionCheck VersionCheck
)); ));
} }
@@ -2928,6 +2928,13 @@ sub distill_verbs {
$query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK"; $query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
$query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1"; $query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
if ( $query =~ m/\A\s*LOAD/i ) {
my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
$tbl ||= '';
$tbl =~ s/`//g;
return "LOAD DATA $tbl";
}
if ( $query =~ m/\Aadministrator command:/ ) { if ( $query =~ m/\Aadministrator command:/ ) {
$query =~ s/administrator command:/ADMIN/; $query =~ s/administrator command:/ADMIN/;
$query = uc $query; $query = uc $query;
@@ -3021,6 +3028,9 @@ sub distill {
map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for; map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
$query = $verbs; $query = $verbs;
} }
elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
return $verbs;
}
else { else {
my @tables = $self->__distill_tables($query, $table, %args); my @tables = $self->__distill_tables($query, $table, %args);
$query = join(q{ }, $verbs, @tables); $query = join(q{ }, $verbs, @tables);
@@ -8259,7 +8269,7 @@ sub get_tables {
return ($tbl); return ($tbl);
} }
$query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig; $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN|DELAYED) / /ig;
if ( $query =~ s/^\s*LOCK TABLES\s+//i ) { if ( $query =~ s/^\s*LOCK TABLES\s+//i ) {
PTDEBUG && _d('Special table type: LOCK TABLES'); PTDEBUG && _d('Special table type: LOCK TABLES');
@@ -8272,6 +8282,10 @@ sub get_tables {
$query =~ s/".*?"/?/sg; # quoted strings $query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings $query =~ s/'.*?'/?/sg; # quoted strings
if ( $query =~ m/\A\s*(?:INSERT|REPLACE)\s+(?!INTO)/i ) {
$query =~ s/\A\s*((?:INSERT|REPLACE))\s+/$1 INTO /i;
}
my @tables; my @tables;
foreach my $tbls ( $query =~ m/$tbl_regex/gio ) { foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {
PTDEBUG && _d('Match tables:', $tbls); PTDEBUG && _d('Match tables:', $tbls);
@@ -9253,59 +9267,79 @@ package Daemon;
use strict; use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0; use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use POSIX qw(setsid); use POSIX qw(setsid);
use Fcntl qw(:DEFAULT);
sub new { sub new {
my ($class, %args) = @_; my ($class, %args) = @_;
foreach my $arg ( qw(o) ) {
die "I need a $arg argument" unless $args{$arg};
}
my $o = $args{o};
my $self = { my $self = {
o => $o, log_file => $args{log_file},
log_file => $o->has('log') ? $o->get('log') : undef, pid_file => $args{pid_file},
PID_file => $o->has('pid') ? $o->get('pid') : undef, daemonize => $args{daemonize},
force_log_file => $args{force_log_file},
parent_exit => $args{parent_exit},
pid_file_owner => 0,
}; };
check_PID_file(undef, $self->{PID_file});
PTDEBUG && _d('Daemonized child will log to', $self->{log_file});
return bless $self, $class; return bless $self, $class;
} }
sub daemonize { sub run {
my ($self) = @_; my ($self) = @_;
PTDEBUG && _d('About to fork and daemonize'); my $daemonize = $self->{daemonize};
defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR"; my $pid_file = $self->{pid_file};
if ( $pid ) { my $log_file = $self->{log_file};
PTDEBUG && _d('Parent PID', $PID, 'exiting after forking child PID',$pid); my $force_log_file = $self->{force_log_file};
exit; my $parent_exit = $self->{parent_exit};
PTDEBUG && _d('Starting daemon');
if ( $pid_file ) {
eval {
$self->_make_pid_file(
pid => $PID, # parent's pid
pid_file => $pid_file,
);
};
die "$EVAL_ERROR\n" if $EVAL_ERROR;
if ( !$daemonize ) {
$self->{pid_file_owner} = $PID; # parent's pid
}
} }
PTDEBUG && _d('Daemonizing child PID', $PID); if ( $daemonize ) {
$self->{PID_owner} = $PID; defined (my $child_pid = fork()) or die "Cannot fork: $OS_ERROR";
$self->{child} = 1; if ( $child_pid ) {
PTDEBUG && _d('Forked child', $child_pid);
$parent_exit->($child_pid) if $parent_exit;
exit 0;
}
POSIX::setsid() or die "Cannot start a new session: $OS_ERROR"; POSIX::setsid() or die "Cannot start a new session: $OS_ERROR";
chdir '/' or die "Cannot chdir to /: $OS_ERROR"; chdir '/' or die "Cannot chdir to /: $OS_ERROR";
$self->_make_PID_file(); if ( $pid_file ) {
$self->_update_pid_file(
$OUTPUT_AUTOFLUSH = 1; pid => $PID, # child's pid
pid_file => $pid_file,
);
$self->{pid_file_owner} = $PID;
}
}
if ( $daemonize || $force_log_file ) {
PTDEBUG && _d('Redirecting STDIN to /dev/null'); PTDEBUG && _d('Redirecting STDIN to /dev/null');
close STDIN; close STDIN;
open STDIN, '/dev/null' open STDIN, '/dev/null'
or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; or die "Cannot reopen STDIN to /dev/null: $OS_ERROR";
if ( $log_file ) {
if ( $self->{log_file} ) { PTDEBUG && _d('Redirecting STDOUT and STDERR to', $log_file);
PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file});
close STDOUT; close STDOUT;
open STDOUT, '>>', $self->{log_file} open STDOUT, '>>', $log_file
or die "Cannot open log file $self->{log_file}: $OS_ERROR"; or die "Cannot open log file $log_file: $OS_ERROR";
close STDERR; close STDERR;
open STDERR, ">&STDOUT" open STDERR, ">&STDOUT"
@@ -9328,82 +9362,119 @@ sub daemonize {
} }
} }
$OUTPUT_AUTOFLUSH = 1;
}
PTDEBUG && _d('Daemon running');
return; return;
} }
sub check_PID_file { sub _make_pid_file {
my ( $self, $file ) = @_; my ($self, %args) = @_;
my $PID_file = $self ? $self->{PID_file} : $file; my @required_args = qw(pid pid_file);
PTDEBUG && _d('Checking PID file', $PID_file); foreach my $arg ( @required_args ) {
if ( $PID_file && -f $PID_file ) { die "I need a $arg argument" unless $args{$arg};
my $pid;
eval {
chomp($pid = (slurp_file($PID_file) || ''));
}; };
if ( $EVAL_ERROR ) { my $pid = $args{pid};
die "The PID file $PID_file already exists but it cannot be read: " my $pid_file = $args{pid_file};
. $EVAL_ERROR;
eval {
sysopen(PID_FH, $pid_file, O_RDWR|O_CREAT|O_EXCL) or die $OS_ERROR;
print PID_FH $PID, "\n";
close PID_FH;
};
if ( my $e = $EVAL_ERROR ) {
if ( $e =~ m/file exists/i ) {
my $old_pid = $self->_check_pid_file(
pid_file => $pid_file,
pid => $PID,
);
if ( $old_pid ) {
warn "Overwriting PID file $pid_file because PID $old_pid "
. "is not running.\n";
} }
PTDEBUG && _d('PID file exists; it contains PID', $pid); $self->_update_pid_file(
if ( $pid ) { pid => $PID,
my $pid_is_alive = kill 0, $pid; pid_file => $pid_file
);
}
else {
die "Error creating PID file $pid_file: $e\n";
}
}
return;
}
sub _check_pid_file {
my ($self, %args) = @_;
my @required_args = qw(pid_file pid);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid_file = $args{pid_file};
my $pid = $args{pid};
PTDEBUG && _d('Checking if PID in', $pid_file, 'is running');
if ( ! -f $pid_file ) {
PTDEBUG && _d('PID file', $pid_file, 'does not exist');
return;
}
open my $fh, '<', $pid_file
or die "Error opening $pid_file: $OS_ERROR";
my $existing_pid = do { local $/; <$fh> };
chomp($existing_pid) if $existing_pid;
close $fh
or die "Error closing $pid_file: $OS_ERROR";
if ( $existing_pid ) {
if ( $existing_pid == $pid ) {
warn "The current PID $pid already holds the PID file $pid_file\n";
return;
}
else {
PTDEBUG && _d('Checking if PID', $existing_pid, 'is running');
my $pid_is_alive = kill 0, $existing_pid;
if ( $pid_is_alive ) { if ( $pid_is_alive ) {
die "The PID file $PID_file already exists " die "PID file $pid_file exists and PID $existing_pid is running\n";
. " and the PID that it contains, $pid, is running";
} }
else {
warn "Overwriting PID file $PID_file because the PID that it "
. "contains, $pid, is not running";
} }
} }
else { else {
die "The PID file $PID_file already exists but it does not " die "PID file $pid_file exists but it is empty. Remove the file "
. "contain a PID"; . "if the process is no longer running.\n";
} }
return $existing_pid;
} }
else {
PTDEBUG && _d('No PID file'); sub _update_pid_file {
} my ($self, %args) = @_;
my @required_args = qw(pid pid_file);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
};
my $pid = $args{pid};
my $pid_file = $args{pid_file};
open my $fh, '>', $pid_file
or die "Cannot open $pid_file: $OS_ERROR";
print { $fh } $pid, "\n"
or die "Cannot print to $pid_file: $OS_ERROR";
close $fh
or warn "Cannot close $pid_file: $OS_ERROR";
return; return;
} }
sub make_PID_file { sub remove_pid_file {
my ( $self ) = @_; my ($self, $pid_file) = @_;
if ( exists $self->{child} ) { $pid_file ||= $self->{pid_file};
die "Do not call Daemon::make_PID_file() for daemonized scripts"; if ( $pid_file && -f $pid_file ) {
} unlink $self->{pid_file}
$self->_make_PID_file(); or warn "Cannot remove PID file $pid_file: $OS_ERROR";
$self->{PID_owner} = $PID;
return;
}
sub _make_PID_file {
my ( $self ) = @_;
my $PID_file = $self->{PID_file};
if ( !$PID_file ) {
PTDEBUG && _d('No PID file to create');
return;
}
$self->check_PID_file();
open my $PID_FH, '>', $PID_file
or die "Cannot open PID file $PID_file: $OS_ERROR";
print $PID_FH $PID
or die "Cannot print to PID file $PID_file: $OS_ERROR";
close $PID_FH
or die "Cannot close PID file $PID_file: $OS_ERROR";
PTDEBUG && _d('Created PID file:', $self->{PID_file});
return;
}
sub _remove_PID_file {
my ( $self ) = @_;
if ( $self->{PID_file} && -f $self->{PID_file} ) {
unlink $self->{PID_file}
or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR";
PTDEBUG && _d('Removed PID file'); PTDEBUG && _d('Removed PID file');
} }
else { else {
@@ -9415,16 +9486,11 @@ sub _remove_PID_file {
sub DESTROY { sub DESTROY {
my ($self) = @_; my ($self) = @_;
$self->_remove_PID_file() if ($self->{PID_owner} || 0) == $PID; if ( $self->{pid_file_owner} == $PID ) {
$self->remove_pid_file();
return;
} }
sub slurp_file { return;
my ($file) = @_;
return unless $file;
open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
return do { local $/; <$fh> };
} }
sub _d { sub _d {
@@ -11479,25 +11545,23 @@ sub _d {
# ########################################################################### # ###########################################################################
# ########################################################################### # ###########################################################################
# HTTPMicro package # HTTP::Micro package
# This package is a copy without comments from the original. The original # This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at, # with comments and its test file can be found in the Bazaar repository at,
# lib/HTTPMicro.pm # lib/HTTP/Micro.pm
# t/lib/HTTPMicro.t # t/lib/HTTP/Micro.t
# See https://launchpad.net/percona-toolkit for more information. # See https://launchpad.net/percona-toolkit for more information.
# ########################################################################### # ###########################################################################
{ {
package HTTP::Micro;
our $VERSION = '0.01';
package HTTPMicro;
BEGIN {
$HTTPMicro::VERSION = '0.001';
}
use strict; use strict;
use warnings; use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Carp (); use Carp ();
my @attributes; my @attributes;
BEGIN { BEGIN {
@attributes = qw(agent timeout); @attributes = qw(agent timeout);
@@ -11568,7 +11632,7 @@ sub _request {
headers => {}, headers => {},
}; };
my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout}); my $handle = HTTP::Micro::Handle->new(timeout => $self->{timeout});
$handle->connect($scheme, $host, $port); $handle->connect($scheme, $host, $port);
@@ -11633,14 +11697,18 @@ sub _split_url {
return ($scheme, $host, $port, $path_query); return ($scheme, $host, $port, $path_query);
} }
package } # HTTP::Micro
HTTPMicro::Handle; # hide from PAUSE/indexers
use strict;
use warnings;
use Carp qw[croak]; {
use Errno qw[EINTR EPIPE]; package HTTP::Micro::Handle;
use IO::Socket qw[SOCK_STREAM];
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Carp qw(croak);
use Errno qw(EINTR EPIPE);
use IO::Socket qw(SOCK_STREAM);
sub BUFSIZE () { 32768 } sub BUFSIZE () { 32768 }
@@ -11683,7 +11751,7 @@ sub connect {
croak(qq/Unsupported URL scheme '$scheme'\n/); croak(qq/Unsupported URL scheme '$scheme'\n/);
} }
$self->{fh} = 'IO::Socket::INET'->new( $self->{fh} = IO::Socket::INET->new(
PeerHost => $host, PeerHost => $host,
PeerPort => $port, PeerPort => $port,
Proto => 'tcp', Proto => 'tcp',
@@ -11947,6 +12015,7 @@ sub can_write {
my $self = shift; my $self = shift;
return $self->_do_timeout('write', @_) return $self->_do_timeout('write', @_)
} }
} # HTTP::Micro::Handle
my $prog = <<'EOP'; my $prog = <<'EOP';
BEGIN { BEGIN {
@@ -11967,6 +12036,7 @@ BEGIN {
} }
} }
{ {
use Carp qw(croak);
my %dispatcher = ( my %dispatcher = (
issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) }, issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) },
subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) }, subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) },
@@ -12122,9 +12192,8 @@ if ( $INC{"IO/Socket/SSL.pm"} ) {
} }
1; 1;
}
# ########################################################################### # ###########################################################################
# End HTTPMicro package # End HTTP::Micro package
# ########################################################################### # ###########################################################################
# ########################################################################### # ###########################################################################

24
t/lib/samples/slowlogs/slow058.txt Normal file
View File

@@ -0,0 +1,24 @@
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 5 Schema: db
# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
LOAD DATA LOCAL INFILE '/tmp/foo.txt' INTO TABLE `foo`;
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
INSERT `foo` VALUES("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
REPLACE `foo` VALUES("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 5 Schema: db
# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
load data local infile '/tmp/foo.txt' into table `foo`;
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
insert `foo` values("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
replace `foo` values("bar");

94
t/pt-query-digest/samples/slow058.txt Normal file
View File

@@ -0,0 +1,94 @@
# Query 1: 0 QPS, 0x concurrency, ID 0x471A0C4BD7A4EE34 at byte 730 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 24 52 26 26 26 26 0 26
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us
# 10us
# 100us
# 1ms
# 10ms ################################################################
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
# SHOW CREATE TABLE `db`.`foo`\G
insert `foo` values("bar")\G
# Query 2: 0 QPS, 0x concurrency, ID 0xF33473286088142B at byte 898 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 25 54 27 27 27 27 0 27
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us
# 10us
# 100us
# 1ms
# 10ms ################################################################
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
# SHOW CREATE TABLE `db`.`foo`\G
replace `foo` values("bar")\G
# Query 3: 0 QPS, 0x concurrency, ID 0xEBAC9C76529E62CE at byte 534 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 0 4us 2us 2us 2us 2us 0 2us
# Lock time 0 0 0 0 0 0 0 0
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 50 108 54 54 54 54 0 54
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us ################################################################
# 10us
# 100us
# 1ms
# 10ms
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'table'\G
# SHOW CREATE TABLE `db`.`table`\G
load data local infile '/tmp/foo.txt' into table `foo`\G
# Profile
# Rank Query ID Response time Calls R/Call V/M Item
# ==== ================== ============= ===== ====== ===== =============
# 1 0x471A0C4BD7A4EE34 0.0376 50.0% 2 0.0188 0.00 INSERT foo
# 2 0xF33473286088142B 0.0376 50.0% 2 0.0188 0.00 REPLACE foo
# 3 0xEBAC9C76529E62CE 0.0000 0.0% 2 0.0000 0.00 LOAD DATA foo

2
t/pt-query-digest/slowlog_analyses.t
View File

@@ -343,7 +343,7 @@ ok(
"t/pt-query-digest/samples/slow051.txt", "t/pt-query-digest/samples/slow051.txt",
), ),
'Analysis for slow051 (issue 918)', 'Analysis for slow051 (issue 918)',
); ) or diag($test_diff);
# ############################################################################# # #############################################################################
# Issue 1124: Make mk-query-digest profile include variance-to-mean ratio # Issue 1124: Make mk-query-digest profile include variance-to-mean ratio