percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-10 13:11:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge lp:~percona-toolkit-dev/percona-toolkit/fix-pqd-distill-bugs.

Browse Source
This commit is contained in:
Daniel Nichter
2013-08-03 12:54:01 -07:00
parent 0ed1fac51d 7c3cd71cb3
commit cd6e41be0e
19 changed files with 712 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

915
bin/pt-query-digest
View File

File diff suppressed because it is too large Load Diff

2
lib/HTTP/Micro.pm
View File

@@ -708,5 +708,5 @@ if ( $INC{"IO/Socket/SSL.pm"} ) {
1;
# ###########################################################################
# End HTTPMicro package
# End HTTP::Micro package
# ###########################################################################

20
lib/QueryParser.pm
View File

@@ -98,7 +98,7 @@ sub get_tables {
# These keywords may appear between UPDATE or SELECT and the table refs.
# They need to be removed so that they are not mistaken for tables.
$query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
$query =~ s/(?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN|DELAYED)\s+/ /ig;
# Another special case: LOCK TABLES tbl [[AS] alias] READ|WRITE, etc.
# We strip the LOCK TABLES stuff and append "FROM" to fake a SELECT
@@ -110,9 +110,21 @@ sub get_tables {
$query = "FROM $query";
}
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
# INSERT and REPLACE without INTO
# https://bugs.launchpad.net/percona-toolkit/+bug/984053
if ( $query =~ m/\A\s*(?:INSERT|REPLACE)(?!\s+INTO)/i ) {
# Add INTO so the reset of the code work as usual.
$query =~ s/\A\s*((?:INSERT|REPLACE))\s+/$1 INTO /i;
}
if ( $query =~ m/\A\s*LOAD DATA/i ) {
my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
return $tbl;
}
my @tables;
foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {

10
lib/QueryRewriter.pm
View File

@@ -246,6 +246,13 @@ sub distill_verbs {
$query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
$query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
if ( $query =~ m/\A\s*LOAD/i ) {
my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
$tbl ||= '';
$tbl =~ s/`//g;
return "LOAD DATA $tbl";
}
if ( $query =~ m/\Aadministrator command:/ ) {
$query =~ s/administrator command:/ADMIN/;
$query = uc $query;
@@ -386,6 +393,9 @@ sub distill {
map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
$query = $verbs;
}
elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
return $verbs;
}
else {
# For everything else, distill the tables.
my @tables = $self->__distill_tables($query, $table, %args);

4
lib/VersionCheck.pm
View File

@@ -45,7 +45,7 @@ use FindBin qw();
eval {
require Percona::Toolkit;
require HTTPMicro;
require HTTP::Micro;
};
# Return the version check file used to keep track of
@@ -335,7 +335,7 @@ sub pingback {
my $instances = $args{instances};
# Optional args
my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
# GET https://upgrade.percona.com, the server will return
# a plaintext list of items/programs it wants the tool

6
t/lib/QueryParser.t
View File

@@ -828,6 +828,12 @@ is_deeply(
[qw(t1 t2)], 'get_tables works for lowercased LOCK TABLES',
);
is_deeply(
[ $qp->get_tables("LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl") ],
[qw(db.tbl)],
"LOAD DATA db.tbl"
);
# #############################################################################
# Done.
# #############################################################################

28
t/lib/QueryRewriter.t
View File

@@ -1412,6 +1412,34 @@ is(
'distills SELECT with REPLACE function (issue 1176)'
);
# LOAD DATA
# https://bugs.launchpad.net/percona-toolkit/+bug/821692
# INSERT and REPLACE without INTO
# https://bugs.launchpad.net/percona-toolkit/+bug/984053
is(
$qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE tbl"),
"LOAD DATA tbl",
"distill LOAD DATA (bug 821692)"
);
is(
$qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE `tbl`"),
"LOAD DATA tbl",
"distill LOAD DATA (bug 821692)"
);
is(
$qr->distill("insert ignore_bar (id) values (4029731)"),
"INSERT ignore_bar",
"distill INSERT without INTO (bug 984053)"
);
is(
$qr->distill("replace ignore_bar (id) values (4029731)"),
"REPLACE ignore_bar",
"distill REPLACE without INTO (bug 984053)"
);
# #############################################################################
# Done.
# #############################################################################

4
t/lib/samples/slowlogs/slow051.txt
View File

@@ -1,6 +1,6 @@
# Time: 071218 11:48:27
# Query_time: 0.000012 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
LOAD DATA INFILE '/tmp/foo.txt' INTO db.tbl;
LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl;
# Time: 071218 11:48:37
# Query_time: 0.000012 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl;
LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl;

24
t/lib/samples/slowlogs/slow058.txt Normal file
View File

@@ -0,0 +1,24 @@
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 5 Schema: db
# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
LOAD DATA LOCAL INFILE '/tmp/foo.txt' INTO TABLE `foo`;
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
INSERT `foo` VALUES("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
REPLACE `foo` VALUES("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 5 Schema: db
# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
load data local infile '/tmp/foo.txt' into table `foo`;
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
insert `foo` values("bar");
# User@Host: meow[meow] @ [1.2.3.8]
# Thread_id: 7 Schema: db
# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
replace `foo` values("bar");

4
t/pt-query-digest/binlog_analyses.t
View File

@@ -28,7 +28,7 @@ ok(
"t/pt-query-digest/samples/binlog001.txt"
),
'Analysis for binlog001',
);
) or diag($test_diff);
ok(
no_diff(
@@ -36,7 +36,7 @@ ok(
"t/pt-query-digest/samples/binlog002.txt"
),
'Analysis for binlog002',
);
) or diag($test_diff);
# #############################################################################
# Done.

4
t/pt-query-digest/daemon.t
View File

@@ -32,8 +32,8 @@ my $pid_file = '/tmp/pt-query-digest.test.pid';
$output = `$trunk/bin/pt-query-digest $trunk/commont/t/samples/slow002.txt --pid $pid_file 2>&1`;
like(
$output,
qr{PID file $pid_file already exists},
'Dies if PID file already exists (--pid without --daemonize) (issue 391)'
qr{PID file $pid_file exists},
'Dies if PID file exists (--pid without --daemonize) (issue 391)'
);
`rm $pid_file >/dev/null 2>&1`;

2
t/pt-query-digest/json.t
View File

@@ -25,7 +25,7 @@ my $results = "t/pt-query-digest/samples/json";
ok(
no_diff(
sub { pt_query_digest::main(@args, "$sample/slowlogs/empty") },
"t/pt-query-digest/samples/empty_report.txt",
"t/pt-query-digest/samples/empty_json_report.txt",
),
'json output for empty log'
) or diag($test_diff);

5
t/pt-query-digest/samples/binlog002.txt
View File

@@ -89,6 +89,9 @@ create table foo (i int)\G
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `d` LIKE 'foo'\G
# SHOW CREATE TABLE `d`.`foo`\G
insert foo values (1) /*... omitted ...*/\G
# Profile
@@ -96,4 +99,4 @@ insert foo values (1) /*... omitted ...*/\G
# ==== ================== ============= ===== ====== ===== ===============
# 1 0xF25D6D5AC7C18FF3 0.0000 0.0% 1 0.0000 0.00 CREATE DATABASE d
# 2 0x03409022EB8A4AE7 0.0000 0.0% 1 0.0000 0.00 CREATE TABLE foo
# 3 0xF579EC4A9633EEA0 0.0000 0.0% 1 0.0000 0.00 INSERT
# 3 0xF579EC4A9633EEA0 0.0000 0.0% 1 0.0000 0.00 INSERT foo

0
t/pt-query-digest/samples/empty_json_report.txt Normal file
View File

2
t/pt-query-digest/samples/empty_report.txt
View File

@@ -0,0 +1,2 @@
# No events processed.

10
t/pt-query-digest/samples/slow051.txt
View File

@@ -1,5 +1,5 @@
# Query 1: 0.20 QPS, 0.00x concurrency, ID 0xD989521B246E945B at byte 146
# Query 1: 0.20 QPS, 0.00x concurrency, ID 0x14354E1D979884B4 at byte 152
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Time range: 2007-12-18 11:48:27 to 11:48:37
@@ -10,7 +10,7 @@
# Lock time 0 0 0 0 0 0 0 0
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 100 86 43 43 43 43 0 43
# Query size 100 98 49 49 49 49 0 49
# Query_time distribution
# 1us
# 10us ################################################################
@@ -23,9 +23,9 @@
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'tbl'\G
# SHOW CREATE TABLE `db`.`tbl`\G
LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl\G
LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl\G
# Profile
# Rank Query ID Response time Calls R/Call V/M Item
# ==== ================== ============= ===== ====== ===== ======
# 1 0xD989521B246E945B 0.0000 100.0% 2 0.0000 0.00 db.tbl
# ==== ================== ============= ===== ====== ===== ===============
# 1 0x14354E1D979884B4 0.0000 100.0% 2 0.0000 0.00 LOAD DATA db.tbl

94
t/pt-query-digest/samples/slow058.txt Normal file
View File

@@ -0,0 +1,94 @@
# Query 1: 0 QPS, 0x concurrency, ID 0x471A0C4BD7A4EE34 at byte 730 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 24 52 26 26 26 26 0 26
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us
# 10us
# 100us
# 1ms
# 10ms ################################################################
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
# SHOW CREATE TABLE `db`.`foo`\G
insert `foo` values("bar")\G
# Query 2: 0 QPS, 0x concurrency, ID 0xF33473286088142B at byte 898 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 25 54 27 27 27 27 0 27
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us
# 10us
# 100us
# 1ms
# 10ms ################################################################
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
# SHOW CREATE TABLE `db`.`foo`\G
replace `foo` values("bar")\G
# Query 3: 0 QPS, 0x concurrency, ID 0xEBAC9C76529E62CE at byte 534 ______
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 33 2
# Exec time 0 4us 2us 2us 2us 2us 0 2us
# Lock time 0 0 0 0 0 0 0 0
# Rows sent 0 0 0 0 0 0 0 0
# Rows examine 0 0 0 0 0 0 0 0
# Query size 50 108 54 54 54 54 0 54
# String:
# Databases db
# Hosts
# Users meow
# Query_time distribution
# 1us ################################################################
# 10us
# 100us
# 1ms
# 10ms
# 100ms
# 1s
# 10s+
# Tables
# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
# SHOW CREATE TABLE `db`.`foo`\G
load data local infile '/tmp/foo.txt' into table `foo`\G
# Profile
# Rank Query ID Response time Calls R/Call V/M Item
# ==== ================== ============= ===== ====== ===== =============
# 1 0x471A0C4BD7A4EE34 0.0376 50.0% 2 0.0188 0.00 INSERT foo
# 2 0xF33473286088142B 0.0376 50.0% 2 0.0188 0.00 REPLACE foo
# 3 0xEBAC9C76529E62CE 0.0000 0.0% 2 0.0000 0.00 LOAD DATA foo

23
t/pt-query-digest/slowlog_analyses.t
View File

@@ -343,7 +343,7 @@ ok(
"t/pt-query-digest/samples/slow051.txt",
),
'Analysis for slow051 (issue 918)',
);
) or diag($test_diff);
# #############################################################################
# Issue 1124: Make mk-query-digest profile include variance-to-mean ratio
@@ -394,9 +394,10 @@ ok(
);
# #############################################################################
# Bug 1176010: pt-query-digest should know how to group quoted and unquoted database names
# Bug 1176010: pt-query-digest should know how to group quoted and unquoted
# database names
# https://bugs.launchpad.net/percona-toolkit/+bug/1176010
#############################################################################
# #############################################################################
ok(
no_diff(
sub { pt_query_digest::main(@args, $sample.'slow057.txt',
@@ -406,6 +407,22 @@ ok(
'Analysis for slow057 (no grouping bug 1176010)'
) or diag($test_diff);
# #############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/821692
# pt-query-digest doesn't distill LOAD DATA correctly
# https://bugs.launchpad.net/percona-toolkit/+bug/984053
# pt-query-digest doesn't distill INSERT/REPLACE without INTO correctly
# #############################################################################
ok(
no_diff(
sub { pt_query_digest::main($sample.'slow058.txt',
'--report-format', 'query_report,profile', '--limit', '100%',
)},
"t/pt-query-digest/samples/slow058.txt",
),
'Analysis for slow058 (bug 821692, bug 984053)'
) or diag($test_diff);
# #############################################################################
# Done.
# #############################################################################

2
util/update-modules
View File

@@ -91,7 +91,7 @@ replace_pkg_in_tool() {
$BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file
if [ "$tool_lang" = "perl" ]; then
if [ "$tool_lang" = "perl" -a $pkg != "HTTP::Micro" ]; then
echo "}" >> $tmp_file
fi