Merge lp:~percona-toolkit-dev/percona-toolkit/fix-pqd-distill-bugs.

lib/HTTP/Micro.pm

@@ -708,5 +708,5 @@ if ( $INC{"IO/Socket/SSL.pm"} ) {
 
 1;
 # ###########################################################################
-# End HTTPMicro package
+# End HTTP::Micro package
 # ###########################################################################

lib/QueryParser.pm

@@ -98,7 +98,7 @@ sub get_tables {
 
    # These keywords may appear between UPDATE or SELECT and the table refs.
    # They need to be removed so that they are not mistaken for tables.
-   $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
+   $query =~ s/(?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN|DELAYED)\s+/ /ig;
 
    # Another special case: LOCK TABLES tbl [[AS] alias] READ|WRITE, etc.
    # We strip the LOCK TABLES stuff and append "FROM" to fake a SELECT
@@ -110,9 +110,21 @@ sub get_tables {
       $query = "FROM $query";
    }
 
-   $query =~ s/\\["']//g;                # quoted strings
+   $query =~ s/\\["']//g;   # quoted strings
-   $query =~ s/".*?"/?/sg;               # quoted strings
+   $query =~ s/".*?"/?/sg;  # quoted strings
-   $query =~ s/'.*?'/?/sg;               # quoted strings
+   $query =~ s/'.*?'/?/sg;  # quoted strings
+
+   # INSERT and REPLACE without INTO
+   # https://bugs.launchpad.net/percona-toolkit/+bug/984053
+   if ( $query =~ m/\A\s*(?:INSERT|REPLACE)(?!\s+INTO)/i ) {
+      # Add INTO so the reset of the code work as usual.
+      $query =~ s/\A\s*((?:INSERT|REPLACE))\s+/$1 INTO /i;
+   }
+
+   if ( $query =~ m/\A\s*LOAD DATA/i ) {
+      my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
+      return $tbl;
+   }
 
    my @tables;
    foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {

lib/QueryRewriter.pm

@@ -246,6 +246,13 @@ sub distill_verbs {
    $query =~ m/\A\s*UNLOCK TABLES/i  && return "UNLOCK";
    $query =~ m/\A\s*xa\s+(\S+)/i     && return "XA_$1";
 
+   if ( $query =~ m/\A\s*LOAD/i ) {
+      my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
+      $tbl ||= '';
+      $tbl =~ s/`//g;
+      return "LOAD DATA $tbl";
+   }
+
    if ( $query =~ m/\Aadministrator command:/ ) {
       $query =~ s/administrator command:/ADMIN/;
       $query = uc $query;
@@ -386,6 +393,9 @@ sub distill {
          map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
          $query = $verbs;
       }
+      elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
+         return $verbs;
+      }
       else {
          # For everything else, distill the tables.
          my @tables = $self->__distill_tables($query, $table, %args);

lib/VersionCheck.pm

@@ -45,7 +45,7 @@ use FindBin qw();
 
 eval {
    require Percona::Toolkit;
-   require HTTPMicro;
+   require HTTP::Micro;
 };
 
 # Return the version check file used to keep track of
@@ -335,7 +335,7 @@ sub pingback {
    my $instances = $args{instances};
 
    # Optional args
-   my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
+   my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
 
    # GET https://upgrade.percona.com, the server will return
    # a plaintext list of items/programs it wants the tool

t/lib/QueryParser.t

@@ -828,6 +828,12 @@ is_deeply(
    [qw(t1 t2)], 'get_tables works for lowercased LOCK TABLES',
 );
 
+is_deeply(
+   [ $qp->get_tables("LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl") ],
+   [qw(db.tbl)],
+   "LOAD DATA db.tbl"
+);
+
 # #############################################################################
 # Done.
 # #############################################################################

t/lib/QueryRewriter.t

@@ -1412,6 +1412,34 @@ is(
    'distills SELECT with REPLACE function (issue 1176)'
 );
 
+# LOAD DATA
+# https://bugs.launchpad.net/percona-toolkit/+bug/821692
+# INSERT and REPLACE without INTO
+# https://bugs.launchpad.net/percona-toolkit/+bug/984053
+is(
+   $qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE tbl"),
+   "LOAD DATA tbl",
+   "distill LOAD DATA (bug 821692)"
+);
+
+is(
+   $qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE `tbl`"),
+   "LOAD DATA tbl",
+   "distill LOAD DATA (bug 821692)"
+);
+
+is(
+   $qr->distill("insert ignore_bar (id) values (4029731)"),
+   "INSERT ignore_bar",
+   "distill INSERT without INTO (bug 984053)"
+);
+
+is(
+   $qr->distill("replace ignore_bar (id) values (4029731)"),
+   "REPLACE ignore_bar",
+   "distill REPLACE without INTO (bug 984053)"
+);
+
 # #############################################################################
 # Done.
 # #############################################################################

t/lib/samples/slowlogs/slow051.txt

@@ -1,6 +1,6 @@
 # Time: 071218 11:48:27
 # Query_time: 0.000012  Lock_time: 0.000000  Rows_sent: 0  Rows_examined: 0
-LOAD DATA INFILE '/tmp/foo.txt' INTO db.tbl;
+LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl;
 # Time: 071218 11:48:37
 # Query_time: 0.000012  Lock_time: 0.000000  Rows_sent: 0  Rows_examined: 0
-LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl;
+LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl;

t/lib/samples/slowlogs/slow058.txt

@@ -0,0 +1,24 @@
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 5  Schema: db
+# Query_time: 0.000002  Lock_time: 0.000000  Rows_sent: 0  Rows_examined: 0
+LOAD DATA LOCAL INFILE '/tmp/foo.txt' INTO TABLE `foo`;
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 7  Schema: db
+# Query_time: 0.018799  Lock_time: 0.009453  Rows_sent: 0  Rows_examined: 0
+INSERT `foo` VALUES("bar");
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 7  Schema: db
+# Query_time: 0.018799  Lock_time: 0.009453  Rows_sent: 0  Rows_examined: 0
+REPLACE `foo` VALUES("bar");
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 5  Schema: db
+# Query_time: 0.000002  Lock_time: 0.000000  Rows_sent: 0  Rows_examined: 0
+load data local infile '/tmp/foo.txt' into table `foo`;
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 7  Schema: db
+# Query_time: 0.018799  Lock_time: 0.009453  Rows_sent: 0  Rows_examined: 0
+insert `foo` values("bar");
+# User@Host: meow[meow] @  [1.2.3.8]
+# Thread_id: 7  Schema: db
+# Query_time: 0.018799  Lock_time: 0.009453  Rows_sent: 0  Rows_examined: 0
+replace `foo` values("bar");

t/pt-query-digest/binlog_analyses.t

@@ -28,7 +28,7 @@ ok(
       "t/pt-query-digest/samples/binlog001.txt"
    ),
    'Analysis for binlog001',
-);
+) or diag($test_diff);
 
 ok(
    no_diff(
@@ -36,7 +36,7 @@ ok(
       "t/pt-query-digest/samples/binlog002.txt"
    ),
    'Analysis for binlog002',
-);
+) or diag($test_diff);
 
 # #############################################################################
 # Done.

t/pt-query-digest/daemon.t

@@ -32,8 +32,8 @@ my $pid_file = '/tmp/pt-query-digest.test.pid';
 $output = `$trunk/bin/pt-query-digest $trunk/commont/t/samples/slow002.txt --pid $pid_file 2>&1`;
 like(
    $output,
-   qr{PID file $pid_file already exists},
+   qr{PID file $pid_file exists},
-   'Dies if PID file already exists (--pid without --daemonize) (issue 391)'
+   'Dies if PID file exists (--pid without --daemonize) (issue 391)'
 );
 `rm $pid_file >/dev/null 2>&1`;
 

t/pt-query-digest/json.t

@@ -25,7 +25,7 @@ my $results = "t/pt-query-digest/samples/json";
 ok(
    no_diff(
       sub { pt_query_digest::main(@args, "$sample/slowlogs/empty") },
-      "t/pt-query-digest/samples/empty_report.txt",
+      "t/pt-query-digest/samples/empty_json_report.txt",
    ),
    'json output for empty log'
 ) or diag($test_diff);

t/pt-query-digest/samples/binlog002.txt

@@ -89,6 +89,9 @@ create table foo (i int)\G
 # 100ms
 #    1s
 #  10s+
+# Tables
+#    SHOW TABLE STATUS FROM `d` LIKE 'foo'\G
+#    SHOW CREATE TABLE `d`.`foo`\G
 insert foo values (1) /*... omitted ...*/\G
 
 # Profile
@@ -96,4 +99,4 @@ insert foo values (1) /*... omitted ...*/\G
 # ==== ================== ============= ===== ====== ===== ===============
 #    1 0xF25D6D5AC7C18FF3  0.0000  0.0%     1 0.0000  0.00 CREATE DATABASE d
 #    2 0x03409022EB8A4AE7  0.0000  0.0%     1 0.0000  0.00 CREATE TABLE foo
-#    3 0xF579EC4A9633EEA0  0.0000  0.0%     1 0.0000  0.00 INSERT
+#    3 0xF579EC4A9633EEA0  0.0000  0.0%     1 0.0000  0.00 INSERT foo

t/pt-query-digest/samples/empty_report.txt

@@ -0,0 +1,2 @@
+
+# No events processed.

t/pt-query-digest/samples/slow051.txt

@@ -1,5 +1,5 @@
 
-# Query 1: 0.20 QPS, 0.00x concurrency, ID 0xD989521B246E945B at byte 146
+# Query 1: 0.20 QPS, 0.00x concurrency, ID 0x14354E1D979884B4 at byte 152
 # This item is included in the report because it matches --limit.
 # Scores: V/M = 0.00
 # Time range: 2007-12-18 11:48:27 to 11:48:37
@@ -10,7 +10,7 @@
 # Lock time      0       0       0       0       0       0       0       0
 # Rows sent      0       0       0       0       0       0       0       0
 # Rows examine   0       0       0       0       0       0       0       0
-# Query size   100      86      43      43      43      43       0      43
+# Query size   100      98      49      49      49      49       0      49
 # Query_time distribution
 #   1us
 #  10us  ################################################################
@@ -23,9 +23,9 @@
 # Tables
 #    SHOW TABLE STATUS FROM `db` LIKE 'tbl'\G
 #    SHOW CREATE TABLE `db`.`tbl`\G
-LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl\G
+LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl\G
 
 # Profile
 # Rank Query ID           Response time Calls R/Call V/M   Item
-# ==== ================== ============= ===== ====== ===== ======
+# ==== ================== ============= ===== ====== ===== ===============
-#    1 0xD989521B246E945B 0.0000 100.0%     2 0.0000  0.00 db.tbl
+#    1 0x14354E1D979884B4 0.0000 100.0%     2 0.0000  0.00 LOAD DATA db.tbl

t/pt-query-digest/samples/slow058.txt

@@ -0,0 +1,94 @@
+
+# Query 1: 0 QPS, 0x concurrency, ID 0x471A0C4BD7A4EE34 at byte 730 ______
+# This item is included in the report because it matches --limit.
+# Scores: V/M = 0.00
+# Attribute    pct   total     min     max     avg     95%  stddev  median
+# ============ === ======= ======= ======= ======= ======= ======= =======
+# Count         33       2
+# Exec time     49    38ms    19ms    19ms    19ms    19ms       0    19ms
+# Lock time     50    19ms     9ms     9ms     9ms     9ms       0     9ms
+# Rows sent      0       0       0       0       0       0       0       0
+# Rows examine   0       0       0       0       0       0       0       0
+# Query size    24      52      26      26      26      26       0      26
+# String:
+# Databases    db
+# Hosts
+# Users        meow
+# Query_time distribution
+#   1us
+#  10us
+# 100us
+#   1ms
+#  10ms  ################################################################
+# 100ms
+#    1s
+#  10s+
+# Tables
+#    SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
+#    SHOW CREATE TABLE `db`.`foo`\G
+insert `foo` values("bar")\G
+
+# Query 2: 0 QPS, 0x concurrency, ID 0xF33473286088142B at byte 898 ______
+# This item is included in the report because it matches --limit.
+# Scores: V/M = 0.00
+# Attribute    pct   total     min     max     avg     95%  stddev  median
+# ============ === ======= ======= ======= ======= ======= ======= =======
+# Count         33       2
+# Exec time     49    38ms    19ms    19ms    19ms    19ms       0    19ms
+# Lock time     50    19ms     9ms     9ms     9ms     9ms       0     9ms
+# Rows sent      0       0       0       0       0       0       0       0
+# Rows examine   0       0       0       0       0       0       0       0
+# Query size    25      54      27      27      27      27       0      27
+# String:
+# Databases    db
+# Hosts
+# Users        meow
+# Query_time distribution
+#   1us
+#  10us
+# 100us
+#   1ms
+#  10ms  ################################################################
+# 100ms
+#    1s
+#  10s+
+# Tables
+#    SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
+#    SHOW CREATE TABLE `db`.`foo`\G
+replace `foo` values("bar")\G
+
+# Query 3: 0 QPS, 0x concurrency, ID 0xEBAC9C76529E62CE at byte 534 ______
+# This item is included in the report because it matches --limit.
+# Scores: V/M = 0.00
+# Attribute    pct   total     min     max     avg     95%  stddev  median
+# ============ === ======= ======= ======= ======= ======= ======= =======
+# Count         33       2
+# Exec time      0     4us     2us     2us     2us     2us       0     2us
+# Lock time      0       0       0       0       0       0       0       0
+# Rows sent      0       0       0       0       0       0       0       0
+# Rows examine   0       0       0       0       0       0       0       0
+# Query size    50     108      54      54      54      54       0      54
+# String:
+# Databases    db
+# Hosts
+# Users        meow
+# Query_time distribution
+#   1us  ################################################################
+#  10us
+# 100us
+#   1ms
+#  10ms
+# 100ms
+#    1s
+#  10s+
+# Tables
+#    SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
+#    SHOW CREATE TABLE `db`.`foo`\G
+load data local infile '/tmp/foo.txt' into table `foo`\G
+
+# Profile
+# Rank Query ID           Response time Calls R/Call V/M   Item
+# ==== ================== ============= ===== ====== ===== =============
+#    1 0x471A0C4BD7A4EE34  0.0376 50.0%     2 0.0188  0.00 INSERT foo
+#    2 0xF33473286088142B  0.0376 50.0%     2 0.0188  0.00 REPLACE foo
+#    3 0xEBAC9C76529E62CE  0.0000  0.0%     2 0.0000  0.00 LOAD DATA foo

t/pt-query-digest/slowlog_analyses.t

@@ -343,7 +343,7 @@ ok(
       "t/pt-query-digest/samples/slow051.txt",
    ),
    'Analysis for slow051 (issue 918)',
-);
+) or diag($test_diff);
 
 # #############################################################################
 # Issue 1124: Make mk-query-digest profile include variance-to-mean ratio
@@ -394,9 +394,10 @@ ok(
 );
 
 # #############################################################################
-# Bug 1176010: pt-query-digest should know how to group quoted and unquoted database names
+# Bug 1176010: pt-query-digest should know how to group quoted and unquoted
+# database names
 # https://bugs.launchpad.net/percona-toolkit/+bug/1176010
-#############################################################################
+# #############################################################################
 ok(
    no_diff(
       sub { pt_query_digest::main(@args, $sample.'slow057.txt',
@@ -406,6 +407,22 @@ ok(
    'Analysis for slow057 (no grouping bug 1176010)'
 ) or diag($test_diff);
 
+# #############################################################################
+# https://bugs.launchpad.net/percona-toolkit/+bug/821692
+# pt-query-digest doesn't distill LOAD DATA correctly 
+# https://bugs.launchpad.net/percona-toolkit/+bug/984053
+# pt-query-digest doesn't distill INSERT/REPLACE without INTO correctly
+# #############################################################################
+ok(
+   no_diff(
+      sub { pt_query_digest::main($sample.'slow058.txt',
+         '--report-format', 'query_report,profile', '--limit', '100%',
+      )},
+      "t/pt-query-digest/samples/slow058.txt",
+   ),
+   'Analysis for slow058 (bug 821692, bug 984053)'
+) or diag($test_diff);
+
 # #############################################################################
 # Done.
 # #############################################################################

util/update-modules

@@ -91,7 +91,7 @@ replace_pkg_in_tool() {
 
    $BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file
 
-   if [ "$tool_lang" = "perl" ]; then
+   if [ "$tool_lang" = "perl" -a $pkg != "HTTP::Micro" ]; then
       echo "}" >> $tmp_file
    fi