Add copy_paste attrib to each query's hash in the JSON output with SHOW TABLE STATUS, SHOW CREATE TABLE, and non-SELECT converted to SELECT for EXPLAIN. Fix typo: EA type is 'string' not 'str'. Add host, port, and ip type hints to pqd.

bin/pt-query-digest

@@ -7635,11 +7635,29 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0;
 
 my $have_json = eval { require JSON };
 
-our $pretty_json = $ENV{PTTEST_PRETTY_JSON} ? 1 : 0;
+our $pretty_json = 0;
-our $sorted_json = $ENV{PTTEST_PRETTY_JSON} ? 1 : 0;
+our $sorted_json = 0;
 
 extends qw(QueryReportFormatter);
 
+has 'QueryRewriter' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
+has 'QueryParser' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
+has 'Quoter' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
 has _json => (
    is       => 'ro',
    init_arg => undef,
@@ -7677,13 +7695,12 @@ override query_report => sub {
    foreach my $arg ( qw(ea worst orderby groupby) ) {
       die "I need a $arg argument" unless defined $arg;
    }
-
+   my $groupby = $args{groupby};
    my $ea      = $args{ea};
    my $worst   = $args{worst};
 
    my @attribs = @{$ea->get_attributes()};
-
+   my $q       = $self->Quoter;
-   my %string_args = map { $_ => 1 } qw( db host arg user bytes pos_in_log );
 
    my @queries;
    foreach my $worst_info ( @$worst ) {
@@ -7724,7 +7741,7 @@ override query_report => sub {
             $class{ts_min} = $ts->{min};
             $class{ts_max} = $ts->{max};
          }
-         elsif ( $string_args{$attrib} ) {
+         elsif ( ($ea->{type_for}->{$attrib} || '') eq 'string' ) {
             $metrics{$attrib} = { value => $metrics{$attrib}{max} }; 
          }
          elsif ( ($ea->{type_for}->{$attrib} || '') eq 'num' ) {
@@ -7737,9 +7754,55 @@ override query_report => sub {
             }
          }
       }
+
+      my $copy_paste;
+      if ( $groupby eq 'fingerprint' ) {
+         my $default_db = $sample->{db}       ? $sample->{db}
+                        : $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}}
+                        :                       undef;
+         my @table_names = $self->QueryParser->extract_tables(
+            query      => $sample->{arg} || '',
+            default_db => $default_db,
+            Quoter     => $self->Quoter,
+         );
+         my @tables;
+         foreach my $db_tbl ( @table_names ) {
+            my ( $db, $tbl ) = @$db_tbl;
+            my $status
+               = 'SHOW TABLE STATUS'
+               . ($db ? " FROM `$db`" : '')
+               . " LIKE '$tbl'\\G";
+            my $create
+               = "SHOW CREATE TABLE "
+               . $q->quote(grep { $_ } @$db_tbl)
+               . "\\G";
+            push @tables, { status => $status, create => $create };
+         }
+         if ( @tables ) {
+            $copy_paste->{tables} = \@tables;
+         }
+
+         if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
+            if ( $item =~ m/^(?:insert|replace)/ ) {
+            }
+            else {
+
+            }
+         }
+         else {
+            my $converted = $self->QueryRewriter->convert_to_select(
+               $sample->{arg} || '',
+            );
+            if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
+               $copy_paste->{explain} = $converted;
+            }
+         }
+      }
+
       push @queries, {
          class       => \%class,
          attributes  => \%metrics,
+         ($copy_paste ? (copy_paste  => $copy_paste) : ()),
       };
    }
 
@@ -13604,6 +13667,9 @@ open my $events_fh, '>', $events_file or die "Cannot open $events_file: $OS_ERRO
          Last_errno    => 'string',
          Thread_id     => 'string',
          InnoDB_trx_id => 'string',
+         host          => 'string',
+         ip            => 'string',
+         port          => 'string',
          Killed        => 'bool',
       };
 

lib/JSONReportFormatter.pm

@@ -33,6 +33,24 @@ our $sorted_json = 0;
 
 extends qw(QueryReportFormatter);
 
+has 'QueryRewriter' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
+has 'QueryParser' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
+has 'Quoter' => (
+   is       => 'ro',
+   isa      => 'Object',
+   required => 1,
+);
+
 has _json => (
    is       => 'ro',
    init_arg => undef,
@@ -70,11 +88,12 @@ override query_report => sub {
    foreach my $arg ( qw(ea worst orderby groupby) ) {
       die "I need a $arg argument" unless defined $arg;
    }
-
+   my $groupby = $args{groupby};
    my $ea      = $args{ea};
    my $worst   = $args{worst};
 
    my @attribs = @{$ea->get_attributes()};
+   my $q       = $self->Quoter;
 
    my @queries;
    foreach my $worst_info ( @$worst ) {
@@ -115,7 +134,7 @@ override query_report => sub {
             $class{ts_min} = $ts->{min};
             $class{ts_max} = $ts->{max};
          }
-         elsif ( ($ea->{type_for}->{$attrib} || '') eq 'str' ) {
+         elsif ( ($ea->{type_for}->{$attrib} || '') eq 'string' ) {
             $metrics{$attrib} = { value => $metrics{$attrib}{max} }; 
          }
          elsif ( ($ea->{type_for}->{$attrib} || '') eq 'num' ) {
@@ -131,9 +150,91 @@ override query_report => sub {
             }
          }
       }
+
+      # Add "copy-paste" info, i.e. this stuff from the regular report:
+      # 
+      # Tables
+      #    SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\G
+      #    SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\G
+      #    SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\G
+      #    SHOW CREATE TABLE `db1`.`gonzo`\G
+      #     update db2.tuningdetail_21_265507 n
+      #           inner join db1.gonzo a using(gonzo) 
+      #                 set n.column1 = a.column1, n.word3 = a.word3\G
+      # Converted for EXPLAIN
+      # EXPLAIN /*!50100 PARTITIONS*/
+      #    select  n.column1 = a.column1, n.word3 = a.word3
+      #    from db2.tuningdetail_21_265507 n
+      #    inner join db1.gonzo a using(gonzo) \G
+      #
+      # The formatting isn't included, just the useful data, like:
+      #
+      # $copy_paste = {
+      #    tables => {
+      #      create => "SHOW CREATE TABLE db.foo",
+      #      status => "SHOW TABLE STATUS FROM db LIKE foo",
+      #    },
+      #    explain => "select ..."
+      # }
+      #
+      # This is called "copy-paste" because users can copy-paste these
+      # ready-made lines into MySQL.
+      my $copy_paste;
+      if ( $groupby eq 'fingerprint' ) {
+         # Get SHOW CREATE TABLE and SHOW TABLE STATUS.
+         my $default_db = $sample->{db}       ? $sample->{db}
+                        : $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}}
+                        :                       undef;
+         my @table_names = $self->QueryParser->extract_tables(
+            query      => $sample->{arg} || '',
+            default_db => $default_db,
+            Quoter     => $self->Quoter,
+         );
+         my @tables;
+         foreach my $db_tbl ( @table_names ) {
+            my ( $db, $tbl ) = @$db_tbl;
+            my $status
+               = 'SHOW TABLE STATUS'
+               . ($db ? " FROM `$db`" : '')
+               . " LIKE '$tbl'\\G";
+            my $create
+               = "SHOW CREATE TABLE "
+               . $q->quote(grep { $_ } @$db_tbl)
+               . "\\G";
+            push @tables, { status => $status, create => $create };
+         }
+         if ( @tables ) {
+            $copy_paste->{tables} = \@tables;
+         }
+
+         # Convert possible non-SELECTs for EXPLAIN.
+         if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
+            if ( $item =~ m/^(?:insert|replace)/ ) {
+               # Cannot convert or EXPLAIN INSERT or REPLACE queries.
+            }
+            else {
+               # SELECT queries don't need to converted for EXPLAIN.
+
+               # TODO: return the actual EXPLAIN plan
+               # $self->explain_report($query, $vals->{default_db});
+            }
+         }
+         else {
+            # Query is not SELECT, INSERT, or REPLACE, so we can convert
+            # it for EXPLAIN.
+            my $converted = $self->QueryRewriter->convert_to_select(
+               $sample->{arg} || '',
+            );
+            if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
+               $copy_paste->{explain} = $converted;
+            }
+         }
+      }
+
       push @queries, {
          class       => \%class,
          attributes  => \%metrics,
+         ($copy_paste ? (copy_paste  => $copy_paste) : ()),
       };
    }
 

t/pt-query-digest/json.t

@@ -36,7 +36,7 @@ ok(
       "$results/output_json_slow002.txt"
    ),
    'json output for slow002'
-);
+) or diag($test_diff);
 
 # --type tcpdump
 
@@ -47,7 +47,7 @@ ok(
       "$results/output_json_tcpdump021.txt",
    ),
    'json output for for tcpdump021',
-);
+) or diag($test_diff);
 
 # #############################################################################
 # Done.

t/pt-query-digest/samples/output_json_slow002.txt

@@ -135,7 +135,15 @@
             "sum" : 0
          },
          "bytes" : {
-            "value" : 129
+            "avg" : "129.000000",
+            "cnt" : "1.000000",
+            "max" : "129.000000",
+            "median" : "129.000000",
+            "min" : "129.000000",
+            "pct" : "0.12",
+            "pct_95" : "129.000000",
+            "stddev" : 0,
+            "sum" : "129.000000"
          },
          "db" : {
             "value" : "db1"
@@ -144,7 +152,15 @@
             "value" : ""
          },
          "pos_in_log" : {
-            "value" : 338
+            "avg" : "338.000000",
+            "cnt" : "1.000000",
+            "max" : "338.000000",
+            "median" : "338.000000",
+            "min" : "338.000000",
+            "pct" : "0.12",
+            "pct_95" : "338.000000",
+            "stddev" : 0,
+            "sum" : "338.000000"
          },
          "user" : {
             "value" : "[SQL_SLAVE]"
@@ -157,6 +173,19 @@
          "sample" : "update db2.tuningdetail_21_265507 n\n      inner join db1.gonzo a using(gonzo) \n      set n.column1 = a.column1, n.word3 = a.word3",
          "ts_max" : "2007-12-18 11:48:27",
          "ts_min" : "2007-12-18 11:48:27"
+      },
+      "copy_paste" : {
+         "explain" : "select  n.column1 = a.column1, n.word3 = a.word3 from db2.tuningdetail_21_265507 n\n      inner join db1.gonzo a using(gonzo) ",
+         "tables" : [
+            {
+               "create" : "SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\\G",
+               "status" : "SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\\G"
+            },
+            {
+               "create" : "SHOW CREATE TABLE `db1`.`gonzo`\\G",
+               "status" : "SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\\G"
+            }
+         ]
       }
    }
 ]

t/pt-query-digest/samples/output_json_tcpdump021.txt

@@ -47,15 +47,7 @@
             "sum" : 0
          },
          "Statement_id" : {
-            "avg" : 0,
+            "value" : 2
-            "cnt" : 1,
-            "max" : 2,
-            "median" : 0,
-            "min" : 2,
-            "pct" : 0.5,
-            "pct_95" : 0,
-            "stddev" : 0,
-            "sum" : null
          },
          "Warning_count" : {
             "avg" : 0,
@@ -69,13 +61,29 @@
             "sum" : 0
          },
          "bytes" : {
-            "value" : 35
+            "avg" : "35.000000",
+            "cnt" : "1.000000",
+            "max" : "35.000000",
+            "median" : "35.000000",
+            "min" : "35.000000",
+            "pct" : "0.33",
+            "pct_95" : "35.000000",
+            "stddev" : 0,
+            "sum" : "35.000000"
          },
          "host" : {
             "value" : "127.0.0.1"
          },
          "pos_in_log" : {
-            "value" : 0
+            "avg" : 0,
+            "cnt" : "1.000000",
+            "max" : 0,
+            "median" : 0,
+            "min" : 0,
+            "pct" : "0.33",
+            "pct_95" : 0,
+            "stddev" : 0,
+            "sum" : 0
          }
       },
       "class" : {
@@ -85,6 +93,15 @@
          "sample" : "PREPARE SELECT i FROM d.t WHERE i=?",
          "ts_max" : "2009-12-08 09:23:49.637394",
          "ts_min" : "2009-12-08 09:23:49.637394"
+      },
+      "copy_paste" : {
+         "explain" : "SELECT i FROM d.t WHERE i=?",
+         "tables" : [
+            {
+               "create" : "SHOW CREATE TABLE `d`.`t`\\G",
+               "status" : "SHOW TABLE STATUS FROM `d` LIKE 't'\\G"
+            }
+         ]
       }
    },
    {
@@ -134,15 +151,7 @@
             "sum" : 0
          },
          "Statement_id" : {
-            "avg" : 0,
+            "value" : "2"
-            "cnt" : 1,
-            "max" : "2",
-            "median" : 0,
-            "min" : "2",
-            "pct" : 0.5,
-            "pct_95" : 0,
-            "stddev" : 0,
-            "sum" : null
          },
          "Warning_count" : {
             "avg" : 0,
@@ -156,13 +165,29 @@
             "sum" : 0
          },
          "bytes" : {
-            "value" : 37
+            "avg" : "37.000000",
+            "cnt" : "1.000000",
+            "max" : "37.000000",
+            "median" : "37.000000",
+            "min" : "37.000000",
+            "pct" : "0.33",
+            "pct_95" : "37.000000",
+            "stddev" : 0,
+            "sum" : "37.000000"
          },
          "host" : {
             "value" : "127.0.0.1"
          },
          "pos_in_log" : {
-            "value" : 1106
+            "avg" : "1106.000000",
+            "cnt" : "1.000000",
+            "max" : "1106.000000",
+            "median" : "1106.000000",
+            "min" : "1106.000000",
+            "pct" : "0.33",
+            "pct_95" : "1106.000000",
+            "stddev" : 0,
+            "sum" : "1106.000000"
          }
       },
       "class" : {
@@ -172,6 +197,15 @@
          "sample" : "EXECUTE SELECT i FROM d.t WHERE i=\"3\"",
          "ts_max" : "2009-12-08 09:23:49.637892",
          "ts_min" : "2009-12-08 09:23:49.637892"
+      },
+      "copy_paste" : {
+         "explain" : "SELECT i FROM d.t WHERE i=\"3\"",
+         "tables" : [
+            {
+               "create" : "SHOW CREATE TABLE `d`.`t`\\G",
+               "status" : "SHOW TABLE STATUS FROM `d` LIKE 't'\\G"
+            }
+         ]
       }
    },
    {
@@ -232,13 +266,29 @@
             "sum" : 0
          },
          "bytes" : {
-            "value" : 27
+            "avg" : "27.000000",
+            "cnt" : "1.000000",
+            "max" : "27.000000",
+            "median" : "27.000000",
+            "min" : "27.000000",
+            "pct" : "0.33",
+            "pct_95" : "27.000000",
+            "stddev" : 0,
+            "sum" : "27.000000"
          },
          "host" : {
             "value" : "127.0.0.1"
          },
          "pos_in_log" : {
-            "value" : 1850
+            "avg" : "1850.000000",
+            "cnt" : "1.000000",
+            "max" : "1850.000000",
+            "median" : "1850.000000",
+            "min" : "1850.000000",
+            "pct" : "0.33",
+            "pct_95" : "1850.000000",
+            "stddev" : 0,
+            "sum" : "1850.000000"
          }
       },
       "class" : {