From c99cb3ba5881a1304e3df76193f21cc9c3a0610a Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 20 May 2013 16:44:44 -0700 Subject: [PATCH] Add copy_paste attrib to each query's hash in the JSON output with SHOW TABLE STATUS, SHOW CREATE TABLE, and non-SELECT converted to SELECT for EXPLAIN. Fix typo: EA type is 'string' not 'str'. Add host, port, and ip type hints to pqd. --- bin/pt-query-digest | 86 +++++++++++-- lib/JSONReportFormatter.pm | 113 +++++++++++++++++- t/pt-query-digest/json.t | 4 +- .../samples/output_json_slow002.txt | 33 ++++- .../samples/output_json_tcpdump021.txt | 98 +++++++++++---- 5 files changed, 290 insertions(+), 44 deletions(-) diff --git a/bin/pt-query-digest b/bin/pt-query-digest index ac32c9db..f914b526 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -7635,11 +7635,29 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0; my $have_json = eval { require JSON }; -our $pretty_json = $ENV{PTTEST_PRETTY_JSON} ? 1 : 0; -our $sorted_json = $ENV{PTTEST_PRETTY_JSON} ? 1 : 0; +our $pretty_json = 0; +our $sorted_json = 0; extends qw(QueryReportFormatter); +has 'QueryRewriter' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + +has 'QueryParser' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + +has 'Quoter' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + has _json => ( is => 'ro', init_arg => undef, @@ -7677,14 +7695,13 @@ override query_report => sub { foreach my $arg ( qw(ea worst orderby groupby) ) { die "I need a $arg argument" unless defined $arg; } - - my $ea = $args{ea}; - my $worst = $args{worst}; + my $groupby = $args{groupby}; + my $ea = $args{ea}; + my $worst = $args{worst}; my @attribs = @{$ea->get_attributes()}; + my $q = $self->Quoter; - my %string_args = map { $_ => 1 } qw( db host arg user bytes pos_in_log ); - my @queries; foreach my $worst_info ( @$worst ) { my $item = $worst_info->[0]; @@ -7693,14 +7710,14 @@ override query_report => sub { my $all_log_pos = $ea->{result_classes}->{$item}->{pos_in_log}->{all}; my $times_seen = sum values %$all_log_pos; - + my %class = ( sample => $sample->{arg}, fingerprint => $item, checksum => make_checksum($item), cnt => $times_seen, ); - + my %metrics; foreach my $attrib ( @attribs ) { $metrics{$attrib} = $ea->metrics( @@ -7724,7 +7741,7 @@ override query_report => sub { $class{ts_min} = $ts->{min}; $class{ts_max} = $ts->{max}; } - elsif ( $string_args{$attrib} ) { + elsif ( ($ea->{type_for}->{$attrib} || '') eq 'string' ) { $metrics{$attrib} = { value => $metrics{$attrib}{max} }; } elsif ( ($ea->{type_for}->{$attrib} || '') eq 'num' ) { @@ -7737,9 +7754,55 @@ override query_report => sub { } } } + + my $copy_paste; + if ( $groupby eq 'fingerprint' ) { + my $default_db = $sample->{db} ? $sample->{db} + : $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}} + : undef; + my @table_names = $self->QueryParser->extract_tables( + query => $sample->{arg} || '', + default_db => $default_db, + Quoter => $self->Quoter, + ); + my @tables; + foreach my $db_tbl ( @table_names ) { + my ( $db, $tbl ) = @$db_tbl; + my $status + = 'SHOW TABLE STATUS' + . ($db ? " FROM `$db`" : '') + . " LIKE '$tbl'\\G"; + my $create + = "SHOW CREATE TABLE " + . $q->quote(grep { $_ } @$db_tbl) + . "\\G"; + push @tables, { status => $status, create => $create }; + } + if ( @tables ) { + $copy_paste->{tables} = \@tables; + } + + if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { + if ( $item =~ m/^(?:insert|replace)/ ) { + } + else { + + } + } + else { + my $converted = $self->QueryRewriter->convert_to_select( + $sample->{arg} || '', + ); + if ( $converted && $converted =~ m/^[\(\s]*select/i ) { + $copy_paste->{explain} = $converted; + } + } + } + push @queries, { class => \%class, attributes => \%metrics, + ($copy_paste ? (copy_paste => $copy_paste) : ()), }; } @@ -13604,6 +13667,9 @@ open my $events_fh, '>', $events_file or die "Cannot open $events_file: $OS_ERRO Last_errno => 'string', Thread_id => 'string', InnoDB_trx_id => 'string', + host => 'string', + ip => 'string', + port => 'string', Killed => 'bool', }; diff --git a/lib/JSONReportFormatter.pm b/lib/JSONReportFormatter.pm index 807334f0..dcf3c5d6 100644 --- a/lib/JSONReportFormatter.pm +++ b/lib/JSONReportFormatter.pm @@ -33,6 +33,24 @@ our $sorted_json = 0; extends qw(QueryReportFormatter); +has 'QueryRewriter' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + +has 'QueryParser' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + +has 'Quoter' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + has _json => ( is => 'ro', init_arg => undef, @@ -70,11 +88,12 @@ override query_report => sub { foreach my $arg ( qw(ea worst orderby groupby) ) { die "I need a $arg argument" unless defined $arg; } - - my $ea = $args{ea}; - my $worst = $args{worst}; + my $groupby = $args{groupby}; + my $ea = $args{ea}; + my $worst = $args{worst}; my @attribs = @{$ea->get_attributes()}; + my $q = $self->Quoter; my @queries; foreach my $worst_info ( @$worst ) { @@ -84,14 +103,14 @@ override query_report => sub { my $all_log_pos = $ea->{result_classes}->{$item}->{pos_in_log}->{all}; my $times_seen = sum values %$all_log_pos; - + my %class = ( sample => $sample->{arg}, fingerprint => $item, checksum => make_checksum($item), cnt => $times_seen, ); - + my %metrics; foreach my $attrib ( @attribs ) { $metrics{$attrib} = $ea->metrics( @@ -115,7 +134,7 @@ override query_report => sub { $class{ts_min} = $ts->{min}; $class{ts_max} = $ts->{max}; } - elsif ( ($ea->{type_for}->{$attrib} || '') eq 'str' ) { + elsif ( ($ea->{type_for}->{$attrib} || '') eq 'string' ) { $metrics{$attrib} = { value => $metrics{$attrib}{max} }; } elsif ( ($ea->{type_for}->{$attrib} || '') eq 'num' ) { @@ -131,9 +150,91 @@ override query_report => sub { } } } + + # Add "copy-paste" info, i.e. this stuff from the regular report: + # + # Tables + # SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\G + # SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\G + # SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\G + # SHOW CREATE TABLE `db1`.`gonzo`\G + # update db2.tuningdetail_21_265507 n + # inner join db1.gonzo a using(gonzo) + # set n.column1 = a.column1, n.word3 = a.word3\G + # Converted for EXPLAIN + # EXPLAIN /*!50100 PARTITIONS*/ + # select n.column1 = a.column1, n.word3 = a.word3 + # from db2.tuningdetail_21_265507 n + # inner join db1.gonzo a using(gonzo) \G + # + # The formatting isn't included, just the useful data, like: + # + # $copy_paste = { + # tables => { + # create => "SHOW CREATE TABLE db.foo", + # status => "SHOW TABLE STATUS FROM db LIKE foo", + # }, + # explain => "select ..." + # } + # + # This is called "copy-paste" because users can copy-paste these + # ready-made lines into MySQL. + my $copy_paste; + if ( $groupby eq 'fingerprint' ) { + # Get SHOW CREATE TABLE and SHOW TABLE STATUS. + my $default_db = $sample->{db} ? $sample->{db} + : $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}} + : undef; + my @table_names = $self->QueryParser->extract_tables( + query => $sample->{arg} || '', + default_db => $default_db, + Quoter => $self->Quoter, + ); + my @tables; + foreach my $db_tbl ( @table_names ) { + my ( $db, $tbl ) = @$db_tbl; + my $status + = 'SHOW TABLE STATUS' + . ($db ? " FROM `$db`" : '') + . " LIKE '$tbl'\\G"; + my $create + = "SHOW CREATE TABLE " + . $q->quote(grep { $_ } @$db_tbl) + . "\\G"; + push @tables, { status => $status, create => $create }; + } + if ( @tables ) { + $copy_paste->{tables} = \@tables; + } + + # Convert possible non-SELECTs for EXPLAIN. + if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { + if ( $item =~ m/^(?:insert|replace)/ ) { + # Cannot convert or EXPLAIN INSERT or REPLACE queries. + } + else { + # SELECT queries don't need to converted for EXPLAIN. + + # TODO: return the actual EXPLAIN plan + # $self->explain_report($query, $vals->{default_db}); + } + } + else { + # Query is not SELECT, INSERT, or REPLACE, so we can convert + # it for EXPLAIN. + my $converted = $self->QueryRewriter->convert_to_select( + $sample->{arg} || '', + ); + if ( $converted && $converted =~ m/^[\(\s]*select/i ) { + $copy_paste->{explain} = $converted; + } + } + } + push @queries, { class => \%class, attributes => \%metrics, + ($copy_paste ? (copy_paste => $copy_paste) : ()), }; } diff --git a/t/pt-query-digest/json.t b/t/pt-query-digest/json.t index 21513543..e8162b1c 100644 --- a/t/pt-query-digest/json.t +++ b/t/pt-query-digest/json.t @@ -36,7 +36,7 @@ ok( "$results/output_json_slow002.txt" ), 'json output for slow002' -); +) or diag($test_diff); # --type tcpdump @@ -47,7 +47,7 @@ ok( "$results/output_json_tcpdump021.txt", ), 'json output for for tcpdump021', -); +) or diag($test_diff); # ############################################################################# # Done. diff --git a/t/pt-query-digest/samples/output_json_slow002.txt b/t/pt-query-digest/samples/output_json_slow002.txt index 3d2fadbb..c23d9219 100644 --- a/t/pt-query-digest/samples/output_json_slow002.txt +++ b/t/pt-query-digest/samples/output_json_slow002.txt @@ -135,7 +135,15 @@ "sum" : 0 }, "bytes" : { - "value" : 129 + "avg" : "129.000000", + "cnt" : "1.000000", + "max" : "129.000000", + "median" : "129.000000", + "min" : "129.000000", + "pct" : "0.12", + "pct_95" : "129.000000", + "stddev" : 0, + "sum" : "129.000000" }, "db" : { "value" : "db1" @@ -144,7 +152,15 @@ "value" : "" }, "pos_in_log" : { - "value" : 338 + "avg" : "338.000000", + "cnt" : "1.000000", + "max" : "338.000000", + "median" : "338.000000", + "min" : "338.000000", + "pct" : "0.12", + "pct_95" : "338.000000", + "stddev" : 0, + "sum" : "338.000000" }, "user" : { "value" : "[SQL_SLAVE]" @@ -157,6 +173,19 @@ "sample" : "update db2.tuningdetail_21_265507 n\n inner join db1.gonzo a using(gonzo) \n set n.column1 = a.column1, n.word3 = a.word3", "ts_max" : "2007-12-18 11:48:27", "ts_min" : "2007-12-18 11:48:27" + }, + "copy_paste" : { + "explain" : "select n.column1 = a.column1, n.word3 = a.word3 from db2.tuningdetail_21_265507 n\n inner join db1.gonzo a using(gonzo) ", + "tables" : [ + { + "create" : "SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\\G", + "status" : "SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\\G" + }, + { + "create" : "SHOW CREATE TABLE `db1`.`gonzo`\\G", + "status" : "SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\\G" + } + ] } } ] diff --git a/t/pt-query-digest/samples/output_json_tcpdump021.txt b/t/pt-query-digest/samples/output_json_tcpdump021.txt index d3a08e79..23b95212 100644 --- a/t/pt-query-digest/samples/output_json_tcpdump021.txt +++ b/t/pt-query-digest/samples/output_json_tcpdump021.txt @@ -47,15 +47,7 @@ "sum" : 0 }, "Statement_id" : { - "avg" : 0, - "cnt" : 1, - "max" : 2, - "median" : 0, - "min" : 2, - "pct" : 0.5, - "pct_95" : 0, - "stddev" : 0, - "sum" : null + "value" : 2 }, "Warning_count" : { "avg" : 0, @@ -69,13 +61,29 @@ "sum" : 0 }, "bytes" : { - "value" : 35 + "avg" : "35.000000", + "cnt" : "1.000000", + "max" : "35.000000", + "median" : "35.000000", + "min" : "35.000000", + "pct" : "0.33", + "pct_95" : "35.000000", + "stddev" : 0, + "sum" : "35.000000" }, "host" : { "value" : "127.0.0.1" }, "pos_in_log" : { - "value" : 0 + "avg" : 0, + "cnt" : "1.000000", + "max" : 0, + "median" : 0, + "min" : 0, + "pct" : "0.33", + "pct_95" : 0, + "stddev" : 0, + "sum" : 0 } }, "class" : { @@ -85,6 +93,15 @@ "sample" : "PREPARE SELECT i FROM d.t WHERE i=?", "ts_max" : "2009-12-08 09:23:49.637394", "ts_min" : "2009-12-08 09:23:49.637394" + }, + "copy_paste" : { + "explain" : "SELECT i FROM d.t WHERE i=?", + "tables" : [ + { + "create" : "SHOW CREATE TABLE `d`.`t`\\G", + "status" : "SHOW TABLE STATUS FROM `d` LIKE 't'\\G" + } + ] } }, { @@ -134,15 +151,7 @@ "sum" : 0 }, "Statement_id" : { - "avg" : 0, - "cnt" : 1, - "max" : "2", - "median" : 0, - "min" : "2", - "pct" : 0.5, - "pct_95" : 0, - "stddev" : 0, - "sum" : null + "value" : "2" }, "Warning_count" : { "avg" : 0, @@ -156,13 +165,29 @@ "sum" : 0 }, "bytes" : { - "value" : 37 + "avg" : "37.000000", + "cnt" : "1.000000", + "max" : "37.000000", + "median" : "37.000000", + "min" : "37.000000", + "pct" : "0.33", + "pct_95" : "37.000000", + "stddev" : 0, + "sum" : "37.000000" }, "host" : { "value" : "127.0.0.1" }, "pos_in_log" : { - "value" : 1106 + "avg" : "1106.000000", + "cnt" : "1.000000", + "max" : "1106.000000", + "median" : "1106.000000", + "min" : "1106.000000", + "pct" : "0.33", + "pct_95" : "1106.000000", + "stddev" : 0, + "sum" : "1106.000000" } }, "class" : { @@ -172,6 +197,15 @@ "sample" : "EXECUTE SELECT i FROM d.t WHERE i=\"3\"", "ts_max" : "2009-12-08 09:23:49.637892", "ts_min" : "2009-12-08 09:23:49.637892" + }, + "copy_paste" : { + "explain" : "SELECT i FROM d.t WHERE i=\"3\"", + "tables" : [ + { + "create" : "SHOW CREATE TABLE `d`.`t`\\G", + "status" : "SHOW TABLE STATUS FROM `d` LIKE 't'\\G" + } + ] } }, { @@ -232,13 +266,29 @@ "sum" : 0 }, "bytes" : { - "value" : 27 + "avg" : "27.000000", + "cnt" : "1.000000", + "max" : "27.000000", + "median" : "27.000000", + "min" : "27.000000", + "pct" : "0.33", + "pct_95" : "27.000000", + "stddev" : 0, + "sum" : "27.000000" }, "host" : { "value" : "127.0.0.1" }, "pos_in_log" : { - "value" : 1850 + "avg" : "1850.000000", + "cnt" : "1.000000", + "max" : "1850.000000", + "median" : "1850.000000", + "min" : "1850.000000", + "pct" : "0.33", + "pct_95" : "1850.000000", + "stddev" : 0, + "sum" : "1850.000000" } }, "class" : {