diff --git a/bin/pt-query-digest b/bin/pt-query-digest index aca147b6..7c50dd53 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -7900,10 +7900,12 @@ override query_report => sub { distillate => $distill, attribute => $groupby, query_count => $times_seen, - example => { - query => substr($sample->{arg}, 0, $self->max_query_length), - ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, - }, + $args{anon} ? () : ( + example => { + query => substr($sample->{arg}, 0, $self->max_query_length), + ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, + }, + ), }; my %metrics; @@ -7981,19 +7983,21 @@ override query_report => sub { push @tables, { status => $status, create => $create }; } - if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { - if ( $item =~ m/^(?:insert|replace)/ ) { + if ( !$args{anon} ) { + if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { + if ( $item =~ m/^(?:insert|replace)/ ) { + } + else { + + } } else { - - } - } - else { - my $converted = $qr->convert_to_select( - $sample->{arg} || '', - ); - if ( $converted && $converted =~ m/^[\(\s]*select/i ) { - $class->{example}->{as_select} = $converted; + my $converted = $qr->convert_to_select( + $sample->{arg} || '', + ); + if ( $converted && $converted =~ m/^[\(\s]*select/i ) { + $class->{example}->{as_select} = $converted; + } } } } @@ -14155,7 +14159,7 @@ sub print_reports { $print_header = 1; } - my $report_class = $o->get('output') =~ m/\Ajson\z/i + my $report_class = $o->get('output') =~ m/^json/i ? 'JSONReportFormatter' : 'QueryReportFormatter'; my $qrf = $report_class->new( @@ -14182,6 +14186,7 @@ sub print_reports { variations => $o->get('variations'), group => { map { $_=>1 } qw(rusage date hostname files header) }, resume => $resume, + anon => $o->get('output') eq 'json-anon', ); } @@ -15611,11 +15616,12 @@ type: string; default: report How to format and print the query analysis results. Accepted values are: - VALUE FORMAT - ======= ============================== - report Standard query analysis report - slowlog MySQL slow log - json JSON, on array per query class + VALUE FORMAT + ======= ============================== + report Standard query analysis report + slowlog MySQL slow log + json JSON, on array per query class + json-anon JSON without example queries The entire C output can be disabled by specifying C<--no-report> (see L<"--[no]report">), and its sections can be disabled or rearranged diff --git a/lib/JSONReportFormatter.pm b/lib/JSONReportFormatter.pm index 5cfdb850..465127a2 100644 --- a/lib/JSONReportFormatter.pm +++ b/lib/JSONReportFormatter.pm @@ -228,10 +228,12 @@ override query_report => sub { distillate => $distill, attribute => $groupby, query_count => $times_seen, - example => { - query => substr($sample->{arg}, 0, $self->max_query_length), - ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, - }, + $args{anon} ? () : ( + example => { + query => substr($sample->{arg}, 0, $self->max_query_length), + ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, + }, + ), }; my %metrics; @@ -340,26 +342,28 @@ override query_report => sub { push @tables, { status => $status, create => $create }; } - # Convert possible non-SELECTs for EXPLAIN. - if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { - if ( $item =~ m/^(?:insert|replace)/ ) { - # Cannot convert or EXPLAIN INSERT or REPLACE queries. + if ( !$args{anon} ) { + # Convert possible non-SELECTs for EXPLAIN. + if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { + if ( $item =~ m/^(?:insert|replace)/ ) { + # Cannot convert or EXPLAIN INSERT or REPLACE queries. + } + else { + # SELECT queries don't need to converted for EXPLAIN. + + # TODO: return the actual EXPLAIN plan + # $self->explain_report($query, $vals->{default_db}); + } } else { - # SELECT queries don't need to converted for EXPLAIN. - - # TODO: return the actual EXPLAIN plan - # $self->explain_report($query, $vals->{default_db}); - } - } - else { - # Query is not SELECT, INSERT, or REPLACE, so we can convert - # it for EXPLAIN. - my $converted = $qr->convert_to_select( - $sample->{arg} || '', - ); - if ( $converted && $converted =~ m/^[\(\s]*select/i ) { - $class->{example}->{as_select} = $converted; + # Query is not SELECT, INSERT, or REPLACE, so we can convert + # it for EXPLAIN. + my $converted = $qr->convert_to_select( + $sample->{arg} || '', + ); + if ( $converted && $converted =~ m/^[\(\s]*select/i ) { + $class->{example}->{as_select} = $converted; + } } } } diff --git a/t/pt-query-digest/json.t b/t/pt-query-digest/json.t index e8162b1c..fbb1e4d1 100644 --- a/t/pt-query-digest/json.t +++ b/t/pt-query-digest/json.t @@ -20,12 +20,12 @@ local $JSONReportFormatter::pretty_json = 1; my @args = qw(--output json); my $sample = "$trunk/t/lib/samples"; -my $results = "t/pt-query-digest/samples"; +my $results = "t/pt-query-digest/samples/json"; ok( no_diff( sub { pt_query_digest::main(@args, "$sample/slowlogs/empty") }, - "$results/empty_report.txt", + "t/pt-query-digest/samples/empty_report.txt", ), 'json output for empty log' ) or diag($test_diff); @@ -33,18 +33,30 @@ ok( ok( no_diff( sub { pt_query_digest::main(@args, "$sample/slowlogs/slow002.txt") }, - "$results/output_json_slow002.txt" + "$results/slow002.txt", + sed => [ qq/'s!$trunk!TRUNK!'/ ], ), 'json output for slow002' ) or diag($test_diff); +ok( + no_diff( + sub { pt_query_digest::main(qw(--output json-anon), + "$sample/slowlogs/slow002.txt") }, + "$results/slow002-anon.txt", + sed => [ qq/'s!$trunk!TRUNK!'/ ], + ), + 'json-anon output for slow002' +) or diag($test_diff); + # --type tcpdump ok( no_diff( sub { pt_query_digest::main(qw(--type tcpdump --limit 10 --watch-server 127.0.0.1:12345), @args, "$sample/tcpdump/tcpdump021.txt") }, - "$results/output_json_tcpdump021.txt", + "$results/tcpdump021.txt", + sed => [ qq/'s!$trunk!TRUNK!'/ ], ), 'json output for for tcpdump021', ) or diag($test_diff); diff --git a/t/pt-query-digest/samples/empty_report.txt b/t/pt-query-digest/samples/empty_report.txt index a05997d5..e69de29b 100644 --- a/t/pt-query-digest/samples/empty_report.txt +++ b/t/pt-query-digest/samples/empty_report.txt @@ -1,2 +0,0 @@ - -# No events processed. diff --git a/t/pt-query-digest/samples/json/slow002-anon.txt b/t/pt-query-digest/samples/json/slow002-anon.txt new file mode 100644 index 00000000..42791f45 --- /dev/null +++ b/t/pt-query-digest/samples/json/slow002-anon.txt @@ -0,0 +1,257 @@ + +{ + "classes" : [ + { + "attribute" : "fingerprint", + "checksum" : "66825DDC008FFA89", + "distillate" : "UPDATE db?.tuningdetail_?_? db?.gonzo", + "fingerprint" : "update d?tuningdetail_?_? n inner join d?gonzo a using(gonzo) set n.column? = a.column?, n.word? = a.word?", + "metrics" : { + "Filesort" : { + "yes" : "0" + }, + "Filesort_on_disk" : { + "yes" : "0" + }, + "Full_join" : { + "yes" : "0" + }, + "Full_scan" : { + "yes" : "1" + }, + "Lock_time" : { + "avg" : "0.000091", + "max" : "0.000091", + "median" : "0.000091", + "min" : "0.000091", + "pct" : "0.125000", + "pct_95" : "0.000091", + "stddev" : "0.000000", + "sum" : "0.000091" + }, + "Merge_passes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "QC_Hit" : { + "yes" : "0" + }, + "Query_length" : { + "avg" : "129", + "max" : "129", + "median" : "129", + "min" : "129", + "pct" : "0", + "pct_95" : "129", + "stddev" : "0", + "sum" : "129" + }, + "Query_time" : { + "avg" : "0.726052", + "max" : "0.726052", + "median" : "0.726052", + "min" : "0.726052", + "pct" : "0.125000", + "pct_95" : "0.726052", + "stddev" : "0.000000", + "sum" : "0.726052" + }, + "Rows_examined" : { + "avg" : "62951", + "max" : "62951", + "median" : "62951", + "min" : "62951", + "pct" : "0", + "pct_95" : "62951", + "stddev" : "0", + "sum" : "62951" + }, + "Rows_sent" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_table" : { + "yes" : "0" + }, + "Tmp_table_on_disk" : { + "yes" : "0" + }, + "db" : { + "value" : "db1" + }, + "host" : { + "value" : "" + }, + "user" : { + "value" : "[SQL_SLAVE]" + } + }, + "query_count" : 1, + "tables" : [ + { + "create" : "SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\\G", + "status" : "SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\\G" + }, + { + "create" : "SHOW CREATE TABLE `db1`.`gonzo`\\G", + "status" : "SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\\G" + } + ], + "ts_max" : "2007-12-18 11:48:27", + "ts_min" : "2007-12-18 11:48:27" + } + ], + "global" : { + "files" : [ + { + "name" : "TRUNK/t/lib/samples/slowlogs/slow002.txt", + "size" : 3841 + } + ], + "metrics" : { + "Filesort" : { + "cnt" : "0" + }, + "Filesort_on_disk" : { + "cnt" : "0" + }, + "Full_join" : { + "cnt" : "0" + }, + "Full_scan" : { + "cnt" : "1" + }, + "InnoDB_IO_r_bytes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_ops" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_pages_distinct" : { + "avg" : "17", + "max" : "24", + "median" : "17", + "min" : "11", + "pct_95" : "23", + "stddev" : "3", + "sum" : "107" + }, + "InnoDB_queue_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_rec_lock_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "Lock_time" : { + "avg" : "0.000038", + "max" : "0.000091", + "median" : "0.000026", + "min" : "0.000000", + "pct_95" : "0.000089", + "stddev" : "0.000028", + "sum" : "0.000304" + }, + "Merge_passes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "QC_Hit" : { + "cnt" : "0" + }, + "Query_length" : { + "avg" : "62", + "max" : "129", + "median" : "62", + "min" : "5", + "pct_95" : "124", + "stddev" : "34", + "sum" : "502" + }, + "Query_time" : { + "avg" : "0.095260", + "max" : "0.726052", + "median" : "0.000516", + "min" : "0.000012", + "pct_95" : "0.705093", + "stddev" : "0.231765", + "sum" : "0.762080" + }, + "Rows_examined" : { + "avg" : "7868", + "max" : "62951", + "median" : "0", + "min" : "0", + "pct_95" : "61003", + "stddev" : "20174", + "sum" : "62951" + }, + "Rows_sent" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_table" : { + "cnt" : "0" + }, + "Tmp_table_on_disk" : { + "cnt" : "0" + } + }, + "query_count" : 8, + "unique_query_count" : 7 + } +} diff --git a/t/pt-query-digest/samples/output_json_slow002.txt b/t/pt-query-digest/samples/json/slow002.txt similarity index 98% rename from t/pt-query-digest/samples/output_json_slow002.txt rename to t/pt-query-digest/samples/json/slow002.txt index 563c6a88..c6243729 100644 --- a/t/pt-query-digest/samples/output_json_slow002.txt +++ b/t/pt-query-digest/samples/json/slow002.txt @@ -121,7 +121,7 @@ "global" : { "files" : [ { - "name" : "/Users/daniel/p/release-2.2.3/t/lib/samples/slowlogs/slow002.txt", + "name" : "TRUNK/t/lib/samples/slowlogs/slow002.txt", "size" : 3841 } ], diff --git a/t/pt-query-digest/samples/output_json_tcpdump021.txt b/t/pt-query-digest/samples/json/tcpdump021.txt similarity index 98% rename from t/pt-query-digest/samples/output_json_tcpdump021.txt rename to t/pt-query-digest/samples/json/tcpdump021.txt index e0e2310d..6caf4681 100644 --- a/t/pt-query-digest/samples/output_json_tcpdump021.txt +++ b/t/pt-query-digest/samples/json/tcpdump021.txt @@ -187,7 +187,7 @@ "global" : { "files" : [ { - "name" : "/Users/daniel/p/release-2.2.3/t/lib/samples/tcpdump/tcpdump021.txt", + "name" : "TRUNK/t/lib/samples/tcpdump/tcpdump021.txt", "size" : 2827 } ],