From 567c1f11e6049908e3a798e0e7249c75a627798b Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 27 Nov 2013 18:42:20 -0800 Subject: [PATCH] Add rate_limit to JSON output. --- bin/pt-query-digest | 19 + lib/JSONReportFormatter.pm | 19 + t/pt-query-digest/json.t | 15 + .../samples/slow059_report02.txt | 680 ++++++++++++++++++ 4 files changed, 733 insertions(+) create mode 100644 t/pt-query-digest/samples/slow059_report02.txt diff --git a/bin/pt-query-digest b/bin/pt-query-digest index 0c15a3f2..99c4c9a7 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -7850,6 +7850,25 @@ override query_report => sub { $global_data->{queries_per_second} = $qps if $qps; $global_data->{concurrency} = $conc if $conc; + if ( exists $results->{globals}->{rate_limit} ) { + my $rate_limit = $results->{globals}->{rate_limit}->{min} || ''; + my ($type, $limit) = $rate_limit =~ m/^(\w+):(\d+)$/; + if ( $type && $limit ) { + $global_data->{rate_limit} = { + type => $type, + limit => int($limit), + }; + } + else { + $global_data->{rate_limit}->{error} = "Invalid rate limit: $rate_limit"; + } + + if ( ($results->{globals}->{rate_limit}->{min} || '') + ne ($results->{globals}->{rate_limit}->{max} || '') ) { + $global_data->{rate_limit}->{diff} = 1; + } + } + my %hidden_attrib = ( arg => 1, fingerprint => 1, diff --git a/lib/JSONReportFormatter.pm b/lib/JSONReportFormatter.pm index badbc267..35bd91a3 100644 --- a/lib/JSONReportFormatter.pm +++ b/lib/JSONReportFormatter.pm @@ -147,6 +147,25 @@ override query_report => sub { $global_data->{queries_per_second} = $qps if $qps; $global_data->{concurrency} = $conc if $conc; + if ( exists $results->{globals}->{rate_limit} ) { + my $rate_limit = $results->{globals}->{rate_limit}->{min} || ''; + my ($type, $limit) = $rate_limit =~ m/^(\w+):(\d+)$/; + if ( $type && $limit ) { + $global_data->{rate_limit} = { + type => $type, + limit => int($limit), + }; + } + else { + $global_data->{rate_limit}->{error} = "Invalid rate limit: $rate_limit"; + } + + if ( ($results->{globals}->{rate_limit}->{min} || '') + ne ($results->{globals}->{rate_limit}->{max} || '') ) { + $global_data->{rate_limit}->{diff} = 1; + } + } + my %hidden_attrib = ( arg => 1, fingerprint => 1, diff --git a/t/pt-query-digest/json.t b/t/pt-query-digest/json.t index b1b84875..245fbd2b 100644 --- a/t/pt-query-digest/json.t +++ b/t/pt-query-digest/json.t @@ -61,6 +61,21 @@ ok( 'json output for for tcpdump021', ) or diag($test_diff); +# ############################################################################# +# pt-query-digest support for Percona Server slow log rate limiting +# https://blueprints.launchpad.net/percona-toolkit/+spec/pt-query-digest-rate-limit +# ############################################################################# +ok( + no_diff( + sub { pt_query_digest::main(@args, + "$trunk/t/lib/samples/slowlogs/slow059.txt", + qw(--output json)) + }, + "t/pt-query-digest/samples/slow059_report02.txt" + ), + 'json output for slow059' +) or diag($test_diff); + # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-query-digest/samples/slow059_report02.txt b/t/pt-query-digest/samples/slow059_report02.txt new file mode 100644 index 00000000..31fc28fa --- /dev/null +++ b/t/pt-query-digest/samples/slow059_report02.txt @@ -0,0 +1,680 @@ + +{ + "classes" : [ + { + "attribute" : "fingerprint", + "checksum" : "9EA505F4786E7F15", + "distillate" : "SELECT bar", + "example" : { + "Query_time" : "0.000237", + "query" : "SELECT foo FROM bar WHERE id=2", + "ts" : "2013-11-28 01:05:31" + }, + "fingerprint" : "select foo from bar where id=?", + "histograms" : { + "Query_time" : [ + 0, + 0, + 2, + 0, + 0, + 0, + 0, + 0 + ] + }, + "metrics" : { + "Bytes_sent" : { + "avg" : "545", + "max" : "545", + "median" : "545", + "min" : "545", + "pct" : "0", + "pct_95" : "545", + "stddev" : "0", + "sum" : "1090" + }, + "Filesort" : { + "yes" : "0" + }, + "Filesort_on_disk" : { + "yes" : "0" + }, + "Full_join" : { + "yes" : "0" + }, + "Full_scan" : { + "yes" : "0" + }, + "InnoDB_IO_r_bytes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_ops" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.666667", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_pages_distinct" : { + "avg" : "2", + "max" : "2", + "median" : "2", + "min" : "2", + "pct" : "0", + "pct_95" : "2", + "stddev" : "0", + "sum" : "4" + }, + "InnoDB_queue_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.666667", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_rec_lock_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.666667", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_trx_id" : { + "value" : "1A885840" + }, + "Killed" : { + "yes" : "0" + }, + "Last_errno" : { + "value" : "0" + }, + "Lock_time" : { + "avg" : "0.000118", + "max" : "0.000122", + "median" : "0.000118", + "min" : "0.000114", + "pct" : "0.666667", + "pct_95" : "0.000122", + "stddev" : "0.000006", + "sum" : "0.000236" + }, + "Merge_passes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "QC_Hit" : { + "yes" : "0" + }, + "Query_length" : { + "avg" : "30", + "max" : "30", + "median" : "30", + "min" : "30", + "pct" : "0", + "pct_95" : "30", + "stddev" : "0", + "sum" : "60" + }, + "Query_time" : { + "avg" : "0.000233", + "max" : "0.000237", + "median" : "0.000233", + "min" : "0.000228", + "pct" : "0.666667", + "pct_95" : "0.000237", + "stddev" : "0.000006", + "sum" : "0.000465" + }, + "Rows_examined" : { + "avg" : "1", + "max" : "1", + "median" : "1", + "min" : "1", + "pct" : "0", + "pct_95" : "1", + "stddev" : "0", + "sum" : "2" + }, + "Rows_sent" : { + "avg" : "1", + "max" : "1", + "median" : "1", + "min" : "1", + "pct" : "0", + "pct_95" : "1", + "stddev" : "0", + "sum" : "2" + }, + "Tmp_disk_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_table" : { + "yes" : "0" + }, + "Tmp_table_on_disk" : { + "yes" : "0" + }, + "Tmp_table_sizes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "db" : { + "value" : "maindb" + }, + "host" : { + "value" : "localhost" + }, + "rate_limit" : { + "value" : "query:2" + }, + "user" : { + "value" : "user1" + } + }, + "query_count" : 2, + "tables" : [ + { + "create" : "SHOW CREATE TABLE `maindb`.`bar`\\G", + "status" : "SHOW TABLE STATUS FROM `maindb` LIKE 'bar'\\G" + } + ], + "ts_max" : "2013-11-28 01:05:31", + "ts_min" : "2013-11-28 01:05:31" + }, + { + "attribute" : "fingerprint", + "checksum" : "D2BA209E593ABAA7", + "distillate" : "INSERT foo", + "example" : { + "Query_time" : "0.000165", + "query" : "INSERT INTO foo VALUES (NULL, 3)", + "ts" : "2013-11-28 01:05:31" + }, + "fingerprint" : "insert into foo values(?+)", + "histograms" : { + "Query_time" : [ + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0 + ] + }, + "metrics" : { + "Bytes_sent" : { + "avg" : "481", + "max" : "481", + "median" : "481", + "min" : "481", + "pct" : "0", + "pct_95" : "481", + "stddev" : "0", + "sum" : "481" + }, + "Filesort" : { + "yes" : "1" + }, + "Filesort_on_disk" : { + "yes" : "0" + }, + "Full_join" : { + "yes" : "0" + }, + "Full_scan" : { + "yes" : "0" + }, + "InnoDB_IO_r_bytes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_ops" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.333333", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_pages_distinct" : { + "avg" : "3", + "max" : "3", + "median" : "3", + "min" : "3", + "pct" : "0", + "pct_95" : "3", + "stddev" : "0", + "sum" : "3" + }, + "InnoDB_queue_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.333333", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_rec_lock_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct" : "0.333333", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_trx_id" : { + "value" : "1A885842" + }, + "Killed" : { + "yes" : "0" + }, + "Last_errno" : { + "value" : "0" + }, + "Lock_time" : { + "avg" : "0.000048", + "max" : "0.000048", + "median" : "0.000048", + "min" : "0.000048", + "pct" : "0.333333", + "pct_95" : "0.000048", + "stddev" : "0.000000", + "sum" : "0.000048" + }, + "Merge_passes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "QC_Hit" : { + "yes" : "0" + }, + "Query_length" : { + "avg" : "32", + "max" : "32", + "median" : "32", + "min" : "32", + "pct" : "0", + "pct_95" : "32", + "stddev" : "0", + "sum" : "32" + }, + "Query_time" : { + "avg" : "0.000165", + "max" : "0.000165", + "median" : "0.000165", + "min" : "0.000165", + "pct" : "0.333333", + "pct_95" : "0.000165", + "stddev" : "0.000000", + "sum" : "0.000165" + }, + "Rows_affected" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Rows_examined" : { + "avg" : "10", + "max" : "10", + "median" : "10", + "min" : "10", + "pct" : "0", + "pct_95" : "10", + "stddev" : "0", + "sum" : "10" + }, + "Rows_sent" : { + "avg" : "5", + "max" : "5", + "median" : "5", + "min" : "5", + "pct" : "0", + "pct_95" : "5", + "stddev" : "0", + "sum" : "5" + }, + "Tmp_disk_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_table" : { + "yes" : "0" + }, + "Tmp_table_on_disk" : { + "yes" : "0" + }, + "Tmp_table_sizes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "db" : { + "value" : "maindb" + }, + "host" : { + "value" : "localhost" + }, + "rate_limit" : { + "value" : "query:2" + }, + "user" : { + "value" : "user1" + } + }, + "query_count" : 1, + "tables" : [ + { + "create" : "SHOW CREATE TABLE `maindb`.`foo`\\G", + "status" : "SHOW TABLE STATUS FROM `maindb` LIKE 'foo'\\G" + } + ], + "ts_max" : "2013-11-28 01:05:31", + "ts_min" : "2013-11-28 01:05:31" + } + ], + "global" : { + "files" : [ + { + "name" : "/Users/daniel/p/pqd-ps-rate/t/lib/samples/slowlogs/slow059.txt", + "size" : 2152 + } + ], + "metrics" : { + "Bytes_sent" : { + "avg" : "523", + "max" : "545", + "median" : "537", + "min" : "481", + "pct_95" : "537", + "stddev" : "32", + "sum" : "1571" + }, + "Filesort" : { + "cnt" : "1" + }, + "Filesort_on_disk" : { + "cnt" : "0" + }, + "Full_join" : { + "cnt" : "0" + }, + "Full_scan" : { + "cnt" : "0" + }, + "InnoDB_IO_r_bytes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_ops" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "InnoDB_IO_r_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_pages_distinct" : { + "avg" : "2", + "max" : "3", + "median" : "1", + "min" : "2", + "pct_95" : "2", + "stddev" : "0", + "sum" : "7" + }, + "InnoDB_queue_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "InnoDB_rec_lock_wait" : { + "avg" : "0.000000", + "max" : "0.000000", + "median" : "0.000000", + "min" : "0.000000", + "pct_95" : "0.000000", + "stddev" : "0.000000", + "sum" : "0.000000" + }, + "Killed" : { + "cnt" : "0" + }, + "Lock_time" : { + "avg" : "0.000095", + "max" : "0.000122", + "median" : "0.000114", + "min" : "0.000048", + "pct_95" : "0.000119", + "stddev" : "0.000033", + "sum" : "0.000284" + }, + "Merge_passes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "QC_Hit" : { + "cnt" : "0" + }, + "Query_length" : { + "avg" : "30", + "max" : "32", + "median" : "28", + "min" : "30", + "pct_95" : "31", + "stddev" : "1", + "sum" : "92" + }, + "Query_time" : { + "avg" : "0.000210", + "max" : "0.000237", + "median" : "0.000225", + "min" : "0.000165", + "pct_95" : "0.000236", + "stddev" : "0.000034", + "sum" : "0.000630" + }, + "Rows_affected" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Rows_examined" : { + "avg" : "4", + "max" : "10", + "median" : "0", + "min" : "1", + "pct_95" : "9", + "stddev" : "4", + "sum" : "12" + }, + "Rows_sent" : { + "avg" : "2", + "max" : "5", + "median" : "0", + "min" : "1", + "pct_95" : "4", + "stddev" : "1", + "sum" : "7" + }, + "Tmp_disk_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_table" : { + "cnt" : "0" + }, + "Tmp_table_on_disk" : { + "cnt" : "0" + }, + "Tmp_table_sizes" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + }, + "Tmp_tables" : { + "avg" : "0", + "max" : "0", + "median" : "0", + "min" : "0", + "pct_95" : "0", + "stddev" : "0", + "sum" : "0" + } + }, + "query_count" : 3, + "rate_limit" : { + "limit" : 2, + "type" : "query" + }, + "unique_query_count" : 2 + } +}