percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-10 02:49:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add and use max_query_length and max_fingerprint_length for JSONReportFormatter: 10_000 and 5_000 chars. Report Query_length (aka bytes).

Browse Source
This commit is contained in:
Daniel Nichter
2013-06-13 16:33:46 -07:00
parent 652d95248d
commit a593d1dab9
2 changed files with 63 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
bin/pt-query-digest
View File

@@ -6636,7 +6636,7 @@ sub hostname {
sub files { sub files {
my ( $self, %args ) = @_; my ( $self, %args ) = @_;
if ( $args{files} ) { if ( $args{files} ) {
return "# Files: " . join(', ', @{$args{files}}) . "\n"; return "# Files: " . join(', ', map { $_->{name} } @{$args{files}}) . "\n";
} }
return; return;
} }
@@ -7638,6 +7638,7 @@ my $have_json = eval { require JSON };
our $pretty_json = $ENV{PTTEST_PRETTY_JSON} || 0; our $pretty_json = $ENV{PTTEST_PRETTY_JSON} || 0;
our $sorted_json = $ENV{PTTEST_PRETTY_JSON} || 0; our $sorted_json = $ENV{PTTEST_PRETTY_JSON} || 0;
extends qw(QueryReportFormatter); extends qw(QueryReportFormatter);
has 'QueryRewriter' => ( has 'QueryRewriter' => (
@@ -7664,6 +7665,20 @@ has _json => (
builder => '_build_json', builder => '_build_json',
); );
has 'max_query_length' => (
is => 'rw',
isa => 'Int',
required => 0,
default => sub { return 10_000; }, # characters, not bytes
);
has 'max_fingerprint_length' => (
is => 'rw',
isa => 'Int',
required => 0,
default => sub { return 5_000; }, # characters, not bytes
);
sub _build_json { sub _build_json {
return unless $have_json; return unless $have_json;
return JSON->new->utf8 return JSON->new->utf8
@@ -7739,7 +7754,6 @@ override query_report => sub {
fingerprint => 1, fingerprint => 1,
pos_in_log => 1, pos_in_log => 1,
ts => 1, ts => 1,
bytes => 1,
); );
foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) { foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) {
@@ -7751,39 +7765,41 @@ override query_report => sub {
my $metrics = $ea->stats()->{globals}->{$attrib}; my $metrics = $ea->stats()->{globals}->{$attrib};
my $int = $attrib =~ m/(?:time|wait)$/ ? 0 : 1; my $int = $attrib =~ m/(?:time|wait)$/ ? 0 : 1;
my $real_attrib = $attrib eq 'bytes' ? 'Query_length' : $attrib;
if ( $type eq 'num' ) { if ( $type eq 'num' ) {
foreach my $m ( qw(sum min max) ) { foreach my $m ( qw(sum min max) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%d', $store->{$m} || 0); = sprintf('%d', $store->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%.6f', $store->{$m} || 0); = sprintf('%.6f', $store->{$m} || 0);
} }
} }
foreach my $m ( qw(pct_95 stddev median) ) { foreach my $m ( qw(pct_95 stddev median) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%d', $metrics->{$m} || 0); = sprintf('%d', $metrics->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%.6f', $metrics->{$m} || 0); = sprintf('%.6f', $metrics->{$m} || 0);
} }
} }
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{avg} $global_data->{metrics}->{$real_attrib}->{avg}
= sprintf('%d', $store->{sum} / $store->{cnt}); = sprintf('%d', $store->{sum} / $store->{cnt});
} }
else { else {
$global_data->{metrics}->{$attrib}->{avg} $global_data->{metrics}->{$real_attrib}->{avg}
= sprintf('%.6f', $store->{sum} / $store->{cnt}); = sprintf('%.6f', $store->{sum} / $store->{cnt});
} }
} }
elsif ( $type eq 'bool' ) { elsif ( $type eq 'bool' ) {
my $store = $results->{globals}->{$attrib}; my $store = $results->{globals}->{$real_attrib};
$global_data->{metrics}->{$attrib}->{cnt} $global_data->{metrics}->{$real_attrib}->{cnt}
= sprintf('%d', $store->{sum}); = sprintf('%d', $store->{sum});
} }
} }
@@ -7804,19 +7820,20 @@ override query_report => sub {
my $checksum = make_checksum($item); my $checksum = make_checksum($item);
my $class = { my $class = {
checksum => $checksum, checksum => $checksum,
fingerprint => $item, fingerprint => substr($item, 0, $self->max_fingerprint_length),
distillate => $distill, distillate => $distill,
attribute => $groupby, attribute => $groupby,
query_count => $times_seen, query_count => $times_seen,
example => { example => {
query => $sample->{arg}, query => substr($sample->{arg}, 0, $self->max_query_length),
ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
}, },
}; };
my %metrics; my %metrics;
foreach my $attrib ( @attribs ) { foreach my $attrib ( @attribs ) {
$metrics{$attrib} = $ea->metrics( my $real_attrib = $attrib eq 'bytes' ? 'Query_length' : $attrib;
$metrics{$real_attrib} = $ea->metrics(
attrib => $attrib, attrib => $attrib,
where => $item, where => $item,
); );
@@ -7840,7 +7857,7 @@ override query_report => sub {
$class->{ts_max} = $ts->{max}; $class->{ts_max} = $ts->{max};
} }
else { else {
my $type = $ea->type_for($attrib) || 'string'; my $type = $attrib eq 'Query_length' ? 'num' : $ea->type_for($attrib) || 'string';
if ( $type eq 'string' ) { if ( $type eq 'string' ) {
$metrics{$attrib} = { value => $metrics{$attrib}{max} }; $metrics{$attrib} = { value => $metrics{$attrib}{max} };
} }

45
lib/JSONReportFormatter.pm
View File

@@ -31,6 +31,7 @@ my $have_json = eval { require JSON };
our $pretty_json = $ENV{PTTEST_PRETTY_JSON} || 0; our $pretty_json = $ENV{PTTEST_PRETTY_JSON} || 0;
our $sorted_json = $ENV{PTTEST_PRETTY_JSON} || 0; our $sorted_json = $ENV{PTTEST_PRETTY_JSON} || 0;
extends qw(QueryReportFormatter); extends qw(QueryReportFormatter);
has 'QueryRewriter' => ( has 'QueryRewriter' => (
@@ -57,6 +58,20 @@ has _json => (
builder => '_build_json', builder => '_build_json',
); );
has 'max_query_length' => (
is => 'rw',
isa => 'Int',
required => 0,
default => sub { return 10_000; }, # characters, not bytes
);
has 'max_fingerprint_length' => (
is => 'rw',
isa => 'Int',
required => 0,
default => sub { return 5_000; }, # characters, not bytes
);
sub _build_json { sub _build_json {
return unless $have_json; return unless $have_json;
return JSON->new->utf8 return JSON->new->utf8
@@ -137,7 +152,6 @@ override query_report => sub {
fingerprint => 1, fingerprint => 1,
pos_in_log => 1, pos_in_log => 1,
ts => 1, ts => 1,
bytes => 1,
); );
foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) { foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) {
@@ -149,39 +163,43 @@ override query_report => sub {
my $metrics = $ea->stats()->{globals}->{$attrib}; my $metrics = $ea->stats()->{globals}->{$attrib};
my $int = $attrib =~ m/(?:time|wait)$/ ? 0 : 1; my $int = $attrib =~ m/(?:time|wait)$/ ? 0 : 1;
# Be careful of Perl references: changing $attrib really changes it;
# it's not a local copy or copy-on-write.
my $real_attrib = $attrib eq 'bytes' ? 'Query_length' : $attrib;
if ( $type eq 'num' ) { if ( $type eq 'num' ) {
foreach my $m ( qw(sum min max) ) { foreach my $m ( qw(sum min max) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%d', $store->{$m} || 0); = sprintf('%d', $store->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%.6f', $store->{$m} || 0); = sprintf('%.6f', $store->{$m} || 0);
} }
} }
foreach my $m ( qw(pct_95 stddev median) ) { foreach my $m ( qw(pct_95 stddev median) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%d', $metrics->{$m} || 0); = sprintf('%d', $metrics->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib}->{$m} $global_data->{metrics}->{$real_attrib}->{$m}
= sprintf('%.6f', $metrics->{$m} || 0); = sprintf('%.6f', $metrics->{$m} || 0);
} }
} }
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib}->{avg} $global_data->{metrics}->{$real_attrib}->{avg}
= sprintf('%d', $store->{sum} / $store->{cnt}); = sprintf('%d', $store->{sum} / $store->{cnt});
} }
else { else {
$global_data->{metrics}->{$attrib}->{avg} $global_data->{metrics}->{$real_attrib}->{avg}
= sprintf('%.6f', $store->{sum} / $store->{cnt}); = sprintf('%.6f', $store->{sum} / $store->{cnt});
} }
} }
elsif ( $type eq 'bool' ) { elsif ( $type eq 'bool' ) {
my $store = $results->{globals}->{$attrib}; my $store = $results->{globals}->{$real_attrib};
$global_data->{metrics}->{$attrib}->{cnt} $global_data->{metrics}->{$real_attrib}->{cnt}
= sprintf('%d', $store->{sum}); = sprintf('%d', $store->{sum});
} }
} }
@@ -206,19 +224,20 @@ override query_report => sub {
my $checksum = make_checksum($item); my $checksum = make_checksum($item);
my $class = { my $class = {
checksum => $checksum, checksum => $checksum,
fingerprint => $item, fingerprint => substr($item, 0, $self->max_fingerprint_length),
distillate => $distill, distillate => $distill,
attribute => $groupby, attribute => $groupby,
query_count => $times_seen, query_count => $times_seen,
example => { example => {
query => $sample->{arg}, query => substr($sample->{arg}, 0, $self->max_query_length),
ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
}, },
}; };
my %metrics; my %metrics;
foreach my $attrib ( @attribs ) { foreach my $attrib ( @attribs ) {
$metrics{$attrib} = $ea->metrics( my $real_attrib = $attrib eq 'bytes' ? 'Query_length' : $attrib;
$metrics{$real_attrib} = $ea->metrics(
attrib => $attrib, attrib => $attrib,
where => $item, where => $item,
); );
@@ -242,7 +261,7 @@ override query_report => sub {
$class->{ts_max} = $ts->{max}; $class->{ts_max} = $ts->{max};
} }
else { else {
my $type = $ea->type_for($attrib) || 'string'; my $type = $attrib eq 'Query_length' ? 'num' : $ea->type_for($attrib) || 'string';
if ( $type eq 'string' ) { if ( $type eq 'string' ) {
$metrics{$attrib} = { value => $metrics{$attrib}{max} }; $metrics{$attrib} = { value => $metrics{$attrib}{max} };
} }